From f60a2e81524713ae6d95a32d23ea418b95c6d427 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Tue, 11 Jan 2022 11:17:00 +0200 Subject: [PATCH] Chore: Split release pipelines (#43419) * Split release pipelines * Re-add pr and main pipelines --- .drone.yml | 3344 +++++++++++++++----------- scripts/drone/pipelines/release.star | 139 +- scripts/drone/steps/lib.star | 40 +- 3 files changed, 2109 insertions(+), 1414 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7e7492b0d4b..5b2bd514221 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1079,33 +1079,13 @@ type: docker --- depends_on: [] kind: pipeline -name: oss-build-release +name: oss-build-publish-e2e-release node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql +services: [] steps: - commands: - mkdir -p bin @@ -1124,95 +1104,6 @@ steps: - yarn install --immutable image: grafana/build-container:1.4.9 name: initialize -- commands: - - |- - echo -e "unknwon - referer - errorstring - eror - iam - wan" > words_to_ignore.txt - - codespell -I words_to_ignore.txt docs/ - - rm words_to_ignore.txt - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: shellcheck -- commands: - - ./bin/grabpl lint-backend --edition oss - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend -- commands: - - yarn run prettier:check - - yarn run lint - - yarn run i18n:compile - - yarn run typecheck - depends_on: - - initialize - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.4.9 - name: lint-frontend -- commands: - - ./bin/grabpl test-backend --edition oss - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend -- commands: - - ./bin/grabpl integration-tests --edition oss - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration -- commands: - - yarn run ci:test-frontend - depends_on: - - initialize - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.4.9 - name: test-frontend -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - ./bin/grabpl integration-tests --database postgres - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.4.9 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - ./bin/grabpl integration-tests --database mysql - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.4.9 - name: mysql-integration-tests - commands: - ./bin/grabpl build-backend --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} --no-pull-enterprise ${DRONE_TAG} @@ -1288,6 +1179,41 @@ steps: from_secret: grafana_api_key image: grafana/build-container:1.4.9 name: package +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.4.9 + name: copy-packages-for-docker +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition oss --shouldSave + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock - commands: - ./e2e/start-server depends_on: @@ -1361,41 +1287,6 @@ steps: from_secret: github_token image: google/cloud-sdk:367.0.0 name: e2e_tests_artifacts_upload -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock - commands: - yarn storybook:build - ./bin/grabpl verify-storybook @@ -1483,8 +1374,203 @@ volumes: temp: medium: memory --- +depends_on: [] +kind: pipeline +name: oss-test-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl verify-version ${DRONE_TAG} + - ./bin/grabpl gen-version ${DRONE_TAG} + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - |- + echo -e "unknwon + referer + errorstring + eror + iam + wan" > words_to_ignore.txt + - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: codespell +- commands: + - ./bin/grabpl shellcheck + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: shellcheck +- commands: + - ./bin/grabpl lint-backend --edition oss + depends_on: + - initialize + environment: + CGO_ENABLED: "1" + image: grafana/build-container:1.4.9 + name: lint-backend +- commands: + - yarn run prettier:check + - yarn run lint + - yarn run i18n:compile + - yarn run typecheck + depends_on: + - initialize + environment: + TEST_MAX_WORKERS: 50% + image: grafana/build-container:1.4.9 + name: lint-frontend +- commands: + - ./bin/grabpl test-backend --edition oss + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend +- commands: + - ./bin/grabpl integration-tests --edition oss + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-integration +- commands: + - yarn run ci:test-frontend + depends_on: + - initialize + environment: + TEST_MAX_WORKERS: 50% + image: grafana/build-container:1.4.9 + name: test-frontend +trigger: + ref: + - refs/tags/v* + repo: + exclude: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: oss-integration-tests-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.6.48 + name: mysql + volumes: + - name: mysql + path: /var/lib/mysql +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl verify-version ${DRONE_TAG} + - ./bin/grabpl gen-version ${DRONE_TAG} + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - apt-get update + - apt-get install -yq postgresql-client + - dockerize -wait tcp://postgres:5432 -timeout 120s + - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql + - go clean -testcache + - ./bin/grabpl integration-tests --database postgres + depends_on: + - grabpl + environment: + GRAFANA_TEST_DB: postgres + PGPASSWORD: grafanatest + POSTGRES_HOST: postgres + image: grafana/build-container:1.4.9 + name: postgres-integration-tests +- commands: + - apt-get update + - apt-get install -yq default-mysql-client + - dockerize -wait tcp://mysql:3306 -timeout 120s + - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root + -prootpass + - go clean -testcache + - ./bin/grabpl integration-tests --database mysql + depends_on: + - grabpl + environment: + GRAFANA_TEST_DB: mysql + MYSQL_HOST: mysql + image: grafana/build-container:1.4.9 + name: mysql-integration-tests +trigger: + ref: + - refs/tags/v* + repo: + exclude: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- depends_on: -- oss-build-release +- oss-build-publish-e2e-release +- oss-test-release +- oss-integration-tests-release kind: pipeline name: oss-windows-release platform: @@ -1544,39 +1630,346 @@ depends_on: [] image_pull_secrets: - dockerconfigjson kind: pipeline -name: enterprise-build-release +name: enterprise-build-e2e-publish-release node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TAG} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + - make gen-go + - ./bin/grabpl verify-version ${DRONE_TAG} + - ./bin/grabpl gen-version ${DRONE_TAG} + - yarn install --immutable + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - ./bin/grabpl build-backend --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} + --no-pull-enterprise ${DRONE_TAG} + depends_on: + - initialize + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: build-backend +- commands: + - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps + --edition enterprise --no-pull-enterprise ${DRONE_TAG} + depends_on: + - initialize + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.4.9 + name: build-frontend +- commands: + - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign + --signing-admin + depends_on: + - initialize + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: build-plugins +- commands: + - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . + depends_on: + - build-backend + image: grafana/build-container:1.4.9 + name: validate-scuemata +- commands: + - '# Make sure the git tree is clean.' + - '# Stashing changes, since packages that were produced in build-backend step are + needed.' + - git stash + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - '# The above command generates Typescript files (*.gen.ts) from all appropriate + .cue files.' + - '# It is required that the generated Typescript be in sync with the input CUE + files.' + - '# ...Modulo eslint auto-fixes...:' + - yarn run eslint . --ext .gen.ts --fix + - '# If any filenames are emitted by the below script, run the generator command + `grafana-cli cue gen-ts` locally and commit the result.' + - ./scripts/clean-git-or-error.sh + - '# Un-stash changes.' + - git stash pop + depends_on: + - validate-scuemata + image: grafana/build-container:1.4.9 + name: ensure-cuetsified +- commands: + - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} + --no-pull-enterprise ${DRONE_TAG} + depends_on: + - initialize + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: build-backend-enterprise2 +- commands: + - ./bin/grabpl package --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} + --no-pull-enterprise --sign ${DRONE_TAG} + depends_on: + - build-plugins + - build-backend + - build-frontend + - build-backend-enterprise2 + environment: + GITHUB_TOKEN: + from_secret: github_token + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: package +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.4.9 + name: copy-packages-for-docker +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition enterprise --shouldSave + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images-ubuntu volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.1-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached + - name: docker + path: /var/run/docker.sock +- commands: + - ./e2e/start-server + depends_on: + - package + detach: true + environment: + PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz + PORT: 3001 + RUNDIR: e2e/tmp-grafana-enterprise + image: grafana/build-container:1.4.9 + name: end-to-end-tests-server +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-dashboards-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-smoke-tests-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-panels-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-various-suite +- commands: + - apt-get update + - apt-get install -yq zip + - ls -lah ./e2e + - find ./e2e -type f -name "*.mp4" + - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json + - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ + - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip + - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip + - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' + - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} + -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", + \"description\": \"Click on the details to download e2e recording videos\", \"context\": + \"e2e_artifacts\"}"' + depends_on: + - end-to-end-tests-dashboards-suite + - end-to-end-tests-panels-suite + - end-to-end-tests-smoke-tests-suite + - end-to-end-tests-various-suite + environment: + E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev + GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: + from_secret: gcp_upload_artifacts_key + GITHUB_TOKEN: + from_secret: github_token + image: google/cloud-sdk:367.0.0 + name: e2e_tests_artifacts_upload +- commands: + - ./bin/grabpl upload-cdn --edition enterprise --bucket "$${PRERELEASE_BUCKET}/artifacts/static-assets" + depends_on: + - package + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-cdn-assets +- commands: + - ./bin/grabpl upload-packages --edition enterprise --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads/${DRONE_TAG} + depends_on: + - package + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-packages +- commands: + - ./bin/grabpl package --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} + --no-pull-enterprise --sign ${DRONE_TAG} + depends_on: + - build-plugins + - build-backend + - build-frontend + - build-backend-enterprise2 + environment: + GITHUB_TOKEN: + from_secret: github_token + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: package-enterprise2 +- commands: + - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "$${PRERELEASE_BUCKET}/artifacts/static-assets" + depends_on: + - package-enterprise2 + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-cdn-assets-enterprise2 +- commands: + - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads-enterprise2/${DRONE_TAG} + depends_on: + - package-enterprise2 + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-packages-enterprise2 +trigger: + ref: + - refs/tags/v* + repo: + exclude: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +clone: + disable: true +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: enterprise-test-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] steps: - commands: - mkdir -p bin @@ -1676,6 +2069,117 @@ steps: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.4.9 name: test-frontend +- commands: + - ./bin/grabpl lint-backend --edition enterprise2 + depends_on: + - initialize + environment: + CGO_ENABLED: "1" + image: grafana/build-container:1.4.9 + name: lint-backend-enterprise2 +- commands: + - ./bin/grabpl test-backend --edition enterprise2 + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-enterprise2 +- commands: + - ./bin/grabpl integration-tests --edition enterprise2 + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-integration-enterprise2 +trigger: + ref: + - refs/tags/v* + repo: + exclude: + - grafana/grafana +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + disable: true +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: enterprise-integration-tests-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.6.48 + name: mysql + volumes: + - name: mysql + path: /var/lib/mysql +- environment: {} + image: redis:6.2.1-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_TAG} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + ${DRONE_TAG} + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + - make gen-go + - ./bin/grabpl verify-version ${DRONE_TAG} + - ./bin/grabpl gen-version ${DRONE_TAG} + - yarn install --immutable + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -1706,224 +2210,6 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.4.9 name: mysql-integration-tests -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} - --no-pull-enterprise ${DRONE_TAG} - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend -- commands: - - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps - --edition enterprise --no-pull-enterprise ${DRONE_TAG} - depends_on: - - initialize - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.4.9 - name: build-frontend -- commands: - - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign - --signing-admin - depends_on: - - initialize - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: build-plugins -- commands: - - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . - depends_on: - - build-backend - image: grafana/build-container:1.4.9 - name: validate-scuemata -- commands: - - '# Make sure the git tree is clean.' - - '# Stashing changes, since packages that were produced in build-backend step are - needed.' - - git stash - - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . - - '# The above command generates Typescript files (*.gen.ts) from all appropriate - .cue files.' - - '# It is required that the generated Typescript be in sync with the input CUE - files.' - - '# ...Modulo eslint auto-fixes...:' - - yarn run eslint . --ext .gen.ts --fix - - '# If any filenames are emitted by the below script, run the generator command - `grafana-cli cue gen-ts` locally and commit the result.' - - ./scripts/clean-git-or-error.sh - - '# Un-stash changes.' - - git stash pop - depends_on: - - validate-scuemata - image: grafana/build-container:1.4.9 - name: ensure-cuetsified -- commands: - - ./bin/grabpl lint-backend --edition enterprise2 - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend-enterprise2 -- commands: - - ./bin/grabpl test-backend --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-enterprise2 -- commands: - - ./bin/grabpl integration-tests --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration-enterprise2 -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} - --no-pull-enterprise ${DRONE_TAG} - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend-enterprise2 -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} - --no-pull-enterprise --sign ${DRONE_TAG} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - - test-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package -- commands: - - ./e2e/start-server - depends_on: - - package - detach: true - environment: - PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz - PORT: 3001 - RUNDIR: e2e/tmp-grafana-enterprise - image: grafana/build-container:1.4.9 - name: end-to-end-tests-server -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - image: google/cloud-sdk:367.0.0 - name: e2e_tests_artifacts_upload -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - ./bin/grabpl integration-tests @@ -1942,76 +2228,6 @@ steps: MEMCACHED_HOSTS: memcached:11211 image: grafana/build-container:1.4.9 name: memcached-integration-tests -- commands: - - ./bin/grabpl upload-cdn --edition enterprise --bucket "$${PRERELEASE_BUCKET}/artifacts/static-assets" - depends_on: - - package - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets -- commands: - - ./bin/grabpl upload-packages --edition enterprise --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads/${DRONE_TAG} - depends_on: - - package - - redis-integration-tests - - memcached-integration-tests - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} - --no-pull-enterprise --sign ${DRONE_TAG} - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - - test-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package-enterprise2 -- commands: - - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "$${PRERELEASE_BUCKET}/artifacts/static-assets" - depends_on: - - package-enterprise2 - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets-enterprise2 -- commands: - - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads-enterprise2/${DRONE_TAG} - depends_on: - - package-enterprise2 - - redis-integration-tests - - memcached-integration-tests - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages-enterprise2 trigger: ref: - refs/tags/v* @@ -2033,7 +2249,9 @@ volumes: clone: disable: true depends_on: -- enterprise-build-release +- enterprise-build-e2e-publish-release +- enterprise-test-release +- enterprise-integration-tests-release image_pull_secrets: - dockerconfigjson kind: pipeline @@ -2111,33 +2329,287 @@ volumes: --- depends_on: [] kind: pipeline -name: oss-build-test-release +name: oss-build-publish-e2e-test-release node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl verify-version v7.3.0-test + - ./bin/grabpl gen-version v7.3.0-test + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - ./bin/grabpl build-backend --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} + --no-pull-enterprise v7.3.0-test + depends_on: + - initialize + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: build-backend +- commands: + - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps + --edition oss --no-pull-enterprise v7.3.0-test + depends_on: + - initialize + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.4.9 + name: build-frontend +- commands: + - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin + depends_on: + - initialize + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: build-plugins +- commands: + - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . + depends_on: + - build-backend + image: grafana/build-container:1.4.9 + name: validate-scuemata +- commands: + - '# Make sure the git tree is clean.' + - '# Stashing changes, since packages that were produced in build-backend step are + needed.' + - git stash + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - '# The above command generates Typescript files (*.gen.ts) from all appropriate + .cue files.' + - '# It is required that the generated Typescript be in sync with the input CUE + files.' + - '# ...Modulo eslint auto-fixes...:' + - yarn run eslint . --ext .gen.ts --fix + - '# If any filenames are emitted by the below script, run the generator command + `grafana-cli cue gen-ts` locally and commit the result.' + - ./scripts/clean-git-or-error.sh + - '# Un-stash changes.' + - git stash pop + depends_on: + - validate-scuemata + image: grafana/build-container:1.4.9 + name: ensure-cuetsified +- commands: + - ./bin/grabpl package --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} --no-pull-enterprise + --sign v7.3.0-test + depends_on: + - build-plugins + - build-backend + - build-frontend + environment: + GITHUB_TOKEN: + from_secret: github_token + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: package +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.4.9 + name: copy-packages-for-docker +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition oss --shouldSave + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images-ubuntu volumes: - - name: mysql - path: /var/lib/mysql + - name: docker + path: /var/run/docker.sock +- commands: + - ./e2e/start-server + depends_on: + - package + detach: true + environment: + PORT: 3001 + image: grafana/build-container:1.4.9 + name: end-to-end-tests-server +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-dashboards-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-smoke-tests-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-panels-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-various-suite +- commands: + - apt-get update + - apt-get install -yq zip + - ls -lah ./e2e + - find ./e2e -type f -name "*.mp4" + - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json + - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ + - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip + - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip + - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' + - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} + -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", + \"description\": \"Click on the details to download e2e recording videos\", \"context\": + \"e2e_artifacts\"}"' + depends_on: + - end-to-end-tests-dashboards-suite + - end-to-end-tests-panels-suite + - end-to-end-tests-smoke-tests-suite + - end-to-end-tests-various-suite + environment: + E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev + GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: + from_secret: gcp_upload_artifacts_key + GITHUB_TOKEN: + from_secret: github_token + image: google/cloud-sdk:367.0.0 + name: e2e_tests_artifacts_upload +- commands: + - yarn storybook:build + - ./bin/grabpl verify-storybook + depends_on: + - build-frontend + environment: + NODE_OPTIONS: --max_old_space_size=4096 + image: grafana/build-container:1.4.9 + name: build-storybook +- commands: + - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" + depends_on: + - end-to-end-tests-server + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-cdn-assets +- commands: + - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads-test + depends_on: + - end-to-end-tests-dashboards-suite + - end-to-end-tests-panels-suite + - end-to-end-tests-smoke-tests-suite + - end-to-end-tests-various-suite + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-packages +- commands: + - echo Testing release + depends_on: + - build-storybook + - end-to-end-tests-dashboards-suite + - end-to-end-tests-panels-suite + - end-to-end-tests-smoke-tests-suite + - end-to-end-tests-various-suite + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: store-storybook +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +depends_on: [] +kind: pipeline +name: oss-test-test-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] steps: - commands: - mkdir -p bin @@ -2215,6 +2687,62 @@ steps: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.4.9 name: test-frontend +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: oss-integration-tests-test-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.6.48 + name: mysql + volumes: + - name: mysql + path: /var/lib/mysql +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl verify-version v7.3.0-test + - ./bin/grabpl gen-version v7.3.0-test + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -2245,8 +2773,130 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.4.9 name: mysql-integration-tests +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +depends_on: +- oss-build-publish-e2e-test-release +- oss-test-test-release +- oss-integration-tests-test-release +kind: pipeline +name: oss-windows-test-release +platform: + arch: amd64 + os: windows + version: "1809" +services: [] +steps: - commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} + - echo $env:DRONE_RUNNER_NAME + image: mcr.microsoft.com/windows:1809 + name: identify-runner +- commands: + - $$ProgressPreference = "SilentlyContinue" + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/windows/grabpl.exe + -OutFile grabpl.exe + image: grafana/ci-wix:0.1.1 + name: initialize +- commands: + - $$gcpKey = $$env:GCP_KEY + - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) + > gcpkey.json' + - dos2unix gcpkey.json + - gcloud auth activate-service-account --key-file=gcpkey.json + - rm gcpkey.json + - cp C:\App\nssm-2.24.zip . + - .\grabpl.exe windows-installer --edition oss --packages-bucket grafana-downloads-test + v7.3.0-test + - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] + - gsutil cp $$fname gs://grafana-downloads-test/oss/release/ + - gsutil cp "$$fname.sha256" gs://grafana-downloads-test/oss/release/ + depends_on: + - initialize + environment: + GCP_KEY: + from_secret: gcp_key + GITHUB_TOKEN: + from_secret: github_token + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/ci-wix:0.1.1 + name: build-windows-installer +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + disable: true +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: enterprise-build-e2e-publish-test-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout main + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + - make gen-go + - ./bin/grabpl verify-version v7.3.0-test + - ./bin/grabpl gen-version v7.3.0-test + - yarn install --immutable + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - ./bin/grabpl build-backend --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} --no-pull-enterprise v7.3.0-test depends_on: - initialize @@ -2257,7 +2907,7 @@ steps: name: build-backend - commands: - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps - --edition oss --no-pull-enterprise v7.3.0-test + --edition enterprise --no-pull-enterprise v7.3.0-test depends_on: - initialize environment: @@ -2265,7 +2915,8 @@ steps: image: grafana/build-container:1.4.9 name: build-frontend - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin + - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign + --signing-admin depends_on: - initialize environment: @@ -2301,12 +2952,23 @@ steps: image: grafana/build-container:1.4.9 name: ensure-cuetsified - commands: - - ./bin/grabpl package --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} --no-pull-enterprise - --sign v7.3.0-test + - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} + --no-pull-enterprise v7.3.0-test + depends_on: + - initialize + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: build-backend-enterprise2 +- commands: + - ./bin/grabpl package --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} + --no-pull-enterprise --sign v7.3.0-test depends_on: - build-plugins - build-backend - build-frontend + - build-backend-enterprise2 environment: GITHUB_TOKEN: from_secret: github_token @@ -2320,13 +2982,50 @@ steps: from_secret: grafana_api_key image: grafana/build-container:1.4.9 name: package +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.4.9 + name: copy-packages-for-docker +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition enterprise --shouldSave + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock - commands: - ./e2e/start-server depends_on: - package detach: true environment: + PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz PORT: 3001 + RUNDIR: e2e/tmp-grafana-enterprise image: grafana/build-container:1.4.9 name: end-to-end-tests-server - commands: @@ -2394,53 +3093,9 @@ steps: image: google/cloud-sdk:367.0.0 name: e2e_tests_artifacts_upload - commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ + - ./bin/grabpl upload-cdn --edition enterprise --bucket "grafana-static-assets" depends_on: - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - yarn storybook:build - - ./bin/grabpl verify-storybook - depends_on: - - build-frontend - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.4.9 - name: build-storybook -- commands: - - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" - depends_on: - - end-to-end-tests-server environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -2449,12 +3104,9 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads-test + - ./bin/grabpl upload-packages --edition enterprise --packages-bucket grafana-downloads-test depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite + - package environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -2463,20 +3115,48 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-packages - commands: - - echo Testing release + - ./bin/grabpl package --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} + --no-pull-enterprise --sign v7.3.0-test depends_on: - - build-storybook - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite + - build-plugins + - build-backend + - build-frontend + - build-backend-enterprise2 environment: - GCP_KEY: + GITHUB_TOKEN: + from_secret: github_token + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: package-enterprise2 +- commands: + - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "grafana-static-assets" + depends_on: + - package-enterprise2 + environment: + GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 - name: store-storybook + name: upload-cdn-assets-enterprise2 +- commands: + - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket grafana-downloads-test + depends_on: + - package-enterprise2 + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-packages-enterprise2 trigger: event: - custom @@ -2492,98 +3172,19 @@ volumes: temp: medium: memory --- -depends_on: -- oss-build-test-release -kind: pipeline -name: oss-windows-test-release -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: initialize -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --packages-bucket grafana-downloads-test - v7.3.0-test - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://grafana-downloads-test/oss/release/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads-test/oss/release/ - depends_on: - - initialize - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- clone: disable: true depends_on: [] image_pull_secrets: - dockerconfigjson kind: pipeline -name: enterprise-build-test-release +name: enterprise-test-test-release node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.1-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached +services: [] steps: - commands: - mkdir -p bin @@ -2682,6 +3283,113 @@ steps: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.4.9 name: test-frontend +- commands: + - ./bin/grabpl lint-backend --edition enterprise2 + depends_on: + - initialize + environment: + CGO_ENABLED: "1" + image: grafana/build-container:1.4.9 + name: lint-backend-enterprise2 +- commands: + - ./bin/grabpl test-backend --edition enterprise2 + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-enterprise2 +- commands: + - ./bin/grabpl integration-tests --edition enterprise2 + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-integration-enterprise2 +trigger: + event: + - custom +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + disable: true +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: enterprise-integration-tests-test-release +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.6.48 + name: mysql + volumes: + - name: mysql + path: /var/lib/mysql +- environment: {} + image: redis:6.2.1-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout main + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + - make gen-go + - ./bin/grabpl verify-version v7.3.0-test + - ./bin/grabpl gen-version v7.3.0-test + - yarn install --immutable + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -2712,224 +3420,6 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.4.9 name: mysql-integration-tests -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} - --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend -- commands: - - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps - --edition enterprise --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.4.9 - name: build-frontend -- commands: - - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign - --signing-admin - depends_on: - - initialize - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: build-plugins -- commands: - - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . - depends_on: - - build-backend - image: grafana/build-container:1.4.9 - name: validate-scuemata -- commands: - - '# Make sure the git tree is clean.' - - '# Stashing changes, since packages that were produced in build-backend step are - needed.' - - git stash - - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . - - '# The above command generates Typescript files (*.gen.ts) from all appropriate - .cue files.' - - '# It is required that the generated Typescript be in sync with the input CUE - files.' - - '# ...Modulo eslint auto-fixes...:' - - yarn run eslint . --ext .gen.ts --fix - - '# If any filenames are emitted by the below script, run the generator command - `grafana-cli cue gen-ts` locally and commit the result.' - - ./scripts/clean-git-or-error.sh - - '# Un-stash changes.' - - git stash pop - depends_on: - - validate-scuemata - image: grafana/build-container:1.4.9 - name: ensure-cuetsified -- commands: - - ./bin/grabpl lint-backend --edition enterprise2 - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend-enterprise2 -- commands: - - ./bin/grabpl test-backend --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-enterprise2 -- commands: - - ./bin/grabpl integration-tests --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration-enterprise2 -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} - --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend-enterprise2 -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} - --no-pull-enterprise --sign v7.3.0-test - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - - test-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package -- commands: - - ./e2e/start-server - depends_on: - - package - detach: true - environment: - PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz - PORT: 3001 - RUNDIR: e2e/tmp-grafana-enterprise - image: grafana/build-container:1.4.9 - name: end-to-end-tests-server -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - image: google/cloud-sdk:367.0.0 - name: e2e_tests_artifacts_upload -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - ./bin/grabpl integration-tests @@ -2948,76 +3438,6 @@ steps: MEMCACHED_HOSTS: memcached:11211 image: grafana/build-container:1.4.9 name: memcached-integration-tests -- commands: - - ./bin/grabpl upload-cdn --edition enterprise --bucket "grafana-static-assets" - depends_on: - - package - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets -- commands: - - ./bin/grabpl upload-packages --edition enterprise --packages-bucket grafana-downloads-test - depends_on: - - package - - redis-integration-tests - - memcached-integration-tests - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} - --no-pull-enterprise --sign v7.3.0-test - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - - test-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package-enterprise2 -- commands: - - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "grafana-static-assets" - depends_on: - - package-enterprise2 - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets-enterprise2 -- commands: - - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket grafana-downloads-test - depends_on: - - package-enterprise2 - - redis-integration-tests - - memcached-integration-tests - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages-enterprise2 trigger: event: - custom @@ -3036,7 +3456,9 @@ volumes: clone: disable: true depends_on: -- enterprise-build-test-release +- enterprise-build-e2e-publish-test-release +- enterprise-test-test-release +- enterprise-integration-tests-test-release image_pull_secrets: - dockerconfigjson kind: pipeline @@ -3111,9 +3533,13 @@ volumes: name: docker --- depends_on: -- oss-build-test-release +- oss-build-publish-e2e-test-release +- oss-test-test-release +- oss-integration-tests-test-release - oss-windows-test-release -- enterprise-build-test-release +- enterprise-build-e2e-publish-test-release +- enterprise-test-test-release +- enterprise-integration-tests-test-release - enterprise-windows-test-release kind: pipeline name: publish-test-release @@ -3191,9 +3617,13 @@ volumes: name: docker --- depends_on: -- oss-build-test-release +- oss-build-publish-e2e-test-release +- oss-test-test-release +- oss-integration-tests-test-release - oss-windows-test-release -- enterprise-build-test-release +- enterprise-build-e2e-publish-test-release +- enterprise-test-test-release +- enterprise-integration-tests-test-release - enterprise-windows-test-release - publish-test-release kind: pipeline @@ -3221,33 +3651,269 @@ type: docker --- depends_on: [] kind: pipeline -name: oss-build-release-branch +name: oss-build-publish-e2e-release-branch node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + --no-pull-enterprise + depends_on: + - initialize + environment: {} + image: grafana/build-container:1.4.9 + name: build-backend +- commands: + - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id + ${DRONE_BUILD_NUMBER} --no-pull-enterprise + depends_on: + - initialize + environment: + NODE_OPTIONS: --max_old_space_size=8192 + image: grafana/build-container:1.4.9 + name: build-frontend +- commands: + - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin + depends_on: + - initialize + environment: + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: build-plugins +- commands: + - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . + depends_on: + - build-backend + image: grafana/build-container:1.4.9 + name: validate-scuemata +- commands: + - '# Make sure the git tree is clean.' + - '# Stashing changes, since packages that were produced in build-backend step are + needed.' + - git stash + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - '# The above command generates Typescript files (*.gen.ts) from all appropriate + .cue files.' + - '# It is required that the generated Typescript be in sync with the input CUE + files.' + - '# ...Modulo eslint auto-fixes...:' + - yarn run eslint . --ext .gen.ts --fix + - '# If any filenames are emitted by the below script, run the generator command + `grafana-cli cue gen-ts` locally and commit the result.' + - ./scripts/clean-git-or-error.sh + - '# Un-stash changes.' + - git stash pop + depends_on: + - validate-scuemata + image: grafana/build-container:1.4.9 + name: ensure-cuetsified +- commands: + - ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise + --sign + depends_on: + - build-plugins + - build-backend + - build-frontend + environment: + GITHUB_TOKEN: + from_secret: github_token + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: package +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.4.9 + name: copy-packages-for-docker +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition oss --shouldSave + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images-ubuntu volumes: - - name: mysql - path: /var/lib/mysql + - name: docker + path: /var/run/docker.sock +- commands: + - ./e2e/start-server + depends_on: + - package + detach: true + environment: + PORT: 3001 + image: grafana/build-container:1.4.9 + name: end-to-end-tests-server +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-dashboards-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-smoke-tests-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-panels-suite +- commands: + - apt-get install -y netcat + - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 + depends_on: + - package + environment: + HOST: end-to-end-tests-server + image: cypress/included:9.2.0 + name: end-to-end-tests-various-suite +- commands: + - apt-get update + - apt-get install -yq zip + - ls -lah ./e2e + - find ./e2e -type f -name "*.mp4" + - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json + - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ + - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip + - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip + - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' + - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} + -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", + \"description\": \"Click on the details to download e2e recording videos\", \"context\": + \"e2e_artifacts\"}"' + depends_on: + - end-to-end-tests-dashboards-suite + - end-to-end-tests-panels-suite + - end-to-end-tests-smoke-tests-suite + - end-to-end-tests-various-suite + environment: + E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev + GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: + from_secret: gcp_upload_artifacts_key + GITHUB_TOKEN: + from_secret: github_token + image: google/cloud-sdk:367.0.0 + name: e2e_tests_artifacts_upload +- commands: + - yarn storybook:build + - ./bin/grabpl verify-storybook + depends_on: + - build-frontend + environment: + NODE_OPTIONS: --max_old_space_size=4096 + image: grafana/build-container:1.4.9 + name: build-storybook +- commands: + - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" + depends_on: + - end-to-end-tests-server + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-cdn-assets +- commands: + - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads + depends_on: + - end-to-end-tests-dashboards-suite + - end-to-end-tests-panels-suite + - end-to-end-tests-smoke-tests-suite + - end-to-end-tests-various-suite + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-packages +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +depends_on: [] +kind: pipeline +name: oss-test-release-branch +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] steps: - commands: - mkdir -p bin @@ -3324,6 +3990,61 @@ steps: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.4.9 name: test-frontend +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +depends_on: [] +kind: pipeline +name: oss-integration-tests-release-branch +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.6.48 + name: mysql + volumes: + - name: mysql + path: /var/lib/mysql +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - make gen-go + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + - yarn install --immutable + image: grafana/build-container:1.4.9 + name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -3354,8 +4075,124 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.4.9 name: mysql-integration-tests +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory +--- +depends_on: +- oss-build-publish-e2e-release-branch +- oss-test-release-branch +- oss-integration-tests-release-branch +kind: pipeline +name: oss-windows-release-branch +platform: + arch: amd64 + os: windows + version: "1809" +services: [] +steps: - commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - echo $env:DRONE_RUNNER_NAME + image: mcr.microsoft.com/windows:1809 + name: identify-runner +- commands: + - $$ProgressPreference = "SilentlyContinue" + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/windows/grabpl.exe + -OutFile grabpl.exe + image: grafana/ci-wix:0.1.1 + name: initialize +- commands: + - $$gcpKey = $$env:GCP_KEY + - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) + > gcpkey.json' + - dos2unix gcpkey.json + - gcloud auth activate-service-account --key-file=gcpkey.json + - rm gcpkey.json + - cp C:\App\nssm-2.24.zip . + depends_on: + - initialize + environment: + GCP_KEY: + from_secret: gcp_key + GITHUB_TOKEN: + from_secret: github_token + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/ci-wix:0.1.1 + name: build-windows-installer +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + disable: true +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: enterprise-build-e2e-publish-release-branch +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_BRANCH} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + - make gen-go + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + - yarn install --immutable + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: initialize +- commands: + - ./bin/grabpl build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise depends_on: - initialize @@ -3363,7 +4200,7 @@ steps: image: grafana/build-container:1.4.9 name: build-backend - commands: - - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition oss --build-id + - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition enterprise --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise depends_on: - initialize @@ -3372,7 +4209,8 @@ steps: image: grafana/build-container:1.4.9 name: build-frontend - commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin + - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign + --signing-admin depends_on: - initialize environment: @@ -3408,12 +4246,21 @@ steps: image: grafana/build-container:1.4.9 name: ensure-cuetsified - commands: - - ./bin/grabpl package --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} --no-pull-enterprise - --sign + - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + --variants linux-x64 --no-pull-enterprise + depends_on: + - initialize + environment: {} + image: grafana/build-container:1.4.9 + name: build-backend-enterprise2 +- commands: + - ./bin/grabpl package --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + --no-pull-enterprise --sign depends_on: - build-plugins - build-backend - build-frontend + - build-backend-enterprise2 environment: GITHUB_TOKEN: from_secret: github_token @@ -3427,13 +4274,50 @@ steps: from_secret: grafana_api_key image: grafana/build-container:1.4.9 name: package +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.4.9 + name: copy-packages-for-docker +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition enterprise --shouldSave + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json + - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu + depends_on: + - copy-packages-for-docker + environment: + GCP_KEY: + from_secret: gcp_key + image: google/cloud-sdk + name: package-docker-images-ubuntu + volumes: + - name: docker + path: /var/run/docker.sock - commands: - ./e2e/start-server depends_on: - package detach: true environment: + PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz PORT: 3001 + RUNDIR: e2e/tmp-grafana-enterprise image: grafana/build-container:1.4.9 name: end-to-end-tests-server - commands: @@ -3500,41 +4384,6 @@ steps: from_secret: github_token image: google/cloud-sdk:367.0.0 name: e2e_tests_artifacts_upload -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock - commands: - yarn storybook:build - ./bin/grabpl verify-storybook @@ -3545,9 +4394,9 @@ steps: image: grafana/build-container:1.4.9 name: build-storybook - commands: - - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" + - ./bin/grabpl upload-cdn --edition enterprise --bucket "grafana-static-assets" depends_on: - - end-to-end-tests-server + - package environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -3556,12 +4405,9 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads + - ./bin/grabpl upload-packages --edition enterprise --packages-bucket grafana-downloads depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite + - package environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -3569,6 +4415,49 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.1 name: upload-packages +- commands: + - ./bin/grabpl package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + --no-pull-enterprise --variants linux-x64 --sign + depends_on: + - build-plugins + - build-backend + - build-frontend + - build-backend-enterprise2 + environment: + GITHUB_TOKEN: + from_secret: github_token + GPG_KEY_PASSWORD: + from_secret: gpg_key_password + GPG_PRIV_KEY: + from_secret: gpg_priv_key + GPG_PUB_KEY: + from_secret: gpg_pub_key + GRAFANA_API_KEY: + from_secret: grafana_api_key + image: grafana/build-container:1.4.9 + name: package-enterprise2 +- commands: + - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "grafana-static-assets" + depends_on: + - package-enterprise2 + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-cdn-assets-enterprise2 +- commands: + - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket grafana-downloads-enterprise2 + depends_on: + - package-enterprise2 + environment: + GCP_GRAFANA_UPLOAD_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.1 + name: upload-packages-enterprise2 trigger: ref: - refs/heads/v[0-9]* @@ -3584,93 +4473,19 @@ volumes: temp: medium: memory --- -depends_on: -- oss-build-release-branch -kind: pipeline -name: oss-windows-release-branch -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: initialize -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - depends_on: - - initialize - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - ref: - - refs/heads/v[0-9]* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- clone: disable: true depends_on: [] image_pull_secrets: - dockerconfigjson kind: pipeline -name: enterprise-build-release-branch +name: enterprise-test-release-branch node: type: no-parallel platform: arch: amd64 os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.1-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached +services: [] steps: - commands: - mkdir -p bin @@ -3768,6 +4583,112 @@ steps: TEST_MAX_WORKERS: 50% image: grafana/build-container:1.4.9 name: test-frontend +- commands: + - ./bin/grabpl lint-backend --edition enterprise2 + depends_on: + - initialize + environment: + CGO_ENABLED: "1" + image: grafana/build-container:1.4.9 + name: lint-backend-enterprise2 +- commands: + - ./bin/grabpl test-backend --edition enterprise2 + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-enterprise2 +- commands: + - ./bin/grabpl integration-tests --edition enterprise2 + depends_on: + - initialize + image: grafana/build-container:1.4.9 + name: test-backend-integration-enterprise2 +trigger: + ref: + - refs/heads/v[0-9]* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + disable: true +depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: enterprise-integration-tests-release-branch +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: +- environment: + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_DB: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_USER: grafanatest + image: postgres:12.3-alpine + name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata +- environment: + MYSQL_DATABASE: grafana_tests + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_USER: grafana + image: mysql:5.6.48 + name: mysql + volumes: + - name: mysql + path: /var/lib/mysql +- environment: {} + image: redis:6.2.1-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached +steps: +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.9/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - echo $DRONE_RUNNER_NAME + image: alpine:3.15 + name: identify-runner +- commands: + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + - cd grafana-enterprise + - git checkout ${DRONE_BRANCH} + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: clone-enterprise +- commands: + - mv bin/grabpl /tmp/ + - rmdir bin + - mv grafana-enterprise /tmp/ + - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise + - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json + - mkdir bin + - mv /tmp/grabpl bin/ + - make gen-go + - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} + - yarn install --immutable + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: grafana/build-container:1.4.9 + name: initialize - commands: - apt-get update - apt-get install -yq postgresql-client @@ -3798,229 +4719,6 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.4.9 name: mysql-integration-tests -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - --no-pull-enterprise - depends_on: - - initialize - environment: {} - image: grafana/build-container:1.4.9 - name: build-backend -- commands: - - ./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition enterprise --build-id - ${DRONE_BUILD_NUMBER} --no-pull-enterprise - depends_on: - - initialize - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.4.9 - name: build-frontend -- commands: - - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign - --signing-admin - depends_on: - - initialize - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: build-plugins -- commands: - - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . - depends_on: - - build-backend - image: grafana/build-container:1.4.9 - name: validate-scuemata -- commands: - - '# Make sure the git tree is clean.' - - '# Stashing changes, since packages that were produced in build-backend step are - needed.' - - git stash - - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . - - '# The above command generates Typescript files (*.gen.ts) from all appropriate - .cue files.' - - '# It is required that the generated Typescript be in sync with the input CUE - files.' - - '# ...Modulo eslint auto-fixes...:' - - yarn run eslint . --ext .gen.ts --fix - - '# If any filenames are emitted by the below script, run the generator command - `grafana-cli cue gen-ts` locally and commit the result.' - - ./scripts/clean-git-or-error.sh - - '# Un-stash changes.' - - git stash pop - depends_on: - - validate-scuemata - image: grafana/build-container:1.4.9 - name: ensure-cuetsified -- commands: - - ./bin/grabpl lint-backend --edition enterprise2 - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend-enterprise2 -- commands: - - ./bin/grabpl test-backend --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-enterprise2 -- commands: - - ./bin/grabpl integration-tests --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration-enterprise2 -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} - --variants linux-x64 --no-pull-enterprise - depends_on: - - initialize - environment: {} - image: grafana/build-container:1.4.9 - name: build-backend-enterprise2 -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} - --no-pull-enterprise --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - - test-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package -- commands: - - ./e2e/start-server - depends_on: - - package - detach: true - environment: - PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz - PORT: 3001 - RUNDIR: e2e/tmp-grafana-enterprise - image: grafana/build-container:1.4.9 - name: end-to-end-tests-server -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - image: google/cloud-sdk:367.0.0 - name: e2e_tests_artifacts_upload -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - yarn storybook:build - - ./bin/grabpl verify-storybook - depends_on: - - build-frontend - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.4.9 - name: build-storybook - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - ./bin/grabpl integration-tests @@ -4039,76 +4737,6 @@ steps: MEMCACHED_HOSTS: memcached:11211 image: grafana/build-container:1.4.9 name: memcached-integration-tests -- commands: - - ./bin/grabpl upload-cdn --edition enterprise --bucket "grafana-static-assets" - depends_on: - - package - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets -- commands: - - ./bin/grabpl upload-packages --edition enterprise --packages-bucket grafana-downloads - depends_on: - - package - - redis-integration-tests - - memcached-integration-tests - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} - --no-pull-enterprise --variants linux-x64 --sign - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - - test-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package-enterprise2 -- commands: - - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "grafana-static-assets" - depends_on: - - package-enterprise2 - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets-enterprise2 -- commands: - - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket grafana-downloads-enterprise2 - depends_on: - - package-enterprise2 - - redis-integration-tests - - memcached-integration-tests - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages-enterprise2 trigger: ref: - refs/heads/v[0-9]* @@ -4127,7 +4755,9 @@ volumes: clone: disable: true depends_on: -- enterprise-build-release-branch +- enterprise-build-e2e-publish-release-branch +- enterprise-test-release-branch +- enterprise-integration-tests-release-branch image_pull_secrets: - dockerconfigjson kind: pipeline @@ -4341,6 +4971,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 6cfdc1f69fe9dfed20fcd03d97d7bb3bd23d2ddca4540d243609b809cb937e8c +hmac: 7e765f14077cf133a976b894d3940d46c5cf9e1e1dde6116b996a01b2e4ed754 ... diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index a11c6176777..5033cf23f2c 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -1,5 +1,6 @@ load( 'scripts/drone/steps/lib.star', + 'disable_tests', 'download_grabpl_step', 'initialize_step', 'lint_drone_step', @@ -54,6 +55,7 @@ load( ) load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket') + def build_npm_packages_step(edition, ver_mode): if edition == 'enterprise' or ver_mode != 'release': return None @@ -121,16 +123,14 @@ def release_npm_packages_step(edition, ver_mode): def get_steps(edition, ver_mode): - build_steps = [] package_steps = [] - windows_package_steps = [] publish_steps = [] should_publish = ver_mode in ('release', 'test-release',) should_upload = should_publish or ver_mode in ('release-branch',) include_enterprise2 = edition == 'enterprise' edition2 = 'enterprise2' - build_steps = [ + test_steps = [ codespell_step(), shellcheck_step(), lint_backend_step(edition=edition), @@ -138,8 +138,9 @@ def get_steps(edition, ver_mode): test_backend_step(edition=edition), test_backend_integration_step(edition=edition), test_frontend_step(), - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + ] + + build_steps = [ build_backend_step(edition=edition, ver_mode=ver_mode), build_frontend_step(edition=edition, ver_mode=ver_mode), build_plugins_step(edition=edition, sign=True), @@ -147,34 +148,46 @@ def get_steps(edition, ver_mode): ensure_cuetsified_step(), ] + integration_test_steps = [ + postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), + mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + ] + + if include_enterprise2: - build_steps.extend([ + test_steps.extend([ lint_backend_step(edition=edition2), test_backend_step(edition=edition2), test_backend_integration_step(edition=edition2), + ]) + build_steps.extend([ build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']), ]) # Insert remaining steps build_steps.extend([ package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2), - e2e_tests_server_step(edition=edition), - e2e_tests_step('dashboards-suite', edition=edition, tries=3), - e2e_tests_step('smoke-tests-suite', edition=edition, tries=3), - e2e_tests_step('panels-suite', edition=edition, tries=3), - e2e_tests_step('various-suite', edition=edition, tries=3), - e2e_tests_artifacts(edition=edition), copy_packages_for_docker_step(), package_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish), package_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish), + e2e_tests_server_step(edition=edition), ]) + if not disable_tests: + build_steps.extend([ + e2e_tests_step('dashboards-suite', edition=edition, tries=3), + e2e_tests_step('smoke-tests-suite', edition=edition, tries=3), + e2e_tests_step('panels-suite', edition=edition, tries=3), + e2e_tests_step('various-suite', edition=edition, tries=3), + e2e_tests_artifacts(edition=edition), + ]) + build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode) if build_storybook: build_steps.append(build_storybook) if include_enterprise2: - build_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)]) + integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)]) if should_upload: publish_steps.append(upload_cdn_step(edition=edition, ver_mode=ver_mode)) @@ -200,45 +213,102 @@ def get_steps(edition, ver_mode): if step: publish_steps.append(step) - return build_steps, package_steps, windows_package_steps, publish_steps + return test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps def get_oss_pipelines(trigger, ver_mode): edition = 'oss' services = integration_test_services(edition=edition) volumes = integration_test_services_volumes() - build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) - return [ + test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) + windows_pipeline = pipeline( + name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, + steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps, + platform='windows', depends_on=[ + 'oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), + ], + ) + pipelines = [ pipeline( - name='oss-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, + name='oss-build-publish{}-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - build_steps + package_steps + publish_steps, + build_steps + package_steps + publish_steps, volumes=volumes, ), - pipeline( - name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, - steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps, - platform='windows', depends_on=['oss-build-{}'.format(ver_mode)], - ), ] + if not disable_tests: + pipelines.extend([ + pipeline( + name='oss-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[], + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + + test_steps, + volumes=[], + ), + pipeline( + name='oss-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + + integration_test_steps, + volumes=volumes, + ) + ]) + deps = { + 'depends_on': [ + 'oss-build-publish{}-{}'.format(get_e2e_suffix(), ver_mode), + 'oss-test-{}'.format(ver_mode), + 'oss-integration-tests-{}'.format(ver_mode) + ] + } + windows_pipeline.update(deps) + + pipelines.extend([windows_pipeline]) + return pipelines def get_enterprise_pipelines(trigger, ver_mode): edition = 'enterprise' services = integration_test_services(edition=edition) volumes = integration_test_services_volumes() - build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) - return [ + test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) + windows_pipeline = pipeline( + name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, + steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps, + platform='windows', depends_on=[ + 'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), + ], + ) + pipelines = [ pipeline( - name='enterprise-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, + name='enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + - build_steps + package_steps + publish_steps, + build_steps + package_steps + publish_steps, volumes=volumes, ), - pipeline( - name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, - steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps, - platform='windows', depends_on=['enterprise-build-{}'.format(ver_mode)], - ), ] + if not disable_tests: + pipelines.extend([ + pipeline( + name='enterprise-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[], + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + + test_steps, + volumes=[], + ), + pipeline( + name='enterprise-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + + integration_test_steps, + volumes=volumes, + ), + ]) + deps = { + 'depends_on': [ + 'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), + 'enterprise-test-{}'.format(ver_mode), + 'enterprise-integration-tests-{}'.format(ver_mode) + ] + } + windows_pipeline.update(deps) + + pipelines.extend([windows_pipeline]) + + return pipelines def release_pipelines(ver_mode='release', trigger=None): # 'enterprise' edition services contain both OSS and enterprise services @@ -310,3 +380,8 @@ def test_release_pipelines(): )) return pipelines + +def get_e2e_suffix(): + if not disable_tests: + return '-e2e' + return '' diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 2ca91c45b29..8e687f434e7 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -11,6 +11,7 @@ windows_image = 'mcr.microsoft.com/windows:1809' wix_image = 'grafana/ci-wix:0.1.1' test_release_ver = 'v7.3.0-test' +disable_tests = False def slack_step(channel, template, secret): return { @@ -277,13 +278,7 @@ def store_storybook_step(edition, ver_mode): return { 'name': 'store-storybook', 'image': publish_image, - 'depends_on': [ - 'build-storybook', - 'end-to-end-tests-dashboards-suite', - 'end-to-end-tests-panels-suite', - 'end-to-end-tests-smoke-tests-suite', - 'end-to-end-tests-various-suite', - ], + 'depends_on': ['build-storybook',] + end_to_end_tests_deps(edition), 'environment': { 'GCP_KEY': from_secret('gcp_key'), 'PRERELEASE_BUCKET': from_secret(prerelease_bucket) @@ -624,7 +619,6 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is sfx = '-enterprise2' deps.extend([ 'build-backend' + sfx, - 'test-backend' + sfx, ]) variants_str = '' @@ -916,12 +910,7 @@ def release_canary_npm_packages_step(edition): return { 'name': 'release-canary-npm-packages', 'image': build_image, - 'depends_on': [ - 'end-to-end-tests-dashboards-suite', - 'end-to-end-tests-panels-suite', - 'end-to-end-tests-smoke-tests-suite', - 'end-to-end-tests-various-suite', - ], + 'depends_on': end_to_end_tests_deps(edition), 'environment': { 'NPM_TOKEN': from_secret('npm_token'), }, @@ -953,21 +942,12 @@ def upload_packages_step(edition, ver_mode, is_downstream=False): cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads'.format(edition) deps = [] - if edition in 'enterprise2': + if edition in 'enterprise2' or not end_to_end_tests_deps(edition): deps.extend([ 'package' + enterprise2_suffix(edition), ]) else: - deps.extend([ - 'end-to-end-tests-dashboards-suite' + enterprise2_suffix(edition), - 'end-to-end-tests-panels-suite' + enterprise2_suffix(edition), - 'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition), - 'end-to-end-tests-various-suite' + enterprise2_suffix(edition), - ]) - - if edition in ('enterprise', 'enterprise2'): - deps.append('redis-integration-tests') - deps.append('memcached-integration-tests') + deps.extend(end_to_end_tests_deps(edition)) return { 'name': 'upload-packages' + enterprise2_suffix(edition), @@ -1186,3 +1166,13 @@ def ensure_cuetsified_step(): 'git stash pop', ], } + +def end_to_end_tests_deps(edition): + if disable_tests: + return [] + return [ + 'end-to-end-tests-dashboards-suite' + enterprise2_suffix(edition), + 'end-to-end-tests-panels-suite' + enterprise2_suffix(edition), + 'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition), + 'end-to-end-tests-various-suite' + enterprise2_suffix(edition), + ]