diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a47e160aa9..7dda5ab4a65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,19 @@ executors: docker: - image: grafana/wix-toolset-ci:v3 +commands: + install-google-cloud-sdk: + description: "Install Google Cloud SDK" + steps: + - run: + name: "Install Google Cloud SDK" + command: | + VERSION=286.0.0 + CHECKSUM=f199e38c988d53a913fc780b6f57ebda72db5f3848073cea805b39fe3e8e34fd + curl -fLO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${VERSION}-linux-x86_64.tar.gz + echo $CHECKSUM google-cloud-sdk-${VERSION}-linux-x86_64.tar.gz | sha256sum --check --status + tar xf google-cloud-sdk-${VERSION}-linux-x86_64.tar.gz + jobs: install-grabpl: description: Install the Grafana Build Pipeline tool @@ -48,7 +61,7 @@ jobs: - run: name: Install Grafana Build Pipeline command: | - curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.2.5/grabpl + curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.2.7/grabpl chmod +x grabpl mkdir bin mv grabpl bin/ @@ -129,6 +142,13 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout + - restore_cache: + keys: + # Use own cache for front-end builds, since for some reason it can't be unpacked for parts of the + # pipeline + - v1-yarn-build-{{ checksum "yarn.lock" }} + # Used if checksum fails + - v1-yarn-build- - run: name: Increase node memory command: | @@ -136,6 +156,13 @@ jobs: - run: name: CI job started command: ./scripts/ci-job-started.sh + - run: + name: Install dependencies + command: yarn install --frozen-lockfile --no-progress + - save_cache: + key: v1-yarn-build-{{ checksum "yarn.lock" }} + paths: + - node_modules - run: name: Build Grafana frontend command: /tmp/workspace/bin/grabpl build-frontend --edition << parameters.edition >> @@ -212,20 +239,29 @@ jobs: - run: name: CI job started command: ./scripts/ci-job-started.sh + - install-google-cloud-sdk - run: - name: Copy artifacts from workspace - command: cp -r /tmp/workspace/oss/dist . + name: Download packages from GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + mkdir dist + ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}/ dist/ - run: name: Build Windows installer command: ./scripts/build/ci-msi-build/ci-msi-build-oss.sh - run: - name: Move artifacts - command: mkdir oss && mv dist oss/ - - persist_to_workspace: - root: . - paths: - - oss/dist/grafana-*.msi - - oss/dist/grafana-*.msi.sha256 + name: Upload installer to GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + ./google-cloud-sdk/bin/gsutil -m cp dist/grafana-*.msi* gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}/ - run: name: CI job failed command: ./scripts/ci-job-failed.sh @@ -251,20 +287,29 @@ jobs: - run: name: CI job started command: "./scripts/ci-job-started.sh" + - install-google-cloud-sdk - run: - name: Copy artifacts from workspace - command: cp -r /tmp/workspace/enterprise/dist enterprise-dist + name: Download packages from GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + mkdir enterprise-dist + ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/enterprise/packages/${CIRCLE_WORKFLOW_ID}/ enterprise-dist/ - run: name: Build Windows installer command: ./scripts/build/ci-msi-build/ci-msi-build-ee.sh - run: - name: Move artifacts - command: mkdir enterprise && mv enterprise-dist enterprise/dist - - persist_to_workspace: - root: . - paths: - - enterprise/dist/grafana-*.msi - - enterprise/dist/grafana-*.msi.sha256 + name: Upload installer to GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + ./google-cloud-sdk/bin/gsutil -m cp enterprise-dist/grafana-*.msi* gs://grafana-build-pipeline/enterprise/packages/${CIRCLE_WORKFLOW_ID}/ - run: name: CI job failed command: "./scripts/ci-job-failed.sh" @@ -317,6 +362,7 @@ jobs: - run: name: CI job started command: ./scripts/ci-job-started.sh + - install-google-cloud-sdk - run: name: Copy artifacts from workspace command: cp -r /tmp/workspace/oss/* . @@ -343,14 +389,14 @@ jobs: linux-x64,linux-x64-musl,osx64,win64 fi - run: - name: Move artifacts + name: Upload packages to GCS command: | - mkdir -p oss - mv dist oss/ - - persist_to_workspace: - root: . - paths: - - oss/dist/* + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + ./google-cloud-sdk/bin/gsutil -m rsync -r dist gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID} - run: name: CI job failed command: ./scripts/ci-job-failed.sh @@ -376,6 +422,7 @@ jobs: - run: name: CI job started command: ./scripts/ci-job-started.sh + - install-google-cloud-sdk - run: name: Copy artifacts from workspace command: cp -r /tmp/workspace/enterprise/* . @@ -397,14 +444,14 @@ jobs: linux-x64,linux-x64-musl,osx64,win64 fi - run: - name: Move artifacts + name: Upload packages to GCS command: | - mkdir -p enterprise - mv dist enterprise/ - - persist_to_workspace: - root: . - paths: - - enterprise/dist/* + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + ./google-cloud-sdk/bin/gsutil -m rsync -r dist gs://grafana-build-pipeline/enterprise/packages/${CIRCLE_WORKFLOW_ID} - run: name: CI job failed command: ./scripts/ci-job-failed.sh @@ -434,10 +481,20 @@ jobs: - run: name: CI job started command: ./scripts/ci-job-started.sh + - install-google-cloud-sdk + - run: + name: Download packages from GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + mkdir dist + ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/<< parameters.edition >>/packages/${CIRCLE_WORKFLOW_ID}/ dist/ - run: name: Publish packages command: | - cp -r /tmp/workspace/<< parameters.edition >>/dist . if [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then # We're testing the release pipeline /tmp/workspace/bin/grabpl publish-packages --edition << parameters.edition >> \ @@ -518,13 +575,20 @@ jobs: - run: name: CI job started command: ./scripts/ci-job-started.sh + - install-google-cloud-sdk + - run: + name: Download packages from GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + mkdir dist + ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/<< parameters.edition >>/packages/${CIRCLE_WORKFLOW_ID}/ dist/ - run: docker info # XXX: Is this necessary? - run: docker run --privileged linuxkit/binfmt:v0.6 - - run: - name: Copy Grafana archives - command: | - cp -r /tmp/workspace/<< parameters.edition >>/dist . - run: name: Build Docker images command: | @@ -569,14 +633,28 @@ jobs: - attach_workspace: at: /tmp/workspace - checkout + - install-google-cloud-sdk + - run: + name: Download packages from GCS + command: | + # Work around an issue with the gsutil installation + export BOTO_CONFIG=/dev/null + + echo $GCP_GRAFANA_UPLOAD_KEY > /tmp/gcpkey.json + ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json + mkdir dist + ./google-cloud-sdk/bin/gsutil -m rsync -d -r gs://grafana-build-pipeline/oss/packages/${CIRCLE_WORKFLOW_ID}/ dist/ - restore_cache: - key: dependency-cache-{{ checksum "yarn.lock" }} + keys: + - v2-yarn-{{ checksum "yarn.lock" }} + # Used if checksum fails + - v2-yarn- - run: name: yarn install - command: yarn install --pure-lockfile --no-progress + command: yarn install --frozen-lockfile --no-progress no_output_timeout: 5m - save_cache: - key: dependency-cache-{{ checksum "yarn.lock" }} + key: v2-yarn-{{ checksum "yarn.lock" }} paths: - node_modules - run: @@ -718,13 +796,16 @@ jobs: name: CI job started command: "./scripts/ci-job-started.sh" - restore_cache: - key: dependency-cache-{{ checksum "yarn.lock" }} + keys: + - v2-yarn-{{ checksum "yarn.lock" }} + # Used if checksum fails + - v2-yarn- - run: name: yarn install command: "yarn install --frozen-lockfile --no-progress" no_output_timeout: 15m - save_cache: - key: dependency-cache-{{ checksum "yarn.lock" }} + key: v2-yarn-{{ checksum "yarn.lock" }} paths: - node_modules - run: diff --git a/e2e/start-server b/e2e/start-server index d6505578952..35304629e5b 100755 --- a/e2e/start-server +++ b/e2e/start-server @@ -1,4 +1,5 @@ #!/bin/bash +set -eo pipefail . e2e/variables diff --git a/e2e/variables b/e2e/variables index 63e3e538714..1899de36f9a 100644 --- a/e2e/variables +++ b/e2e/variables @@ -2,6 +2,6 @@ RUNDIR=e2e/tmp PIDFILE=$RUNDIR/pid -PACKAGE_FILE=/tmp/workspace/oss/dist/grafana-*linux-amd64.tar.gz +PACKAGE_FILE=dist/grafana-*linux-amd64.tar.gz PROV_DIR=$RUNDIR/conf/provisioning PORT=3001