diff --git a/.circleci/config.yml b/.circleci/config.yml index ec1fcfb411f..876db1d4823 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -323,7 +323,7 @@ jobs: deploy-enterprise-master: docker: - - image: grafana/grafana-ci-deploy:1.1.0 + - image: grafana/grafana-ci-deploy:1.2.0 steps: - attach_workspace: at: . @@ -346,7 +346,7 @@ jobs: deploy-enterprise-release: docker: - - image: grafana/grafana-ci-deploy:1.1.0 + - image: grafana/grafana-ci-deploy:1.2.0 steps: - attach_workspace: at: . @@ -370,15 +370,15 @@ jobs: command: './scripts/build/load-signing-key.sh' - run: name: Update Debian repository - command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"' + command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"' - run: name: Update RPM repository - command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"' + command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"' deploy-master: docker: - - image: grafana/grafana-ci-deploy:1.1.0 + - image: grafana/grafana-ci-deploy:1.2.0 steps: - attach_workspace: at: . @@ -408,7 +408,7 @@ jobs: deploy-release: docker: - - image: grafana/grafana-ci-deploy:1.1.0 + - image: grafana/grafana-ci-deploy:1.2.0 steps: - checkout - attach_workspace: @@ -433,10 +433,10 @@ jobs: command: './scripts/build/load-signing-key.sh' - run: name: Update Debian repository - command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"' + command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"' - run: name: Update RPM repository - command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"' + command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"' workflows: version: 2 diff --git a/scripts/build/build-all.sh b/scripts/build/build-all.sh index 980ef5cc4c2..dd69f8139c8 100755 --- a/scripts/build/build-all.sh +++ b/scripts/build/build-all.sh @@ -30,10 +30,12 @@ fi echo "Build arguments: $OPT" -go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build -go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build -go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build - +# build only amd64 for enterprise +if echo "$EXTRA_OPTS" | grep -vq enterprise ; then + go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build + go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build + go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build +fi go run build.go -goos windows -cc ${CCWIN64} ${OPT} build # Do not remove CC from the linux build, its there for compatibility with Centos6 @@ -62,22 +64,26 @@ echo "Packaging" go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only #removing amd64 phantomjs bin for armv7/arm64 packages rm tools/phantomjs/phantomjs -go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only -go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only -if [ -d '/tmp/phantomjs/darwin' ]; then - cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs -else - echo 'PhantomJS binaries for darwin missing!' +# build only amd64 for enterprise +if echo "$EXTRA_OPTS" | grep -vq enterprise ; then + go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only + go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only fi -go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only if [ -d '/tmp/phantomjs/windows' ]; then cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe - rm tools/phantomjs/phantomjs + rm tools/phantomjs/phantomjs || true else echo 'PhantomJS binaries for Windows missing!' fi go run build.go -goos windows -pkg-arch amd64 ${OPT} package-only -go run build.go latest \ No newline at end of file +go run build.go latest diff --git a/scripts/build/update_repo/README.md b/scripts/build/update_repo/README.md new file mode 100644 index 00000000000..22f2ea1241d --- /dev/null +++ b/scripts/build/update_repo/README.md @@ -0,0 +1,28 @@ +# Repository updates deb/rpm + + + +## Testing + +It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing. + +```bash +docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing +# in the container: +mkdir -p /go/src/github.com/grafana/dist +cd /go/src/github.com/grafana + +#outside of container: +cd /.. +docker cp grafana :/go/src/github.com/grafana/. +docker cp :/private.key + +#in container: +gpg --batch --allow-secret-key-import --import /private.key +cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd .. + +#run these scripts: +./script/build/update_repo/test-update-deb-repo.sh +./script/build/update_repo/test-update-rpm-repo.sh + +``` diff --git a/scripts/build/update_repo/init-deb-repo.sh b/scripts/build/update_repo/init-deb-repo.sh new file mode 100755 index 00000000000..2b245dc2d42 --- /dev/null +++ b/scripts/build/update_repo/init-deb-repo.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Run this if you need to recreate the debian repository for some reason + +# Setup environment +cp scripts/build/update_repo/aptly.conf /etc/aptly.conf +mkdir -p /deb-repo/db \ + /deb-repo/repo \ + /deb-repo/tmp + +aptly repo create -distribution=stable -component=main grafana +aptly repo create -distribution=beta -component=main beta diff --git a/scripts/build/update_repo/test-update-deb-repo.sh b/scripts/build/update_repo/test-update-deb-repo.sh new file mode 100755 index 00000000000..f27e9bec265 --- /dev/null +++ b/scripts/build/update_repo/test-update-deb-repo.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +GPG_PASS=${1:-} + +./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo" diff --git a/scripts/build/update_repo/test-update-rpm-repo.sh b/scripts/build/update_repo/test-update-rpm-repo.sh new file mode 100755 index 00000000000..de85f000de8 --- /dev/null +++ b/scripts/build/update_repo/test-update-rpm-repo.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +GPG_PASS=${1:-} + +./scripts/build/update_repo/update-rpm.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-repo" diff --git a/scripts/build/update_repo/update-deb.sh b/scripts/build/update_repo/update-deb.sh index 89c5937b064..a70af5b8b5c 100755 --- a/scripts/build/update_repo/update-deb.sh +++ b/scripts/build/update_repo/update-deb.sh @@ -3,10 +3,14 @@ RELEASE_TYPE="${1:-}" GPG_PASS="${2:-}" RELEASE_TAG="${3:-}" +DIST_PATH="${4:-}" +GCP_DB_BUCKET="${5:-grafana-aptly-db}" +GCP_REPO_BUCKET="${6:-grafana-repo}" + REPO="grafana" -if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then - echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set" +if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then + echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set" exit 1 fi @@ -22,36 +26,42 @@ fi set -e # Setup environment -cp scripts/build/update_repo/aptly.conf /etc/aptly.conf +cp scripts/build/update_repo/aptly.conf ~/.aptly.conf + mkdir -p /deb-repo/db \ /deb-repo/repo \ /deb-repo/tmp # Download the database -gsutil -m rsync -r "gs://grafana-aptly-db/$RELEASE_TYPE" /deb-repo/db +gsutil -m rsync -r -d "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" /deb-repo/db # Add the new release to the repo -aptly publish drop grafana filesystem:repo:grafana || true -aptly publish drop beta filesystem:repo:grafana || true -cp ./dist/*.deb /deb-repo/tmp +cp $DIST_PATH/*.deb /deb-repo/tmp rm /deb-repo/tmp/grafana_latest*.deb || true -aptly repo add "$REPO" ./dist +aptly repo add "$REPO" /deb-repo/tmp #adds too many packages in enterprise # Setup signing and sign the repo echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf +pkill gpg-agent || true touch /tmp/sign-this +rm /tmp/sign-this.asc || true ./scripts/build/update_repo/unlock-gpg-key.sh "$GPG_PASS" rm /tmp/sign-this /tmp/sign-this.asc -aptly publish repo grafana filesystem:repo:grafana -aptly publish repo beta filesystem:repo:grafana +aptly publish update stable filesystem:repo:grafana +aptly publish update beta filesystem:repo:grafana # Update the repo and db on gcp -gsutil -m rsync -r -d /deb-repo/db "gs://grafana-aptly-db/$RELEASE_TYPE" -gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://grafana-repo/$RELEASE_TYPE/deb" + +gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" + +# Uploads the binaries before the metadata (to prevent 404's for debs) +gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool" + +gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb" # usage: # diff --git a/scripts/build/update_repo/update-rpm.sh b/scripts/build/update_repo/update-rpm.sh index caed3918216..f02647f55a0 100755 --- a/scripts/build/update_repo/update-rpm.sh +++ b/scripts/build/update_repo/update-rpm.sh @@ -2,12 +2,14 @@ RELEASE_TYPE="${1:-}" GPG_PASS="${2:-}" - RELEASE_TAG="${3:-}" +DIST_PATH="${4:-}" +GCP_REPO_BUCKET="${5:-grafana-repo}" + REPO="rpm" -if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then - echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set" +if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then + echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set" exit 1 fi @@ -23,17 +25,17 @@ fi set -e # Setup environment -BUCKET="gs://grafana-repo/$RELEASE_TYPE/$REPO" +BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO" + mkdir -p /rpm-repo # Download the database gsutil -m rsync -r "$BUCKET" /rpm-repo # Add the new release to the repo -cp ./dist/*.rpm /rpm-repo +cp $DIST_PATH/*.rpm /rpm-repo # adds to many files for enterprise rm /rpm-repo/grafana-latest-1*.rpm || true -cd /rpm-repo -createrepo . +createrepo /rpm-repo # Setup signing and sign the repo @@ -45,6 +47,7 @@ pkill gpg-agent || true ./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS" # Update the repo and db on gcp +gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses gsutil -m rsync -r -d /rpm-repo "$BUCKET" # usage: @@ -56,4 +59,4 @@ gsutil -m rsync -r -d /rpm-repo "$BUCKET" # gpgcheck=1 # gpgkey=https://packages.grafana.com/gpg.key # sslverify=1 -# sslcacert=/etc/pki/tls/certs/ca-bundle.crt \ No newline at end of file +# sslcacert=/etc/pki/tls/certs/ca-bundle.crt