public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build: build improvements from master.

Browse Source 
- docker for arm
- upgrade to latest golang
- automated repo deploy
This commit is contained in:
Leonard Gram
2019-01-14 14:11:58 +01:00
parent 5743ceaa93
commit 17f4a03d6f
18 changed files with 331 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+49 -29
View File
@@ -127,7 +127,7 @@ jobs:
build-all:
docker:
- image: grafana/build-container:1.2.1
- image: grafana/build-container:1.2.2
working_directory: /go/src/github.com/grafana/grafana
steps:
- checkout
@@ -200,51 +200,51 @@ jobs:
- dist/grafana*
grafana-docker-master:
docker:
- image: docker:stable-git
machine:
image: circleci/classic:201808-01
steps:
- checkout
- attach_workspace:
at: .
- setup_remote_docker
- run: docker info
- run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker
- run: docker run --privileged linuxkit/binfmt:v0.6
- run: cp dist/grafana-latest.linux-*.tar.gz packaging/docker
- run: cd packaging/docker && ./build-deploy.sh "master-${CIRCLE_SHA1}"
- run: rm packaging/docker/grafana-latest.linux-x64.tar.gz
- run: rm packaging/docker/grafana-latest.linux-*.tar.gz
- run: cp enterprise-dist/grafana-enterprise-*.linux-amd64.tar.gz packaging/docker/grafana-latest.linux-x64.tar.gz
- run: cd packaging/docker && ./build-enterprise.sh "master"
grafana-docker-pr:
docker:
- image: docker:stable-git
machine:
image: circleci/classic:201808-01
steps:
- checkout
- attach_workspace:
at: .
- setup_remote_docker
- run: docker info
- run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker
- run: docker run --privileged linuxkit/binfmt:v0.6
- run: cp dist/grafana-latest.linux-*.tar.gz packaging/docker
- run: cd packaging/docker && ./build.sh "${CIRCLE_SHA1}"
grafana-docker-release:
docker:
- image: docker:stable-git
steps:
- checkout
- attach_workspace:
at: .
- setup_remote_docker
- run: docker info
- run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker
- run: cd packaging/docker && ./build-deploy.sh "${CIRCLE_TAG}"
- run: rm packaging/docker/grafana-latest.linux-x64.tar.gz
- run: cp enterprise-dist/grafana-enterprise-*.linux-amd64.tar.gz packaging/docker/grafana-latest.linux-x64.tar.gz
- run: cd packaging/docker && ./build-enterprise.sh "${CIRCLE_TAG}"
machine:
image: circleci/classic:201808-01
steps:
- checkout
- attach_workspace:
at: .
- run: docker info
- run: docker run --privileged linuxkit/binfmt:v0.6
- run: cp dist/grafana-latest.linux-*.tar.gz packaging/docker
- run: cd packaging/docker && ./build-deploy.sh "${CIRCLE_TAG}"
- run: rm packaging/docker/grafana-latest.linux-*.tar.gz
- run: cp enterprise-dist/grafana-enterprise-*.linux-amd64.tar.gz packaging/docker/grafana-latest.linux-x64.tar.gz
- run: cd packaging/docker && ./build-enterprise.sh "${CIRCLE_TAG}"
build-enterprise:
docker:
- image: grafana/build-container:1.2.1
- image: grafana/build-container:1.2.2
working_directory: /go/src/github.com/grafana/grafana
steps:
- checkout
@@ -276,7 +276,7 @@ jobs:
build-all-enterprise:
docker:
- image: grafana/build-container:1.2.1
- image: grafana/build-container:1.2.2
working_directory: /go/src/github.com/grafana/grafana
steps:
- checkout
@@ -323,7 +323,7 @@ jobs:
deploy-enterprise-master:
docker:
- image: grafana/grafana-ci-deploy:1.0.0
- image: grafana/grafana-ci-deploy:1.1.0
steps:
- attach_workspace:
at: .
@@ -346,7 +346,7 @@ jobs:
deploy-enterprise-release:
docker:
- image: grafana/grafana-ci-deploy:1.0.0
- image: grafana/grafana-ci-deploy:1.1.0
steps:
- attach_workspace:
at: .
@@ -365,10 +365,20 @@ jobs:
- run:
name: Deploy to Grafana.com
command: './scripts/build/publish.sh --enterprise'
- run:
name: Load GPG private key
comand: './scripts/build/load-signing-key.sh'
- run:
name: Update Debian repository
command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
- run:
name: Update RPM repository
command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
deploy-master:
docker:
- image: grafana/grafana-ci-deploy:1.0.0
- image: grafana/grafana-ci-deploy:1.1.0
steps:
- attach_workspace:
at: .
@@ -398,8 +408,9 @@ jobs:
deploy-release:
docker:
- image: grafana/grafana-ci-deploy:1.0.0
- image: grafana/grafana-ci-deploy:1.1.0
steps:
- checkout
- attach_workspace:
at: .
- run:
@@ -417,6 +428,15 @@ jobs:
- run:
name: Deploy to Grafana.com
command: './scripts/build/publish.sh'
- run:
name: Load GPG private key
comand: './scripts/build/load-signing-key.sh'
- run:
name: Update Debian repository
command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
- run:
name: Update RPM repository
command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
workflows:
version: 2
build.go
+2
View File
@@ -164,6 +164,8 @@ func makeLatestDistCopies() {
"_amd64.deb": "dist/grafana_latest_amd64.deb",
".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm",
".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
".linux-armv7.tar.gz": "dist/grafana-latest.linux-armv7.tar.gz",
".linux-arm64.tar.gz": "dist/grafana-latest.linux-arm64.tar.gz",
}
for _, file := range files {
packaging/docker/Dockerfile
+4 -2
View File
@@ -1,4 +1,5 @@
FROM debian:stretch-slim
ARG BASE_IMAGE=debian:stretch-slim
FROM ${BASE_IMAGE}
ARG GRAFANA_TGZ="grafana-latest.linux-x64.tar.gz"
@@ -10,7 +11,8 @@ COPY ${GRAFANA_TGZ} /tmp/grafana.tar.gz
RUN mkdir /tmp/grafana && tar xfvz /tmp/grafana.tar.gz --strip-components=1 -C /tmp/grafana
FROM debian:stretch-slim
ARG BASE_IMAGE=debian:stretch-slim
FROM ${BASE_IMAGE}
ARG GF_UID="472"
ARG GF_GID="472"
packaging/docker/build-deploy.sh
-1
View File
@@ -8,6 +8,5 @@ docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
./push_to_docker_hub.sh "$_grafana_version"
if echo "$_grafana_version" | grep -q "^master-"; then
apk add --no-cache curl
./deploy_to_k8s.sh "grafana/grafana-dev:$_grafana_version"
fi
packaging/docker/build.sh
+32 -8
View File
@@ -1,25 +1,49 @@
#!/bin/sh
_grafana_tag=$1
_grafana_tag=${1:-}
_docker_repo=${2:-grafana/grafana}
# If the tag starts with v, treat this as a official release
if echo "$_grafana_tag" | grep -q "^v"; then
_grafana_version=$(echo "${_grafana_tag}" | cut -d "v" -f 2)
_docker_repo=${2:-grafana/grafana}
else
_grafana_version=$_grafana_tag
_docker_repo=${2:-grafana/grafana-dev}
fi
echo "Building ${_docker_repo}:${_grafana_version}"
docker build \
--tag "${_docker_repo}:${_grafana_version}" \
--no-cache=true .
export DOCKER_CLI_EXPERIMENTAL=enabled
# Build grafana image for a specific arch
docker_build () {
base_image=$1
grafana_tgz=$2
tag=$3
docker build \
--build-arg BASE_IMAGE=${base_image} \
--build-arg GRAFANA_TGZ=${grafana_tgz} \
--tag "${tag}" \
--no-cache=true .
}
# Tag docker images of all architectures
docker_tag_all () {
repo=$1
tag=$2
docker tag "${_docker_repo}:${_grafana_version}" "${repo}:${tag}"
docker tag "${_docker_repo}-arm32v7-linux:${_grafana_version}" "${repo}-arm32v7-linux:${tag}"
docker tag "${_docker_repo}-arm64v8-linux:${_grafana_version}" "${repo}-arm64v8-linux:${tag}"
}
docker_build "debian:stretch-slim" "grafana-latest.linux-x64.tar.gz" "${_docker_repo}:${_grafana_version}"
docker_build "arm32v7/debian:stretch-slim" "grafana-latest.linux-armv7.tar.gz" "${_docker_repo}-arm32v7-linux:${_grafana_version}"
docker_build "arm64v8/debian:stretch-slim" "grafana-latest.linux-arm64.tar.gz" "${_docker_repo}-arm64v8-linux:${_grafana_version}"
# Tag as 'latest' for official release; otherwise tag as grafana/grafana:master
if echo "$_grafana_tag" | grep -q "^v"; then
docker tag "${_docker_repo}:${_grafana_version}" "${_docker_repo}:latest"
docker_tag_all "${_docker_repo}" "latest"
else
docker tag "${_docker_repo}:${_grafana_version}" "grafana/grafana:master"
docker_tag_all "${_docker_repo}" "master"
docker tag "${_docker_repo}:${_grafana_version}" "grafana/grafana-dev:${_grafana_version}"
fi
packaging/docker/push_to_docker_hub.sh
+29 -7
View File
@@ -1,24 +1,46 @@
#!/bin/sh
set -e
_grafana_tag=$1
_grafana_tag=${1:-}
_docker_repo=${2:-grafana/grafana}
# If the tag starts with v, treat this as a official release
if echo "$_grafana_tag" | grep -q "^v"; then
_grafana_version=$(echo "${_grafana_tag}" | cut -d "v" -f 2)
_docker_repo=${2:-grafana/grafana}
else
_grafana_version=$_grafana_tag
_docker_repo=${2:-grafana/grafana-dev}
fi
export DOCKER_CLI_EXPERIMENTAL=enabled
echo "pushing ${_docker_repo}:${_grafana_version}"
docker push "${_docker_repo}:${_grafana_version}"
docker_push_all () {
repo=$1
tag=$2
# Push each image individually
docker push "${repo}:${tag}"
docker push "${repo}-arm32v7-linux:${tag}"
docker push "${repo}-arm64v8-linux:${tag}"
# Create and push a multi-arch manifest
docker manifest create "${repo}:${tag}" \
"${repo}:${tag}" \
"${repo}-arm32v7-linux:${tag}" \
"${repo}-arm64v8-linux:${tag}"
docker manifest push "${repo}:${tag}"
}
if echo "$_grafana_tag" | grep -q "^v" && echo "$_grafana_tag" | grep -vq "beta"; then
echo "pushing ${_docker_repo}:latest"
docker push "${_docker_repo}:latest"
docker_push_all "${_docker_repo}" "latest"
docker_push_all "${_docker_repo}" "${_grafana_version}"
elif echo "$_grafana_tag" | grep -q "^v" && echo "$_grafana_tag" | grep -q "beta"; then
docker_push_all "${_docker_repo}" "${_grafana_version}"
elif echo "$_grafana_tag" | grep -q "master"; then
echo "pushing grafana/grafana:master"
docker push grafana/grafana:master
docker_push_all "${_docker_repo}" "master"
docker push "grafana/grafana-dev:${_grafana_version}"
fi
packaging/publish/publish_both.sh
+11 -11
View File
@@ -1,17 +1,17 @@
#! /usr/bin/env bash
version=5.0.2
version=5.4.3
wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_${version}_amd64.deb
# wget https://dl.grafana.com/oss/release/grafana_${version}_amd64.deb
#
# package_cloud push grafana/stable/debian/jessie grafana_${version}_amd64.deb
# package_cloud push grafana/stable/debian/wheezy grafana_${version}_amd64.deb
# package_cloud push grafana/stable/debian/stretch grafana_${version}_amd64.deb
#
# package_cloud push grafana/testing/debian/jessie grafana_${version}_amd64.deb
# package_cloud push grafana/testing/debian/wheezy grafana_${version}_amd64.deb --verbose
# package_cloud push grafana/testing/debian/stretch grafana_${version}_amd64.deb --verbose
package_cloud push grafana/stable/debian/jessie grafana_${version}_amd64.deb
package_cloud push grafana/stable/debian/wheezy grafana_${version}_amd64.deb
package_cloud push grafana/stable/debian/stretch grafana_${version}_amd64.deb
package_cloud push grafana/testing/debian/jessie grafana_${version}_amd64.deb
package_cloud push grafana/testing/debian/wheezy grafana_${version}_amd64.deb --verbose
package_cloud push grafana/testing/debian/stretch grafana_${version}_amd64.deb --verbose
wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-${version}-1.x86_64.rpm
wget https://dl.grafana.com/oss/release/grafana-${version}-1.x86_64.rpm
package_cloud push grafana/testing/el/6 grafana-${version}-1.x86_64.rpm --verbose
package_cloud push grafana/testing/el/7 grafana-${version}-1.x86_64.rpm --verbose
scripts/build/build-all.sh
+4 -1
View File
@@ -35,6 +35,8 @@ go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
go run build.go -goos windows -cc ${CCWIN64} ${OPT} build
# Do not remove CC from the linux build, its there for compatibility with Centos6
CC=${CCX64} go run build.go ${OPT} build
yarn install --pure-lockfile --no-progress
@@ -57,7 +59,7 @@ go run build.go ${OPT} build-frontend
source /etc/profile.d/rvm.sh
echo "Packaging"
go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only latest
go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
#removing amd64 phantomjs bin for armv7/arm64 packages
rm tools/phantomjs/phantomjs
go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
@@ -78,3 +80,4 @@ else
fi
go run build.go -goos windows -pkg-arch amd64 ${OPT} package-only
go run build.go latest
scripts/build/build.sh
+10 -1
View File
@@ -8,6 +8,8 @@ set -e
EXTRA_OPTS="$@"
CCARMV7=arm-linux-gnueabihf-gcc
CCARM64=aarch64-linux-gnu-gcc
CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
GOPATH=/go
@@ -26,6 +28,9 @@ fi
echo "Build arguments: $OPT"
go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
CC=${CCX64} go run build.go ${OPT} build
yarn install --pure-lockfile --no-progress
@@ -43,4 +48,8 @@ go run build.go ${OPT} build-frontend
source /etc/profile.d/rvm.sh
echo "Packaging"
go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only latest
go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
go run build.go latest
scripts/build/ci-deploy/Dockerfile
+23 -3
View File
@@ -1,5 +1,25 @@
FROM circleci/golang:1.11
RUN git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly && \
cd $GOPATH/src/github.com/aptly-dev/aptly && \
# pin aptly to a specific commit after 1.3.0 that contains gpg2 support
git reset --hard a64807efdaf5e380bfa878c71bc88eae10d62be1 && \
make install
FROM circleci/python:2.7-stretch
RUN sudo pip install awscli && \
curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-222.0.0-linux-x86_64.tar.gz | \
sudo tar xvzf - -C /opt
ENV PATH=$PATH:/opt/google-cloud-sdk/bin
USER root
RUN pip install awscli && \
curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-222.0.0-linux-x86_64.tar.gz | \
tar xvzf - -C /opt && \
apt update && \
apt install -y createrepo expect && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
COPY --from=0 /go/bin/aptly /usr/local/bin/aptly
USER circleci
scripts/build/ci-deploy/build-deploy.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
_version="1.0.0"
_version="1.1.0"
_tag="grafana/grafana-ci-deploy:${_version}"
docker build -t $_tag .
scripts/build/load-signing-key.sh
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -e
git clone git@github.com:torkelo/private.git ~/private-repo
gpg --batch --allow-secret-key-import --import ~/private-repo/signing/private.key
pkill gpg-agent
scripts/build/publish_test.go
+1 -1
View File
@@ -105,6 +105,6 @@ func TestFileWalker(t *testing.T) {
incorrectPackageName := "grafana_5.2.0-474pre1_armfoo.deb"
_, err := mapPackage(incorrectPackageName, incorrectPackageName, []byte{})
if err == nil {
t.Errorf("Testing (%v), expected to fail due to an unrecognized arch, but signalled no error.", incorrectPackageName)
t.Errorf("Testing (%v), expected to fail due to an unrecognized arch, but signaled no error.", incorrectPackageName)
}
}
scripts/build/update_repo/aptly.conf
+27
View File
@@ -0,0 +1,27 @@
{
"rootDir": "/deb-repo/db",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgProvider": "gpg2",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"skipContentsPublishing": false,
"FileSystemPublishEndpoints": {
"repo": {
"rootDir": "/deb-repo/repo",
"linkMethod": "copy"
}
},
"S3PublishEndpoints": {},
"SwiftPublishEndpoints": {}
}
scripts/build/update_repo/sign-rpm-repo.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env expect
set password [lindex $argv 0]
spawn gpg --detach-sign --armor /rpm-repo/repodata/repomd.xml
expect "Enter passphrase: "
send -- "$password\r"
expect eof
scripts/build/update_repo/unlock-gpg-key.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env expect
set password [lindex $argv 0]
spawn gpg --detach-sign --armor /tmp/sign-this
expect "Enter passphrase: "
send -- "$password\r"
expect eof
scripts/build/update_repo/update-deb.sh
Executable
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
RELEASE_TYPE="${1:-}"
GPG_PASS="${2:-}"
RELEASE_TAG="${3:-}"
REPO="grafana"
if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
exit 1
fi
if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
exit 1
fi
if echo "$RELEASE_TAG" | grep -q "beta"; then
REPO="beta"
fi
set -e
# Setup environment
cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
mkdir -p /deb-repo/db \
/deb-repo/repo \
/deb-repo/tmp
# Download the database
gsutil -m rsync -r "gs://grafana-aptly-db/$RELEASE_TYPE" /deb-repo/db
# Add the new release to the repo
aptly publish drop grafana filesystem:repo:grafana || true
aptly publish drop beta filesystem:repo:grafana || true
cp ./dist/*.deb /deb-repo/tmp
rm /deb-repo/tmp/grafana_latest*.deb || true
aptly repo add "$REPO" ./dist
# Setup signing and sign the repo
echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf
touch /tmp/sign-this
./scripts/build/update_repo/unlock-gpg-key.sh "$GPG_PASS"
rm /tmp/sign-this /tmp/sign-this.asc
aptly publish repo grafana filesystem:repo:grafana
aptly publish repo beta filesystem:repo:grafana
# Update the repo and db on gcp
gsutil -m rsync -r -d /deb-repo/db "gs://grafana-aptly-db/$RELEASE_TYPE"
gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://grafana-repo/$RELEASE_TYPE/deb"
# usage:
#
# deb https://packages.grafana.com/oss/deb stable main
scripts/build/update_repo/update-rpm.sh
Executable
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
RELEASE_TYPE="${1:-}"
GPG_PASS="${2:-}"
RELEASE_TAG="${3:-}"
REPO="rpm"
if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
exit 1
fi
if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
exit 1
fi
if echo "$RELEASE_TAG" | grep -q "beta"; then
REPO="rpm-beta"
fi
set -e
# Setup environment
BUCKET="gs://grafana-repo/$RELEASE_TYPE/$REPO"
mkdir -p /rpm-repo
# Download the database
gsutil -m rsync -r "$BUCKET" /rpm-repo
# Add the new release to the repo
cp ./dist/*.rpm /rpm-repo
rm /rpm-repo/grafana-latest-1*.rpm || true
cd /rpm-repo
createrepo .
# Setup signing and sign the repo
echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf
rm /rpm-repo/repodata/repomd.xml.asc || true
pkill gpg-agent || true
./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"
# Update the repo and db on gcp
gsutil -m rsync -r -d /rpm-repo "$BUCKET"
# usage:
# [grafana]
# name=grafana
# baseurl=https://packages.grafana.com/oss/rpm
# repo_gpgcheck=1
# enabled=1
# gpgcheck=1
# gpgkey=https://packages.grafana.com/gpg.key
# sslverify=1
# sslcacert=/etc/pki/tls/certs/ca-bundle.crt