From 9123e0fca8befc314570a5ca99d2e857cf046b95 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Tue, 8 May 2018 09:42:20 +0200 Subject: [PATCH 1/8] build: crossplatform build with packages. Big thanks to @fg2it who created a POC as well as the build container that this work is based on. --- .circleci/config.yml | 5 ++- Gruntfile.js | 4 +++ build.go | 61 ++++++++++++++++++++++++----------- scripts/build/build.sh | 42 ++++++++++++++++++++---- scripts/grunt/release_task.js | 11 +++++-- 5 files changed, 95 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bad5a7c1cd0..1a0be0575e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,10 +70,13 @@ jobs: build: docker: - - image: grafana/build-container:v0.1 + - image: grafana/build-container:crosscompile working_directory: /go/src/github.com/grafana/grafana steps: - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' - run: name: build and package grafana command: './scripts/build/build.sh' diff --git a/Gruntfile.js b/Gruntfile.js index a0607ef49dc..23276e8a122 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -12,6 +12,10 @@ module.exports = function (grunt) { platform: process.platform.replace('win32', 'windows'), }; + if (grunt.option('platform')) { + config.platform = grunt.option('platform'); + } + if (grunt.option('arch')) { config.arch = grunt.option('arch'); } else { diff --git a/build.go b/build.go index 7e7183b8b83..f9611c99a47 100644 --- a/build.go +++ b/build.go @@ -27,8 +27,7 @@ var ( goarch string goos string gocc string - gocxx string - cgo string + cgo bool pkgArch string version string = "v1" // deb & rpm does not support semver so have to handle their version a little differently @@ -53,8 +52,7 @@ func main() { flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH") flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS") flag.StringVar(&gocc, "cc", "", "CC") - flag.StringVar(&gocxx, "cxx", "", "CXX") - flag.StringVar(&cgo, "cgo-enabled", "", "CGO_ENABLED") + flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo") flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH") flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary") flag.BoolVar(&race, "race", race, "Use race detector") @@ -93,20 +91,24 @@ func main() { build("grafana-server", "./pkg/cmd/grafana-server", []string{}) case "build": - clean() + //clean() for _, binary := range binaries { build(binary, "./pkg/cmd/"+binary, []string{}) } + case "build-frontend": + grunt(gruntBuildArg("build")...) + case "test": test("./pkg/...") grunt("test") case "package": - grunt(gruntBuildArg("release")...) - if runtime.GOOS != "windows" { - createLinuxPackages() - } + grunt(gruntBuildArg("build")...) + packageGrafana() + + case "package-only": + packageGrafana() case "pkg-rpm": grunt(gruntBuildArg("release")...) @@ -131,6 +133,22 @@ func main() { } } +func packageGrafana() { + platformArg := fmt.Sprintf("--platform=%v", goos) + previousPkgArch := pkgArch + if pkgArch == "" { + pkgArch = goarch + } + postProcessArgs := gruntBuildArg("package") + postProcessArgs = append(postProcessArgs, platformArg) + grunt(postProcessArgs...) + pkgArch = previousPkgArch + + if goos == "linux" && goarch == "amd64"{ + createLinuxPackages() + } +} + func makeLatestDistCopies() { files, err := ioutil.ReadDir("dist") if err != nil { @@ -138,9 +156,9 @@ func makeLatestDistCopies() { } latestMapping := map[string]string{ - ".deb": "dist/grafana_latest_amd64.deb", - ".rpm": "dist/grafana-latest-1.x86_64.rpm", - ".tar.gz": "dist/grafana-latest.linux-x64.tar.gz", + "_amd64.deb": "dist/grafana_latest_amd64.deb", + ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm", + ".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz", } for _, file := range files { @@ -386,7 +404,8 @@ func test(pkg string) { } func build(binaryName, pkg string, tags []string) { - binary := "./bin/" + binaryName + binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName) + if goos == "windows" { binary += ".exe" } @@ -408,6 +427,7 @@ func build(binaryName, pkg string, tags []string) { if !isDev { setBuildEnv() runPrint("go", "version") + fmt.Printf("Targeting %s/%s\n", goos, goarch) } runPrint("go", args...) @@ -451,6 +471,14 @@ func clean() { func setBuildEnv() { os.Setenv("GOOS", goos) + if goos == "windows" { + // require windows >=7 + os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601") + } + if goarch != "amd64" || goos != "linux" { + // needed for all other archs + cgo = true + } if strings.HasPrefix(goarch, "armv") { os.Setenv("GOARCH", "arm") os.Setenv("GOARM", goarch[4:]) @@ -460,15 +488,12 @@ func setBuildEnv() { if goarch == "386" { os.Setenv("GO386", "387") } - if cgo != "" { - os.Setenv("CGO_ENABLED", cgo) + if cgo { + os.Setenv("CGO_ENABLED", "1") } if gocc != "" { os.Setenv("CC", gocc) } - if gocxx != "" { - os.Setenv("CXX", gocxx) - } } func getGitSha() string { diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 369d434d4f5..ab021d57692 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -4,6 +4,12 @@ # This script is executed from within the container. # +CCARMV7=arm-linux-gnueabihf-gcc +CCARM64=aarch64-linux-gnu-gcc +CCOSX64=/tmp/osxcross/target/bin/o64-clang +CCWIN64=x86_64-w64-mingw32-gcc +CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc + GOPATH=/go REPO_PATH=$GOPATH/src/github.com/grafana/grafana @@ -11,23 +17,45 @@ cd /go/src/github.com/grafana/grafana echo "current dir: $(pwd)" if [ "$CIRCLE_TAG" != "" ]; then - echo "Building a release from tag $CIRCLE_TAG" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false build + echo "Building releases from tag $CIRCLE_TAG" + go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build + go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build + go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build + go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build + CC=${CCX64} go run build.go -includeBuildNumber=false build else echo "Building incremental build for $CIRCLE_BRANCH" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build + CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build fi yarn install --pure-lockfile --no-progress -source /etc/profile.d/rvm.sh - echo "current dir: $(pwd)" +if [ -d "dist" ]; then + rm -rf dist +fi + if [ "$CIRCLE_TAG" != "" ]; then + echo "Building frontend from tag $CIRCLE_TAG" + go run build.go -includeBuildNumber=false build-frontend echo "Packaging a release from tag $CIRCLE_TAG" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false package latest + go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest + go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only + go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only + go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only + go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only else + echo "Building frontend for $CIRCLE_BRANCH" + go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend echo "Packaging incremental build for $CIRCLE_BRANCH" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} package latest + go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest + go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only fi diff --git a/scripts/grunt/release_task.js b/scripts/grunt/release_task.js index 4fbc41cfb08..f24ec64203a 100644 --- a/scripts/grunt/release_task.js +++ b/scripts/grunt/release_task.js @@ -3,13 +3,20 @@ var path = require('path'); module.exports = function(grunt) { "use strict"; - // build, then zip and upload to s3 + // build then zip grunt.registerTask('release', [ 'build', 'build-post-process', 'compress:release' ]); + // package into archives + grunt.registerTask('package', [ + 'clean:temp', + 'build-post-process', + 'compress:release' + ]); + grunt.registerTask('build-post-process', function() { grunt.config('copy.public_to_temp', { expand: true, @@ -18,7 +25,7 @@ module.exports = function(grunt) { dest: '<%= tempDir %>/public/', }); grunt.config('copy.backend_bin', { - cwd: 'bin', + cwd: 'bin/<%= platform %>-<%= arch %>', expand: true, src: ['*'], options: { mode: true}, From 803694f41bb606756a15c0e776f4e12b341f8e48 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Thu, 10 May 2018 10:39:29 +0200 Subject: [PATCH 2/8] build: saves artifacts with the build --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a0be0575e1..4b074688c97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,6 +95,8 @@ jobs: - dist/grafana* - scripts/*.sh - scripts/publish + - store_artifacts: + path: dist build-enterprise: docker: From 3933cb6bf6cacb5c1db04d5d86c1488283ceb4dc Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Fri, 11 May 2018 17:39:37 +0200 Subject: [PATCH 3/8] build: publisher updated to support more architectures and OSs. --- scripts/build/publish.go | 84 ++++++++++++++++++--------- scripts/build/publish_test.go | 104 ++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 27 deletions(-) create mode 100644 scripts/build/publish_test.go diff --git a/scripts/build/publish.go b/scripts/build/publish.go index 500cb0d48ab..0d7217bfb9f 100644 --- a/scripts/build/publish.go +++ b/scripts/build/publish.go @@ -18,8 +18,15 @@ import ( var apiUrl = flag.String("apiUrl", "https://grafana.com/api", "api url") var apiKey = flag.String("apiKey", "", "api key") var version = "" -var versionRe = regexp.MustCompile(`grafana-(.*)\.(linux|windows)`) +var versionRe = regexp.MustCompile(`grafana-(.*)(\.|_)(arm64|armv7|darwin|linux|windows|x86_64)`) +var debVersionRe = regexp.MustCompile(`grafana_(.*)_(arm64|armv7|amd64)\.deb`) var builds = []build{} +var architectureMapping = map[string]string{ + "amd64":"amd64", + "armv7":"armv7", + "arm64":"arm64", + "x86_64":"amd64", +} func main() { flag.Parse() @@ -60,17 +67,54 @@ func main() { } } -func packageWalker(path string, f os.FileInfo, err error) error { - if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") { - return nil - } - - log.Printf("Finding package file %s", f.Name()) - result := versionRe.FindSubmatch([]byte(f.Name())) +func mapPackage(path string, name string, shaBytes []byte) (build, error) { + log.Printf("Finding package file %s", name) + result := versionRe.FindSubmatch([]byte(name)) + debResult := debVersionRe.FindSubmatch([]byte(name)) if len(result) > 0 { version = string(result[1]) log.Printf("Version detected: %v", version) + } else if (len(debResult) > 0) { + version = string(debResult[1]) + } + + os := "" + if strings.Contains(name, "linux") { + os = "linux" + } + if strings.HasSuffix(name, "windows-amd64.zip") { + os = "win" + } + if strings.HasSuffix(name, "darwin-amd64.tar.gz") { + os = "darwin" + } + if strings.HasSuffix(name, ".rpm") { + os = "rhel" + } + if strings.HasSuffix(name, ".deb") { + os = "deb" + } + + arch := "" + for archListed, archReal := range architectureMapping { + if strings.Contains(name, archListed) { + arch = archReal + break + } + } + + return build{ + Os: os, + Arch: arch, + Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + name, + Sha256: string(shaBytes), + }, nil +} + +func packageWalker(path string, f os.FileInfo, err error) error { + if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") { + return nil } shaBytes, err := ioutil.ReadFile(path + ".sha256") @@ -78,27 +122,13 @@ func packageWalker(path string, f os.FileInfo, err error) error { log.Fatalf("Failed to read sha256 file %v", err) } - os := "" - if strings.Contains(f.Name(), "linux-x64.tar.gz") { - os = "linux" - } - if strings.HasSuffix(f.Name(), "windows-x64.zip") { - os = "win" - } - if strings.HasSuffix(f.Name(), ".rpm") { - os = "rhel" - } - if strings.HasSuffix(f.Name(), ".deb") { - os = "deb" + build, err := mapPackage(path, f.Name(), shaBytes) + + if err != nil { + return err } - builds = append(builds, build{ - Os: os, - Arch: "amd64", - Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + f.Name(), - Sha256: string(shaBytes), - }) - + builds = append(builds, build) return nil } diff --git a/scripts/build/publish_test.go b/scripts/build/publish_test.go new file mode 100644 index 00000000000..658dc54be9b --- /dev/null +++ b/scripts/build/publish_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "testing" +) + +type testPackage struct { + path string + version string + os string + arch string +} + +var testData = []testPackage{ + { + path: "grafana-5.2.0-474pre1.arm64.rpm", + version: "5.2.0-474pre1", + os: "rhel", + arch: "arm64", + }, + { + path: "grafana-5.2.0-474pre1.armv7.rpm", + version: "5.2.0-474pre1", + os: "rhel", + arch: "armv7", + }, + { + path: "grafana-5.2.0-474pre1.darwin-amd64.tar.gz", + version: "5.2.0-474pre1", + os: "darwin", + arch: "amd64", + }, + { + path: "grafana-5.2.0-474pre1.linux-amd64.tar.gz", + version: "5.2.0-474pre1", + os: "linux", + arch: "amd64", + }, + { + path: "grafana-5.2.0-474pre1.linux-arm64.tar.gz", + version: "5.2.0-474pre1", + os: "linux", + arch: "arm64", + }, + { + path: "grafana-5.2.0-474pre1.linux-armv7.tar.gz", + version: "5.2.0-474pre1", + os: "linux", + arch: "armv7", + }, + { + path: "grafana-5.2.0-474pre1.windows-amd64.zip", + version: "5.2.0-474pre1", + os: "win", + arch: "amd64", + }, + { + path: "grafana-5.2.0-474pre1.x86_64.rpm", + version: "5.2.0-474pre1", + os: "rhel", + arch: "amd64", + }, + { + path: "grafana_5.2.0-474pre1_amd64.deb", + version: "5.2.0-474pre1", + os: "deb", + arch: "amd64", + }, + { + path: "grafana_5.2.0-474pre1_arm64.deb", + version: "5.2.0-474pre1", + os: "deb", + arch: "arm64", + }, + { + path: "grafana_5.2.0-474pre1_armv7.deb", + version: "5.2.0-474pre1", + os: "deb", + arch: "armv7", + }, +} + +func TestFileWalker(t *testing.T) { + for _, packageInfo := range testData { + version = "" + actualPackageInfo, err := mapPackage(packageInfo.path, packageInfo.path, []byte{}) + if err != nil { + t.Error(err) + continue + } + + if version != packageInfo.version { + t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, version, packageInfo.version) + } + + if actualPackageInfo.Os != packageInfo.os { + t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, actualPackageInfo.Os, packageInfo.os) + } + + if actualPackageInfo.Arch != packageInfo.arch { + t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, actualPackageInfo.Arch, packageInfo.arch) + } + } +} From 7ca346e9f03b7d9d1e49b2fc81c3b0729c695ea3 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Fri, 18 May 2018 11:38:31 +0200 Subject: [PATCH 4/8] build: downloads and bundles phantomjs for darwin and windows. --- .circleci/config.yml | 9 +++++++++ scripts/build/build.sh | 26 ++++++++++++++++++++++++++ scripts/build/download-phantomjs.sh | 17 +++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100755 scripts/build/download-phantomjs.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b074688c97..e4b123073a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,6 +77,15 @@ jobs: - run: name: prepare build tools command: '/tmp/bootstrap.sh' + - restore_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + - run: + name: download phantomjs binaries + command: './scripts/build/download-phantomjs.sh' + - save_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + paths: + - /tmp/phantomjs - run: name: build and package grafana command: './scripts/build/build.sh' diff --git a/scripts/build/build.sh b/scripts/build/build.sh index ab021d57692..7c08d2c333c 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -47,7 +47,20 @@ if [ "$CIRCLE_TAG" != "" ]; then go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only + + if [ -d '/tmp/phantomjs/windows' ]; then + cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe + rm tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only else echo "Building frontend for $CIRCLE_BRANCH" @@ -56,6 +69,19 @@ else go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + + if [ -d '/tmp/phantomjs/windows' ]; then + cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe + rm tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for windows missing!' + fi go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only fi diff --git a/scripts/build/download-phantomjs.sh b/scripts/build/download-phantomjs.sh new file mode 100755 index 00000000000..e346faf8765 --- /dev/null +++ b/scripts/build/download-phantomjs.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +if [ ! -d '/tmp/phantomjs' ]; then + _version="2.1.1" + + curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-windows.zip > /tmp/phantomjs-win.zip + curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-macosx.zip > /tmp/phantomjs-mac.zip + + cd /tmp + unzip /tmp/phantomjs-win.zip + unzip /tmp/phantomjs-mac.zip + + mkdir -p /tmp/phantomjs/windows /tmp/phantomjs/darwin + + cp /tmp/phantomjs-$_version-windows/bin/phantomjs.exe /tmp/phantomjs/windows/phantomjs.exe + cp /tmp/phantomjs-$_version-macosx/bin/phantomjs /tmp/phantomjs/darwin/phantomjs +fi From 7b1e41abc6d3621bb6f8b5f025c8fc5e5903406e Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 21 May 2018 14:06:01 +0200 Subject: [PATCH 5/8] build: crosscompilation for nightlies and releases. --- .circleci/config.yml | 126 +++++++++++++++++++++++----- scripts/build/build-all.sh | 87 +++++++++++++++++++ scripts/build/build.sh | 46 ---------- scripts/build/download-phantomjs.sh | 2 +- 4 files changed, 193 insertions(+), 68 deletions(-) create mode 100755 scripts/build/build-all.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index e4b123073a4..3bdfe2efdcb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,36 @@ jobs: command: './scripts/circle-test-backend.sh' build: + docker: + - image: grafana/build-container:crosscompile + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' + - run: + name: build and package grafana + command: './scripts/build/build.sh' + - run: + name: sign packages + command: './scripts/build/sign_packages.sh' + - run: + name: sha-sum packages + command: 'go run build.go sha-dist' + - run: + name: Build Grafana.com publisher + command: 'go build -o scripts/publish scripts/build/publish.go' + - persist_to_workspace: + root: . + paths: + - dist/grafana* + - scripts/*.sh + - scripts/publish + - store_artifacts: + path: dist + + build-all: docker: - image: grafana/build-container:crosscompile working_directory: /go/src/github.com/grafana/grafana @@ -88,7 +118,7 @@ jobs: - /tmp/phantomjs - run: name: build and package grafana - command: './scripts/build/build.sh' + command: './scripts/build/build-all.sh' - run: name: sign packages command: './scripts/build/sign_packages.sh' @@ -168,45 +198,99 @@ workflows: version: 2 test-and-build: jobs: - - codespell: - filters: - tags: - only: /.*/ - - gometalinter: - filters: - tags: - only: /.*/ - build: filters: tags: - only: /.*/ + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - codespell: + filters: + tags: + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - gometalinter: + filters: + tags: + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - test-frontend: filters: tags: - only: /.*/ + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - test-backend: filters: tags: - only: /.*/ + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - deploy-master: requires: - test-backend - test-frontend - build + - codespell + - gometalinter filters: branches: only: master - - deploy-release: - requires: - - test-backend - - test-frontend - - build + release: + jobs: + - build-all: filters: branches: ignore: /.*/ tags: only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - # - build-enterprise: - # filters: - # tags: - # only: /.*/ + - codespell: + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - gometalinter: + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - test-frontend: + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - test-backend: + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - deploy-release: + requires: + - build-all + - test-backend + - test-frontend + - codespell + - gometalinter + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + nightly-master: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + - crosscompile + jobs: + - build-all + - codespell + - gometalinter + - test-frontend + - test-backend + - deploy-master: + requires: + - build-all + - codespell + - gometalinter + - test-frontend + - test-backend diff --git a/scripts/build/build-all.sh b/scripts/build/build-all.sh new file mode 100755 index 00000000000..7c08d2c333c --- /dev/null +++ b/scripts/build/build-all.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# +# This script is executed from within the container. +# + +CCARMV7=arm-linux-gnueabihf-gcc +CCARM64=aarch64-linux-gnu-gcc +CCOSX64=/tmp/osxcross/target/bin/o64-clang +CCWIN64=x86_64-w64-mingw32-gcc +CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc + +GOPATH=/go +REPO_PATH=$GOPATH/src/github.com/grafana/grafana + +cd /go/src/github.com/grafana/grafana +echo "current dir: $(pwd)" + +if [ "$CIRCLE_TAG" != "" ]; then + echo "Building releases from tag $CIRCLE_TAG" + go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build + go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build + go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build + go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build + CC=${CCX64} go run build.go -includeBuildNumber=false build +else + echo "Building incremental build for $CIRCLE_BRANCH" + go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build + CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build +fi + +yarn install --pure-lockfile --no-progress + +echo "current dir: $(pwd)" + +if [ -d "dist" ]; then + rm -rf dist +fi + +if [ "$CIRCLE_TAG" != "" ]; then + echo "Building frontend from tag $CIRCLE_TAG" + go run build.go -includeBuildNumber=false build-frontend + echo "Packaging a release from tag $CIRCLE_TAG" + go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest + go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only + go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only + + if [ -d '/tmp/phantomjs/windows' ]; then + cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe + rm tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only +else + echo "Building frontend for $CIRCLE_BRANCH" + go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend + echo "Packaging incremental build for $CIRCLE_BRANCH" + go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest + go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + + if [ -d '/tmp/phantomjs/windows' ]; then + cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe + rm tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for windows missing!' + fi + go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only +fi diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 7c08d2c333c..cee80822cac 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -4,10 +4,6 @@ # This script is executed from within the container. # -CCARMV7=arm-linux-gnueabihf-gcc -CCARM64=aarch64-linux-gnu-gcc -CCOSX64=/tmp/osxcross/target/bin/o64-clang -CCWIN64=x86_64-w64-mingw32-gcc CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc GOPATH=/go @@ -18,17 +14,9 @@ echo "current dir: $(pwd)" if [ "$CIRCLE_TAG" != "" ]; then echo "Building releases from tag $CIRCLE_TAG" - go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build - go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build - go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build - go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build CC=${CCX64} go run build.go -includeBuildNumber=false build else echo "Building incremental build for $CIRCLE_BRANCH" - go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build - go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build - go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build - go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build fi @@ -45,43 +33,9 @@ if [ "$CIRCLE_TAG" != "" ]; then go run build.go -includeBuildNumber=false build-frontend echo "Packaging a release from tag $CIRCLE_TAG" go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest - go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only - go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only - - if [ -d '/tmp/phantomjs/darwin' ]; then - cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs - else - echo 'PhantomJS binaries for darwin missing!' - fi - go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only - - if [ -d '/tmp/phantomjs/windows' ]; then - cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe - rm tools/phantomjs/phantomjs - else - echo 'PhantomJS binaries for darwin missing!' - fi - go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only else echo "Building frontend for $CIRCLE_BRANCH" go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend echo "Packaging incremental build for $CIRCLE_BRANCH" go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest - go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only - go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only - - if [ -d '/tmp/phantomjs/darwin' ]; then - cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs - else - echo 'PhantomJS binaries for darwin missing!' - fi - go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only - - if [ -d '/tmp/phantomjs/windows' ]; then - cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe - rm tools/phantomjs/phantomjs - else - echo 'PhantomJS binaries for windows missing!' - fi - go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only fi diff --git a/scripts/build/download-phantomjs.sh b/scripts/build/download-phantomjs.sh index e346faf8765..5738469cd2b 100755 --- a/scripts/build/download-phantomjs.sh +++ b/scripts/build/download-phantomjs.sh @@ -1,7 +1,7 @@ #!/bin/bash -e if [ ! -d '/tmp/phantomjs' ]; then - _version="2.1.1" + _version="2.1.16" curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-windows.zip > /tmp/phantomjs-win.zip curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-macosx.zip > /tmp/phantomjs-mac.zip From 5f98982a09ccb21fa519a1196e0316ad12fda89b Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 21 May 2018 15:57:57 +0200 Subject: [PATCH 6/8] build: clean up the workflow filters. --- .circleci/config.yml | 67 +++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bdfe2efdcb..e9660c18d58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,14 @@ +aliases: + # Workflow filters + - &filter-only-release + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - &filter-not-release + tags: + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + version: 2 jobs: @@ -199,25 +210,15 @@ workflows: test-and-build: jobs: - build: - filters: - tags: - ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-not-release - codespell: - filters: - tags: - ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-not-release - gometalinter: - filters: - tags: - ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-not-release - test-frontend: - filters: - tags: - ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-not-release - test-backend: - filters: - tags: - ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-not-release - deploy-master: requires: - test-backend @@ -231,35 +232,15 @@ workflows: release: jobs: - build-all: - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-only-release - codespell: - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-only-release - gometalinter: - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-only-release - test-frontend: - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-only-release - test-backend: - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-only-release - deploy-release: requires: - build-all @@ -267,11 +248,7 @@ workflows: - test-frontend - codespell - gometalinter - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + filters: *filter-only-release nightly-master: triggers: - schedule: From 1b85b2fca7b821bb8826e0926703bb5580cbb4b5 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 21 May 2018 16:11:53 +0200 Subject: [PATCH 7/8] build: removes deploy from nightly while testing it. --- .circleci/config.yml | 10 +--------- scripts/build/download-phantomjs.sh | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9660c18d58..c56df8e9cb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,6 @@ jobs: command: 'sudo npm install -g yarn --quiet' - restore_cache: key: dependency-cache-{{ checksum "yarn.lock" }} - # Could we skip this step if the cache has been restored? `[ -d node_modules ] || yarn install ...` should be able to apply to build step as well - run: name: yarn install command: 'yarn install --pure-lockfile --no-progress' @@ -221,9 +220,9 @@ workflows: filters: *filter-not-release - deploy-master: requires: + - build - test-backend - test-frontend - - build - codespell - gometalinter filters: @@ -264,10 +263,3 @@ workflows: - gometalinter - test-frontend - test-backend - - deploy-master: - requires: - - build-all - - codespell - - gometalinter - - test-frontend - - test-backend diff --git a/scripts/build/download-phantomjs.sh b/scripts/build/download-phantomjs.sh index 5738469cd2b..e346faf8765 100755 --- a/scripts/build/download-phantomjs.sh +++ b/scripts/build/download-phantomjs.sh @@ -1,7 +1,7 @@ #!/bin/bash -e if [ ! -d '/tmp/phantomjs' ]; then - _version="2.1.16" + _version="2.1.1" curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-windows.zip > /tmp/phantomjs-win.zip curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-macosx.zip > /tmp/phantomjs-mac.zip From bea4741a12c51335b7ac5ed817e83b1f755d8be6 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Wed, 23 May 2018 16:03:21 +0200 Subject: [PATCH 8/8] build: always build for all platforms. --- .circleci/config.yml | 51 +++----------------------------------------- appveyor.yml | 13 ----------- 2 files changed, 3 insertions(+), 61 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c56df8e9cb5..c92a68bf99d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,39 +78,9 @@ jobs: name: build backend and run go tests command: './scripts/circle-test-backend.sh' - build: - docker: - - image: grafana/build-container:crosscompile - working_directory: /go/src/github.com/grafana/grafana - steps: - - checkout - - run: - name: prepare build tools - command: '/tmp/bootstrap.sh' - - run: - name: build and package grafana - command: './scripts/build/build.sh' - - run: - name: sign packages - command: './scripts/build/sign_packages.sh' - - run: - name: sha-sum packages - command: 'go run build.go sha-dist' - - run: - name: Build Grafana.com publisher - command: 'go build -o scripts/publish scripts/build/publish.go' - - persist_to_workspace: - root: . - paths: - - dist/grafana* - - scripts/*.sh - - scripts/publish - - store_artifacts: - path: dist - build-all: docker: - - image: grafana/build-container:crosscompile + - image: grafana/build-container:1.0.0 working_directory: /go/src/github.com/grafana/grafana steps: - checkout @@ -208,7 +178,7 @@ workflows: version: 2 test-and-build: jobs: - - build: + - build-all: filters: *filter-not-release - codespell: filters: *filter-not-release @@ -220,7 +190,7 @@ workflows: filters: *filter-not-release - deploy-master: requires: - - build + - build-all - test-backend - test-frontend - codespell @@ -248,18 +218,3 @@ workflows: - codespell - gometalinter filters: *filter-only-release - nightly-master: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - - crosscompile - jobs: - - build-all - - codespell - - gometalinter - - test-frontend - - test-backend diff --git a/appveyor.yml b/appveyor.yml index a71eb9f81b4..5cdec1b8bf5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,16 +38,3 @@ artifacts: - path: grafana-*windows-*.* name: binzip type: zip - -deploy: - - provider: Environment - name: GrafanaReleaseMaster - on: - buildType: master - - - provider: Environment - name: GrafanaReleaseRelease - on: - buildType: release - -