diff --git a/.circleci/config.yml b/.circleci/config.yml index bad5a7c1cd0..c92a68bf99d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,14 @@ +aliases: + # Workflow filters + - &filter-only-release + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - &filter-not-release + tags: + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + version: 2 jobs: @@ -46,7 +57,6 @@ jobs: command: 'sudo npm install -g yarn --quiet' - restore_cache: key: dependency-cache-{{ checksum "yarn.lock" }} - # Could we skip this step if the cache has been restored? `[ -d node_modules ] || yarn install ...` should be able to apply to build step as well - run: name: yarn install command: 'yarn install --pure-lockfile --no-progress' @@ -68,15 +78,27 @@ jobs: name: build backend and run go tests command: './scripts/circle-test-backend.sh' - build: + build-all: docker: - - image: grafana/build-container:v0.1 + - image: grafana/build-container:1.0.0 working_directory: /go/src/github.com/grafana/grafana steps: - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' + - restore_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + - run: + name: download phantomjs binaries + command: './scripts/build/download-phantomjs.sh' + - save_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + paths: + - /tmp/phantomjs - run: name: build and package grafana - command: './scripts/build/build.sh' + command: './scripts/build/build-all.sh' - run: name: sign packages command: './scripts/build/sign_packages.sh' @@ -92,6 +114,8 @@ jobs: - dist/grafana* - scripts/*.sh - scripts/publish + - store_artifacts: + path: dist build-enterprise: docker: @@ -154,45 +178,43 @@ workflows: version: 2 test-and-build: jobs: + - build-all: + filters: *filter-not-release - codespell: - filters: - tags: - only: /.*/ + filters: *filter-not-release - gometalinter: - filters: - tags: - only: /.*/ - - build: - filters: - tags: - only: /.*/ + filters: *filter-not-release - test-frontend: - filters: - tags: - only: /.*/ + filters: *filter-not-release - test-backend: - filters: - tags: - only: /.*/ + filters: *filter-not-release - deploy-master: requires: + - build-all - test-backend - test-frontend - - build + - codespell + - gometalinter filters: branches: only: master + release: + jobs: + - build-all: + filters: *filter-only-release + - codespell: + filters: *filter-only-release + - gometalinter: + filters: *filter-only-release + - test-frontend: + filters: *filter-only-release + - test-backend: + filters: *filter-only-release - deploy-release: requires: + - build-all - test-backend - test-frontend - - build - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - # - build-enterprise: - # filters: - # tags: - # only: /.*/ + - codespell + - gometalinter + filters: *filter-only-release diff --git a/Gruntfile.js b/Gruntfile.js index a0607ef49dc..23276e8a122 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -12,6 +12,10 @@ module.exports = function (grunt) { platform: process.platform.replace('win32', 'windows'), }; + if (grunt.option('platform')) { + config.platform = grunt.option('platform'); + } + if (grunt.option('arch')) { config.arch = grunt.option('arch'); } else { diff --git a/appveyor.yml b/appveyor.yml index a71eb9f81b4..5cdec1b8bf5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,16 +38,3 @@ artifacts: - path: grafana-*windows-*.* name: binzip type: zip - -deploy: - - provider: Environment - name: GrafanaReleaseMaster - on: - buildType: master - - - provider: Environment - name: GrafanaReleaseRelease - on: - buildType: release - - diff --git a/build.go b/build.go index 7e7183b8b83..f9611c99a47 100644 --- a/build.go +++ b/build.go @@ -27,8 +27,7 @@ var ( goarch string goos string gocc string - gocxx string - cgo string + cgo bool pkgArch string version string = "v1" // deb & rpm does not support semver so have to handle their version a little differently @@ -53,8 +52,7 @@ func main() { flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH") flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS") flag.StringVar(&gocc, "cc", "", "CC") - flag.StringVar(&gocxx, "cxx", "", "CXX") - flag.StringVar(&cgo, "cgo-enabled", "", "CGO_ENABLED") + flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo") flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH") flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary") flag.BoolVar(&race, "race", race, "Use race detector") @@ -93,20 +91,24 @@ func main() { build("grafana-server", "./pkg/cmd/grafana-server", []string{}) case "build": - clean() + //clean() for _, binary := range binaries { build(binary, "./pkg/cmd/"+binary, []string{}) } + case "build-frontend": + grunt(gruntBuildArg("build")...) + case "test": test("./pkg/...") grunt("test") case "package": - grunt(gruntBuildArg("release")...) - if runtime.GOOS != "windows" { - createLinuxPackages() - } + grunt(gruntBuildArg("build")...) + packageGrafana() + + case "package-only": + packageGrafana() case "pkg-rpm": grunt(gruntBuildArg("release")...) @@ -131,6 +133,22 @@ func main() { } } +func packageGrafana() { + platformArg := fmt.Sprintf("--platform=%v", goos) + previousPkgArch := pkgArch + if pkgArch == "" { + pkgArch = goarch + } + postProcessArgs := gruntBuildArg("package") + postProcessArgs = append(postProcessArgs, platformArg) + grunt(postProcessArgs...) + pkgArch = previousPkgArch + + if goos == "linux" && goarch == "amd64"{ + createLinuxPackages() + } +} + func makeLatestDistCopies() { files, err := ioutil.ReadDir("dist") if err != nil { @@ -138,9 +156,9 @@ func makeLatestDistCopies() { } latestMapping := map[string]string{ - ".deb": "dist/grafana_latest_amd64.deb", - ".rpm": "dist/grafana-latest-1.x86_64.rpm", - ".tar.gz": "dist/grafana-latest.linux-x64.tar.gz", + "_amd64.deb": "dist/grafana_latest_amd64.deb", + ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm", + ".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz", } for _, file := range files { @@ -386,7 +404,8 @@ func test(pkg string) { } func build(binaryName, pkg string, tags []string) { - binary := "./bin/" + binaryName + binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName) + if goos == "windows" { binary += ".exe" } @@ -408,6 +427,7 @@ func build(binaryName, pkg string, tags []string) { if !isDev { setBuildEnv() runPrint("go", "version") + fmt.Printf("Targeting %s/%s\n", goos, goarch) } runPrint("go", args...) @@ -451,6 +471,14 @@ func clean() { func setBuildEnv() { os.Setenv("GOOS", goos) + if goos == "windows" { + // require windows >=7 + os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601") + } + if goarch != "amd64" || goos != "linux" { + // needed for all other archs + cgo = true + } if strings.HasPrefix(goarch, "armv") { os.Setenv("GOARCH", "arm") os.Setenv("GOARM", goarch[4:]) @@ -460,15 +488,12 @@ func setBuildEnv() { if goarch == "386" { os.Setenv("GO386", "387") } - if cgo != "" { - os.Setenv("CGO_ENABLED", cgo) + if cgo { + os.Setenv("CGO_ENABLED", "1") } if gocc != "" { os.Setenv("CC", gocc) } - if gocxx != "" { - os.Setenv("CXX", gocxx) - } } func getGitSha() string { diff --git a/scripts/build/build-all.sh b/scripts/build/build-all.sh new file mode 100755 index 00000000000..7c08d2c333c --- /dev/null +++ b/scripts/build/build-all.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# +# This script is executed from within the container. +# + +CCARMV7=arm-linux-gnueabihf-gcc +CCARM64=aarch64-linux-gnu-gcc +CCOSX64=/tmp/osxcross/target/bin/o64-clang +CCWIN64=x86_64-w64-mingw32-gcc +CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc + +GOPATH=/go +REPO_PATH=$GOPATH/src/github.com/grafana/grafana + +cd /go/src/github.com/grafana/grafana +echo "current dir: $(pwd)" + +if [ "$CIRCLE_TAG" != "" ]; then + echo "Building releases from tag $CIRCLE_TAG" + go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build + go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build + go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build + go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build + CC=${CCX64} go run build.go -includeBuildNumber=false build +else + echo "Building incremental build for $CIRCLE_BRANCH" + go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build + CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build +fi + +yarn install --pure-lockfile --no-progress + +echo "current dir: $(pwd)" + +if [ -d "dist" ]; then + rm -rf dist +fi + +if [ "$CIRCLE_TAG" != "" ]; then + echo "Building frontend from tag $CIRCLE_TAG" + go run build.go -includeBuildNumber=false build-frontend + echo "Packaging a release from tag $CIRCLE_TAG" + go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest + go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only + go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only + + if [ -d '/tmp/phantomjs/windows' ]; then + cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe + rm tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only +else + echo "Building frontend for $CIRCLE_BRANCH" + go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend + echo "Packaging incremental build for $CIRCLE_BRANCH" + go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest + go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + + if [ -d '/tmp/phantomjs/darwin' ]; then + cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for darwin missing!' + fi + go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + + if [ -d '/tmp/phantomjs/windows' ]; then + cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe + rm tools/phantomjs/phantomjs + else + echo 'PhantomJS binaries for windows missing!' + fi + go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only +fi diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 369d434d4f5..cee80822cac 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -4,6 +4,8 @@ # This script is executed from within the container. # +CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc + GOPATH=/go REPO_PATH=$GOPATH/src/github.com/grafana/grafana @@ -11,23 +13,29 @@ cd /go/src/github.com/grafana/grafana echo "current dir: $(pwd)" if [ "$CIRCLE_TAG" != "" ]; then - echo "Building a release from tag $CIRCLE_TAG" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false build + echo "Building releases from tag $CIRCLE_TAG" + CC=${CCX64} go run build.go -includeBuildNumber=false build else echo "Building incremental build for $CIRCLE_BRANCH" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build + CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build fi yarn install --pure-lockfile --no-progress -source /etc/profile.d/rvm.sh - echo "current dir: $(pwd)" +if [ -d "dist" ]; then + rm -rf dist +fi + if [ "$CIRCLE_TAG" != "" ]; then + echo "Building frontend from tag $CIRCLE_TAG" + go run build.go -includeBuildNumber=false build-frontend echo "Packaging a release from tag $CIRCLE_TAG" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false package latest + go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest else + echo "Building frontend for $CIRCLE_BRANCH" + go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend echo "Packaging incremental build for $CIRCLE_BRANCH" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} package latest + go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest fi diff --git a/scripts/build/download-phantomjs.sh b/scripts/build/download-phantomjs.sh new file mode 100755 index 00000000000..e346faf8765 --- /dev/null +++ b/scripts/build/download-phantomjs.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +if [ ! -d '/tmp/phantomjs' ]; then + _version="2.1.1" + + curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-windows.zip > /tmp/phantomjs-win.zip + curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-macosx.zip > /tmp/phantomjs-mac.zip + + cd /tmp + unzip /tmp/phantomjs-win.zip + unzip /tmp/phantomjs-mac.zip + + mkdir -p /tmp/phantomjs/windows /tmp/phantomjs/darwin + + cp /tmp/phantomjs-$_version-windows/bin/phantomjs.exe /tmp/phantomjs/windows/phantomjs.exe + cp /tmp/phantomjs-$_version-macosx/bin/phantomjs /tmp/phantomjs/darwin/phantomjs +fi diff --git a/scripts/build/publish.go b/scripts/build/publish.go index 500cb0d48ab..0d7217bfb9f 100644 --- a/scripts/build/publish.go +++ b/scripts/build/publish.go @@ -18,8 +18,15 @@ import ( var apiUrl = flag.String("apiUrl", "https://grafana.com/api", "api url") var apiKey = flag.String("apiKey", "", "api key") var version = "" -var versionRe = regexp.MustCompile(`grafana-(.*)\.(linux|windows)`) +var versionRe = regexp.MustCompile(`grafana-(.*)(\.|_)(arm64|armv7|darwin|linux|windows|x86_64)`) +var debVersionRe = regexp.MustCompile(`grafana_(.*)_(arm64|armv7|amd64)\.deb`) var builds = []build{} +var architectureMapping = map[string]string{ + "amd64":"amd64", + "armv7":"armv7", + "arm64":"arm64", + "x86_64":"amd64", +} func main() { flag.Parse() @@ -60,17 +67,54 @@ func main() { } } -func packageWalker(path string, f os.FileInfo, err error) error { - if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") { - return nil - } - - log.Printf("Finding package file %s", f.Name()) - result := versionRe.FindSubmatch([]byte(f.Name())) +func mapPackage(path string, name string, shaBytes []byte) (build, error) { + log.Printf("Finding package file %s", name) + result := versionRe.FindSubmatch([]byte(name)) + debResult := debVersionRe.FindSubmatch([]byte(name)) if len(result) > 0 { version = string(result[1]) log.Printf("Version detected: %v", version) + } else if (len(debResult) > 0) { + version = string(debResult[1]) + } + + os := "" + if strings.Contains(name, "linux") { + os = "linux" + } + if strings.HasSuffix(name, "windows-amd64.zip") { + os = "win" + } + if strings.HasSuffix(name, "darwin-amd64.tar.gz") { + os = "darwin" + } + if strings.HasSuffix(name, ".rpm") { + os = "rhel" + } + if strings.HasSuffix(name, ".deb") { + os = "deb" + } + + arch := "" + for archListed, archReal := range architectureMapping { + if strings.Contains(name, archListed) { + arch = archReal + break + } + } + + return build{ + Os: os, + Arch: arch, + Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + name, + Sha256: string(shaBytes), + }, nil +} + +func packageWalker(path string, f os.FileInfo, err error) error { + if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") { + return nil } shaBytes, err := ioutil.ReadFile(path + ".sha256") @@ -78,27 +122,13 @@ func packageWalker(path string, f os.FileInfo, err error) error { log.Fatalf("Failed to read sha256 file %v", err) } - os := "" - if strings.Contains(f.Name(), "linux-x64.tar.gz") { - os = "linux" - } - if strings.HasSuffix(f.Name(), "windows-x64.zip") { - os = "win" - } - if strings.HasSuffix(f.Name(), ".rpm") { - os = "rhel" - } - if strings.HasSuffix(f.Name(), ".deb") { - os = "deb" + build, err := mapPackage(path, f.Name(), shaBytes) + + if err != nil { + return err } - builds = append(builds, build{ - Os: os, - Arch: "amd64", - Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + f.Name(), - Sha256: string(shaBytes), - }) - + builds = append(builds, build) return nil } diff --git a/scripts/build/publish_test.go b/scripts/build/publish_test.go new file mode 100644 index 00000000000..658dc54be9b --- /dev/null +++ b/scripts/build/publish_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "testing" +) + +type testPackage struct { + path string + version string + os string + arch string +} + +var testData = []testPackage{ + { + path: "grafana-5.2.0-474pre1.arm64.rpm", + version: "5.2.0-474pre1", + os: "rhel", + arch: "arm64", + }, + { + path: "grafana-5.2.0-474pre1.armv7.rpm", + version: "5.2.0-474pre1", + os: "rhel", + arch: "armv7", + }, + { + path: "grafana-5.2.0-474pre1.darwin-amd64.tar.gz", + version: "5.2.0-474pre1", + os: "darwin", + arch: "amd64", + }, + { + path: "grafana-5.2.0-474pre1.linux-amd64.tar.gz", + version: "5.2.0-474pre1", + os: "linux", + arch: "amd64", + }, + { + path: "grafana-5.2.0-474pre1.linux-arm64.tar.gz", + version: "5.2.0-474pre1", + os: "linux", + arch: "arm64", + }, + { + path: "grafana-5.2.0-474pre1.linux-armv7.tar.gz", + version: "5.2.0-474pre1", + os: "linux", + arch: "armv7", + }, + { + path: "grafana-5.2.0-474pre1.windows-amd64.zip", + version: "5.2.0-474pre1", + os: "win", + arch: "amd64", + }, + { + path: "grafana-5.2.0-474pre1.x86_64.rpm", + version: "5.2.0-474pre1", + os: "rhel", + arch: "amd64", + }, + { + path: "grafana_5.2.0-474pre1_amd64.deb", + version: "5.2.0-474pre1", + os: "deb", + arch: "amd64", + }, + { + path: "grafana_5.2.0-474pre1_arm64.deb", + version: "5.2.0-474pre1", + os: "deb", + arch: "arm64", + }, + { + path: "grafana_5.2.0-474pre1_armv7.deb", + version: "5.2.0-474pre1", + os: "deb", + arch: "armv7", + }, +} + +func TestFileWalker(t *testing.T) { + for _, packageInfo := range testData { + version = "" + actualPackageInfo, err := mapPackage(packageInfo.path, packageInfo.path, []byte{}) + if err != nil { + t.Error(err) + continue + } + + if version != packageInfo.version { + t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, version, packageInfo.version) + } + + if actualPackageInfo.Os != packageInfo.os { + t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, actualPackageInfo.Os, packageInfo.os) + } + + if actualPackageInfo.Arch != packageInfo.arch { + t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, actualPackageInfo.Arch, packageInfo.arch) + } + } +} diff --git a/scripts/grunt/release_task.js b/scripts/grunt/release_task.js index 4fbc41cfb08..f24ec64203a 100644 --- a/scripts/grunt/release_task.js +++ b/scripts/grunt/release_task.js @@ -3,13 +3,20 @@ var path = require('path'); module.exports = function(grunt) { "use strict"; - // build, then zip and upload to s3 + // build then zip grunt.registerTask('release', [ 'build', 'build-post-process', 'compress:release' ]); + // package into archives + grunt.registerTask('package', [ + 'clean:temp', + 'build-post-process', + 'compress:release' + ]); + grunt.registerTask('build-post-process', function() { grunt.config('copy.public_to_temp', { expand: true, @@ -18,7 +25,7 @@ module.exports = function(grunt) { dest: '<%= tempDir %>/public/', }); grunt.config('copy.backend_bin', { - cwd: 'bin', + cwd: 'bin/<%= platform %>-<%= arch %>', expand: true, src: ['*'], options: { mode: true},