Merge branch 'master' into azure-monitor

This commit is contained in:
Daniel Lee
2019-01-28 19:23:34 +01:00
378 changed files with 8050 additions and 3821 deletions
+21 -12
View File
@@ -30,9 +30,13 @@ fi
echo "Build arguments: $OPT"
go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
# build only amd64 for enterprise
if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
go run build.go -goarch armv6 -cc ${CCARMV7} ${OPT} build
go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
fi
go run build.go -goos windows -cc ${CCWIN64} ${OPT} build
@@ -62,22 +66,27 @@ echo "Packaging"
go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
#removing amd64 phantomjs bin for armv7/arm64 packages
rm tools/phantomjs/phantomjs
go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
if [ -d '/tmp/phantomjs/darwin' ]; then
cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs
else
echo 'PhantomJS binaries for darwin missing!'
# build only amd64 for enterprise
if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
go run build.go -goos linux -pkg-arch armv6 ${OPT} -skipRpm package-only
go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
if [ -d '/tmp/phantomjs/darwin' ]; then
cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs
else
echo 'PhantomJS binaries for darwin missing!'
fi
go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only
fi
go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only
if [ -d '/tmp/phantomjs/windows' ]; then
cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe
rm tools/phantomjs/phantomjs
rm tools/phantomjs/phantomjs || true
else
echo 'PhantomJS binaries for Windows missing!'
fi
go run build.go -goos windows -pkg-arch amd64 ${OPT} package-only
go run build.go latest
go run build.go latest
+2
View File
@@ -28,6 +28,7 @@ fi
echo "Build arguments: $OPT"
go run build.go -goarch armv6 -cc ${CCARMV7} ${OPT} build
go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
@@ -49,6 +50,7 @@ source /etc/profile.d/rvm.sh
echo "Packaging"
go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
go run build.go -goos linux -pkg-arch armv6 ${OPT} -skipRpm package-only
go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
+3 -3
View File
@@ -8,8 +8,6 @@ RUN git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-de
FROM circleci/python:2.7-stretch
ENV PATH=$PATH:/opt/google-cloud-sdk/bin
USER root
RUN pip install awscli && \
@@ -18,7 +16,9 @@ RUN pip install awscli && \
apt update && \
apt install -y createrepo expect && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
rm -rf /var/lib/apt/lists/* && \
ln -s /opt/google-cloud-sdk/bin/gsutil /usr/bin/gsutil && \
ln -s /opt/google-cloud-sdk/bin/gcloud /usr/bin/gcloud
COPY --from=0 /go/bin/aptly /usr/local/bin/aptly
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
_version="1.1.0"
_version="1.2.0"
_tag="grafana/grafana-ci-deploy:${_version}"
docker build -t $_tag .
-194
View File
@@ -1,194 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
var apiUrl = flag.String("apiUrl", "https://grafana.com/api", "api url")
var apiKey = flag.String("apiKey", "", "api key")
var version = ""
var versionRe = regexp.MustCompile(`grafana-(.*)(\.|_)(arm64|armhfp|aarch64|armv7|darwin|linux|windows|x86_64)`)
var debVersionRe = regexp.MustCompile(`grafana_(.*)_(arm64|armv7|armhf|amd64)\.deb`)
var builds = []build{}
var architectureMapping = map[string]string{
"armv7": "armv7",
"armhfp": "armv7",
"armhf": "armv7",
"arm64": "arm64",
"aarch64": "arm64",
"amd64": "amd64",
"x86_64": "amd64",
}
func main() {
flag.Parse()
if *apiKey == "" {
log.Fatalf("Require apiKey command line parameters")
}
err := filepath.Walk("dist", packageWalker)
if err != nil {
log.Fatalf("Cannot find any packages to publish, %v", err)
}
if version == "" {
log.Fatalf("No version found")
}
if len(builds) == 0 {
log.Fatalf("No builds found")
}
nightly := release{
Version: version,
ReleaseDate: time.Now(),
Stable: false,
Nightly: true,
Beta: false,
WhatsNewUrl: "",
ReleaseNotesUrl: "",
Builds: builds,
}
postRequest("/grafana/versions", nightly, fmt.Sprintf("Create Release %s", nightly.Version))
postRequest("/grafana/versions/"+nightly.Version, nightly, fmt.Sprintf("Update Release %s", nightly.Version))
for _, b := range nightly.Builds {
postRequest(fmt.Sprintf("/grafana/versions/%s/packages", nightly.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch))
postRequest(fmt.Sprintf("/grafana/versions/%s/packages/%s/%s", nightly.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch))
}
}
func mapPackage(path string, name string, shaBytes []byte) (build, error) {
log.Printf("Finding package file %s", name)
result := versionRe.FindSubmatch([]byte(name))
debResult := debVersionRe.FindSubmatch([]byte(name))
if len(result) > 0 {
version = string(result[1])
log.Printf("Version detected: %v", version)
} else if len(debResult) > 0 {
version = string(debResult[1])
} else {
return build{}, fmt.Errorf("Unable to figure out version from '%v'", name)
}
os := ""
if strings.Contains(name, "linux") {
os = "linux"
}
if strings.HasSuffix(name, "windows-amd64.zip") {
os = "win"
}
if strings.HasSuffix(name, "darwin-amd64.tar.gz") {
os = "darwin"
}
if strings.HasSuffix(name, ".rpm") {
os = "rhel"
}
if strings.HasSuffix(name, ".deb") {
os = "deb"
}
if os == "" {
return build{}, fmt.Errorf("Unable to figure out os from '%v'", name)
}
arch := ""
for archListed, archReal := range architectureMapping {
if strings.Contains(name, archListed) {
arch = archReal
break
}
}
if arch == "" {
return build{}, fmt.Errorf("Unable to figure out arch from '%v'", name)
}
return build{
Os: os,
Arch: arch,
Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + name,
Sha256: string(shaBytes),
}, nil
}
func packageWalker(path string, f os.FileInfo, err error) error {
if err != nil {
log.Printf("error: %v", err)
}
if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") {
return nil
}
shaBytes, err := ioutil.ReadFile(path + ".sha256")
if err != nil {
log.Fatalf("Failed to read sha256 file %v", err)
}
build, err := mapPackage(path, f.Name(), shaBytes)
if err != nil {
log.Printf("Could not map metadata from package: %v", err)
return nil
}
builds = append(builds, build)
return nil
}
func postRequest(url string, obj interface{}, desc string) {
jsonBytes, _ := json.Marshal(obj)
req, _ := http.NewRequest(http.MethodPost, (*apiUrl)+url, bytes.NewReader(jsonBytes))
req.Header.Add("Authorization", "Bearer "+(*apiKey))
req.Header.Add("Content-Type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatalf("error: %v", err)
}
if res.StatusCode == http.StatusOK {
log.Printf("Action: %s \t OK", desc)
} else {
if res.Body != nil {
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if strings.Contains(string(body), "already exists") || strings.Contains(string(body), "Nothing to update") {
log.Printf("Action: %s \t Already exists", desc)
} else {
log.Printf("Action: %s \t Failed - Status: %v", desc, res.Status)
log.Printf("Resp: %s", body)
log.Fatalf("Quitting")
}
}
}
}
type release struct {
Version string `json:"version"`
ReleaseDate time.Time `json:"releaseDate"`
Stable bool `json:"stable"`
Beta bool `json:"beta"`
Nightly bool `json:"nightly"`
WhatsNewUrl string `json:"whatsNewUrl"`
ReleaseNotesUrl string `json:"releaseNotesUrl"`
Builds []build `json:"-"`
}
type build struct {
Os string `json:"os"`
Url string `json:"url"`
Sha256 string `json:"sha256"`
Arch string `json:"arch"`
}
@@ -14,10 +14,10 @@ type releaseFromExternalContent struct {
artifactConfigurations []buildArtifact
}
func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, nightly bool) (*release, error) {
func (re releaseFromExternalContent) prepareRelease(baseArchiveURL, whatsNewURL string, releaseNotesURL string, nightly bool) (*release, error) {
version := re.rawVersion[1:]
beta := strings.Contains(version, "beta")
var rt ReleaseType
var rt releaseType
if beta {
rt = BETA
} else if nightly {
@@ -28,11 +28,11 @@ func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl
builds := []build{}
for _, ba := range re.artifactConfigurations {
sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, rt)))
sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getURL(baseArchiveURL, version, rt)))
if err != nil {
return nil, err
}
builds = append(builds, newBuild(baseArchiveUrl, ba, version, rt, sha256))
builds = append(builds, newBuild(baseArchiveURL, ba, version, rt, sha256))
}
r := release{
@@ -41,8 +41,8 @@ func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl
Stable: rt.stable(),
Beta: rt.beta(),
Nightly: rt.nightly(),
WhatsNewUrl: whatsNewUrl,
ReleaseNotesUrl: releaseNotesUrl,
WhatsNewURL: whatsNewURL,
ReleaseNotesURL: releaseNotesURL,
Builds: builds,
}
return &r, nil
@@ -52,9 +52,9 @@ type urlGetter interface {
getContents(url string) (string, error)
}
type getHttpContents struct{}
type getHTTPContents struct{}
func (getHttpContents) getContents(url string) (string, error) {
func (getHTTPContents) getContents(url string) (string, error) {
response, err := http.Get(url)
if err != nil {
return "", err
+12 -11
View File
@@ -2,7 +2,6 @@ package main
import (
"fmt"
"github.com/pkg/errors"
"io/ioutil"
"log"
"os"
@@ -10,6 +9,8 @@ import (
"regexp"
"strings"
"time"
"github.com/pkg/errors"
)
type releaseLocalSources struct {
@@ -17,11 +18,11 @@ type releaseLocalSources struct {
artifactConfigurations []buildArtifact
}
func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, nightly bool) (*release, error) {
func (r releaseLocalSources) prepareRelease(baseArchiveURL, whatsNewURL string, releaseNotesURL string, nightly bool) (*release, error) {
if !nightly {
return nil, errors.New("Local releases only supported for nightly builds.")
return nil, errors.New("Local releases only supported for nightly builds")
}
buildData := r.findBuilds(baseArchiveUrl)
buildData := r.findBuilds(baseArchiveURL)
rel := release{
Version: buildData.version,
@@ -29,8 +30,8 @@ func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string,
Stable: false,
Beta: false,
Nightly: nightly,
WhatsNewUrl: whatsNewUrl,
ReleaseNotesUrl: releaseNotesUrl,
WhatsNewURL: whatsNewURL,
ReleaseNotesURL: releaseNotesURL,
Builds: buildData.builds,
}
@@ -42,13 +43,13 @@ type buildData struct {
builds []build
}
func (r releaseLocalSources) findBuilds(baseArchiveUrl string) buildData {
func (r releaseLocalSources) findBuilds(baseArchiveURL string) buildData {
data := buildData{}
filepath.Walk(r.path, createBuildWalker(r.path, &data, r.artifactConfigurations, baseArchiveUrl))
filepath.Walk(r.path, createBuildWalker(r.path, &data, r.artifactConfigurations, baseArchiveURL))
return data
}
func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifact, baseArchiveUrl string) func(path string, f os.FileInfo, err error) error {
func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifact, baseArchiveURL string) func(path string, f os.FileInfo, err error) error {
return func(path string, f os.FileInfo, err error) error {
if err != nil {
log.Printf("error: %v", err)
@@ -73,7 +74,7 @@ func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifac
data.version = version
data.builds = append(data.builds, build{
Os: archive.os,
Url: archive.getUrl(baseArchiveUrl, version, NIGHTLY),
URL: archive.getURL(baseArchiveURL, version, NIGHTLY),
Sha256: string(shaBytes),
Arch: archive.arch,
})
@@ -90,5 +91,5 @@ func grabVersion(name string, suffix string) (string, error) {
return string(match[2]), nil
}
return "", errors.New("No version found.")
return "", errors.New("No version found")
}
+12 -12
View File
@@ -9,8 +9,8 @@ import (
func main() {
var version string
var whatsNewUrl string
var releaseNotesUrl string
var whatsNewURL string
var releaseNotesURL string
var dryRun bool
var enterprise bool
var fromLocal bool
@@ -18,8 +18,8 @@ func main() {
var apiKey string
flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)")
flag.StringVar(&whatsNewUrl, "wn", "", "What's new url (ex: --wn http://docs.grafana.org/guides/whats-new-in-v5-2/)")
flag.StringVar(&releaseNotesUrl, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)")
flag.StringVar(&whatsNewURL, "wn", "", "What's new url (ex: --wn http://docs.grafana.org/guides/whats-new-in-v5-2/)")
flag.StringVar(&releaseNotesURL, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)")
flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)")
flag.BoolVar(&dryRun, "dry-run", false, "--dry-run")
flag.BoolVar(&enterprise, "enterprise", false, "--enterprise")
@@ -37,7 +37,7 @@ func main() {
if dryRun {
log.Println("Dry-run has been enabled.")
}
var baseUrl string
var baseURL string
var builder releaseBuilder
var product string
@@ -46,7 +46,7 @@ func main() {
if enterprise {
product = "grafana-enterprise"
baseUrl = createBaseUrl(archiveProviderRoot, "enterprise", product, nightly)
baseURL = createBaseURL(archiveProviderRoot, "enterprise", product, nightly)
var err error
buildArtifacts, err = filterBuildArtifacts([]artifactFilter{
{os: "deb", arch: "amd64"},
@@ -61,7 +61,7 @@ func main() {
} else {
product = "grafana"
baseUrl = createBaseUrl(archiveProviderRoot, "oss", product, nightly)
baseURL = createBaseURL(archiveProviderRoot, "oss", product, nightly)
}
if fromLocal {
@@ -72,7 +72,7 @@ func main() {
}
} else {
builder = releaseFromExternalContent{
getter: getHttpContents{},
getter: getHTTPContents{},
rawVersion: version,
artifactConfigurations: buildArtifacts,
}
@@ -80,18 +80,18 @@ func main() {
p := publisher{
apiKey: apiKey,
apiUri: "https://grafana.com/api",
apiURI: "https://grafana.com/api",
product: product,
dryRun: dryRun,
enterprise: enterprise,
baseArchiveUrl: baseUrl,
baseArchiveURL: baseURL,
builder: builder,
}
if err := p.doRelease(whatsNewUrl, releaseNotesUrl, nightly); err != nil {
if err := p.doRelease(whatsNewURL, releaseNotesURL, nightly); err != nil {
log.Fatalf("error: %v", err)
}
}
func createBaseUrl(root string, bucketName string, product string, nightly bool) string {
func createBaseURL(root string, bucketName string, product string, nightly bool) string {
var subPath string
if nightly {
subPath = "master"
+35 -23
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/pkg/errors"
"io/ioutil"
"log"
"net/http"
@@ -14,20 +13,20 @@ import (
type publisher struct {
apiKey string
apiUri string
apiURI string
product string
dryRun bool
enterprise bool
baseArchiveUrl string
baseArchiveURL string
builder releaseBuilder
}
type releaseBuilder interface {
prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, nightly bool) (*release, error)
prepareRelease(baseArchiveURL, whatsNewURL string, releaseNotesURL string, nightly bool) (*release, error)
}
func (p *publisher) doRelease(whatsNewUrl string, releaseNotesUrl string, nightly bool) error {
currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl, nightly)
func (p *publisher) doRelease(whatsNewURL string, releaseNotesURL string, nightly bool) error {
currentRelease, err := p.builder.prepareRelease(p.baseArchiveURL, whatsNewURL, releaseNotesURL, nightly)
if err != nil {
return err
}
@@ -62,23 +61,26 @@ func (p *publisher) postRelease(r *release) error {
return nil
}
type ReleaseType int
type releaseType int
const (
STABLE ReleaseType = iota + 1
// STABLE is a release type constant
STABLE releaseType = iota + 1
// BETA is a release type constant
BETA
// NIGHTLY is a release type constant
NIGHTLY
)
func (rt ReleaseType) beta() bool {
func (rt releaseType) beta() bool {
return rt == BETA
}
func (rt ReleaseType) stable() bool {
func (rt releaseType) stable() bool {
return rt == STABLE
}
func (rt ReleaseType) nightly() bool {
func (rt releaseType) nightly() bool {
return rt == NIGHTLY
}
@@ -88,7 +90,7 @@ type buildArtifact struct {
urlPostfix string
}
func (t buildArtifact) getUrl(baseArchiveUrl, version string, releaseType ReleaseType) string {
func (t buildArtifact) getURL(baseArchiveURL, version string, releaseType releaseType) string {
prefix := "-"
rhelReleaseExtra := ""
@@ -100,7 +102,7 @@ func (t buildArtifact) getUrl(baseArchiveUrl, version string, releaseType Releas
rhelReleaseExtra = "-1"
}
url := strings.Join([]string{baseArchiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "")
url := strings.Join([]string{baseArchiveURL, prefix, version, rhelReleaseExtra, t.urlPostfix}, "")
return url
}
@@ -125,11 +127,21 @@ var completeBuildArtifactConfigurations = []buildArtifact{
arch: "armv7",
urlPostfix: "_armhf.deb",
},
{
os: "deb",
arch: "armv6",
urlPostfix: "_armel.deb",
},
{
os: "rhel",
arch: "armv7",
urlPostfix: ".armhfp.rpm",
},
{
os: "linux",
arch: "armv6",
urlPostfix: ".linux-armv6.tar.gz",
},
{
os: "linux",
arch: "armv7",
@@ -181,23 +193,23 @@ func filterBuildArtifacts(filters []artifactFilter) ([]buildArtifact, error) {
}
if !matched {
return nil, errors.New(fmt.Sprintf("No buildArtifact for os=%v, arch=%v", f.os, f.arch))
return nil, fmt.Errorf("No buildArtifact for os=%v, arch=%v", f.os, f.arch)
}
}
return artifacts, nil
}
func newBuild(baseArchiveUrl string, ba buildArtifact, version string, rt ReleaseType, sha256 string) build {
func newBuild(baseArchiveURL string, ba buildArtifact, version string, rt releaseType, sha256 string) build {
return build{
Os: ba.os,
Url: ba.getUrl(baseArchiveUrl, version, rt),
URL: ba.getURL(baseArchiveURL, version, rt),
Sha256: sha256,
Arch: ba.arch,
}
}
func (p *publisher) apiUrl(url string) string {
return fmt.Sprintf("%s/%s%s", p.apiUri, p.product, url)
func (p *publisher) apiURL(url string) string {
return fmt.Sprintf("%s/%s%s", p.apiURI, p.product, url)
}
func (p *publisher) postRequest(url string, obj interface{}, desc string) error {
@@ -207,12 +219,12 @@ func (p *publisher) postRequest(url string, obj interface{}, desc string) error
}
if p.dryRun {
log.Println(fmt.Sprintf("POST to %s:", p.apiUrl(url)))
log.Println(fmt.Sprintf("POST to %s:", p.apiURL(url)))
log.Println(string(jsonBytes))
return nil
}
req, err := http.NewRequest(http.MethodPost, p.apiUrl(url), bytes.NewReader(jsonBytes))
req, err := http.NewRequest(http.MethodPost, p.apiURL(url), bytes.NewReader(jsonBytes))
if err != nil {
return err
}
@@ -254,14 +266,14 @@ type release struct {
Stable bool `json:"stable"`
Beta bool `json:"beta"`
Nightly bool `json:"nightly"`
WhatsNewUrl string `json:"whatsNewUrl"`
ReleaseNotesUrl string `json:"releaseNotesUrl"`
WhatsNewURL string `json:"whatsNewUrl"`
ReleaseNotesURL string `json:"releaseNotesUrl"`
Builds []build `json:"-"`
}
type build struct {
Os string `json:"os"`
Url string `json:"url"`
URL string `json:"url"`
Sha256 string `json:"sha256"`
Arch string `json:"arch"`
}
@@ -7,69 +7,69 @@ func TestPreparingReleaseFromRemote(t *testing.T) {
cases := []struct {
version string
expectedVersion string
whatsNewUrl string
relNotesUrl string
whatsNewURL string
relNotesURL string
nightly bool
expectedBeta bool
expectedStable bool
expectedArch string
expectedOs string
expectedUrl string
baseArchiveUrl string
expectedURL string
baseArchiveURL string
buildArtifacts []buildArtifact
}{
{
version: "v5.2.0-beta1",
expectedVersion: "5.2.0-beta1",
whatsNewUrl: "https://whatsnews.foo/",
relNotesUrl: "https://relnotes.foo/",
whatsNewURL: "https://whatsnews.foo/",
relNotesURL: "https://relnotes.foo/",
nightly: false,
expectedBeta: true,
expectedStable: false,
expectedArch: "amd64",
expectedOs: "linux",
expectedUrl: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.2.0-beta1.linux-amd64.tar.gz",
baseArchiveUrl: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana",
expectedURL: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.2.0-beta1.linux-amd64.tar.gz",
baseArchiveURL: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana",
buildArtifacts: []buildArtifact{{"linux", "amd64", ".linux-amd64.tar.gz"}},
},
{
version: "v5.2.3",
expectedVersion: "5.2.3",
whatsNewUrl: "https://whatsnews.foo/",
relNotesUrl: "https://relnotes.foo/",
whatsNewURL: "https://whatsnews.foo/",
relNotesURL: "https://relnotes.foo/",
nightly: false,
expectedBeta: false,
expectedStable: true,
expectedArch: "amd64",
expectedOs: "rhel",
expectedUrl: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.2.3-1.x86_64.rpm",
baseArchiveUrl: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana",
expectedURL: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.2.3-1.x86_64.rpm",
baseArchiveURL: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana",
buildArtifacts: []buildArtifact{{"rhel", "amd64", ".x86_64.rpm"}},
},
{
version: "v5.4.0-pre1asdf",
expectedVersion: "5.4.0-pre1asdf",
whatsNewUrl: "https://whatsnews.foo/",
relNotesUrl: "https://relnotes.foo/",
whatsNewURL: "https://whatsnews.foo/",
relNotesURL: "https://relnotes.foo/",
nightly: true,
expectedBeta: false,
expectedStable: false,
expectedArch: "amd64",
expectedOs: "rhel",
expectedUrl: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.4.0-pre1asdf.x86_64.rpm",
baseArchiveUrl: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana",
expectedURL: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.4.0-pre1asdf.x86_64.rpm",
baseArchiveURL: "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana",
buildArtifacts: []buildArtifact{{"rhel", "amd64", ".x86_64.rpm"}},
},
}
for _, test := range cases {
builder := releaseFromExternalContent{
getter: mockHttpGetter{},
getter: mockHTTPGetter{},
rawVersion: test.version,
artifactConfigurations: test.buildArtifacts,
}
rel, _ := builder.prepareRelease(test.baseArchiveUrl, test.whatsNewUrl, test.relNotesUrl, test.nightly)
rel, _ := builder.prepareRelease(test.baseArchiveURL, test.whatsNewURL, test.relNotesURL, test.nightly)
if rel.Beta != test.expectedBeta || rel.Stable != test.expectedStable {
t.Errorf("%s should have been tagged as beta=%v, stable=%v.", test.version, test.expectedBeta, test.expectedStable)
@@ -93,21 +93,21 @@ func TestPreparingReleaseFromRemote(t *testing.T) {
t.Errorf("Expected os to be %v, but it was %v", test.expectedOs, build.Os)
}
if build.Url != test.expectedUrl {
t.Errorf("Expected url to be %v, but it was %v", test.expectedUrl, build.Url)
if build.URL != test.expectedURL {
t.Errorf("Expected url to be %v, but it was %v", test.expectedURL, build.URL)
}
}
}
type mockHttpGetter struct{}
type mockHTTPGetter struct{}
func (mockHttpGetter) getContents(url string) (string, error) {
func (mockHTTPGetter) getContents(url string) (string, error) {
return url, nil
}
func TestPreparingReleaseFromLocal(t *testing.T) {
whatsNewUrl := "https://whatsnews.foo/"
relNotesUrl := "https://relnotes.foo/"
whatsNewURL := "https://whatsnews.foo/"
relNotesURL := "https://relnotes.foo/"
expectedVersion := "5.4.0-123pre1"
expectedBuilds := 4
@@ -118,17 +118,17 @@ func TestPreparingReleaseFromLocal(t *testing.T) {
artifactConfigurations: completeBuildArtifactConfigurations,
}
relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl, true)
relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewURL, relNotesURL, true)
if relAll.Stable || !relAll.Nightly {
t.Error("Expected a nightly release but wasn't.")
}
if relAll.ReleaseNotesUrl != relNotesUrl {
t.Errorf("expected releaseNotesUrl to be %s, but it was %s", relNotesUrl, relAll.ReleaseNotesUrl)
if relAll.ReleaseNotesURL != relNotesURL {
t.Errorf("expected releaseNotesURL to be %s, but it was %s", relNotesURL, relAll.ReleaseNotesURL)
}
if relAll.WhatsNewUrl != whatsNewUrl {
t.Errorf("expected whatsNewUrl to be %s, but it was %s", whatsNewUrl, relAll.WhatsNewUrl)
if relAll.WhatsNewURL != whatsNewURL {
t.Errorf("expected whatsNewURL to be %s, but it was %s", whatsNewURL, relAll.WhatsNewURL)
}
if relAll.Beta {
@@ -155,7 +155,7 @@ func TestPreparingReleaseFromLocal(t *testing.T) {
}},
}
relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl, true)
relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewURL, relNotesURL, true)
if len(relOne.Builds) != 1 {
t.Errorf("Expected 1 artifact, but was %v", len(relOne.Builds))
+28
View File
@@ -0,0 +1,28 @@
# Repository updates deb/rpm
## Testing
It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.
```bash
docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing
# in the container:
mkdir -p /go/src/github.com/grafana/dist
cd /go/src/github.com/grafana
#outside of container:
cd <grafana project dir>/..
docker cp grafana <container_name>:/go/src/github.com/grafana/.
docker cp <gpg.key used for signing> <container_name>:/private.key
#in container:
gpg --batch --allow-secret-key-import --import /private.key
cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..
#run these scripts:
./script/build/update_repo/test-update-deb-repo.sh <gpg key password>
./script/build/update_repo/test-update-rpm-repo.sh <gpg key password>
```
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# Run this if you need to recreate the debian repository for some reason
# Setup environment
cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
mkdir -p /deb-repo/db \
/deb-repo/repo \
/deb-repo/tmp
aptly repo create -distribution=stable -component=main grafana
aptly repo create -distribution=beta -component=main beta
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
GPG_PASS=${1:-}
./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
GPG_PASS=${1:-}
./scripts/build/update_repo/update-rpm.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-repo"
+22 -12
View File
@@ -3,10 +3,14 @@
RELEASE_TYPE="${1:-}"
GPG_PASS="${2:-}"
RELEASE_TAG="${3:-}"
DIST_PATH="${4:-}"
GCP_DB_BUCKET="${5:-grafana-aptly-db}"
GCP_REPO_BUCKET="${6:-grafana-repo}"
REPO="grafana"
if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
exit 1
fi
@@ -22,36 +26,42 @@ fi
set -e
# Setup environment
cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
cp scripts/build/update_repo/aptly.conf ~/.aptly.conf
mkdir -p /deb-repo/db \
/deb-repo/repo \
/deb-repo/tmp
# Download the database
gsutil -m rsync -r "gs://grafana-aptly-db/$RELEASE_TYPE" /deb-repo/db
gsutil -m rsync -r -d "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" /deb-repo/db
# Add the new release to the repo
aptly publish drop grafana filesystem:repo:grafana || true
aptly publish drop beta filesystem:repo:grafana || true
cp ./dist/*.deb /deb-repo/tmp
cp $DIST_PATH/*.deb /deb-repo/tmp
rm /deb-repo/tmp/grafana_latest*.deb || true
aptly repo add "$REPO" ./dist
aptly repo add "$REPO" /deb-repo/tmp #adds too many packages in enterprise
# Setup signing and sign the repo
echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf
pkill gpg-agent || true
touch /tmp/sign-this
rm /tmp/sign-this.asc || true
./scripts/build/update_repo/unlock-gpg-key.sh "$GPG_PASS"
rm /tmp/sign-this /tmp/sign-this.asc
aptly publish repo grafana filesystem:repo:grafana
aptly publish repo beta filesystem:repo:grafana
aptly publish update stable filesystem:repo:grafana
aptly publish update beta filesystem:repo:grafana
# Update the repo and db on gcp
gsutil -m rsync -r -d /deb-repo/db "gs://grafana-aptly-db/$RELEASE_TYPE"
gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://grafana-repo/$RELEASE_TYPE/deb"
gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
# Uploads the binaries before the metadata (to prevent 404's for debs)
gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
# usage:
#
+11 -8
View File
@@ -2,12 +2,14 @@
RELEASE_TYPE="${1:-}"
GPG_PASS="${2:-}"
RELEASE_TAG="${3:-}"
DIST_PATH="${4:-}"
GCP_REPO_BUCKET="${5:-grafana-repo}"
REPO="rpm"
if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
exit 1
fi
@@ -23,17 +25,17 @@ fi
set -e
# Setup environment
BUCKET="gs://grafana-repo/$RELEASE_TYPE/$REPO"
BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO"
mkdir -p /rpm-repo
# Download the database
gsutil -m rsync -r "$BUCKET" /rpm-repo
# Add the new release to the repo
cp ./dist/*.rpm /rpm-repo
cp $DIST_PATH/*.rpm /rpm-repo # adds to many files for enterprise
rm /rpm-repo/grafana-latest-1*.rpm || true
cd /rpm-repo
createrepo .
createrepo /rpm-repo
# Setup signing and sign the repo
@@ -45,6 +47,7 @@ pkill gpg-agent || true
./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"
# Update the repo and db on gcp
gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
gsutil -m rsync -r -d /rpm-repo "$BUCKET"
# usage:
@@ -56,4 +59,4 @@ gsutil -m rsync -r -d /rpm-repo "$BUCKET"
# gpgcheck=1
# gpgkey=https://packages.grafana.com/gpg.key
# sslverify=1
# sslcacert=/etc/pki/tls/certs/ca-bundle.crt
# sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
function exit_if_fail {
command=$@
echo "Executing '$command'"
eval $command
rc=$?
if [ $rc -ne 0 ]; then
echo "'$command' returned $rc."
exit $rc
fi
}
go get -u github.com/alecthomas/gometalinter
go get -u github.com/tsenart/deadcode
go get -u github.com/jgautheron/goconst/cmd/goconst
go get -u github.com/gordonklaus/ineffassign
go get -u github.com/opennota/check/cmd/structcheck
go get -u github.com/mdempsky/unconvert
go get -u github.com/opennota/check/cmd/varcheck
go get -u honnef.co/go/tools/cmd/staticcheck
exit_if_fail gometalinter --enable-gc --vendor --deadline 10m --disable-all \
--enable=deadcode \
--enable=goconst \
--enable=gofmt \
--enable=ineffassign \
--enable=structcheck \
--enable=unconvert \
--enable=varcheck \
--enable=staticcheck
exit_if_fail go vet ./pkg/...