From 7e093a32a282f25b4922c856d9cf21e12689c7ad Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Wed, 31 Oct 2018 16:59:30 +0100 Subject: [PATCH 1/7] build: improved release publisher dry-run. --- scripts/build/release_publisher/main.go | 11 +++-- scripts/build/release_publisher/publisher.go | 46 ++++++++++---------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/scripts/build/release_publisher/main.go b/scripts/build/release_publisher/main.go index fde4317bbb9..40430aebf2b 100644 --- a/scripts/build/release_publisher/main.go +++ b/scripts/build/release_publisher/main.go @@ -7,8 +7,6 @@ import ( "os" ) -var baseUri string = "https://grafana.com/api" - func main() { var version string var whatsNewUrl string @@ -33,8 +31,13 @@ func main() { log.Println("Dry-run has been enabled.") } - p := publisher{apiKey: apiKey} - if err := p.doRelease(version, whatsNewUrl, releaseNotesUrl, dryRun); err != nil { + p := publisher{ + apiKey: apiKey, + baseUri: "https://grafana.com/api", + product: "grafana", + dryRun: dryRun, + } + if err := p.doRelease(version, whatsNewUrl, releaseNotesUrl); err != nil { log.Fatalf("error: %v", err) } } diff --git a/scripts/build/release_publisher/publisher.go b/scripts/build/release_publisher/publisher.go index 60b60ca55f7..0466aaa5687 100644 --- a/scripts/build/release_publisher/publisher.go +++ b/scripts/build/release_publisher/publisher.go @@ -13,52 +13,39 @@ import ( type publisher struct { apiKey string + baseUri string + product string + dryRun bool } -func (p *publisher) doRelease(version string, whatsNewUrl string, releaseNotesUrl string, dryRun bool) error { +func (p *publisher) doRelease(version string, whatsNewUrl string, releaseNotesUrl string) error { currentRelease, err := newRelease(version, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations, getHttpContents{}) if err != nil { return err } - if dryRun { - relJson, err := json.Marshal(currentRelease) - if err != nil { - return err - } - log.Println(string(relJson)) - - for _, b := range currentRelease.Builds { - artifactJson, err := json.Marshal(b) - if err != nil { - return err - } - log.Println(string(artifactJson)) - } - } else { - if err := p.postRelease(currentRelease); err != nil { - return err - } + if err := p.postRelease(currentRelease); err != nil { + return err } return nil } func (p *publisher) postRelease(r *release) error { - err := p.postRequest("/grafana/versions", r, fmt.Sprintf("Create Release %s", r.Version)) + err := p.postRequest("/versions", r, fmt.Sprintf("Create Release %s", r.Version)) if err != nil { return err } - err = p.postRequest("/grafana/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version)) + err = p.postRequest("/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version)) if err != nil { return err } for _, b := range r.Builds { - err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch)) + err = p.postRequest(fmt.Sprintf("/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch)) if err != nil { return err } - err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch)) + err = p.postRequest(fmt.Sprintf("/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch)) if err != nil { return err } @@ -185,12 +172,23 @@ func newBuild(ba buildArtifact, version string, isBeta bool, sha256 string) buil } } +func (p *publisher) apiUrl(url string) string { + return fmt.Sprintf("%s/%s%s", p.baseUri, p.product, url) +} + func (p *publisher) postRequest(url string, obj interface{}, desc string) error { jsonBytes, err := json.Marshal(obj) if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, baseUri+url, bytes.NewReader(jsonBytes)) + + if p.dryRun { + log.Println(fmt.Sprintf("POST to %s:", p.apiUrl(url))) + log.Println(string(jsonBytes)) + return nil + } + + req, err := http.NewRequest(http.MethodPost, p.apiUrl(url), bytes.NewReader(jsonBytes)) if err != nil { return err } From e2d3382470a3bc3cf536254cf860907ce3fa727a Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Thu, 1 Nov 2018 13:35:31 +0100 Subject: [PATCH 2/7] build: prepares release tool for finding local releases. --- scripts/build/release_publisher/main.go | 29 +++++++++---- scripts/build/release_publisher/publisher.go | 42 ++++++++++++------- .../build/release_publisher/publisher_test.go | 9 +++- 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/scripts/build/release_publisher/main.go b/scripts/build/release_publisher/main.go index 40430aebf2b..8b3ec8da74e 100644 --- a/scripts/build/release_publisher/main.go +++ b/scripts/build/release_publisher/main.go @@ -12,6 +12,7 @@ func main() { var whatsNewUrl string var releaseNotesUrl string var dryRun bool + var enterprise bool var apiKey string flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)") @@ -19,25 +20,39 @@ func main() { flag.StringVar(&releaseNotesUrl, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)") flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)") flag.BoolVar(&dryRun, "dry-run", false, "--dry-run") + flag.BoolVar(&enterprise, "enterprise", false, "--enterprise") flag.Parse() if len(os.Args) == 1 { - fmt.Println("Usage: go run publisher.go main.go --version --wn --rn --apikey --dry-run false") - fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run true") + fmt.Println("Usage: go run publisher.go main.go --version --wn --rn --apikey --dry-run false --enterprise false") + fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run --enterprise") os.Exit(1) } if dryRun { log.Println("Dry-run has been enabled.") } + var baseUrl string + + if enterprise { + baseUrl = fmt.Sprintf("https://s3-us-west-2.amazonaws.com/%s", "grafana-enterprise-releases/release/grafana-enterprise") + } else { + baseUrl = fmt.Sprintf("https://s3-us-west-2.amazonaws.com/%s", "grafana-releases/release/grafana") + } p := publisher{ - apiKey: apiKey, - baseUri: "https://grafana.com/api", - product: "grafana", - dryRun: dryRun, + apiKey: apiKey, + baseUri: "https://grafana.com/api", + product: "grafana", + dryRun: dryRun, + enterprise: enterprise, + baseArchiveUrl: baseUrl, + builder: releaseFromExternalContent{ + getter: getHttpContents{}, + rawVersion: version, + }, } - if err := p.doRelease(version, whatsNewUrl, releaseNotesUrl); err != nil { + if err := p.doRelease(whatsNewUrl, releaseNotesUrl); err != nil { log.Fatalf("error: %v", err) } } diff --git a/scripts/build/release_publisher/publisher.go b/scripts/build/release_publisher/publisher.go index 0466aaa5687..2bd84462daa 100644 --- a/scripts/build/release_publisher/publisher.go +++ b/scripts/build/release_publisher/publisher.go @@ -12,14 +12,21 @@ import ( ) type publisher struct { - apiKey string - baseUri string - product string - dryRun bool + apiKey string + baseUri string + product string + dryRun bool + enterprise bool + baseArchiveUrl string + builder releaseBuilder } -func (p *publisher) doRelease(version string, whatsNewUrl string, releaseNotesUrl string) error { - currentRelease, err := newRelease(version, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations, getHttpContents{}) +type releaseBuilder interface { + prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) +} + +func (p *publisher) doRelease(whatsNewUrl string, releaseNotesUrl string) error { + currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations) if err != nil { return err } @@ -54,15 +61,13 @@ func (p *publisher) postRelease(r *release) error { return nil } -const baseArhiveUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana" - type buildArtifact struct { os string arch string urlPostfix string } -func (t buildArtifact) getUrl(version string, isBeta bool) string { +func (t buildArtifact) getUrl(baseArchiveUrl, version string, isBeta bool) string { prefix := "-" rhelReleaseExtra := "" @@ -74,7 +79,7 @@ func (t buildArtifact) getUrl(version string, isBeta bool) string { rhelReleaseExtra = "-1" } - url := strings.Join([]string{baseArhiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "") + url := strings.Join([]string{baseArchiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "") return url } @@ -136,18 +141,23 @@ var buildArtifactConfigurations = []buildArtifact{ }, } -func newRelease(rawVersion string, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact, getter urlGetter) (*release, error) { - version := rawVersion[1:] +type releaseFromExternalContent struct { + getter urlGetter + rawVersion string +} + +func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) { + version := re.rawVersion[1:] now := time.Now() isBeta := strings.Contains(version, "beta") builds := []build{} for _, ba := range artifactConfigurations { - sha256, err := getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(version, isBeta))) + sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta))) if err != nil { return nil, err } - builds = append(builds, newBuild(ba, version, isBeta, sha256)) + builds = append(builds, newBuild(baseArchiveUrl, ba, version, isBeta, sha256)) } r := release{ @@ -163,10 +173,10 @@ func newRelease(rawVersion string, whatsNewUrl string, releaseNotesUrl string, a return &r, nil } -func newBuild(ba buildArtifact, version string, isBeta bool, sha256 string) build { +func newBuild(baseArchiveUrl string, ba buildArtifact, version string, isBeta bool, sha256 string) build { return build{ Os: ba.os, - Url: ba.getUrl(version, isBeta), + Url: ba.getUrl(baseArchiveUrl, version, isBeta), Sha256: sha256, Arch: ba.arch, } diff --git a/scripts/build/release_publisher/publisher_test.go b/scripts/build/release_publisher/publisher_test.go index 9bc350e6a54..38a9d972380 100644 --- a/scripts/build/release_publisher/publisher_test.go +++ b/scripts/build/release_publisher/publisher_test.go @@ -9,9 +9,14 @@ func TestNewRelease(t *testing.T) { relNotesUrl := "https://relnotes.foo/" expectedArch := "amd64" expectedOs := "linux" - buildArtifacts := []buildArtifact{{expectedOs, expectedArch, ".linux-amd64.tar.gz"}} + buildArtifacts := []buildArtifact{{expectedOs,expectedArch, ".linux-amd64.tar.gz"}} - rel, _ := newRelease(versionIn, whatsNewUrl, relNotesUrl, buildArtifacts, mockHttpGetter{}) + builder := releaseFromExternalContent{ + getter: mockHttpGetter{}, + rawVersion: versionIn, + } + + rel, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana", whatsNewUrl, relNotesUrl, buildArtifacts) if !rel.Beta || rel.Stable { t.Errorf("%s should have been tagged as beta (not stable), but wasn't .", versionIn) From c5c3e08442fb2f8cb3eedceb4fc30ff7708b67ab Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Thu, 1 Nov 2018 14:49:32 +0100 Subject: [PATCH 3/7] build: refactor releaser. --- .../release_publisher/externalrelease.go | 62 +++++++++++++++++++ scripts/build/release_publisher/publisher.go | 53 ---------------- .../build/release_publisher/publisher_test.go | 7 ++- 3 files changed, 68 insertions(+), 54 deletions(-) create mode 100644 scripts/build/release_publisher/externalrelease.go diff --git a/scripts/build/release_publisher/externalrelease.go b/scripts/build/release_publisher/externalrelease.go new file mode 100644 index 00000000000..795e3bc999b --- /dev/null +++ b/scripts/build/release_publisher/externalrelease.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" +) + +type releaseFromExternalContent struct { + getter urlGetter + rawVersion string +} + +func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) { + version := re.rawVersion[1:] + now := time.Now() + isBeta := strings.Contains(version, "beta") + + builds := []build{} + for _, ba := range artifactConfigurations { + sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta))) + if err != nil { + return nil, err + } + builds = append(builds, newBuild(baseArchiveUrl, ba, version, isBeta, sha256)) + } + + r := release{ + Version: version, + ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local), + Stable: !isBeta, + Beta: isBeta, + Nightly: false, + WhatsNewUrl: whatsNewUrl, + ReleaseNotesUrl: releaseNotesUrl, + Builds: builds, + } + return &r, nil +} + +type urlGetter interface { + getContents(url string) (string, error) +} + +type getHttpContents struct{} + +func (getHttpContents) getContents(url string) (string, error) { + response, err := http.Get(url) + if err != nil { + return "", err + } + + defer response.Body.Close() + all, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", err + } + + return string(all), nil +} diff --git a/scripts/build/release_publisher/publisher.go b/scripts/build/release_publisher/publisher.go index 2bd84462daa..e8c3b676e59 100644 --- a/scripts/build/release_publisher/publisher.go +++ b/scripts/build/release_publisher/publisher.go @@ -141,38 +141,6 @@ var buildArtifactConfigurations = []buildArtifact{ }, } -type releaseFromExternalContent struct { - getter urlGetter - rawVersion string -} - -func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) { - version := re.rawVersion[1:] - now := time.Now() - isBeta := strings.Contains(version, "beta") - - builds := []build{} - for _, ba := range artifactConfigurations { - sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta))) - if err != nil { - return nil, err - } - builds = append(builds, newBuild(baseArchiveUrl, ba, version, isBeta, sha256)) - } - - r := release{ - Version: version, - ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local), - Stable: !isBeta, - Beta: isBeta, - Nightly: false, - WhatsNewUrl: whatsNewUrl, - ReleaseNotesUrl: releaseNotesUrl, - Builds: builds, - } - return &r, nil -} - func newBuild(baseArchiveUrl string, ba buildArtifact, version string, isBeta bool, sha256 string) build { return build{ Os: ba.os, @@ -251,24 +219,3 @@ type build struct { Sha256 string `json:"sha256"` Arch string `json:"arch"` } - -type urlGetter interface { - getContents(url string) (string, error) -} - -type getHttpContents struct{} - -func (getHttpContents) getContents(url string) (string, error) { - response, err := http.Get(url) - if err != nil { - return "", err - } - - defer response.Body.Close() - all, err := ioutil.ReadAll(response.Body) - if err != nil { - return "", err - } - - return string(all), nil -} diff --git a/scripts/build/release_publisher/publisher_test.go b/scripts/build/release_publisher/publisher_test.go index 38a9d972380..04a9c0ca54c 100644 --- a/scripts/build/release_publisher/publisher_test.go +++ b/scripts/build/release_publisher/publisher_test.go @@ -2,7 +2,7 @@ package main import "testing" -func TestNewRelease(t *testing.T) { +func TestPreparingReleaseFromRemote(t *testing.T) { versionIn := "v5.2.0-beta1" expectedVersion := "5.2.0-beta1" whatsNewUrl := "https://whatsnews.foo/" @@ -46,3 +46,8 @@ type mockHttpGetter struct{} func (mockHttpGetter) getContents(url string) (string, error) { return url, nil } + + +func TestPreparingReleaseFromLocal(t *testing.T) { + +} From d9eaec99e2f04eea03fa9ab0a0971d9d1290f91a Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Fri, 2 Nov 2018 14:56:46 +0100 Subject: [PATCH 4/7] build: publisher can find artifacts from local sources. --- ...nterprise-5.4.0-123pre1.linux-amd64.tar.gz | 0 ...se-5.4.0-123pre1.linux-amd64.tar.gz.sha256 | 1 + ...enterprise-5.4.0-123pre1.windows-amd64.zip | 0 ...ise-5.4.0-123pre1.windows-amd64.zip.sha256 | 1 + ...rafana-enterprise-5.4.0-123pre1.x86_64.rpm | 0 ...enterprise-5.4.0-123pre1.x86_64.rpm.sha256 | 1 + ...grafana-enterprise_5.4.0-123pre1_amd64.deb | 0 ...-enterprise_5.4.0-123pre1_amd64.deb.sha256 | 1 + .../build/release_publisher/localrelease.go | 91 +++++++++++++++++++ .../build/release_publisher/publisher_test.go | 59 +++++++++++- 10 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb create mode 100644 scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 create mode 100644 scripts/build/release_publisher/localrelease.go diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb b/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/localrelease.go b/scripts/build/release_publisher/localrelease.go new file mode 100644 index 00000000000..45bebc524e4 --- /dev/null +++ b/scripts/build/release_publisher/localrelease.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "github.com/pkg/errors" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +type releaseLocalSources struct { + path string +} + +func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) { + buildData := r.findBuilds(artifactConfigurations, baseArchiveUrl) + + rel := release{ + Version: buildData.version, + ReleaseDate: time.Time{}, + Stable: false, + Beta: false, + Nightly: true, + WhatsNewUrl: whatsNewUrl, + ReleaseNotesUrl: releaseNotesUrl, + Builds: buildData.builds, + } + + return &rel, nil +} + +type buildData struct { + version string + builds []build +} + +func (r releaseLocalSources) findBuilds(buildArtifacts []buildArtifact, baseArchiveUrl string) buildData { + data := buildData{} + filepath.Walk(r.path, createBuildWalker(r.path, &data, buildArtifacts, baseArchiveUrl)) + return data +} + +func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifact, baseArchiveUrl string) func(path string, f os.FileInfo, err error) error { + return func(path string, f os.FileInfo, err error) error { + if err != nil { + log.Printf("error: %v", err) + } + + if f.Name() == path || strings.HasSuffix(f.Name(), ".sha256") { + return nil + } + + shaBytes, err := ioutil.ReadFile(path + ".sha256") + if err != nil { + log.Fatalf("Failed to read sha256 file %v", err) + } + + + for _, archive := range archiveTypes { + if strings.HasSuffix(f.Name(), archive.urlPostfix) { + version, err := grabVersion(f.Name(), archive.urlPostfix) + if err != nil { + log.Println(err) + continue + } + data.version = version + data.builds = append(data.builds, build{ + Os: archive.os, + Url: archive.getUrl(baseArchiveUrl, version, false), + Sha256: string(shaBytes), + Arch: archive.arch, + }) + return nil + } + } + return nil + } + +} +func grabVersion(name string, suffix string) (string, error) { + match := regexp.MustCompile(fmt.Sprintf(`grafana(-enterprise)?[-_](.*)%s`, suffix)).FindSubmatch([]byte(name)) + if len(match) > 0 { + return string(match[2]), nil + } + + return "", errors.New("No version found.") +} diff --git a/scripts/build/release_publisher/publisher_test.go b/scripts/build/release_publisher/publisher_test.go index 04a9c0ca54c..14e92f01bc0 100644 --- a/scripts/build/release_publisher/publisher_test.go +++ b/scripts/build/release_publisher/publisher_test.go @@ -11,7 +11,9 @@ func TestPreparingReleaseFromRemote(t *testing.T) { expectedOs := "linux" buildArtifacts := []buildArtifact{{expectedOs,expectedArch, ".linux-amd64.tar.gz"}} - builder := releaseFromExternalContent{ + var builder releaseBuilder + + builder = releaseFromExternalContent{ getter: mockHttpGetter{}, rawVersion: versionIn, } @@ -49,5 +51,60 @@ func (mockHttpGetter) getContents(url string) (string, error) { func TestPreparingReleaseFromLocal(t *testing.T) { + whatsNewUrl := "https://whatsnews.foo/" + relNotesUrl := "https://relnotes.foo/" + expectedVersion := "5.4.0-123pre1" + expectedBuilds := 4 + var builder releaseBuilder + builder = releaseLocalSources{ + path: "local_test_data", + } + + relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl, buildArtifactConfigurations) + + if relAll.Stable || !relAll.Nightly { + t.Error("Expected a nightly release but wasn't.") + } + + if relAll.ReleaseNotesUrl != relNotesUrl { + t.Errorf("expected releaseNotesUrl to be %s, but it was %s", relNotesUrl, relAll.ReleaseNotesUrl) + } + if relAll.WhatsNewUrl != whatsNewUrl { + t.Errorf("expected whatsNewUrl to be %s, but it was %s", whatsNewUrl, relAll.WhatsNewUrl) + } + + if relAll.Beta { + t.Errorf("Expected release to be nightly, not beta.") + } + + if relAll.Version != expectedVersion { + t.Errorf("Expected version=%s, but got=%s", expectedVersion, relAll.Version) + } + + if len(relAll.Builds) != expectedBuilds { + t.Errorf("Expected %v builds, but was %v", expectedBuilds, len(relAll.Builds)) + } + + expectedArch := "amd64" + expectedOs := "win" + relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl, []buildArtifact{{ + os: expectedOs, + arch: expectedArch, + urlPostfix: ".windows-amd64.zip", + }}) + + if len(relOne.Builds) != 1 { + t.Errorf("Expected 1 artifact, but was %v", len(relOne.Builds)) + } + + build := relOne.Builds[0] + + if build.Arch != expectedArch { + t.Fatalf("Expected arch to be %s, but was %s", expectedArch, build.Arch) + } + + if build.Os != expectedOs { + t.Fatalf("Expected os to be %s, but was %s", expectedOs, build.Os) + } } From 5da9760aebb19138c69bf7316e359730543b749a Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 5 Nov 2018 09:51:54 +0100 Subject: [PATCH 5/7] build: publisher supports both local and remote. --- .../release_publisher/externalrelease.go | 5 +-- .../build/release_publisher/localrelease.go | 23 ++++++------- scripts/build/release_publisher/main.go | 33 ++++++++++++++----- scripts/build/release_publisher/publisher.go | 8 ++--- .../build/release_publisher/publisher_test.go | 25 +++++++++----- 5 files changed, 61 insertions(+), 33 deletions(-) diff --git a/scripts/build/release_publisher/externalrelease.go b/scripts/build/release_publisher/externalrelease.go index 795e3bc999b..2d69fa604f8 100644 --- a/scripts/build/release_publisher/externalrelease.go +++ b/scripts/build/release_publisher/externalrelease.go @@ -11,15 +11,16 @@ import ( type releaseFromExternalContent struct { getter urlGetter rawVersion string + artifactConfigurations []buildArtifact } -func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) { +func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) { version := re.rawVersion[1:] now := time.Now() isBeta := strings.Contains(version, "beta") builds := []build{} - for _, ba := range artifactConfigurations { + for _, ba := range re.artifactConfigurations { sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta))) if err != nil { return nil, err diff --git a/scripts/build/release_publisher/localrelease.go b/scripts/build/release_publisher/localrelease.go index 45bebc524e4..1fb266aa041 100644 --- a/scripts/build/release_publisher/localrelease.go +++ b/scripts/build/release_publisher/localrelease.go @@ -14,14 +14,16 @@ import ( type releaseLocalSources struct { path string + artifactConfigurations []buildArtifact } -func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) { - buildData := r.findBuilds(artifactConfigurations, baseArchiveUrl) +func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) { + buildData := r.findBuilds(baseArchiveUrl) + now := time.Now() rel := release{ Version: buildData.version, - ReleaseDate: time.Time{}, + ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local), Stable: false, Beta: false, Nightly: true, @@ -38,9 +40,9 @@ type buildData struct { builds []build } -func (r releaseLocalSources) findBuilds(buildArtifacts []buildArtifact, baseArchiveUrl string) buildData { +func (r releaseLocalSources) findBuilds(baseArchiveUrl string) buildData { data := buildData{} - filepath.Walk(r.path, createBuildWalker(r.path, &data, buildArtifacts, baseArchiveUrl)) + filepath.Walk(r.path, createBuildWalker(r.path, &data, r.artifactConfigurations, baseArchiveUrl)) return data } @@ -54,14 +56,13 @@ func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifac return nil } - shaBytes, err := ioutil.ReadFile(path + ".sha256") - if err != nil { - log.Fatalf("Failed to read sha256 file %v", err) - } - - for _, archive := range archiveTypes { if strings.HasSuffix(f.Name(), archive.urlPostfix) { + shaBytes, err := ioutil.ReadFile(path + ".sha256") + if err != nil { + log.Fatalf("Failed to read sha256 file %v", err) + } + version, err := grabVersion(f.Name(), archive.urlPostfix) if err != nil { log.Println(err) diff --git a/scripts/build/release_publisher/main.go b/scripts/build/release_publisher/main.go index 8b3ec8da74e..66ab38ab00e 100644 --- a/scripts/build/release_publisher/main.go +++ b/scripts/build/release_publisher/main.go @@ -13,6 +13,7 @@ func main() { var releaseNotesUrl string var dryRun bool var enterprise bool + var fromLocal bool var apiKey string flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)") @@ -21,6 +22,7 @@ func main() { flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)") flag.BoolVar(&dryRun, "dry-run", false, "--dry-run") flag.BoolVar(&enterprise, "enterprise", false, "--enterprise") + flag.BoolVar(&fromLocal, "from-local", false, "--from-local") flag.Parse() if len(os.Args) == 1 { @@ -33,24 +35,39 @@ func main() { log.Println("Dry-run has been enabled.") } var baseUrl string + var builder releaseBuilder + var product string + + if fromLocal { + path, _ := os.Getwd() + builder = releaseLocalSources{ + path: path, + artifactConfigurations: buildArtifactConfigurations, + } + } else { + builder = releaseFromExternalContent{ + getter: getHttpContents{}, + rawVersion: version, + artifactConfigurations: buildArtifactConfigurations, + } + } if enterprise { - baseUrl = fmt.Sprintf("https://s3-us-west-2.amazonaws.com/%s", "grafana-enterprise-releases/release/grafana-enterprise") + baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/release/grafana-enterprise" + product = "grafana-enterprise" } else { - baseUrl = fmt.Sprintf("https://s3-us-west-2.amazonaws.com/%s", "grafana-releases/release/grafana") + baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana" + product = "grafana" } p := publisher{ apiKey: apiKey, - baseUri: "https://grafana.com/api", - product: "grafana", + apiUri: "https://grafana.com/api", + product: product, dryRun: dryRun, enterprise: enterprise, baseArchiveUrl: baseUrl, - builder: releaseFromExternalContent{ - getter: getHttpContents{}, - rawVersion: version, - }, + builder: builder, } if err := p.doRelease(whatsNewUrl, releaseNotesUrl); err != nil { log.Fatalf("error: %v", err) diff --git a/scripts/build/release_publisher/publisher.go b/scripts/build/release_publisher/publisher.go index e8c3b676e59..0874c1357b6 100644 --- a/scripts/build/release_publisher/publisher.go +++ b/scripts/build/release_publisher/publisher.go @@ -13,7 +13,7 @@ import ( type publisher struct { apiKey string - baseUri string + apiUri string product string dryRun bool enterprise bool @@ -22,11 +22,11 @@ type publisher struct { } type releaseBuilder interface { - prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact) (*release, error) + prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) } func (p *publisher) doRelease(whatsNewUrl string, releaseNotesUrl string) error { - currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations) + currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl) if err != nil { return err } @@ -151,7 +151,7 @@ func newBuild(baseArchiveUrl string, ba buildArtifact, version string, isBeta bo } func (p *publisher) apiUrl(url string) string { - return fmt.Sprintf("%s/%s%s", p.baseUri, p.product, url) + return fmt.Sprintf("%s/%s%s", p.apiUri, p.product, url) } func (p *publisher) postRequest(url string, obj interface{}, desc string) error { diff --git a/scripts/build/release_publisher/publisher_test.go b/scripts/build/release_publisher/publisher_test.go index 14e92f01bc0..fed17007bb6 100644 --- a/scripts/build/release_publisher/publisher_test.go +++ b/scripts/build/release_publisher/publisher_test.go @@ -16,9 +16,10 @@ func TestPreparingReleaseFromRemote(t *testing.T) { builder = releaseFromExternalContent{ getter: mockHttpGetter{}, rawVersion: versionIn, + artifactConfigurations: buildArtifactConfigurations, } - rel, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana", whatsNewUrl, relNotesUrl, buildArtifacts) + rel, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana", whatsNewUrl, relNotesUrl) if !rel.Beta || rel.Stable { t.Errorf("%s should have been tagged as beta (not stable), but wasn't .", versionIn) @@ -57,11 +58,13 @@ func TestPreparingReleaseFromLocal(t *testing.T) { expectedBuilds := 4 var builder releaseBuilder + testDataPath := "local_test_data" builder = releaseLocalSources{ - path: "local_test_data", + path: testDataPath, + artifactConfigurations: buildArtifactConfigurations, } - relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl, buildArtifactConfigurations) + relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl) if relAll.Stable || !relAll.Nightly { t.Error("Expected a nightly release but wasn't.") @@ -88,11 +91,17 @@ func TestPreparingReleaseFromLocal(t *testing.T) { expectedArch := "amd64" expectedOs := "win" - relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl, []buildArtifact{{ - os: expectedOs, - arch: expectedArch, - urlPostfix: ".windows-amd64.zip", - }}) + + builder = releaseLocalSources{ + path: testDataPath, + artifactConfigurations: []buildArtifact{{ + os: expectedOs, + arch: expectedArch, + urlPostfix: ".windows-amd64.zip", + }}, + } + + relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl) if len(relOne.Builds) != 1 { t.Errorf("Expected 1 artifact, but was %v", len(relOne.Builds)) From d728a3c521683ec5007fc09b5dd7b62e480c6686 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 5 Nov 2018 14:52:23 +0100 Subject: [PATCH 6/7] build: publisher uses local time. Previously the local day was used but the timestamp was set to midnight. --- scripts/build/release_publisher/externalrelease.go | 3 +-- scripts/build/release_publisher/localrelease.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/build/release_publisher/externalrelease.go b/scripts/build/release_publisher/externalrelease.go index 2d69fa604f8..2c9f9e1631a 100644 --- a/scripts/build/release_publisher/externalrelease.go +++ b/scripts/build/release_publisher/externalrelease.go @@ -16,7 +16,6 @@ type releaseFromExternalContent struct { func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) { version := re.rawVersion[1:] - now := time.Now() isBeta := strings.Contains(version, "beta") builds := []build{} @@ -30,7 +29,7 @@ func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl r := release{ Version: version, - ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local), + ReleaseDate: time.Now(), Stable: !isBeta, Beta: isBeta, Nightly: false, diff --git a/scripts/build/release_publisher/localrelease.go b/scripts/build/release_publisher/localrelease.go index 1fb266aa041..bc2e95f7cd8 100644 --- a/scripts/build/release_publisher/localrelease.go +++ b/scripts/build/release_publisher/localrelease.go @@ -20,10 +20,9 @@ type releaseLocalSources struct { func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) { buildData := r.findBuilds(baseArchiveUrl) - now := time.Now() rel := release{ Version: buildData.version, - ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local), + ReleaseDate: time.Now(), Stable: false, Beta: false, Nightly: true, From 1de35c43a79d913cbb59cd8531df48bb267f8b61 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 5 Nov 2018 16:26:19 +0100 Subject: [PATCH 7/7] build: publishes grafana enterprise to grafana.com --- .circleci/config.yml | 3 +++ scripts/build/publish.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af5acef4bea..5907e8d4862 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -335,6 +335,9 @@ jobs: - run: name: deploy to gcp command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/master' + - run: + name: Deploy to grafana.com + command: 'cd enterprise-dist && scripts/build/release_publisher/release_publisher -apikey ${GRAFANA_COM_API_KEY} -enterprise -from-local' deploy-enterprise-release: diff --git a/scripts/build/publish.sh b/scripts/build/publish.sh index b3fab180ac9..c03146eb910 100755 --- a/scripts/build/publish.sh +++ b/scripts/build/publish.sh @@ -1,4 +1,4 @@ -#/bin/sh +#!/bin/sh # no relation to publish.go