diff --git a/.circleci/config.yml b/.circleci/config.yml index af5acef4bea..5907e8d4862 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -335,6 +335,9 @@ jobs: - run: name: deploy to gcp command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/master' + - run: + name: Deploy to grafana.com + command: 'cd enterprise-dist && scripts/build/release_publisher/release_publisher -apikey ${GRAFANA_COM_API_KEY} -enterprise -from-local' deploy-enterprise-release: diff --git a/scripts/build/publish.sh b/scripts/build/publish.sh index b3fab180ac9..c03146eb910 100755 --- a/scripts/build/publish.sh +++ b/scripts/build/publish.sh @@ -1,4 +1,4 @@ -#/bin/sh +#!/bin/sh # no relation to publish.go diff --git a/scripts/build/release_publisher/externalrelease.go b/scripts/build/release_publisher/externalrelease.go new file mode 100644 index 00000000000..2c9f9e1631a --- /dev/null +++ b/scripts/build/release_publisher/externalrelease.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" +) + +type releaseFromExternalContent struct { + getter urlGetter + rawVersion string + artifactConfigurations []buildArtifact +} + +func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) { + version := re.rawVersion[1:] + isBeta := strings.Contains(version, "beta") + + builds := []build{} + for _, ba := range re.artifactConfigurations { + sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta))) + if err != nil { + return nil, err + } + builds = append(builds, newBuild(baseArchiveUrl, ba, version, isBeta, sha256)) + } + + r := release{ + Version: version, + ReleaseDate: time.Now(), + Stable: !isBeta, + Beta: isBeta, + Nightly: false, + WhatsNewUrl: whatsNewUrl, + ReleaseNotesUrl: releaseNotesUrl, + Builds: builds, + } + return &r, nil +} + +type urlGetter interface { + getContents(url string) (string, error) +} + +type getHttpContents struct{} + +func (getHttpContents) getContents(url string) (string, error) { + response, err := http.Get(url) + if err != nil { + return "", err + } + + defer response.Body.Close() + all, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", err + } + + return string(all), nil +} diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.linux-amd64.tar.gz.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.windows-amd64.zip.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise-5.4.0-123pre1.x86_64.rpm.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb b/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 b/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 new file mode 100644 index 00000000000..c3068040269 --- /dev/null +++ b/scripts/build/release_publisher/local_test_data/grafana-enterprise_5.4.0-123pre1_amd64.deb.sha256 @@ -0,0 +1 @@ +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/scripts/build/release_publisher/localrelease.go b/scripts/build/release_publisher/localrelease.go new file mode 100644 index 00000000000..bc2e95f7cd8 --- /dev/null +++ b/scripts/build/release_publisher/localrelease.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "github.com/pkg/errors" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +type releaseLocalSources struct { + path string + artifactConfigurations []buildArtifact +} + +func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) { + buildData := r.findBuilds(baseArchiveUrl) + + rel := release{ + Version: buildData.version, + ReleaseDate: time.Now(), + Stable: false, + Beta: false, + Nightly: true, + WhatsNewUrl: whatsNewUrl, + ReleaseNotesUrl: releaseNotesUrl, + Builds: buildData.builds, + } + + return &rel, nil +} + +type buildData struct { + version string + builds []build +} + +func (r releaseLocalSources) findBuilds(baseArchiveUrl string) buildData { + data := buildData{} + filepath.Walk(r.path, createBuildWalker(r.path, &data, r.artifactConfigurations, baseArchiveUrl)) + return data +} + +func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifact, baseArchiveUrl string) func(path string, f os.FileInfo, err error) error { + return func(path string, f os.FileInfo, err error) error { + if err != nil { + log.Printf("error: %v", err) + } + + if f.Name() == path || strings.HasSuffix(f.Name(), ".sha256") { + return nil + } + + for _, archive := range archiveTypes { + if strings.HasSuffix(f.Name(), archive.urlPostfix) { + shaBytes, err := ioutil.ReadFile(path + ".sha256") + if err != nil { + log.Fatalf("Failed to read sha256 file %v", err) + } + + version, err := grabVersion(f.Name(), archive.urlPostfix) + if err != nil { + log.Println(err) + continue + } + data.version = version + data.builds = append(data.builds, build{ + Os: archive.os, + Url: archive.getUrl(baseArchiveUrl, version, false), + Sha256: string(shaBytes), + Arch: archive.arch, + }) + return nil + } + } + return nil + } + +} +func grabVersion(name string, suffix string) (string, error) { + match := regexp.MustCompile(fmt.Sprintf(`grafana(-enterprise)?[-_](.*)%s`, suffix)).FindSubmatch([]byte(name)) + if len(match) > 0 { + return string(match[2]), nil + } + + return "", errors.New("No version found.") +} diff --git a/scripts/build/release_publisher/main.go b/scripts/build/release_publisher/main.go index fde4317bbb9..66ab38ab00e 100644 --- a/scripts/build/release_publisher/main.go +++ b/scripts/build/release_publisher/main.go @@ -7,13 +7,13 @@ import ( "os" ) -var baseUri string = "https://grafana.com/api" - func main() { var version string var whatsNewUrl string var releaseNotesUrl string var dryRun bool + var enterprise bool + var fromLocal bool var apiKey string flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)") @@ -21,20 +21,55 @@ func main() { flag.StringVar(&releaseNotesUrl, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)") flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)") flag.BoolVar(&dryRun, "dry-run", false, "--dry-run") + flag.BoolVar(&enterprise, "enterprise", false, "--enterprise") + flag.BoolVar(&fromLocal, "from-local", false, "--from-local") flag.Parse() if len(os.Args) == 1 { - fmt.Println("Usage: go run publisher.go main.go --version --wn --rn --apikey --dry-run false") - fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run true") + fmt.Println("Usage: go run publisher.go main.go --version --wn --rn --apikey --dry-run false --enterprise false") + fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run --enterprise") os.Exit(1) } if dryRun { log.Println("Dry-run has been enabled.") } + var baseUrl string + var builder releaseBuilder + var product string - p := publisher{apiKey: apiKey} - if err := p.doRelease(version, whatsNewUrl, releaseNotesUrl, dryRun); err != nil { + if fromLocal { + path, _ := os.Getwd() + builder = releaseLocalSources{ + path: path, + artifactConfigurations: buildArtifactConfigurations, + } + } else { + builder = releaseFromExternalContent{ + getter: getHttpContents{}, + rawVersion: version, + artifactConfigurations: buildArtifactConfigurations, + } + } + + if enterprise { + baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/release/grafana-enterprise" + product = "grafana-enterprise" + } else { + baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana" + product = "grafana" + } + + p := publisher{ + apiKey: apiKey, + apiUri: "https://grafana.com/api", + product: product, + dryRun: dryRun, + enterprise: enterprise, + baseArchiveUrl: baseUrl, + builder: builder, + } + if err := p.doRelease(whatsNewUrl, releaseNotesUrl); err != nil { log.Fatalf("error: %v", err) } } diff --git a/scripts/build/release_publisher/publisher.go b/scripts/build/release_publisher/publisher.go index 60b60ca55f7..0874c1357b6 100644 --- a/scripts/build/release_publisher/publisher.go +++ b/scripts/build/release_publisher/publisher.go @@ -12,53 +12,47 @@ import ( ) type publisher struct { - apiKey string + apiKey string + apiUri string + product string + dryRun bool + enterprise bool + baseArchiveUrl string + builder releaseBuilder } -func (p *publisher) doRelease(version string, whatsNewUrl string, releaseNotesUrl string, dryRun bool) error { - currentRelease, err := newRelease(version, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations, getHttpContents{}) +type releaseBuilder interface { + prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) +} + +func (p *publisher) doRelease(whatsNewUrl string, releaseNotesUrl string) error { + currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl) if err != nil { return err } - if dryRun { - relJson, err := json.Marshal(currentRelease) - if err != nil { - return err - } - log.Println(string(relJson)) - - for _, b := range currentRelease.Builds { - artifactJson, err := json.Marshal(b) - if err != nil { - return err - } - log.Println(string(artifactJson)) - } - } else { - if err := p.postRelease(currentRelease); err != nil { - return err - } + if err := p.postRelease(currentRelease); err != nil { + return err } return nil } func (p *publisher) postRelease(r *release) error { - err := p.postRequest("/grafana/versions", r, fmt.Sprintf("Create Release %s", r.Version)) + err := p.postRequest("/versions", r, fmt.Sprintf("Create Release %s", r.Version)) if err != nil { return err } - err = p.postRequest("/grafana/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version)) + err = p.postRequest("/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version)) if err != nil { return err } for _, b := range r.Builds { - err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch)) + err = p.postRequest(fmt.Sprintf("/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch)) if err != nil { return err } - err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch)) + err = p.postRequest(fmt.Sprintf("/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch)) if err != nil { return err } @@ -67,15 +61,13 @@ func (p *publisher) postRelease(r *release) error { return nil } -const baseArhiveUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana" - type buildArtifact struct { os string arch string urlPostfix string } -func (t buildArtifact) getUrl(version string, isBeta bool) string { +func (t buildArtifact) getUrl(baseArchiveUrl, version string, isBeta bool) string { prefix := "-" rhelReleaseExtra := "" @@ -87,7 +79,7 @@ func (t buildArtifact) getUrl(version string, isBeta bool) string { rhelReleaseExtra = "-1" } - url := strings.Join([]string{baseArhiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "") + url := strings.Join([]string{baseArchiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "") return url } @@ -149,48 +141,32 @@ var buildArtifactConfigurations = []buildArtifact{ }, } -func newRelease(rawVersion string, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact, getter urlGetter) (*release, error) { - version := rawVersion[1:] - now := time.Now() - isBeta := strings.Contains(version, "beta") - - builds := []build{} - for _, ba := range artifactConfigurations { - sha256, err := getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(version, isBeta))) - if err != nil { - return nil, err - } - builds = append(builds, newBuild(ba, version, isBeta, sha256)) - } - - r := release{ - Version: version, - ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local), - Stable: !isBeta, - Beta: isBeta, - Nightly: false, - WhatsNewUrl: whatsNewUrl, - ReleaseNotesUrl: releaseNotesUrl, - Builds: builds, - } - return &r, nil -} - -func newBuild(ba buildArtifact, version string, isBeta bool, sha256 string) build { +func newBuild(baseArchiveUrl string, ba buildArtifact, version string, isBeta bool, sha256 string) build { return build{ Os: ba.os, - Url: ba.getUrl(version, isBeta), + Url: ba.getUrl(baseArchiveUrl, version, isBeta), Sha256: sha256, Arch: ba.arch, } } +func (p *publisher) apiUrl(url string) string { + return fmt.Sprintf("%s/%s%s", p.apiUri, p.product, url) +} + func (p *publisher) postRequest(url string, obj interface{}, desc string) error { jsonBytes, err := json.Marshal(obj) if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, baseUri+url, bytes.NewReader(jsonBytes)) + + if p.dryRun { + log.Println(fmt.Sprintf("POST to %s:", p.apiUrl(url))) + log.Println(string(jsonBytes)) + return nil + } + + req, err := http.NewRequest(http.MethodPost, p.apiUrl(url), bytes.NewReader(jsonBytes)) if err != nil { return err } @@ -243,24 +219,3 @@ type build struct { Sha256 string `json:"sha256"` Arch string `json:"arch"` } - -type urlGetter interface { - getContents(url string) (string, error) -} - -type getHttpContents struct{} - -func (getHttpContents) getContents(url string) (string, error) { - response, err := http.Get(url) - if err != nil { - return "", err - } - - defer response.Body.Close() - all, err := ioutil.ReadAll(response.Body) - if err != nil { - return "", err - } - - return string(all), nil -} diff --git a/scripts/build/release_publisher/publisher_test.go b/scripts/build/release_publisher/publisher_test.go index 9bc350e6a54..fed17007bb6 100644 --- a/scripts/build/release_publisher/publisher_test.go +++ b/scripts/build/release_publisher/publisher_test.go @@ -2,16 +2,24 @@ package main import "testing" -func TestNewRelease(t *testing.T) { +func TestPreparingReleaseFromRemote(t *testing.T) { versionIn := "v5.2.0-beta1" expectedVersion := "5.2.0-beta1" whatsNewUrl := "https://whatsnews.foo/" relNotesUrl := "https://relnotes.foo/" expectedArch := "amd64" expectedOs := "linux" - buildArtifacts := []buildArtifact{{expectedOs, expectedArch, ".linux-amd64.tar.gz"}} + buildArtifacts := []buildArtifact{{expectedOs,expectedArch, ".linux-amd64.tar.gz"}} - rel, _ := newRelease(versionIn, whatsNewUrl, relNotesUrl, buildArtifacts, mockHttpGetter{}) + var builder releaseBuilder + + builder = releaseFromExternalContent{ + getter: mockHttpGetter{}, + rawVersion: versionIn, + artifactConfigurations: buildArtifactConfigurations, + } + + rel, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana", whatsNewUrl, relNotesUrl) if !rel.Beta || rel.Stable { t.Errorf("%s should have been tagged as beta (not stable), but wasn't .", versionIn) @@ -41,3 +49,71 @@ type mockHttpGetter struct{} func (mockHttpGetter) getContents(url string) (string, error) { return url, nil } + + +func TestPreparingReleaseFromLocal(t *testing.T) { + whatsNewUrl := "https://whatsnews.foo/" + relNotesUrl := "https://relnotes.foo/" + expectedVersion := "5.4.0-123pre1" + expectedBuilds := 4 + + var builder releaseBuilder + testDataPath := "local_test_data" + builder = releaseLocalSources{ + path: testDataPath, + artifactConfigurations: buildArtifactConfigurations, + } + + relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl) + + if relAll.Stable || !relAll.Nightly { + t.Error("Expected a nightly release but wasn't.") + } + + if relAll.ReleaseNotesUrl != relNotesUrl { + t.Errorf("expected releaseNotesUrl to be %s, but it was %s", relNotesUrl, relAll.ReleaseNotesUrl) + } + if relAll.WhatsNewUrl != whatsNewUrl { + t.Errorf("expected whatsNewUrl to be %s, but it was %s", whatsNewUrl, relAll.WhatsNewUrl) + } + + if relAll.Beta { + t.Errorf("Expected release to be nightly, not beta.") + } + + if relAll.Version != expectedVersion { + t.Errorf("Expected version=%s, but got=%s", expectedVersion, relAll.Version) + } + + if len(relAll.Builds) != expectedBuilds { + t.Errorf("Expected %v builds, but was %v", expectedBuilds, len(relAll.Builds)) + } + + expectedArch := "amd64" + expectedOs := "win" + + builder = releaseLocalSources{ + path: testDataPath, + artifactConfigurations: []buildArtifact{{ + os: expectedOs, + arch: expectedArch, + urlPostfix: ".windows-amd64.zip", + }}, + } + + relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl) + + if len(relOne.Builds) != 1 { + t.Errorf("Expected 1 artifact, but was %v", len(relOne.Builds)) + } + + build := relOne.Builds[0] + + if build.Arch != expectedArch { + t.Fatalf("Expected arch to be %s, but was %s", expectedArch, build.Arch) + } + + if build.Os != expectedOs { + t.Fatalf("Expected os to be %s, but was %s", expectedOs, build.Os) + } +}