From 29bb039c942b421d8f78d93c9bb68538e7322a04 Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Mon, 3 Oct 2022 13:30:11 +0200 Subject: [PATCH] CI: Move `grafanacom` command to OSS (#55853) (#56168) * Move publish-packages command over from * Fix lint * Move grafanacom command to OSS * Add GetLatestMainBuild to gsutil * Fix lint * More lint fixes * Add tests for grafanacom * Fix lint (cherry picked from commit 947838cca08386d4e7554e0a3fcf9584907098e7) Co-authored-by: Dimitris Sotirakis --- pkg/build/cmd/artifacts.go | 118 +++++++++++ pkg/build/cmd/grafanacom.go | 323 +++++++++++++++++++++++++++++ pkg/build/cmd/grafanacom_test.go | 35 ++++ pkg/build/cmd/main.go | 15 ++ pkg/build/config/version_mode.go | 2 + pkg/build/gcloud/storage/gsutil.go | 36 ++++ 6 files changed, 529 insertions(+) create mode 100644 pkg/build/cmd/grafanacom.go create mode 100644 pkg/build/cmd/grafanacom_test.go diff --git a/pkg/build/cmd/artifacts.go b/pkg/build/cmd/artifacts.go index 53807878518..f93d002b107 100644 --- a/pkg/build/cmd/artifacts.go +++ b/pkg/build/cmd/artifacts.go @@ -1,13 +1,24 @@ package main import ( + "fmt" + "strings" + "github.com/grafana/grafana/pkg/build/config" ) const ReleaseFolder = "release" +const MainFolder = "main" const EnterpriseSfx = "-enterprise" const CacheSettings = "Cache-Control:public, max-age=" +type buildArtifact struct { + Os string + Arch string + urlPostfix string + packagePostfix string +} + type PublishConfig struct { config.Config @@ -20,3 +31,110 @@ type PublishConfig struct { TTL string SimulateRelease bool } + +const rhelOS = "rhel" +const debOS = "deb" + +func (t buildArtifact) GetURL(baseArchiveURL string, cfg PublishConfig) string { + rev := "" + prefix := "-" + if t.Os == debOS { + prefix = "_" + } else if t.Os == rhelOS { + rev = "-1" + } + + version := cfg.Version + verComponents := strings.Split(version, "-") + if len(verComponents) > 2 { + panic(fmt.Sprintf("Version string contains more than one hyphen: %q", version)) + } + + switch t.Os { + case debOS, rhelOS: + if len(verComponents) > 1 { + // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this + // is considered of lower lexical value than the empty character, and this way pre-release versions are + // considered to be of a lower version than the final version (which lacks this suffix). + version = fmt.Sprintf("%s~%s", verComponents[0], verComponents[1]) + } + } + + // https://dl.grafana.com/oss/main/grafana_8.5.0~54094pre_armhf.deb: 404 Not Found + url := fmt.Sprintf("%s%s%s%s%s%s", baseArchiveURL, t.packagePostfix, prefix, version, rev, t.urlPostfix) + return url +} + +var ArtifactConfigs = []buildArtifact{ + { + Os: debOS, + Arch: "arm64", + urlPostfix: "_arm64.deb", + }, + { + Os: rhelOS, + Arch: "arm64", + urlPostfix: ".aarch64.rpm", + }, + { + Os: "linux", + Arch: "arm64", + urlPostfix: ".linux-arm64.tar.gz", + }, + { + Os: debOS, + Arch: "armv7", + urlPostfix: "_armhf.deb", + }, + { + Os: debOS, + Arch: "armv6", + packagePostfix: "-rpi", + urlPostfix: "_armhf.deb", + }, + { + Os: rhelOS, + Arch: "armv7", + urlPostfix: ".armhfp.rpm", + }, + { + Os: "linux", + Arch: "armv6", + urlPostfix: ".linux-armv6.tar.gz", + }, + { + Os: "linux", + Arch: "armv7", + urlPostfix: ".linux-armv7.tar.gz", + }, + { + Os: "darwin", + Arch: "amd64", + urlPostfix: ".darwin-amd64.tar.gz", + }, + { + Os: "deb", + Arch: "amd64", + urlPostfix: "_amd64.deb", + }, + { + Os: rhelOS, + Arch: "amd64", + urlPostfix: ".x86_64.rpm", + }, + { + Os: "linux", + Arch: "amd64", + urlPostfix: ".linux-amd64.tar.gz", + }, + { + Os: "win", + Arch: "amd64", + urlPostfix: ".windows-amd64.zip", + }, + { + Os: "win-installer", + Arch: "amd64", + urlPostfix: ".windows-amd64.msi", + }, +} diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go new file mode 100644 index 00000000000..87358419b22 --- /dev/null +++ b/pkg/build/cmd/grafanacom.go @@ -0,0 +1,323 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "os" + + "path" + "path/filepath" + "strings" + "time" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/urfave/cli/v2" +) + +const grafanaAPI = "https://grafana.com/api" + +// GrafanaCom implements the sub-command "grafana-com". +func GrafanaCom(c *cli.Context) error { + bucketStr := c.String("src-bucket") + edition := config.Edition(c.String("edition")) + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("couldn't activate service account, err: %w", err) + } + + metadata, err := GenerateMetadata(c) + if err != nil { + return err + } + + releaseMode, err := metadata.GetReleaseMode() + if err != nil { + return err + } + + version := metadata.GrafanaVersion + if releaseMode.Mode == config.Cronjob { + gcs, err := storage.New() + if err != nil { + return err + } + bucket := gcs.Bucket(bucketStr) + latestMainVersion, err := storage.GetLatestMainBuild(c.Context, bucket, filepath.Join(string(edition), "main")) + if err != nil { + return err + } + version = latestMainVersion + } + + dryRun := c.Bool("dry-run") + simulateRelease := c.Bool("simulate-release") + // Test release mode and dryRun imply simulateRelease + if releaseMode.IsTest || dryRun { + simulateRelease = true + } + + grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY")) + if grafanaAPIKey == "" { + return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1) + } + whatsNewURL, releaseNotesURL, err := getReleaseURLs() + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + // TODO: Verify config values + cfg := PublishConfig{ + Config: config.Config{ + Version: version, + }, + Edition: edition, + ReleaseMode: releaseMode, + GrafanaAPIKey: grafanaAPIKey, + WhatsNewURL: whatsNewURL, + ReleaseNotesURL: releaseNotesURL, + DryRun: dryRun, + TTL: c.String("ttl"), + SimulateRelease: simulateRelease, + } + + if err := publishPackages(cfg); err != nil { + return cli.NewExitError(err.Error(), 1) + } + + log.Println("Successfully published packages to grafana.com!") + return nil +} + +func getReleaseURLs() (string, string, error) { + type grafanaConf struct { + WhatsNewURL string `json:"whatsNewUrl"` + ReleaseNotesURL string `json:"releaseNotesUrl"` + } + type packageConf struct { + Grafana grafanaConf `json:"grafana"` + } + + pkgB, err := os.ReadFile("package.json") + if err != nil { + return "", "", fmt.Errorf("failed to read package.json: %w", err) + } + + var pconf packageConf + if err := json.Unmarshal(pkgB, &pconf); err != nil { + return "", "", fmt.Errorf("failed to decode package.json: %w", err) + } + if _, err := url.ParseRequestURI(pconf.Grafana.WhatsNewURL); err != nil { + return "", "", fmt.Errorf("grafana.whatsNewUrl is invalid in package.json: %q", pconf.Grafana.WhatsNewURL) + } + if _, err := url.ParseRequestURI(pconf.Grafana.ReleaseNotesURL); err != nil { + return "", "", fmt.Errorf("grafana.releaseNotesUrl is invalid in package.json: %q", + pconf.Grafana.ReleaseNotesURL) + } + + return pconf.Grafana.WhatsNewURL, pconf.Grafana.ReleaseNotesURL, nil +} + +// publishPackages publishes packages to grafana.com. +func publishPackages(cfg PublishConfig) error { + log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n", + cfg.Version, cfg.Edition, cfg.ReleaseMode.Mode, cfg.DryRun, cfg.SimulateRelease) + + versionStr := fmt.Sprintf("v%s", cfg.Version) + log.Printf("Creating release %s at grafana.com...\n", versionStr) + + var sfx string + var pth string + switch cfg.Edition { + case config.EditionOSS: + pth = "oss" + case config.EditionEnterprise: + pth = "enterprise" + sfx = EnterpriseSfx + default: + return fmt.Errorf("unrecognized edition %q", cfg.Edition) + } + + switch cfg.ReleaseMode.Mode { + case config.MainMode, config.CustomMode, config.CronjobMode: + pth = path.Join(pth, MainFolder) + default: + pth = path.Join(pth, ReleaseFolder) + } + + product := fmt.Sprintf("grafana%s", sfx) + pth = path.Join(pth, product) + baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth) + + var builds []buildRepr + for _, ba := range ArtifactConfigs { + u := ba.GetURL(baseArchiveURL, cfg) + + sha256, err := getSHA256(u) + if err != nil { + return err + } + + builds = append(builds, buildRepr{ + OS: ba.Os, + URL: u, + SHA256: string(sha256), + Arch: ba.Arch, + }) + } + + r := releaseRepr{ + Version: cfg.Version, + ReleaseDate: time.Now().UTC(), + Builds: builds, + Stable: cfg.ReleaseMode.Mode == config.TagMode, + Beta: cfg.ReleaseMode.IsBeta, + Nightly: cfg.ReleaseMode.Mode == config.CronjobMode, + } + if cfg.ReleaseMode.Mode == config.TagMode || r.Beta { + r.WhatsNewURL = cfg.WhatsNewURL + r.ReleaseNotesURL = cfg.ReleaseNotesURL + } + + if err := postRequest(cfg, "versions", r, fmt.Sprintf("create release %s", r.Version)); err != nil { + return err + } + + if err := postRequest(cfg, fmt.Sprintf("versions/%s", cfg.Version), r, + fmt.Sprintf("update release %s", cfg.Version)); err != nil { + return err + } + + for _, b := range r.Builds { + if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages", cfg.Version), b, + fmt.Sprintf("create build %s %s", b.OS, b.Arch)); err != nil { + return err + } + if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages/%s/%s", cfg.Version, b.Arch, b.OS), b, + fmt.Sprintf("update build %s %s", b.OS, b.Arch)); err != nil { + return err + } + } + + return nil +} + +func getSHA256(u string) ([]byte, error) { + shaURL := fmt.Sprintf("%s.sha256", u) + // nolint:gosec + resp, err := http.Get(shaURL) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Println("failed to close response body, err: %w", err) + } + }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("failed downloading %s: %s", u, resp.Status) + } + + var sha256 []byte + if err := json.NewDecoder(resp.Body).Decode(&sha256); err != nil { + return nil, err + } + return sha256, nil +} + +func postRequest(cfg PublishConfig, pth string, obj interface{}, descr string) error { + var sfx string + switch cfg.Edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = EnterpriseSfx + default: + return fmt.Errorf("unrecognized edition %q", cfg.Edition) + } + product := fmt.Sprintf("grafana%s", sfx) + + jsonB, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("failed to JSON encode release: %w", err) + } + + u, err := constructURL(product, pth) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(jsonB)) + if err != nil { + return err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cfg.GrafanaAPIKey)) + req.Header.Add("Content-Type", "application/json") + + log.Printf("Posting to grafana.com API, %s - JSON: %s\n", u, string(jsonB)) + if cfg.SimulateRelease { + log.Println("Only simulating request") + return nil + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed posting to %s (%s): %s", u, descr, err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Println("failed to close response body, err: %w", err) + } + }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + var body []byte + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return err + } + if err != nil { + return err + } + + if strings.Contains(string(body), "already exists") || strings.Contains(string(body), "Nothing to update") { + log.Printf("Already exists: %s\n", descr) + return nil + } + + return fmt.Errorf("failed posting to %s (%s): %s", u, descr, resp.Status) + } + + log.Printf("Successfully posted to grafana.com API, %s\n", u) + + return nil +} + +func constructURL(product string, pth string) (string, error) { + productPath := filepath.Clean(filepath.Join("/", product, pth)) + u, err := url.Parse(grafanaAPI) + if err != nil { + return "", err + } + u.Path = path.Join(u.Path, productPath) + return u.String(), err +} + +type buildRepr struct { + OS string `json:"os"` + URL string `json:"url"` + SHA256 string `json:"sha256"` + Arch string `json:"arch"` +} + +type releaseRepr struct { + Version string `json:"version"` + ReleaseDate time.Time `json:"releaseDate"` + Stable bool `json:"stable"` + Beta bool `json:"beta"` + Nightly bool `json:"nightly"` + WhatsNewURL string `json:"whatsNewUrl"` + ReleaseNotesURL string `json:"releaseNotesUrl"` + Builds []buildRepr `json:"-"` +} diff --git a/pkg/build/cmd/grafanacom_test.go b/pkg/build/cmd/grafanacom_test.go new file mode 100644 index 00000000000..bf90874b1c9 --- /dev/null +++ b/pkg/build/cmd/grafanacom_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "testing" +) + +func Test_constructURL(t *testing.T) { + type args struct { + product string + pth string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "cleans .. sequence", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false}, + {name: "doesn't clean anything - non malicious url", args: args{"foo", "bar"}, want: "https://grafana.com/api/foo/bar", wantErr: false}, + {name: "doesn't clean anything - three dots", args: args{"...", "..."}, want: "https://grafana.com/api/.../...", wantErr: false}, + {name: "cleans .", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := constructURL(tt.args.product, tt.args.pth) + if (err != nil) != tt.wantErr { + t.Errorf("constructURL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("constructURL() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 66c5e801d07..71e3c06c70c 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -216,6 +216,21 @@ func main() { }, }, }, + { + Name: "grafana-com", + Usage: "Publish packages to grafana.com", + Action: GrafanaCom, + Flags: []cli.Flag{ + &editionFlag, + &buildIDFlag, + &dryRunFlag, + &cli.StringFlag{ + Name: "src-bucket", + Value: "grafana-downloads", + Usage: "Google Cloud Storage bucket", + }, + }, + }, }, }, } diff --git a/pkg/build/config/version_mode.go b/pkg/build/config/version_mode.go index 7f1c0c635ce..f7aa6578fe4 100644 --- a/pkg/build/config/version_mode.go +++ b/pkg/build/config/version_mode.go @@ -9,6 +9,7 @@ const ( ReleaseBranchMode VersionMode = "branch" PullRequestMode VersionMode = "pull_request" CustomMode VersionMode = "custom" + CronjobMode VersionMode = "cron" ) const ( @@ -17,6 +18,7 @@ const ( Push = "push" Custom = "custom" Promote = "promote" + Cronjob = "cron" ) const ( diff --git a/pkg/build/gcloud/storage/gsutil.go b/pkg/build/gcloud/storage/gsutil.go index d9865f5fa98..ca3484c5df7 100644 --- a/pkg/build/gcloud/storage/gsutil.go +++ b/pkg/build/gcloud/storage/gsutil.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "sync" "time" @@ -358,6 +359,41 @@ func (client *Client) DownloadDirectory(ctx context.Context, bucket *storage.Buc return nil } +// GetLatestMainBuild gets the latest main build which is successfully uploaded to the gcs bucket. +func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path string) (string, error) { + if bucket == nil { + return "", ErrorNilBucket + } + + it := bucket.Objects(ctx, &storage.Query{ + Prefix: path, + }) + + var files []string + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return "", fmt.Errorf("failed to iterate through bucket, err: %w", err) + } + + files = append(files, attrs.Name) + } + + var latestVersion string + for i := len(files) - 1; i >= 0; i-- { + captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`) + if captureVersion.MatchString(files[i]) { + latestVersion = captureVersion.FindString(files[i]) + break + } + } + + return latestVersion, nil +} + // downloadFile downloads an object to a file. func (client *Client) downloadFile(ctx context.Context, bucket *storage.BucketHandle, objectName, destFileName string) error { if bucket == nil {