From f23be415c5276a4d328bc70b3d2c0b8bd802839f Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Mon, 30 Jan 2023 17:24:10 +0100 Subject: [PATCH] CI: Add `artifacts publish` build command (#62445) * CI: Add `artifacts publish` build command * Lint release.star --- .drone.yml | 52 +++++--- pkg/build/cmd/main.go | 53 ++++++++ pkg/build/cmd/publishartifacts.go | 211 ++++++++++++++++++++++++++++++ scripts/drone/events/release.star | 9 +- 4 files changed, 303 insertions(+), 22 deletions(-) create mode 100644 pkg/build/cmd/publishartifacts.go diff --git a/.drone.yml b/.drone.yml index 1879dd0a72b..00584d6a234 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4322,20 +4322,27 @@ platform: services: [] steps: - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.20/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - ./bin/grabpl artifacts publish --security --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} - depends_on: - - grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts publish --security --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + depends_on: + - compile-build-cmd + environment: + ENTERPRISE2_SECURITY_PREFIX: + from_secret: enterprise2_security_prefix GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket + SECURITY_DEST_BUCKET: + from_secret: security_dest_bucket + STATIC_ASSET_EDITIONS: + from_secret: static_asset_editions image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts trigger: @@ -4366,20 +4373,27 @@ platform: services: [] steps: - commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.20/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - ./bin/grabpl artifacts publish --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} - depends_on: - - grabpl + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: + CGO_ENABLED: 0 + image: golang:1.19.4 + name: compile-build-cmd +- commands: + - ./bin/build artifacts publish --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + depends_on: + - compile-build-cmd + environment: + ENTERPRISE2_SECURITY_PREFIX: + from_secret: enterprise2_security_prefix GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket + SECURITY_DEST_BUCKET: + from_secret: security_dest_bucket + STATIC_ASSET_EDITIONS: + from_secret: static_asset_editions image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts trigger: @@ -6502,6 +6516,6 @@ kind: secret name: aws_secret_access_key --- kind: signature -hmac: 6e76bf175f2c58fd4ffdc42e2120c558345a71a45011279b14092acb67252b28 +hmac: eba6c445aae6d75df0a2963d5e1e90c44474587a9e1d11e21bf2ba4d99f14da8 ... diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 1ee1270a65d..2613c50bff0 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -190,6 +190,59 @@ func main() { Name: "artifacts", Usage: "Handle Grafana artifacts", Subcommands: cli.Commands{ + { + Name: "publish", + Usage: "Publish Grafana artifacts", + Action: PublishArtifactsAction, + Flags: []cli.Flag{ + &editionFlag, + &cli.BoolFlag{ + Name: "security", + Usage: "Security release", + }, + &cli.StringFlag{ + Name: "security-dest-bucket", + Usage: "Google Cloud Storage bucket for security packages (or $SECURITY_DEST_BUCKET)", + }, + &cli.StringFlag{ + Name: "tag", + Usage: "Grafana version tag", + }, + &cli.StringFlag{ + Name: "src-bucket", + Value: "grafana-prerelease", + Usage: "Google Cloud Storage bucket", + }, + &cli.StringFlag{ + Name: "dest-bucket", + Value: "grafana-downloads", + Usage: "Google Cloud Storage bucket for published packages", + }, + &cli.StringFlag{ + Name: "enterprise2-dest-bucket", + Value: "grafana-downloads-enterprise2", + Usage: "Google Cloud Storage bucket for published packages", + }, + &cli.StringFlag{ + Name: "enterprise2-security-prefix", + Usage: "Bucket path prefix for enterprise2 security releases (or $ENTERPRISE2_SECURITY_PREFIX)", + }, + &cli.StringFlag{ + Name: "static-assets-bucket", + Value: "grafana-static-assets", + Usage: "Google Cloud Storage bucket for static assets", + }, + &cli.StringSliceFlag{ + Name: "static-asset-editions", + Usage: "All the editions of the static assets (or $STATIC_ASSET_EDITIONS)", + }, + &cli.StringFlag{ + Name: "storybook-bucket", + Value: "grafana-storybook", + Usage: "Google Cloud Storage bucket for storybooks", + }, + }, + }, { Name: "docker", Usage: "Handle Grafana Docker images", diff --git a/pkg/build/cmd/publishartifacts.go b/pkg/build/cmd/publishartifacts.go new file mode 100644 index 00000000000..bd158d6d3ad --- /dev/null +++ b/pkg/build/cmd/publishartifacts.go @@ -0,0 +1,211 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/grafana/grafana/pkg/build/versions" + "github.com/urfave/cli/v2" +) + +type publishConfig struct { + tag string + srcBucket string + destBucket string + enterprise2DestBucket string + enterprise2SecurityPrefix string + staticAssetsBucket string + staticAssetEditions []string + storybookBucket string + security bool +} + +// requireListWithEnvFallback first checks the CLI for a flag with the required +// name. If this is empty, it falls back to taking the environment variable. +// Sadly, we cannot use cli.Flag.EnvVars for this due to it potentially leaking +// environment variables as default values in usage-errors. +func requireListWithEnvFallback(cctx *cli.Context, name string, envName string) ([]string, error) { + result := cctx.StringSlice(name) + if len(result) == 0 { + for _, v := range strings.Split(os.Getenv(envName), ",") { + value := strings.TrimSpace(v) + if value != "" { + result = append(result, value) + } + } + } + if len(result) == 0 { + return nil, cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1) + } + return result, nil +} + +func requireStringWithEnvFallback(cctx *cli.Context, name string, envName string) (string, error) { + result := cctx.String(name) + if result == "" { + result = os.Getenv(envName) + } + if result == "" { + return "", cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1) + } + return result, nil +} + +// Action implements the sub-command "publish-artifacts". +func PublishArtifactsAction(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + staticAssetEditions, err := requireListWithEnvFallback(c, "static-asset-editions", "STATIC_ASSET_EDITIONS") + if err != nil { + return err + } + securityDestBucket, err := requireStringWithEnvFallback(c, "security-dest-bucket", "SECURITY_DEST_BUCKET") + if err != nil { + return err + } + enterprise2SecurityPrefix, err := requireStringWithEnvFallback(c, "enterprise2-security-prefix", "ENTERPRISE2_SECURITY_PREFIX") + if err != nil { + return err + } + + if err := gcloud.ActivateServiceAccount(); err != nil { + return fmt.Errorf("error connecting to gcp, %q", err) + } + + cfg := publishConfig{ + srcBucket: c.String("src-bucket"), + destBucket: c.String("dest-bucket"), + enterprise2DestBucket: c.String("enterprise2-dest-bucket"), + enterprise2SecurityPrefix: enterprise2SecurityPrefix, + staticAssetsBucket: c.String("static-assets-bucket"), + staticAssetEditions: staticAssetEditions, + storybookBucket: c.String("storybook-bucket"), + security: c.Bool("security"), + tag: strings.TrimPrefix(c.String("tag"), "v"), + } + + if cfg.security { + cfg.destBucket = securityDestBucket + } + + err = copyStaticAssets(cfg) + if err != nil { + return err + } + err = copyStorybook(cfg) + if err != nil { + return err + } + err = copyDownloads(cfg) + if err != nil { + return err + } + err = copyEnterprise2Downloads(cfg) + if err != nil { + return err + } + return nil +} + +func copyStaticAssets(cfg publishConfig) error { + for _, edition := range cfg.staticAssetEditions { + log.Printf("Copying static assets for %s", edition) + srcURL := fmt.Sprintf("%s/artifacts/static-assets/%s/%s/*", cfg.srcBucket, edition, cfg.tag) + destURL := fmt.Sprintf("%s/%s/%s/", cfg.staticAssetsBucket, edition, cfg.tag) + err := gcsCopy("static assets", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying static assets, %q", err) + } + } + log.Printf("Successfully copied static assets!") + return nil +} + +func copyStorybook(cfg publishConfig) error { + if cfg.security { + log.Printf("skipping storybook copy - not needed for a security release") + return nil + } + log.Printf("Copying storybooks...") + srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag) + destURL := fmt.Sprintf("%s/%s", cfg.storybookBucket, cfg.tag) + err := gcsCopy("storybook", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying storybook. %q", err) + } + stableVersion, err := versions.GetLatestVersion(versions.LatestStableVersionURL) + if err != nil { + return err + } + isLatest, err := versions.IsGreaterThanOrEqual(cfg.tag, stableVersion) + if err != nil { + return err + } + if isLatest { + log.Printf("Copying storybooks to latest...") + srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag) + destURL := fmt.Sprintf("%s/latest", cfg.storybookBucket) + err := gcsCopy("storybook (latest)", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying storybook to latest. %q", err) + } + } + + log.Printf("Successfully copied storybook!") + return nil +} + +func copyDownloads(cfg publishConfig) error { + for _, edition := range []string{ + "oss", "enterprise", + } { + destURL := fmt.Sprintf("%s/%s/", cfg.destBucket, edition) + srcURL := fmt.Sprintf("%s/artifacts/downloads/v%s/%s/release/*", cfg.srcBucket, cfg.tag, edition) + if !cfg.security { + destURL = filepath.Join(destURL, "release") + } + log.Printf("Copying downloads for %s, from %s bucket to %s bucket", edition, srcURL, destURL) + err := gcsCopy("downloads", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying downloads, %q", err) + } + } + log.Printf("Successfully copied downloads.") + return nil +} + +func copyEnterprise2Downloads(cfg publishConfig) error { + var prefix string + if cfg.security { + prefix = cfg.enterprise2SecurityPrefix + } + srcURL := fmt.Sprintf("%s/artifacts/downloads-enterprise2/v%s/enterprise2/release/*", cfg.srcBucket, cfg.tag) + destURL := fmt.Sprintf("%s/enterprise2/%srelease", cfg.enterprise2DestBucket, prefix) + log.Printf("Copying downloads for enterprise2, from %s bucket to %s bucket", srcURL, destURL) + err := gcsCopy("enterprise2 downloads", srcURL, destURL) + if err != nil { + return fmt.Errorf("error copying ") + } + return nil +} + +func gcsCopy(desc, src, dest string) error { + args := strings.Split(fmt.Sprintf("-m cp -r gs://%s gs://%s", src, dest), " ") + // nolint:gosec + cmd := exec.Command("gsutil", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to publish %s: %w\n%s", desc, err, out) + } + return nil +} diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index d9edb1c9bee..8db03e1caec 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -532,13 +532,16 @@ def publish_artifacts_step(mode): "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), + "ENTERPRISE2_SECURITY_PREFIX": from_secret("enterprise2_security_prefix"), + "SECURITY_DEST_BUCKET": from_secret("security_dest_bucket"), + "STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"), }, "commands": [ - "./bin/grabpl artifacts publish {}--tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}".format( + "./bin/build artifacts publish {}--tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}".format( security, ), ], - "depends_on": ["grabpl"], + "depends_on": ["compile-build-cmd"], } def publish_artifacts_pipelines(mode): @@ -547,7 +550,7 @@ def publish_artifacts_pipelines(mode): "target": [mode], } steps = [ - download_grabpl_step(), + compile_build_cmd(), publish_artifacts_step(mode), ]