CI: Split /bin/build artifacts * subcommand (#66005)

* Split go files

* Modify publishartifacts.go

* Split main

* FIXME: Temp add GCSCopy to gcloud/storage

* Fix lint

* Exported flags

* Update starlark - make drone

* Further backend lint fixes

* Add fallback_test.go

* Fix go imports order

* make drone

* Remove GCSCopy from static assets

* Remove GCSCopy from storybook

* Fix lint

* Remove GCSCopy

* Remove success logline

* drone yaml cleanup

* Add artifacts-editions flag

* Fix starlark

* Add default sting slice in artifacts packages command

pkg/build/cmd/flags.go

@@ -51,4 +51,22 @@ var (
 		Name:  "tag",
 		Usage: "Grafana version tag",
 	}
+	securityFlag = cli.BoolFlag{
+		Name:  "security",
+		Usage: "Security release",
+	}
+	srcFlag = cli.StringFlag{
+		Name:  "src-bucket",
+		Value: "grafana-prerelease",
+		Usage: "Google Cloud Storage bucket",
+	}
+	securityDestBucketFlag = cli.StringFlag{
+		Name:  "security-dest-bucket",
+		Usage: "Google Cloud Storage bucket for security packages (or $SECURITY_DEST_BUCKET)",
+	}
+	destFlag = cli.StringFlag{
+		Name:  "dest-bucket",
+		Value: "grafana-downloads",
+		Usage: "Google Cloud Storage bucket for published packages",
+	}
 )

pkg/build/cmd/main.go

@@ -198,42 +198,31 @@ func main() {
 			Usage: "Handle Grafana artifacts",
 			Subcommands: cli.Commands{
 				{
-					Name:   "publish",
-					Usage:  "Publish Grafana artifacts",
-					Action: PublishArtifactsAction,
+					Name:   "storybook",
+					Usage:  "Publish Grafana storybook",
+					Action: PublishStorybookAction,
 					Flags: []cli.Flag{
 						&editionFlag,
-						&cli.BoolFlag{
-							Name:  "security",
-							Usage: "Security release",
-						},
+						&tagFlag,
+						&srcFlag,
 						&cli.StringFlag{
-							Name:  "security-dest-bucket",
-							Usage: "Google Cloud Storage bucket for security packages (or $SECURITY_DEST_BUCKET)",
-						},
-						&cli.StringFlag{
-							Name:  "tag",
-							Usage: "Grafana version tag",
-						},
-						&cli.StringFlag{
-							Name:  "src-bucket",
-							Value: "grafana-prerelease",
-							Usage: "Google Cloud Storage bucket",
-						},
-						&cli.StringFlag{
-							Name:  "dest-bucket",
-							Value: "grafana-downloads",
-							Usage: "Google Cloud Storage bucket for published packages",
-						},
-						&cli.StringFlag{
-							Name:  "enterprise2-dest-bucket",
-							Value: "grafana-downloads-enterprise2",
-							Usage: "Google Cloud Storage bucket for published packages",
-						},
-						&cli.StringFlag{
-							Name:  "enterprise2-security-prefix",
-							Usage: "Bucket path prefix for enterprise2 security releases (or $ENTERPRISE2_SECURITY_PREFIX)",
+							Name:  "storybook-bucket",
+							Value: "grafana-storybook",
+							Usage: "Google Cloud Storage bucket for storybooks",
 						},
+					},
+				},
+				{
+					Name:   "static-assets",
+					Usage:  "Publish Grafana static assets",
+					Action: PublishStaticAssetsAction,
+					Flags: []cli.Flag{
+						&editionFlag,
+						&securityFlag,
+						&securityDestBucketFlag,
+						&tagFlag,
+						&srcFlag,
+						&destFlag,
 						&cli.StringFlag{
 							Name:  "static-assets-bucket",
 							Value: "grafana-static-assets",
@@ -243,10 +232,32 @@ func main() {
 							Name:  "static-asset-editions",
 							Usage: "All the editions of the static assets (or $STATIC_ASSET_EDITIONS)",
 						},
+					},
+				},
+				{
+					Name:   "packages",
+					Usage:  "Publish Grafana packages",
+					Action: PublishArtifactsAction,
+					Flags: []cli.Flag{
+						&editionFlag,
+						&securityFlag,
+						&securityDestBucketFlag,
+						&tagFlag,
+						&srcFlag,
+						&destFlag,
+						&cli.StringSliceFlag{
+							Name:  "artifacts-editions",
+							Value: cli.NewStringSlice("oss", "enterprise", "enterprise2"),
+							Usage: "Editions for which the artifacts should be delivered (oss,enterprise,enterprise2), (or $ARTIFACTS_EDITIONS)",
+						},
 						&cli.StringFlag{
-							Name:  "storybook-bucket",
-							Value: "grafana-storybook",
-							Usage: "Google Cloud Storage bucket for storybooks",
+							Name:  "enterprise2-dest-bucket",
+							Value: "grafana-downloads-enterprise2",
+							Usage: "Google Cloud Storage bucket for published packages",
+						},
+						&cli.StringFlag{
+							Name:  "enterprise2-security-prefix",
+							Usage: "Bucket path prefix for enterprise2 security releases (or $ENTERPRISE2_SECURITY_PREFIX)",
 						},
 					},
 				},

pkg/build/cmd/publishartifacts.go

@@ -3,60 +3,16 @@ package main
 import (
 	"fmt"
 	"log"
-	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 
+	"github.com/grafana/grafana/pkg/build/env"
 	"github.com/grafana/grafana/pkg/build/gcloud"
-	"github.com/grafana/grafana/pkg/build/versions"
+	"github.com/grafana/grafana/pkg/build/gcloud/storage"
 	"github.com/urfave/cli/v2"
 )
 
-type publishConfig struct {
-	tag                       string
-	srcBucket                 string
-	destBucket                string
-	enterprise2DestBucket     string
-	enterprise2SecurityPrefix string
-	staticAssetsBucket        string
-	staticAssetEditions       []string
-	storybookBucket           string
-	security                  bool
-}
-
-// requireListWithEnvFallback first checks the CLI for a flag with the required
-// name. If this is empty, it  falls back to taking the environment variable.
-// Sadly, we cannot use cli.Flag.EnvVars for this due to it potentially leaking
-// environment variables as default values in usage-errors.
-func requireListWithEnvFallback(cctx *cli.Context, name string, envName string) ([]string, error) {
-	result := cctx.StringSlice(name)
-	if len(result) == 0 {
-		for _, v := range strings.Split(os.Getenv(envName), ",") {
-			value := strings.TrimSpace(v)
-			if value != "" {
-				result = append(result, value)
-			}
-		}
-	}
-	if len(result) == 0 {
-		return nil, cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1)
-	}
-	return result, nil
-}
-
-func requireStringWithEnvFallback(cctx *cli.Context, name string, envName string) (string, error) {
-	result := cctx.String(name)
-	if result == "" {
-		result = os.Getenv(envName)
-	}
-	if result == "" {
-		return "", cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1)
-	}
-	return result, nil
-}
-
-// Action implements the sub-command "publish-artifacts".
+// PublishArtifactsAction Action implements the sub-command "publish-artifacts".
 func PublishArtifactsAction(c *cli.Context) error {
 	if c.NArg() > 0 {
 		if err := cli.ShowSubcommandHelp(c); err != nil {
@@ -65,17 +21,23 @@ func PublishArtifactsAction(c *cli.Context) error {
 		return cli.Exit("", 1)
 	}
 
-	staticAssetEditions, err := requireListWithEnvFallback(c, "static-asset-editions", "STATIC_ASSET_EDITIONS")
+	security := c.Bool("security")
+	var securityDestBucket, enterprise2SecurityPrefix string
+
+	artifactsEditions, err := env.RequireListWithEnvFallback(c, "artifacts-editions", "ARTIFACTS_EDITIONS")
 	if err != nil {
 		return err
 	}
-	securityDestBucket, err := requireStringWithEnvFallback(c, "security-dest-bucket", "SECURITY_DEST_BUCKET")
-	if err != nil {
-		return err
-	}
-	enterprise2SecurityPrefix, err := requireStringWithEnvFallback(c, "enterprise2-security-prefix", "ENTERPRISE2_SECURITY_PREFIX")
-	if err != nil {
-		return err
+
+	if security {
+		securityDestBucket, err = env.RequireStringWithEnvFallback(c, "security-dest-bucket", "SECURITY_DEST_BUCKET")
+		if err != nil {
+			return err
+		}
+		enterprise2SecurityPrefix, err = env.RequireStringWithEnvFallback(c, "enterprise2-security-prefix", "ENTERPRISE2_SECURITY_PREFIX")
+		if err != nil {
+			return err
+		}
 	}
 
 	if err := gcloud.ActivateServiceAccount(); err != nil {
@@ -87,10 +49,7 @@ func PublishArtifactsAction(c *cli.Context) error {
 		destBucket:                c.String("dest-bucket"),
 		enterprise2DestBucket:     c.String("enterprise2-dest-bucket"),
 		enterprise2SecurityPrefix: enterprise2SecurityPrefix,
-		staticAssetsBucket:        c.String("static-assets-bucket"),
-		staticAssetEditions:       staticAssetEditions,
-		storybookBucket:           c.String("storybook-bucket"),
-		security:                  c.Bool("security"),
+		security:                  security,
 		tag:                       strings.TrimPrefix(c.String("tag"), "v"),
 	}
 
@@ -98,114 +57,56 @@ func PublishArtifactsAction(c *cli.Context) error {
 		cfg.destBucket = securityDestBucket
 	}
 
-	err = copyStaticAssets(cfg)
+	gcs, err := storage.New()
 	if err != nil {
 		return err
 	}
-	err = copyStorybook(cfg)
-	if err != nil {
-		return err
-	}
-	err = copyDownloads(cfg)
-	if err != nil {
-		return err
-	}
-	err = copyEnterprise2Downloads(cfg)
-	if err != nil {
-		return err
+
+	for _, edition := range artifactsEditions {
+		switch edition {
+		case "oss", "enterprise":
+			err = copyArtifacts(c, gcs, cfg, edition)
+			if err != nil {
+				return err
+			}
+		case "enterprise2":
+			err = copyEnterprise2Artifacts(c, gcs, cfg)
+			if err != nil {
+				return err
+			}
+		default:
+			log.Printf("unrecognised artifacts edition: %s\n", edition)
+		}
 	}
+
 	return nil
 }
 
-func copyStaticAssets(cfg publishConfig) error {
-	for _, edition := range cfg.staticAssetEditions {
-		log.Printf("Copying static assets for %s", edition)
-		srcURL := fmt.Sprintf("%s/artifacts/static-assets/%s/%s/*", cfg.srcBucket, edition, cfg.tag)
-		destURL := fmt.Sprintf("%s/%s/%s/", cfg.staticAssetsBucket, edition, cfg.tag)
-		err := gcsCopy("static assets", srcURL, destURL)
-		if err != nil {
-			return fmt.Errorf("error copying static assets, %q", err)
-		}
+func copyArtifacts(c *cli.Context, gcs *storage.Client, cfg publishConfig, edition string) error {
+	bucket := gcs.Bucket(cfg.destBucket)
+	destURL := edition
+	if !cfg.security {
+		destURL = filepath.Join(destURL, "release")
 	}
-	log.Printf("Successfully copied static assets!")
-	return nil
-}
-
-func copyStorybook(cfg publishConfig) error {
-	if cfg.security {
-		log.Printf("skipping storybook copy - not needed for a security release")
-		return nil
-	}
-	log.Printf("Copying storybooks...")
-	srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag)
-	destURL := fmt.Sprintf("%s/%s", cfg.storybookBucket, cfg.tag)
-	err := gcsCopy("storybook", srcURL, destURL)
-	if err != nil {
-		return fmt.Errorf("error copying storybook. %q", err)
-	}
-	stableVersion, err := versions.GetLatestVersion(versions.LatestStableVersionURL)
-	if err != nil {
+	log.Printf("Copying downloads for %s, from %s bucket to %s bucket", edition, cfg.srcBucket, destURL)
+	if err := gcs.CopyRemoteDir(c.Context, gcs.Bucket(cfg.srcBucket), fmt.Sprintf("artifacts/downloads/v%s/%s/release", cfg.tag, edition), bucket, destURL); err != nil {
 		return err
 	}
-	isLatest, err := versions.IsGreaterThanOrEqual(cfg.tag, stableVersion)
-	if err != nil {
-		return err
-	}
-	if isLatest {
-		log.Printf("Copying storybooks to latest...")
-		srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag)
-		destURL := fmt.Sprintf("%s/latest", cfg.storybookBucket)
-		err := gcsCopy("storybook (latest)", srcURL, destURL)
-		if err != nil {
-			return fmt.Errorf("error copying storybook to latest. %q", err)
-		}
-	}
 
-	log.Printf("Successfully copied storybook!")
-	return nil
-}
-
-func copyDownloads(cfg publishConfig) error {
-	for _, edition := range []string{
-		"oss", "enterprise",
-	} {
-		destURL := fmt.Sprintf("%s/%s/", cfg.destBucket, edition)
-		srcURL := fmt.Sprintf("%s/artifacts/downloads/v%s/%s/release/*", cfg.srcBucket, cfg.tag, edition)
-		if !cfg.security {
-			destURL = filepath.Join(destURL, "release")
-		}
-		log.Printf("Copying downloads for %s, from %s bucket to %s bucket", edition, srcURL, destURL)
-		err := gcsCopy("downloads", srcURL, destURL)
-		if err != nil {
-			return fmt.Errorf("error copying downloads, %q", err)
-		}
-	}
 	log.Printf("Successfully copied downloads.")
 	return nil
 }
 
-func copyEnterprise2Downloads(cfg publishConfig) error {
+func copyEnterprise2Artifacts(c *cli.Context, gcs *storage.Client, cfg publishConfig) error {
+	bucket := gcs.Bucket(cfg.enterprise2DestBucket)
 	var prefix string
 	if cfg.security {
 		prefix = cfg.enterprise2SecurityPrefix
 	}
-	srcURL := fmt.Sprintf("%s/artifacts/downloads-enterprise2/v%s/enterprise2/release/*", cfg.srcBucket, cfg.tag)
-	destURL := fmt.Sprintf("%s/enterprise2/%srelease", cfg.enterprise2DestBucket, prefix)
-	log.Printf("Copying downloads for enterprise2, from %s bucket to %s bucket", srcURL, destURL)
-	err := gcsCopy("enterprise2 downloads", srcURL, destURL)
-	if err != nil {
-		return fmt.Errorf("error copying ")
-	}
-	return nil
-}
-
-func gcsCopy(desc, src, dest string) error {
-	args := strings.Split(fmt.Sprintf("-m cp -r gs://%s gs://%s", src, dest), " ")
-	// nolint:gosec
-	cmd := exec.Command("gsutil", args...)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("failed to publish %s: %w\n%s", desc, err, out)
+	destURL := fmt.Sprintf("enterprise2/%srelease", prefix)
+	log.Printf("Copying downloads for enterprise2, from %s bucket to %s bucket", cfg.srcBucket, destURL)
+	if err := gcs.CopyRemoteDir(c.Context, gcs.Bucket(cfg.srcBucket), fmt.Sprintf("artifacts/downloads-enterprise2/v%s/enterprise2/release", cfg.tag), bucket, destURL); err != nil {
+		return err
 	}
 	return nil
 }

pkg/build/cmd/publishconfig.go

@@ -0,0 +1,13 @@
+package main
+
+type publishConfig struct {
+	tag                       string
+	srcBucket                 string
+	destBucket                string
+	enterprise2DestBucket     string
+	enterprise2SecurityPrefix string
+	staticAssetsBucket        string
+	staticAssetEditions       []string
+	storybookBucket           string
+	security                  bool
+}

pkg/build/cmd/publishstaticassets.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/build/env"
+	"github.com/grafana/grafana/pkg/build/gcloud"
+	"github.com/grafana/grafana/pkg/build/gcloud/storage"
+	"github.com/urfave/cli/v2"
+)
+
+// PublishStaticAssetsAction Action implements the sub-command "artifacts static-assets".
+func PublishStaticAssetsAction(c *cli.Context) error {
+	if c.NArg() > 0 {
+		if err := cli.ShowSubcommandHelp(c); err != nil {
+			return cli.Exit(err.Error(), 1)
+		}
+		return cli.Exit("", 1)
+	}
+
+	staticAssetEditions, err := env.RequireListWithEnvFallback(c, "static-asset-editions", "STATIC_ASSET_EDITIONS")
+	if err != nil {
+		return err
+	}
+
+	if err := gcloud.ActivateServiceAccount(); err != nil {
+		return fmt.Errorf("error connecting to gcp, %q", err)
+	}
+
+	cfg := publishConfig{
+		srcBucket:           c.String("src-bucket"),
+		staticAssetsBucket:  c.String("static-assets-bucket"),
+		staticAssetEditions: staticAssetEditions,
+		tag:                 strings.TrimPrefix(c.String("tag"), "v"),
+	}
+
+	gcs, err := storage.New()
+	if err != nil {
+		return err
+	}
+	bucket := gcs.Bucket(cfg.staticAssetsBucket)
+
+	for _, edition := range staticAssetEditions {
+		if err := gcs.CopyRemoteDir(c.Context, gcs.Bucket(cfg.srcBucket), fmt.Sprintf("artifacts/static-assets/%s/%s", edition, cfg.tag), bucket, fmt.Sprintf("%s/%s", edition, cfg.tag)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/build/cmd/publishstorybook.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/build/gcloud"
+	"github.com/grafana/grafana/pkg/build/gcloud/storage"
+	"github.com/grafana/grafana/pkg/build/versions"
+	"github.com/urfave/cli/v2"
+)
+
+// PublishStorybookAction Action implements the sub-command "publish-artifacts".
+func PublishStorybookAction(c *cli.Context) error {
+	if c.NArg() > 0 {
+		if err := cli.ShowSubcommandHelp(c); err != nil {
+			return cli.Exit(err.Error(), 1)
+		}
+		return cli.Exit("", 1)
+	}
+
+	if err := gcloud.ActivateServiceAccount(); err != nil {
+		return fmt.Errorf("error connecting to gcp, %q", err)
+	}
+
+	cfg := publishConfig{
+		srcBucket:       c.String("src-bucket"),
+		storybookBucket: c.String("storybook-bucket"),
+		tag:             strings.TrimPrefix(c.String("tag"), "v"),
+	}
+
+	gcs, err := storage.New()
+	if err != nil {
+		return err
+	}
+	bucket := gcs.Bucket(cfg.storybookBucket)
+	if err := gcs.CopyRemoteDir(c.Context, gcs.Bucket(cfg.srcBucket), fmt.Sprintf("artifacts/storybook/v%s", cfg.tag), bucket, cfg.tag); err != nil {
+		return err
+	}
+
+	if latest, err := isLatest(cfg); err != nil && latest {
+		log.Printf("Copying storybooks to latest...")
+		if err := gcs.CopyRemoteDir(c.Context, gcs.Bucket(cfg.srcBucket), fmt.Sprintf("artifacts/storybook/v%s", cfg.tag), bucket, "latest"); err != nil {
+			return err
+		}
+	} else {
+		return err
+	}
+
+	return nil
+}
+
+func isLatest(cfg publishConfig) (bool, error) {
+	stableVersion, err := versions.GetLatestVersion(versions.LatestStableVersionURL)
+	if err != nil {
+		return false, err
+	}
+	isLatest, err := versions.IsGreaterThanOrEqual(cfg.tag, stableVersion)
+	if err != nil {
+		return false, err
+	}
+	return isLatest, nil
+}