From 93e78e21466d457da5fe04bb4a07b4c88cdf7b65 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Mon, 19 Sep 2022 11:19:31 +0300 Subject: [PATCH] CI: Move `fetch-images` sub-command from `build-pipeline` repo (#55298) * Move fetch images from build-pipeline * Modify drone * Replace zerolog with builtin log * Fix lint --- .drone.yml | 50 +++++++++---- pkg/build/cmd/fetchimages.go | 116 ++++++++++++++++++++++++++++++ pkg/build/cmd/main.go | 15 ++++ pkg/build/cmd/storestorybook.go | 3 - pkg/build/config/version.go | 7 +- pkg/build/config/versions.go | 1 + pkg/build/docker/dockerconfig.go | 15 ++++ scripts/drone/events/release.star | 17 ++--- 8 files changed, 197 insertions(+), 27 deletions(-) create mode 100644 pkg/build/cmd/fetchimages.go create mode 100644 pkg/build/docker/dockerconfig.go diff --git a/.drone.yml b/.drone.yml index 3fced8c18a5..649ae8e2cb3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3038,10 +3038,16 @@ steps: image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition oss --base - alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.1 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition oss depends_on: - - grabpl + - compile-build-cmd environment: DOCKER_PASSWORD: from_secret: docker_password @@ -3118,10 +3124,16 @@ steps: image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition enterprise - --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.1 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition enterprise depends_on: - - grabpl + - compile-build-cmd environment: DOCKER_PASSWORD: from_secret: docker_password @@ -3181,10 +3193,16 @@ steps: image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition oss --base - alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.1 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition oss depends_on: - - grabpl + - compile-build-cmd environment: DOCKER_PASSWORD: from_secret: docker_password @@ -3262,10 +3280,16 @@ steps: image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - - ./bin/grabpl artifacts docker fetch --version-tag ${TAG} --edition enterprise - --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7 + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] + environment: + CGO_ENABLED: 0 + image: golang:1.19.1 + name: compile-build-cmd +- commands: + - ./bin/build artifacts docker fetch --edition enterprise depends_on: - - grabpl + - compile-build-cmd environment: DOCKER_PASSWORD: from_secret: docker_password @@ -5060,6 +5084,6 @@ kind: secret name: packages_secret_access_key --- kind: signature -hmac: 1bbfd995ded7c2d1c0330ef2009691577a6613ec98df296fc8ec4388b9898e2d +hmac: f5d430e8ea6ad2f889f24ffcaa5a2555e77b45fab364b6db85efa591bedc0a6b ... diff --git a/pkg/build/cmd/fetchimages.go b/pkg/build/cmd/fetchimages.go new file mode 100644 index 00000000000..3845daed5a7 --- /dev/null +++ b/pkg/build/cmd/fetchimages.go @@ -0,0 +1,116 @@ +package main + +import ( + "fmt" + "log" + "os/exec" + "strings" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/docker" + "github.com/grafana/grafana/pkg/build/gcloud" + "github.com/urfave/cli/v2" +) + +const ( + alpine = "alpine" + ubuntu = "ubuntu" +) + +func FetchImages(c *cli.Context) error { + if c.NArg() > 0 { + if err := cli.ShowSubcommandHelp(c); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return cli.NewExitError("", 1) + } + + metadata, err := GenerateMetadata(c) + if err != nil { + return err + } + + buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) + if err != nil { + return err + } + + cfg := docker.Config{ + Archs: buildConfig.Docker.Architectures, + Distribution: buildConfig.Docker.Distribution, + Bucket: buildConfig.Docker.PrereleaseBucket, + Edition: c.String("edition"), + Tag: metadata.GrafanaVersion, + } + + edition := fmt.Sprintf("-%s", cfg.Edition) + + err = gcloud.ActivateServiceAccount() + if err != nil { + return err + } + + var basesStr []string + for _, base := range cfg.Distribution { + switch base { + case alpine: + basesStr = append(basesStr, "") + case ubuntu: + basesStr = append(basesStr, "-ubuntu") + default: + return fmt.Errorf("unrecognized base %q", base) + } + } + + err = downloadFromGCS(cfg, basesStr, edition) + if err != nil { + return err + } + + err = loadImages(cfg, basesStr, edition) + if err != nil { + return err + } + return nil +} + +func loadImages(cfg docker.Config, basesStr []string, edition string) error { + log.Println("Loading fetched image files to local docker registry...") + log.Printf("Number of images to be loaded: %d\n", len(basesStr)*len(cfg.Archs)) + for _, base := range basesStr { + for _, arch := range cfg.Archs { + imageFilename := fmt.Sprintf("grafana%s-%s%s-%s.img", edition, cfg.Tag, base, arch) + log.Printf("image file name: %s\n", imageFilename) + //nolint:gosec + cmd := exec.Command("docker", "load", "-i", imageFilename) + cmd.Dir = "." + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("out: %s\n", out) + return fmt.Errorf("error loading image: %q", err) + } + log.Printf("Successfully loaded %s!\n %s\n", fmt.Sprintf("grafana%s-%s%s-%s", edition, cfg.Tag, base, arch), out) + } + } + log.Println("Images successfully loaded!") + return nil +} + +func downloadFromGCS(cfg docker.Config, basesStr []string, edition string) error { + log.Printf("Downloading Docker images from GCS bucket: %s\n", cfg.Bucket) + + for _, base := range basesStr { + for _, arch := range cfg.Archs { + src := fmt.Sprintf("gs://%s/%s/grafana%s-%s%s-%s.img", cfg.Bucket, cfg.Tag, edition, cfg.Tag, base, arch) + args := strings.Split(fmt.Sprintf("-m cp -r %s .", src), " ") + //nolint:gosec + cmd := exec.Command("gsutil", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to download: %w\n%s", err, out) + } + } + } + log.Printf("Successfully fetched image files from %s bucket!\n", cfg.Bucket) + return nil +} diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 94feb90676b..e606923755f 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -135,6 +135,21 @@ func main() { }, }, }, + { + Name: "docker", + Usage: "Handle Grafana Docker images", + Subcommands: cli.Commands{ + { + Name: "fetch", + Usage: "Fetch Grafana Docker images", + ArgsUsage: "[version]", + Action: ArgCountWrapper(1, FetchImages), + Flags: []cli.Flag{ + &editionFlag, + }, + }, + }, + }, } if err := app.Run(os.Args); err != nil { diff --git a/pkg/build/cmd/storestorybook.go b/pkg/build/cmd/storestorybook.go index 84403548c37..0daeb7080f6 100644 --- a/pkg/build/cmd/storestorybook.go +++ b/pkg/build/cmd/storestorybook.go @@ -22,9 +22,6 @@ func StoreStorybook(c *cli.Context) error { if err != nil { return err } - if err != nil { - return err - } storybookBucket := buildConfig.Buckets.Storybook srcPath := buildConfig.Buckets.StorybookSrcDir diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go index 22252acc676..fa8c8b3555b 100644 --- a/pkg/build/config/version.go +++ b/pkg/build/config/version.go @@ -28,9 +28,10 @@ type PluginSignature struct { } type Docker struct { - ShouldSave bool `json:"shouldSave,omitempty"` - Distribution []Distribution `json:"distribution,omitempty"` - Architectures []Architecture `json:"archs,omitempty"` + ShouldSave bool `json:"shouldSave,omitempty"` + Distribution []Distribution `json:"distribution,omitempty"` + Architectures []Architecture `json:"archs,omitempty"` + PrereleaseBucket string `json:"prereleaseBucket,omitempty"` } type Buckets struct { diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go index a3d435b6601..6b7ee7339cd 100644 --- a/pkg/build/config/versions.go +++ b/pkg/build/config/versions.go @@ -151,6 +151,7 @@ var Versions = VersionMap{ Alpine, Ubuntu, }, + PrereleaseBucket: "grafana-prerelease/artifacts/docker", }, Buckets: Buckets{ Artifacts: "grafana-prerelease/artifacts/downloads", diff --git a/pkg/build/docker/dockerconfig.go b/pkg/build/docker/dockerconfig.go new file mode 100644 index 00000000000..7a971dc2ba5 --- /dev/null +++ b/pkg/build/docker/dockerconfig.go @@ -0,0 +1,15 @@ +package docker + +import ( + "github.com/grafana/grafana/pkg/build/config" +) + +type Config struct { + Bucket string + Edition string + Tag string + Distribution []config.Distribution + Archs []config.Architecture + DockerHubRepo string + Security bool +} diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index d3f01b5c8a9..aff972bccb6 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -120,22 +120,23 @@ def fetch_images_step(edition): 'DOCKER_USER': from_secret('docker_username'), 'DOCKER_PASSWORD': from_secret('docker_password'), }, - 'commands': ['./bin/grabpl artifacts docker fetch --version-tag ${{TAG}} --edition {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(edition)], - 'depends_on': ['grabpl'], + 'commands': ['./bin/build artifacts docker fetch --edition {}'.format(edition)], + 'depends_on': ['compile-build-cmd'], 'volumes': [{ 'name': 'docker', 'path': '/var/run/docker.sock' }], } -def publish_image_steps(version, mode, docker_repo, additional_docker_repo=""): +def publish_image_steps(edition, mode, docker_repo, additional_docker_repo=""): steps = [ download_grabpl_step(), - fetch_images_step(version), - publish_images_step(version, 'release', mode, docker_repo), + compile_build_cmd(), + fetch_images_step(edition), + publish_images_step(edition, 'release', mode, docker_repo), ] if additional_docker_repo != "": - steps.extend([publish_images_step(version, 'release', mode, additional_docker_repo)]) + steps.extend([publish_images_step(edition, 'release', mode, additional_docker_repo)]) return steps @@ -146,9 +147,9 @@ def publish_image_pipelines(mode): } return [pipeline( - name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='oss', mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition="" + name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition="" ), pipeline( - name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="" + name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="" ),] def get_oss_pipelines(trigger, ver_mode):