CI: move grabpl build-docker from grabpl to grafana (#53077)

* add grabpl build-docker
2022-08-09 09:43:28 +03:00
parent ed5dac7a75
commit 59ce564d49
7 changed files with 354 additions and 15 deletions
@@ -0,0 +1,51 @@
+package main
+
+import (
+	"log"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/build/docker"
+	"github.com/grafana/grafana/pkg/build/gcloud"
+	"github.com/urfave/cli/v2"
+)
+
+func BuildDocker(c *cli.Context) error {
+	if err := docker.Init(); err != nil {
+		return err
+	}
+
+	metadata, err := config.GetMetadata(filepath.Join("dist", "version.json"))
+	if err != nil {
+		return err
+	}
+
+	useUbuntu := c.Bool("ubuntu")
+	verMode, err := config.GetVersion(metadata.ReleaseMode)
+	if err != nil {
+		return err
+	}
+
+	shouldSave := verMode.Docker.ShouldSave
+	if shouldSave {
+		if err := gcloud.ActivateServiceAccount(); err != nil {
+			return err
+		}
+	}
+
+	edition := config.Edition(c.String("edition"))
+
+	version := metadata.GrafanaVersion
+
+	log.Printf("Building Docker images, version %s, %s edition, Ubuntu based: %v...", version, edition,
+		useUbuntu)
+
+	for _, arch := range verMode.Docker.Architectures {
+		if _, err := docker.BuildImage(version, arch, ".", useUbuntu, shouldSave, edition); err != nil {
+			return cli.Exit(err.Error(), 1)
+		}
+	}
+
+	log.Println("Successfully built Docker images!")
+	return nil
+}
@@ -3,7 +3,9 @@ package main
 import (
 	"log"
 	"os"
+	"strings"

+	"github.com/grafana/grafana/pkg/build/docker"
 	"github.com/urfave/cli/v2"
 )

@@ -23,11 +25,34 @@ func main() {
 				&buildIDFlag,
 			},
 		},
+		{
+			Name:   "build-docker",
+			Usage:  "Build Grafana Docker images",
+			Action: ArgCountWrapper(1, BuildDocker),
+			Flags: []cli.Flag{
+				&jobsFlag,
+				&editionFlag,
+				&cli.BoolFlag{
+					Name:  "ubuntu",
+					Usage: "Use Ubuntu base image",
+				},
+				&cli.BoolFlag{
+					Name:  "shouldSave",
+					Usage: "Should save docker image to tarball",
+				},
+				&cli.StringFlag{
+					Name:  "archs",
+					Value: strings.Join(docker.AllArchs, ","),
+					Usage: "Comma separated architectures to build",
+				},
+			},
+		},
 		{
 			Name:   "shellcheck",
 			Usage:  "Run shellcheck on shell scripts",
 			Action: Shellcheck,
-		}, {
+		},
+		{
 			Name:   "build-plugins",
 			Usage:  "Build internal plug-ins",
 			Action: ArgCountWrapper(1, BuildInternalPlugins),
@@ -0,0 +1,156 @@
+package docker
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/build/config"
+)
+
+// verifyArchive verifies the integrity of an archive file.
+func verifyArchive(archive string) error {
+	log.Printf("Verifying checksum of %q", archive)
+
+	//nolint:gosec
+	shaB, err := ioutil.ReadFile(archive + ".sha256")
+	if err != nil {
+		return err
+	}
+
+	exp := strings.TrimSpace(string(shaB))
+
+	//nolint:gosec
+	f, err := os.Open(archive)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Println("error closing file:", err)
+		}
+	}()
+
+	h := sha256.New()
+	_, err = io.Copy(h, f)
+	if err != nil {
+		return err
+	}
+
+	chksum := hex.EncodeToString(h.Sum(nil))
+	if chksum != exp {
+		return fmt.Errorf("archive checksum is different than expected: %q", archive)
+	}
+
+	log.Printf("Archive %q has expected checksum: %s", archive, exp)
+
+	return nil
+}
+
+// BuildImage builds a Docker image.
+// The image tag is returned.
+func BuildImage(version string, arch config.Architecture, grafanaDir string, useUbuntu, shouldSave bool, edition config.Edition) ([]string, error) {
+	var baseArch string
+
+	switch arch {
+	case "amd64":
+	case "armv7":
+		baseArch = "arm32v7/"
+	case "arm64":
+		baseArch = "arm64v8/"
+	default:
+		return []string{}, fmt.Errorf("unrecognized architecture %q", arch)
+	}
+
+	libc := "-musl"
+	dockerfile := "Dockerfile"
+	baseImage := fmt.Sprintf("%salpine:3.15", baseArch)
+	tagSuffix := ""
+	if useUbuntu {
+		libc = ""
+		dockerfile = "ubuntu.Dockerfile"
+		baseImage = fmt.Sprintf("%subuntu:20.04", baseArch)
+		tagSuffix = "-ubuntu"
+	}
+
+	var editionStr string
+	var dockerRepo string
+	var additionalDockerRepo string
+	var tags []string
+	var imageFileBase string
+	switch edition {
+	case config.EditionOSS:
+		dockerRepo = "grafana/grafana-image-tags"
+		additionalDockerRepo = "grafana/grafana-oss-image-tags"
+		imageFileBase = "grafana-oss"
+	case config.EditionEnterprise:
+		dockerRepo = "grafana/grafana-enterprise-image-tags"
+		imageFileBase = "grafana-enterprise"
+		editionStr = "-enterprise"
+	default:
+		return []string{}, fmt.Errorf("unrecognized edition %s", edition)
+	}
+
+	buildDir := filepath.Join(grafanaDir, "packaging/docker")
+	// For example: grafana-8.5.0-52819pre.linux-amd64-musl.tar.gz
+	archive := fmt.Sprintf("grafana%s-%s.linux-%s%s.tar.gz", editionStr, version, arch, libc)
+	if err := verifyArchive(filepath.Join(buildDir, archive)); err != nil {
+		return []string{}, err
+	}
+
+	tag := fmt.Sprintf("%s:%s%s-%s", dockerRepo, version, tagSuffix, arch)
+	tags = append(tags, tag)
+
+	args := []string{
+		"build", "--build-arg", fmt.Sprintf("BASE_IMAGE=%s", baseImage),
+		"--build-arg", fmt.Sprintf("GRAFANA_TGZ=%s", archive), "--tag", tag, "--no-cache",
+		"-f", dockerfile, ".",
+	}
+
+	log.Printf("Running Docker: docker %s", strings.Join(args, " "))
+	//nolint:gosec
+	cmd := exec.Command("docker", args...)
+	cmd.Dir = buildDir
+	cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled")
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return []string{}, fmt.Errorf("building Docker image failed: %w\n%s", err, output)
+	}
+	if shouldSave {
+		imageFile := fmt.Sprintf("%s-%s%s-%s.img", imageFileBase, version, tagSuffix, arch)
+		//nolint:gosec
+		cmd = exec.Command("docker", "save", tag, "-o", imageFile)
+		cmd.Dir = buildDir
+		if output, err := cmd.CombinedOutput(); err != nil {
+			return []string{}, fmt.Errorf("saving Docker image failed: %w\n%s", err, output)
+		}
+		gcsURL := fmt.Sprintf("gs://grafana-prerelease/artifacts/docker/%s/%s", version, imageFile)
+		//nolint:gosec
+		cmd = exec.Command("gsutil", "cp", imageFile, gcsURL)
+		cmd.Dir = buildDir
+		if output, err := cmd.CombinedOutput(); err != nil {
+			return []string{}, fmt.Errorf("storing Docker image failed: %w\n%s", err, output)
+		}
+		log.Printf("Docker image %s stored to grafana-prerelease GCS bucket", imageFile)
+	}
+	if additionalDockerRepo != "" {
+		additionalTag := fmt.Sprintf("%s:%s%s-%s", additionalDockerRepo, version, tagSuffix, arch)
+
+		//nolint:gosec
+		cmd = exec.Command("docker", "tag", tag, additionalTag)
+		cmd.Dir = buildDir
+		if output, err := cmd.CombinedOutput(); err != nil {
+			return []string{}, fmt.Errorf("tagging Docker image failed: %w\n%s", err, output)
+		}
+		tags = append(tags, additionalTag)
+	}
+
+	return tags, nil
+}
@@ -0,0 +1,28 @@
+package docker
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+)
+
+// AllArchs is a list of all supported Docker image architectures.
+var AllArchs = []string{"amd64", "armv7", "arm64"}
+
+// Init initializes the OS for Docker image building.
+func Init() error {
+	// Necessary for cross-platform builds
+	if err := os.Setenv("DOCKER_BUILDKIT", "1"); err != nil {
+		log.Println("error setting DOCKER_BUILDKIT environment variable:", err)
+	}
+
+	// Enable execution of Docker images for other architectures
+	cmd := exec.Command("docker", "run", "--privileged", "--rm",
+		"docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to enable execution of cross-platform Docker images: %w\n%s", err, output)
+	}
+	return nil
+}
@@ -0,0 +1,65 @@
+package gcloud
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func GetDecodedKey() ([]byte, error) {
+	gcpKey := strings.TrimSpace(os.Getenv("GCP_KEY"))
+	if gcpKey == "" {
+		return nil, fmt.Errorf("the environment variable GCP_KEY must be set")
+	}
+
+	gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKey)
+	if err != nil {
+		// key is not always base64 encoded
+		validKey := []byte(gcpKey)
+		if json.Valid(validKey) {
+			return validKey, nil
+		}
+		return nil, fmt.Errorf("error decoding the gcp_key, err: %q", err)
+	}
+
+	return gcpKeyB, nil
+}
+
+func ActivateServiceAccount() error {
+	byteKey, err := GetDecodedKey()
+	if err != nil {
+		return err
+	}
+
+	f, err := os.CreateTemp("", "*.json")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := os.Remove(f.Name()); err != nil {
+			log.Printf("error removing %s: %s", f.Name(), err)
+		}
+	}()
+
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Println("error closing file:", err)
+		}
+	}()
+
+	if _, err := f.Write(byteKey); err != nil {
+		return fmt.Errorf("failed to write GCP key file: %w", err)
+	}
+	keyArg := fmt.Sprintf("--key-file=%s", f.Name())
+	//nolint:gosec
+	cmd := exec.Command("gcloud", "auth", "activate-service-account", keyArg)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to sign into GCP: %w\n%s", err, output)
+	}
+	return nil
+}