public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: move grabpl build-docker from grabpl to grafana (#53077)

Browse Source 
* add grabpl build-docker
This commit is contained in:
Kevin Minehart
2022-08-09 09:43:28 +03:00
committed by GitHub
parent ed5dac7a75
commit 59ce564d49
7 changed files with 354 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+23 -12
View File
@@ -465,9 +465,10 @@ steps:
image: grafana/docker-puppeteer:1.1.0
name: test-a11y-frontend
- commands:
- ./bin/grabpl build-docker --edition oss -archs amd64
- ./bin/build build-docker --edition oss -archs amd64
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -1295,9 +1296,10 @@ steps:
repo:
- grafana/grafana
- commands:
- ./bin/grabpl build-docker --edition oss
- ./bin/build build-docker --edition oss
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -1307,9 +1309,10 @@ steps:
- name: docker
path: /var/run/docker.sock
- commands:
- ./bin/grabpl build-docker --edition oss --ubuntu
- ./bin/build build-docker --edition oss --ubuntu
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -1867,9 +1870,10 @@ steps:
image: grafana/build-container:1.5.9
name: copy-packages-for-docker
- commands:
- ./bin/grabpl build-docker --edition oss --shouldSave
- ./bin/build build-docker --edition oss --shouldSave
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -1879,9 +1883,10 @@ steps:
- name: docker
path: /var/run/docker.sock
- commands:
- ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu
- ./bin/build build-docker --edition oss --shouldSave --ubuntu
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -2514,9 +2519,10 @@ steps:
image: grafana/build-container:1.5.9
name: copy-packages-for-docker
- commands:
- ./bin/grabpl build-docker --edition enterprise --shouldSave
- ./bin/build build-docker --edition enterprise --shouldSave
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -2526,9 +2532,10 @@ steps:
- name: docker
path: /var/run/docker.sock
- commands:
- ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu
- ./bin/build build-docker --edition enterprise --shouldSave --ubuntu
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -3813,9 +3820,10 @@ steps:
image: grafana/build-container:1.5.9
name: copy-packages-for-docker
- commands:
- ./bin/grabpl build-docker --edition oss --shouldSave
- ./bin/build build-docker --edition oss --shouldSave
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -3825,9 +3833,10 @@ steps:
- name: docker
path: /var/run/docker.sock
- commands:
- ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu
- ./bin/build build-docker --edition oss --shouldSave --ubuntu
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -4408,9 +4417,10 @@ steps:
image: grafana/build-container:1.5.9
name: copy-packages-for-docker
- commands:
- ./bin/grabpl build-docker --edition enterprise --shouldSave
- ./bin/build build-docker --edition enterprise --shouldSave
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -4420,9 +4430,10 @@ steps:
- name: docker
path: /var/run/docker.sock
- commands:
- ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu
- ./bin/build build-docker --edition enterprise --shouldSave --ubuntu
depends_on:
- copy-packages-for-docker
- compile-build-cmd
environment:
GCP_KEY:
from_secret: gcp_key
@@ -5134,6 +5145,6 @@ kind: secret
name: gcp_upload_artifacts_key
---
kind: signature
hmac: e7c5243dd58f49260d0dce8bf6c221901ecd3eb8545f9b624e4a8998ee96910e
hmac: 000a4fa4dc0c88d9ee3d5096a9ded360037e23f4ba91896c02419cd4ccce38c3
...
pkg/build/cmd/builddocker.go
+51
View File
@@ -0,0 +1,51 @@
package main
import (
"log"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/urfave/cli/v2"
)
func BuildDocker(c *cli.Context) error {
if err := docker.Init(); err != nil {
return err
}
metadata, err := config.GetMetadata(filepath.Join("dist", "version.json"))
if err != nil {
return err
}
useUbuntu := c.Bool("ubuntu")
verMode, err := config.GetVersion(metadata.ReleaseMode)
if err != nil {
return err
}
shouldSave := verMode.Docker.ShouldSave
if shouldSave {
if err := gcloud.ActivateServiceAccount(); err != nil {
return err
}
}
edition := config.Edition(c.String("edition"))
version := metadata.GrafanaVersion
log.Printf("Building Docker images, version %s, %s edition, Ubuntu based: %v...", version, edition,
useUbuntu)
for _, arch := range verMode.Docker.Architectures {
if _, err := docker.BuildImage(version, arch, ".", useUbuntu, shouldSave, edition); err != nil {
return cli.Exit(err.Error(), 1)
}
}
log.Println("Successfully built Docker images!")
return nil
}
pkg/build/cmd/main.go
+26 -1
View File
@@ -3,7 +3,9 @@ package main
import (
"log"
"os"
"strings"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/urfave/cli/v2"
)
@@ -23,11 +25,34 @@ func main() {
&buildIDFlag,
},
},
{
Name: "build-docker",
Usage: "Build Grafana Docker images",
Action: ArgCountWrapper(1, BuildDocker),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
&cli.BoolFlag{
Name: "ubuntu",
Usage: "Use Ubuntu base image",
},
&cli.BoolFlag{
Name: "shouldSave",
Usage: "Should save docker image to tarball",
},
&cli.StringFlag{
Name: "archs",
Value: strings.Join(docker.AllArchs, ","),
Usage: "Comma separated architectures to build",
},
},
},
{
Name: "shellcheck",
Usage: "Run shellcheck on shell scripts",
Action: Shellcheck,
}, {
},
{
Name: "build-plugins",
Usage: "Build internal plug-ins",
Action: ArgCountWrapper(1, BuildInternalPlugins),
pkg/build/docker/build.go
+156
View File
@@ -0,0 +1,156 @@
package docker
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
)
// verifyArchive verifies the integrity of an archive file.
func verifyArchive(archive string) error {
log.Printf("Verifying checksum of %q", archive)
//nolint:gosec
shaB, err := ioutil.ReadFile(archive + ".sha256")
if err != nil {
return err
}
exp := strings.TrimSpace(string(shaB))
//nolint:gosec
f, err := os.Open(archive)
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
log.Println("error closing file:", err)
}
}()
h := sha256.New()
_, err = io.Copy(h, f)
if err != nil {
return err
}
chksum := hex.EncodeToString(h.Sum(nil))
if chksum != exp {
return fmt.Errorf("archive checksum is different than expected: %q", archive)
}
log.Printf("Archive %q has expected checksum: %s", archive, exp)
return nil
}
// BuildImage builds a Docker image.
// The image tag is returned.
func BuildImage(version string, arch config.Architecture, grafanaDir string, useUbuntu, shouldSave bool, edition config.Edition) ([]string, error) {
var baseArch string
switch arch {
case "amd64":
case "armv7":
baseArch = "arm32v7/"
case "arm64":
baseArch = "arm64v8/"
default:
return []string{}, fmt.Errorf("unrecognized architecture %q", arch)
}
libc := "-musl"
dockerfile := "Dockerfile"
baseImage := fmt.Sprintf("%salpine:3.15", baseArch)
tagSuffix := ""
if useUbuntu {
libc = ""
dockerfile = "ubuntu.Dockerfile"
baseImage = fmt.Sprintf("%subuntu:20.04", baseArch)
tagSuffix = "-ubuntu"
}
var editionStr string
var dockerRepo string
var additionalDockerRepo string
var tags []string
var imageFileBase string
switch edition {
case config.EditionOSS:
dockerRepo = "grafana/grafana-image-tags"
additionalDockerRepo = "grafana/grafana-oss-image-tags"
imageFileBase = "grafana-oss"
case config.EditionEnterprise:
dockerRepo = "grafana/grafana-enterprise-image-tags"
imageFileBase = "grafana-enterprise"
editionStr = "-enterprise"
default:
return []string{}, fmt.Errorf("unrecognized edition %s", edition)
}
buildDir := filepath.Join(grafanaDir, "packaging/docker")
// For example: grafana-8.5.0-52819pre.linux-amd64-musl.tar.gz
archive := fmt.Sprintf("grafana%s-%s.linux-%s%s.tar.gz", editionStr, version, arch, libc)
if err := verifyArchive(filepath.Join(buildDir, archive)); err != nil {
return []string{}, err
}
tag := fmt.Sprintf("%s:%s%s-%s", dockerRepo, version, tagSuffix, arch)
tags = append(tags, tag)
args := []string{
"build", "--build-arg", fmt.Sprintf("BASE_IMAGE=%s", baseImage),
"--build-arg", fmt.Sprintf("GRAFANA_TGZ=%s", archive), "--tag", tag, "--no-cache",
"-f", dockerfile, ".",
}
log.Printf("Running Docker: docker %s", strings.Join(args, " "))
//nolint:gosec
cmd := exec.Command("docker", args...)
cmd.Dir = buildDir
cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled")
if output, err := cmd.CombinedOutput(); err != nil {
return []string{}, fmt.Errorf("building Docker image failed: %w\n%s", err, output)
}
if shouldSave {
imageFile := fmt.Sprintf("%s-%s%s-%s.img", imageFileBase, version, tagSuffix, arch)
//nolint:gosec
cmd = exec.Command("docker", "save", tag, "-o", imageFile)
cmd.Dir = buildDir
if output, err := cmd.CombinedOutput(); err != nil {
return []string{}, fmt.Errorf("saving Docker image failed: %w\n%s", err, output)
}
gcsURL := fmt.Sprintf("gs://grafana-prerelease/artifacts/docker/%s/%s", version, imageFile)
//nolint:gosec
cmd = exec.Command("gsutil", "cp", imageFile, gcsURL)
cmd.Dir = buildDir
if output, err := cmd.CombinedOutput(); err != nil {
return []string{}, fmt.Errorf("storing Docker image failed: %w\n%s", err, output)
}
log.Printf("Docker image %s stored to grafana-prerelease GCS bucket", imageFile)
}
if additionalDockerRepo != "" {
additionalTag := fmt.Sprintf("%s:%s%s-%s", additionalDockerRepo, version, tagSuffix, arch)
//nolint:gosec
cmd = exec.Command("docker", "tag", tag, additionalTag)
cmd.Dir = buildDir
if output, err := cmd.CombinedOutput(); err != nil {
return []string{}, fmt.Errorf("tagging Docker image failed: %w\n%s", err, output)
}
tags = append(tags, additionalTag)
}
return tags, nil
}
pkg/build/docker/init.go
+28
View File
@@ -0,0 +1,28 @@
package docker
import (
"fmt"
"log"
"os"
"os/exec"
)
// AllArchs is a list of all supported Docker image architectures.
var AllArchs = []string{"amd64", "armv7", "arm64"}
// Init initializes the OS for Docker image building.
func Init() error {
// Necessary for cross-platform builds
if err := os.Setenv("DOCKER_BUILDKIT", "1"); err != nil {
log.Println("error setting DOCKER_BUILDKIT environment variable:", err)
}
// Enable execution of Docker images for other architectures
cmd := exec.Command("docker", "run", "--privileged", "--rm",
"docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64")
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to enable execution of cross-platform Docker images: %w\n%s", err, output)
}
return nil
}
pkg/build/gcloud/auth.go
+65
View File
@@ -0,0 +1,65 @@
package gcloud
import (
"encoding/base64"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"strings"
)
func GetDecodedKey() ([]byte, error) {
gcpKey := strings.TrimSpace(os.Getenv("GCP_KEY"))
if gcpKey == "" {
return nil, fmt.Errorf("the environment variable GCP_KEY must be set")
}
gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKey)
if err != nil {
// key is not always base64 encoded
validKey := []byte(gcpKey)
if json.Valid(validKey) {
return validKey, nil
}
return nil, fmt.Errorf("error decoding the gcp_key, err: %q", err)
}
return gcpKeyB, nil
}
func ActivateServiceAccount() error {
byteKey, err := GetDecodedKey()
if err != nil {
return err
}
f, err := os.CreateTemp("", "*.json")
if err != nil {
return err
}
defer func() {
if err := os.Remove(f.Name()); err != nil {
log.Printf("error removing %s: %s", f.Name(), err)
}
}()
defer func() {
if err := f.Close(); err != nil {
log.Println("error closing file:", err)
}
}()
if _, err := f.Write(byteKey); err != nil {
return fmt.Errorf("failed to write GCP key file: %w", err)
}
keyArg := fmt.Sprintf("--key-file=%s", f.Name())
//nolint:gosec
cmd := exec.Command("gcloud", "auth", "activate-service-account", keyArg)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to sign into GCP: %w\n%s", err, output)
}
return nil
}
scripts/drone/steps/lib.star
+5 -2
View File
@@ -799,7 +799,7 @@ def copy_packages_for_docker_step():
def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
cmd = './bin/grabpl build-docker --edition {}'.format(edition)
cmd = './bin/build build-docker --edition {}'.format(edition)
if publish:
cmd += ' --shouldSave'
@@ -814,7 +814,10 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis
return {
'name': 'build-docker-images' + ubuntu_sfx,
'image': 'google/cloud-sdk',
'depends_on': ['copy-packages-for-docker'],
'depends_on': [
'copy-packages-for-docker',
'compile-build-cmd',
],
'commands': [
cmd
],