public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v8.5.x] CI: Backport grafana-com publishing (#65168)

Browse Source 
[v8.5.x] Backport grafana-com publishing

This should complete #56797.
This commit is contained in:
Horst Gutmann
2023-05-12 09:33:12 +03:00
committed by GitHub
parent b06e795c27
commit 8135ff40d6
53 changed files with 5533 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+5 -1
View File
@@ -3233,6 +3233,8 @@ steps:
- publish-linux-packages-deb
- publish-linux-packages-rpm
environment:
GCP_KEY:
from_secret: gcp_key
GRAFANA_COM_API_KEY:
from_secret: grafana_api_key
image: grafana/grafana-ci-deploy:1.3.1
@@ -3326,6 +3328,8 @@ steps:
- publish-linux-packages-deb
- publish-linux-packages-rpm
environment:
GCP_KEY:
from_secret: gcp_key
GRAFANA_COM_API_KEY:
from_secret: grafana_api_key
image: grafana/grafana-ci-deploy:1.3.1
@@ -4792,6 +4796,6 @@ kind: secret
name: aws_secret_access_key
---
kind: signature
hmac: bf3d2885e476653b0f3bcecb4e0335b2d243285d90e3d74492bf794a90615e06
hmac: 702107d1f512b722de5103aa8c80ff0f0048b3e42acbc3af17064301414def49
...
pkg/build/cmd/argcount_wrapper.go
+31
View File
@@ -0,0 +1,31 @@
package main
import "github.com/urfave/cli/v2"
// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided.
func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() != num {
if err := cli.ShowSubcommandHelp(ctx); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
return action(ctx)
}
}
// ArgCountWrapper will cause the action to fail if there were more than `num` args provided.
func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() > max {
if err := cli.ShowSubcommandHelp(ctx); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
return action(ctx)
}
}
pkg/build/cmd/flags.go
+59
View File
@@ -0,0 +1,59 @@
package main
import "github.com/urfave/cli/v2"
var (
jobsFlag = cli.IntFlag{
Name: "jobs",
Usage: "Number of parallel jobs",
}
buildIDFlag = cli.StringFlag{
Name: "build-id",
Usage: "Optionally supply a build ID to be part of the version",
}
editionFlag = cli.StringFlag{
Name: "edition",
Usage: "The edition of Grafana to build (oss or enterprise)",
Value: "oss",
}
variantsFlag = cli.StringFlag{
Name: "variants",
Usage: "Comma-separated list of variants to build",
}
triesFlag = cli.IntFlag{
Name: "tries",
Usage: "Specify number of tries before failing",
Value: 1,
}
noInstallDepsFlag = cli.BoolFlag{
Name: "no-install-deps",
Usage: "Don't install dependencies",
}
signingAdminFlag = cli.BoolFlag{
Name: "signing-admin",
Usage: "Use manifest signing admin API endpoint?",
}
signFlag = cli.BoolFlag{
Name: "sign",
Usage: "Enable plug-in signing (you must set GRAFANA_API_KEY)",
}
dryRunFlag = cli.BoolFlag{
Name: "dry-run",
Usage: "Only simulate actions",
}
gcpKeyFlag = cli.StringFlag{
Name: "gcp-key",
Usage: "Google Cloud Platform key file",
Required: true,
}
gitHubTokenFlag = cli.StringFlag{
Name: "github-token",
Value: "",
EnvVars: []string{"GITHUB_TOKEN"},
Usage: "GitHub token",
}
tagFlag = cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
}
)
pkg/build/cmd/grafanacom.go
+323
View File
@@ -0,0 +1,323 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/grafana/grafana/pkg/build/packaging"
)
const grafanaAPI = "https://grafana.com/api"
// GrafanaCom implements the sub-command "grafana-com".
func GrafanaCom(c *cli.Context) error {
bucketStr := c.String("src-bucket")
edition := config.Edition(c.String("edition"))
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
releaseMode, err := metadata.GetReleaseMode()
if err != nil {
return err
}
version := metadata.GrafanaVersion
if releaseMode.Mode == config.Cronjob {
gcs, err := storage.New()
if err != nil {
return err
}
bucket := gcs.Bucket(bucketStr)
latestMainVersion, err := storage.GetLatestMainBuild(c.Context, bucket, filepath.Join(string(edition), "main"))
if err != nil {
return err
}
version = latestMainVersion
}
dryRun := c.Bool("dry-run")
simulateRelease := c.Bool("simulate-release")
// Test release mode and dryRun imply simulateRelease
if releaseMode.IsTest || dryRun {
simulateRelease = true
}
grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY"))
if grafanaAPIKey == "" {
return cli.Exit("the environment variable GRAFANA_COM_API_KEY must be set", 1)
}
whatsNewURL, releaseNotesURL, err := getReleaseURLs()
if err != nil {
return cli.Exit(err.Error(), 1)
}
// TODO: Verify config values
cfg := packaging.PublishConfig{
Config: config.Config{
Version: version,
},
Edition: edition,
ReleaseMode: releaseMode,
GrafanaAPIKey: grafanaAPIKey,
WhatsNewURL: whatsNewURL,
ReleaseNotesURL: releaseNotesURL,
DryRun: dryRun,
TTL: c.String("ttl"),
SimulateRelease: simulateRelease,
}
if err := publishPackages(cfg); err != nil {
return cli.Exit(err.Error(), 1)
}
log.Println("Successfully published packages to grafana.com!")
return nil
}
func getReleaseURLs() (string, string, error) {
type grafanaConf struct {
WhatsNewURL string `json:"whatsNewUrl"`
ReleaseNotesURL string `json:"releaseNotesUrl"`
}
type packageConf struct {
Grafana grafanaConf `json:"grafana"`
}
pkgB, err := os.ReadFile("package.json")
if err != nil {
return "", "", fmt.Errorf("failed to read package.json: %w", err)
}
var pconf packageConf
if err := json.Unmarshal(pkgB, &pconf); err != nil {
return "", "", fmt.Errorf("failed to decode package.json: %w", err)
}
if _, err := url.ParseRequestURI(pconf.Grafana.WhatsNewURL); err != nil {
return "", "", fmt.Errorf("grafana.whatsNewUrl is invalid in package.json: %q", pconf.Grafana.WhatsNewURL)
}
if _, err := url.ParseRequestURI(pconf.Grafana.ReleaseNotesURL); err != nil {
return "", "", fmt.Errorf("grafana.releaseNotesUrl is invalid in package.json: %q",
pconf.Grafana.ReleaseNotesURL)
}
return pconf.Grafana.WhatsNewURL, pconf.Grafana.ReleaseNotesURL, nil
}
// publishPackages publishes packages to grafana.com.
func publishPackages(cfg packaging.PublishConfig) error {
log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n",
cfg.Version, cfg.Edition, cfg.ReleaseMode.Mode, cfg.DryRun, cfg.SimulateRelease)
versionStr := fmt.Sprintf("v%s", cfg.Version)
log.Printf("Creating release %s at grafana.com...\n", versionStr)
var sfx string
var pth string
switch cfg.Edition {
case config.EditionOSS:
pth = "oss"
case config.EditionEnterprise:
pth = "enterprise"
sfx = packaging.EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
switch cfg.ReleaseMode.Mode {
case config.MainMode, config.DownstreamMode, config.CronjobMode:
pth = path.Join(pth, packaging.MainFolder)
default:
pth = path.Join(pth, packaging.ReleaseFolder)
}
product := fmt.Sprintf("grafana%s", sfx)
pth = path.Join(pth, product)
baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth)
var builds []buildRepr
for _, ba := range packaging.ArtifactConfigs {
u := ba.GetURL(baseArchiveURL, cfg)
sha256, err := getSHA256(u)
if err != nil {
return err
}
builds = append(builds, buildRepr{
OS: ba.Os,
URL: u,
SHA256: string(sha256),
Arch: ba.Arch,
})
}
r := releaseRepr{
Version: cfg.Version,
ReleaseDate: time.Now().UTC(),
Builds: builds,
Stable: cfg.ReleaseMode.Mode == config.TagMode && !cfg.ReleaseMode.IsBeta && !cfg.ReleaseMode.IsTest,
Beta: cfg.ReleaseMode.IsBeta,
Nightly: cfg.ReleaseMode.Mode == config.CronjobMode,
}
if cfg.ReleaseMode.Mode == config.TagMode || r.Beta {
r.WhatsNewURL = cfg.WhatsNewURL
r.ReleaseNotesURL = cfg.ReleaseNotesURL
}
if err := postRequest(cfg, "versions", r, fmt.Sprintf("create release %s", r.Version)); err != nil {
return err
}
if err := postRequest(cfg, fmt.Sprintf("versions/%s", cfg.Version), r,
fmt.Sprintf("update release %s", cfg.Version)); err != nil {
return err
}
for _, b := range r.Builds {
if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages", cfg.Version), b,
fmt.Sprintf("create build %s %s", b.OS, b.Arch)); err != nil {
return err
}
if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages/%s/%s", cfg.Version, b.Arch, b.OS), b,
fmt.Sprintf("update build %s %s", b.OS, b.Arch)); err != nil {
return err
}
}
return nil
}
func getSHA256(u string) ([]byte, error) {
shaURL := fmt.Sprintf("%s.sha256", u)
// nolint:gosec
resp, err := http.Get(shaURL)
if err != nil {
return nil, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println("failed to close response body, err: %w", err)
}
}()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("failed downloading %s: %s", u, resp.Status)
}
sha256, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return sha256, nil
}
func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = packaging.EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
product := fmt.Sprintf("grafana%s", sfx)
jsonB, err := json.Marshal(obj)
if err != nil {
return fmt.Errorf("failed to JSON encode release: %w", err)
}
u, err := constructURL(product, pth)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(jsonB))
if err != nil {
return err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cfg.GrafanaAPIKey))
req.Header.Add("Content-Type", "application/json")
log.Printf("Posting to grafana.com API, %s - JSON: %s\n", u, string(jsonB))
if cfg.SimulateRelease {
log.Println("Only simulating request")
return nil
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("failed posting to %s (%s): %s", u, descr, err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println("failed to close response body, err: %w", err)
}
}()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if strings.Contains(string(body), "already exists") || strings.Contains(string(body), "Nothing to update") {
log.Printf("Already exists: %s\n", descr)
return nil
}
return fmt.Errorf("failed posting to %s (%s): %s", u, descr, resp.Status)
}
log.Printf("Successfully posted to grafana.com API, %s\n", u)
return nil
}
func constructURL(product string, pth string) (string, error) {
productPath := filepath.Clean(filepath.Join("/", product, pth))
u, err := url.Parse(grafanaAPI)
if err != nil {
return "", err
}
u.Path = path.Join(u.Path, productPath)
return u.String(), err
}
type buildRepr struct {
OS string `json:"os"`
URL string `json:"url"`
SHA256 string `json:"sha256"`
Arch string `json:"arch"`
}
type releaseRepr struct {
Version string `json:"version"`
ReleaseDate time.Time `json:"releaseDate"`
Stable bool `json:"stable"`
Beta bool `json:"beta"`
Nightly bool `json:"nightly"`
WhatsNewURL string `json:"whatsNewUrl"`
ReleaseNotesURL string `json:"releaseNotesUrl"`
Builds []buildRepr `json:"-"`
}
pkg/build/cmd/grafanacom_test.go
+35
View File
@@ -0,0 +1,35 @@
package main
import (
"testing"
)
func Test_constructURL(t *testing.T) {
type args struct {
product string
pth string
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{name: "cleans .. sequence", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false},
{name: "doesn't clean anything - non malicious url", args: args{"foo", "bar"}, want: "https://grafana.com/api/foo/bar", wantErr: false},
{name: "doesn't clean anything - three dots", args: args{"...", "..."}, want: "https://grafana.com/api/.../...", wantErr: false},
{name: "cleans .", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := constructURL(tt.args.product, tt.args.pth)
if (err != nil) != tt.wantErr {
t.Errorf("constructURL() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("constructURL() got = %v, want %v", got, tt.want)
}
})
}
}
pkg/build/cmd/main.go
+48
View File
@@ -0,0 +1,48 @@
package main
import (
"log"
"os"
"github.com/urfave/cli/v2"
)
var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5)
//nolint:unused
func registerAppCommand(c *cli.Command) {
additionalCommands = append(additionalCommands, c)
}
func main() {
app := cli.NewApp()
app.Commands = cli.Commands{
{
Name: "publish",
Usage: "Publish packages to Grafana com and repositories",
Subcommands: cli.Commands{
{
Name: "grafana-com",
Usage: "Publish packages to grafana.com",
Action: GrafanaCom,
Flags: []cli.Flag{
&editionFlag,
&buildIDFlag,
&dryRunFlag,
&cli.StringFlag{
Name: "src-bucket",
Value: "grafana-downloads",
Usage: "Google Cloud Storage bucket",
},
},
},
},
},
}
app.Commands = append(app.Commands, additionalCommands...)
if err := app.Run(os.Args); err != nil {
log.Fatalln(err)
}
}
pkg/build/compilers/install.go
+50
View File
@@ -0,0 +1,50 @@
package compilers
import (
"fmt"
"os"
"os/exec"
"path/filepath"
)
const (
ArmV6 = "/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc"
Armv7 = "arm-linux-gnueabihf-gcc"
Armv7Musl = "/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc"
Arm64 = "aarch64-linux-gnu-gcc"
Arm64Musl = "/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc"
Osx64 = "/tmp/osxcross/target/bin/o64-clang"
Win64 = "x86_64-w64-mingw32-gcc"
LinuxX64 = "/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc"
LinuxX64Musl = "/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc"
)
func Install() error {
// From the os.TempDir documentation:
// On Unix systems, it returns $TMPDIR if non-empty,
// else /tmp. On Windows, it uses GetTempPath,
// returning the first non-empty value from %TMP%, %TEMP%, %USERPROFILE%,
// or the Windows directory. On Plan 9, it returns /tmp.
tmp := os.TempDir()
var (
centosArchive = "x86_64-centos6-linux-gnu.tar.xz"
osxArchive = "osxcross.tar.xz"
)
for _, fname := range []string{centosArchive, osxArchive} {
path := filepath.Join(tmp, fname)
if _, err := os.Stat(path); err != nil {
return fmt.Errorf("stat error: %w", err)
}
// Ignore gosec G204 as this function is only used in the build process.
//nolint:gosec
cmd := exec.Command("tar", "xfJ", fname)
cmd.Dir = tmp
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to unpack %q: %q, %w", fname, output, err)
}
}
return nil
}
pkg/build/config/config.go
+18
View File
@@ -0,0 +1,18 @@
package config
type Config struct {
Version string
Bucket string
DebRepoBucket string
DebDBBucket string
RPMRepoBucket string
GPGPassPath string
GPGPrivateKey string
GPGPublicKey string
NumWorkers int
GitHubUser string
GitHubToken string
PullEnterprise bool
PackageVersion string
SignPackages bool
}
pkg/build/config/edition.go
+9
View File
@@ -0,0 +1,9 @@
package config
type Edition string
const (
EditionOSS Edition = "oss"
EditionEnterprise Edition = "enterprise"
EditionEnterprise2 Edition = "enterprise2"
)
pkg/build/config/genmetadata.go
+102
View File
@@ -0,0 +1,102 @@
package config
import (
"fmt"
"os"
"strings"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/urfave/cli/v2"
)
func GenerateMetadata(c *cli.Context) (Metadata, error) {
var metadata Metadata
version := ""
event, err := droneutil.GetDroneEventFromEnv()
if err != nil {
return Metadata{}, err
}
tag, ok := os.LookupEnv("DRONE_TAG")
if !ok {
fmt.Println("DRONE_TAG envvar not present, %w", err)
}
var releaseMode ReleaseMode
switch event {
case string(PullRequestMode):
releaseMode = ReleaseMode{Mode: PullRequestMode}
case Push:
mode, err := CheckDroneTargetBranch()
if err != nil {
return Metadata{}, err
}
releaseMode = ReleaseMode{Mode: mode}
case Custom:
if edition, _ := os.LookupEnv("EDITION"); edition == string(EditionEnterprise2) {
releaseMode = ReleaseMode{Mode: Enterprise2Mode}
if tag != "" {
version = strings.TrimPrefix(tag, "v")
}
break
}
mode, err := CheckDroneTargetBranch()
if err != nil {
return Metadata{}, err
}
// if there is a custom event targeting the main branch, that's an enterprise downstream build
if mode == MainBranch {
releaseMode = ReleaseMode{Mode: DownstreamMode}
} else {
releaseMode = ReleaseMode{Mode: mode}
}
case Tag, Promote:
if tag == "" {
return Metadata{}, fmt.Errorf("DRONE_TAG envvar not present for a tag/promotion event, %w", err)
}
version = strings.TrimPrefix(tag, "v")
mode, err := CheckSemverSuffix()
if err != nil {
return Metadata{}, err
}
releaseMode = mode
case Cronjob:
releaseMode = ReleaseMode{Mode: CronjobMode}
}
if version == "" {
version, err = generateVersionFromBuildID()
if err != nil {
return Metadata{}, err
}
}
currentCommit, err := GetDroneCommit()
if err != nil {
return Metadata{}, err
}
metadata = Metadata{
GrafanaVersion: version,
ReleaseMode: releaseMode,
GrabplVersion: c.App.Version,
CurrentCommit: currentCommit,
}
fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode)
return metadata, nil
}
func generateVersionFromBuildID() (string, error) {
buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER")
if !ok {
return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable")
}
var err error
version, err := GetGrafanaVersion(buildID, ".")
if err != nil {
return "", err
}
return version, nil
}
pkg/build/config/genmetadata_test.go
+81
View File
@@ -0,0 +1,81 @@
package config
import (
"flag"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
)
const (
DroneBuildEvent = "DRONE_BUILD_EVENT"
DroneTargetBranch = "DRONE_TARGET_BRANCH"
DroneTag = "DRONE_TAG"
DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE"
DroneBuildNumber = "DRONE_BUILD_NUMBER"
)
const (
hashedGrafanaVersion = "9.2.0-12345pre"
versionedBranch = "v9.2.x"
)
func TestGetMetadata(t *testing.T) {
tcs := []struct {
envMap map[string]string
expVersion string
mode ReleaseMode
}{
{map[string]string{DroneBuildEvent: PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: PullRequestMode}},
{map[string]string{DroneBuildEvent: Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: Push, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: MainMode}},
{map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: DownstreamMode}},
{map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345", "EDITION": string(EditionEnterprise2)}, hashedGrafanaVersion, ReleaseMode{Mode: Enterprise2Mode}},
{map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}},
{map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}},
}
ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil)
for _, tc := range tcs {
t.Run("Should return valid metadata, ", func(t *testing.T) {
setUpEnv(t, tc.envMap)
testMetadata(t, ctx, tc.expVersion, tc.mode)
})
}
}
func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode ReleaseMode) {
t.Helper()
metadata, err := GenerateMetadata(ctx)
require.NoError(t, err)
t.Run("with a valid version", func(t *testing.T) {
expVersion := metadata.GrafanaVersion
require.Equal(t, expVersion, version)
})
t.Run("with a valid release mode from the built-in list", func(t *testing.T) {
expMode := metadata.ReleaseMode
require.NoError(t, err)
require.Equal(t, expMode, releaseMode)
})
}
func setUpEnv(t *testing.T, envMap map[string]string) {
t.Helper()
os.Clearenv()
err := os.Setenv("DRONE_COMMIT", "abcd12345")
require.NoError(t, err)
for k, v := range envMap {
err := os.Setenv(k, v)
require.NoError(t, err)
}
}
pkg/build/config/package.json
+3
View File
@@ -0,0 +1,3 @@
{
"version": "9.2.0-pre"
}
pkg/build/config/revision.go
+55
View File
@@ -0,0 +1,55 @@
package config
import (
"context"
"fmt"
"strconv"
"time"
"github.com/grafana/grafana/pkg/build/executil"
)
type Revision struct {
Timestamp int64
SHA256 string
Branch string
}
func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) {
out, err := executil.OutputAt(ctx, dir, "git", "show", "-s", "--format=%ct")
if err != nil {
return time.Now().Unix(), nil
}
stamp, err := strconv.ParseInt(out, 10, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse output from git show: %q", out)
}
return stamp, nil
}
// GrafanaRevision uses git commands to get information about the checked out Grafana code located at 'grafanaDir'.
// This could maybe be a more generic "Describe" function in the "git" package.
func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) {
stamp, err := GrafanaTimestamp(ctx, grafanaDir)
if err != nil {
return Revision{}, err
}
sha, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--short", "HEAD")
if err != nil {
return Revision{}, err
}
branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD")
if err != nil {
return Revision{}, err
}
return Revision{
SHA256: sha,
Branch: branch,
Timestamp: stamp,
}, nil
}
pkg/build/config/variant.go
+63
View File
@@ -0,0 +1,63 @@
package config
// Variant is the OS / Architecture combination that Grafana can be compiled for.
type Variant string
const (
VariantLinuxAmd64 Variant = "linux-amd64"
VariantLinuxAmd64Musl Variant = "linux-amd64-musl"
VariantArmV6 Variant = "linux-armv6"
VariantArmV7 Variant = "linux-armv7"
VariantArmV7Musl Variant = "linux-armv7-musl"
VariantArm64 Variant = "linux-arm64"
VariantArm64Musl Variant = "linux-arm64-musl"
VariantDarwinAmd64 Variant = "darwin-amd64"
VariantWindowsAmd64 Variant = "windows-amd64"
)
var AllVariants = []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
}
// Architecture is an allowed value in the GOARCH environment variable.
type Architecture string
const (
ArchAMD64 Architecture = "amd64"
ArchARMv6 Architecture = "armv6"
ArchARMv7 Architecture = "armv7"
ArchARM64 Architecture = "arm64"
ArchARMHF Architecture = "armhf"
ArchARMHFP Architecture = "armhfp"
ArchARM Architecture = "arm"
)
type OS string
const (
OSWindows OS = "windows"
OSDarwin OS = "darwin"
OSLinux OS = "linux"
)
type LibC string
const (
LibCMusl = "musl"
)
// Distribution is the base os image where the Grafana image is built on.
type Distribution string
const (
Ubuntu Distribution = "ubuntu"
Alpine Distribution = "alpine"
)
pkg/build/config/version.go
+157
View File
@@ -0,0 +1,157 @@
package config
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/grafana/grafana/pkg/build/git"
)
type Metadata struct {
GrafanaVersion string `json:"version,omitempty"`
ReleaseMode ReleaseMode `json:"releaseMode,omitempty"`
GrabplVersion string `json:"grabplVersion,omitempty"`
CurrentCommit string `json:"currentCommit,omitempty"`
}
type ReleaseMode struct {
Mode VersionMode `json:"mode,omitempty"`
IsBeta bool `json:"isBeta,omitempty"`
IsTest bool `json:"isTest,omitempty"`
}
type PluginSignature struct {
Sign bool `json:"sign,omitempty"`
AdminSign bool `json:"adminSign,omitempty"`
}
type Docker struct {
ShouldSave bool `json:"shouldSave,omitempty"`
Distribution []Distribution `json:"distribution,omitempty"`
Architectures []Architecture `json:"archs,omitempty"`
PrereleaseBucket string `json:"prereleaseBucket,omitempty"`
}
type Buckets struct {
Artifacts string `json:"artifacts,omitempty"`
ArtifactsEnterprise2 string `json:"artifactsEnterprise2,omitempty"`
CDNAssets string `json:"CDNAssets,omitempty"`
CDNAssetsDir string `json:"CDNAssetsDir,omitempty"`
Storybook string `json:"storybook,omitempty"`
StorybookSrcDir string `json:"storybookSrcDir,omitempty"`
}
// BuildConfig represents the struct that defines all of the different variables used to build Grafana
type BuildConfig struct {
Variants []Variant `json:"variants,omitempty"`
PluginSignature PluginSignature `json:"pluginSignature,omitempty"`
Docker Docker `json:"docker,omitempty"`
Buckets Buckets `json:"buckets,omitempty"`
}
func (md *Metadata) GetReleaseMode() (ReleaseMode, error) {
return md.ReleaseMode, nil
}
// VersionMap is a map of versions. Each key of the Versions map is an event that uses the the config as the value for that key.
// For example, the 'pull_request' key will have data in it that might cause Grafana to be built differently in a pull request,
// than the way it will be built in 'main'
type VersionMap map[VersionMode]BuildConfig
// GetBuildConfig reads the embedded config.json and decodes it.
func GetBuildConfig(mode VersionMode) (*BuildConfig, error) {
if v, ok := Versions[mode]; ok {
return &v, nil
}
return nil, fmt.Errorf("mode '%s' not found in version list", mode)
}
// GetGrafanaVersion gets the Grafana version from the package.json
func GetGrafanaVersion(buildID, grafanaDir string) (string, error) {
pkgJSONPath := filepath.Join(grafanaDir, "package.json")
//nolint:gosec
pkgJSONB, err := os.ReadFile(pkgJSONPath)
if err != nil {
return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err)
}
pkgObj := map[string]interface{}{}
if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil {
return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err)
}
version := pkgObj["version"].(string)
if version == "" {
return "", fmt.Errorf("failed to read version from %q", pkgJSONPath)
}
if buildID != "" {
buildID = shortenBuildID(buildID)
verComponents := strings.Split(version, "-")
version = verComponents[0]
if len(verComponents) > 1 {
buildID = fmt.Sprintf("%s%s", buildID, verComponents[1])
}
version = fmt.Sprintf("%s-%s", version, buildID)
}
return version, nil
}
func CheckDroneTargetBranch() (VersionMode, error) {
rePRCheckBranch := git.PRCheckRegexp()
reRlsBranch := regexp.MustCompile(`^v\d+\.\d+\.x$`)
target := os.Getenv("DRONE_TARGET_BRANCH")
if target == "" {
return "", fmt.Errorf("failed to get DRONE_TARGET_BRANCH environmental variable")
} else if target == string(MainMode) {
return MainMode, nil
}
if reRlsBranch.MatchString(target) {
return ReleaseBranchMode, nil
}
if rePRCheckBranch.MatchString(target) {
return PullRequestMode, nil
}
fmt.Printf("unrecognized target branch: %s, defaulting to %s", target, PullRequestMode)
return PullRequestMode, nil
}
func CheckSemverSuffix() (ReleaseMode, error) {
reBetaRls := regexp.MustCompile(`beta.*`)
reTestRls := regexp.MustCompile(`test.*`)
tagSuffix, ok := os.LookupEnv("DRONE_SEMVER_PRERELEASE")
if !ok || tagSuffix == "" {
fmt.Println("DRONE_SEMVER_PRERELEASE doesn't exist for a tag, this is a release event...")
return ReleaseMode{Mode: TagMode}, nil
}
switch {
case reBetaRls.MatchString(tagSuffix):
return ReleaseMode{Mode: TagMode, IsBeta: true}, nil
case reTestRls.MatchString(tagSuffix):
return ReleaseMode{Mode: TagMode, IsTest: true}, nil
default:
fmt.Printf("DRONE_SEMVER_PRERELEASE is custom string, release event with %s suffix\n", tagSuffix)
return ReleaseMode{Mode: TagMode}, nil
}
}
func GetDroneCommit() (string, error) {
commit := strings.TrimSpace(os.Getenv("DRONE_COMMIT"))
if commit == "" {
return "", fmt.Errorf("the environment variable DRONE_COMMIT is missing")
}
return commit, nil
}
func shortenBuildID(buildID string) string {
buildID = strings.ReplaceAll(buildID, "-", "")
if len(buildID) < 9 {
return buildID
}
return buildID[0:8]
}
pkg/build/config/version_mode.go
+27
View File
@@ -0,0 +1,27 @@
package config
// VersionMode defines the source event that created a release or published version
type VersionMode string
const (
MainMode VersionMode = "main"
TagMode VersionMode = "release"
ReleaseBranchMode VersionMode = "branch"
PullRequestMode VersionMode = "pull_request"
DownstreamMode VersionMode = "downstream"
Enterprise2Mode VersionMode = "enterprise2"
CronjobMode VersionMode = "cron"
)
const (
Tag = "tag"
PullRequest = "pull_request"
Push = "push"
Custom = "custom"
Promote = "promote"
Cronjob = "cron"
)
const (
MainBranch = "main"
)
pkg/build/config/versions.go
+206
View File
@@ -0,0 +1,206 @@
package config
const PublicBucket = "grafana-downloads"
var Versions = VersionMap{
PullRequestMode: {
Variants: []Variant{
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
},
PluginSignature: PluginSignature{
Sign: false,
AdminSign: false,
},
Docker: Docker{
ShouldSave: false,
Architectures: []Architecture{
ArchAMD64,
},
Distribution: []Distribution{
Alpine,
},
},
},
MainMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
},
PluginSignature: PluginSignature{
Sign: true,
AdminSign: true,
},
Docker: Docker{
ShouldSave: false,
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
},
Distribution: []Distribution{
Alpine,
Ubuntu,
},
},
Buckets: Buckets{
Artifacts: "grafana-downloads",
ArtifactsEnterprise2: "grafana-downloads-enterprise2",
CDNAssets: "grafana-static-assets",
Storybook: "grafana-storybook",
},
},
DownstreamMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
},
PluginSignature: PluginSignature{
Sign: true,
AdminSign: true,
},
Docker: Docker{
ShouldSave: true,
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
},
Distribution: []Distribution{
Alpine,
Ubuntu,
},
},
Buckets: Buckets{
Artifacts: "grafana-downloads",
ArtifactsEnterprise2: "grafana-downloads-enterprise2",
CDNAssets: "grafana-static-assets",
},
},
ReleaseBranchMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
},
PluginSignature: PluginSignature{
Sign: true,
AdminSign: true,
},
Docker: Docker{
ShouldSave: true,
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7,
},
Distribution: []Distribution{
Alpine,
Ubuntu,
},
},
Buckets: Buckets{
Artifacts: "grafana-downloads",
ArtifactsEnterprise2: "grafana-downloads-enterprise2",
CDNAssets: "grafana-static-assets",
},
},
TagMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
},
PluginSignature: PluginSignature{
Sign: true,
AdminSign: true,
},
Docker: Docker{
ShouldSave: true,
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7,
},
Distribution: []Distribution{
Alpine,
Ubuntu,
},
PrereleaseBucket: "grafana-prerelease/artifacts/docker",
},
Buckets: Buckets{
Artifacts: "grafana-prerelease/artifacts/downloads",
ArtifactsEnterprise2: "grafana-prerelease/artifacts/downloads-enterprise2",
CDNAssets: "grafana-prerelease",
CDNAssetsDir: "artifacts/static-assets",
Storybook: "grafana-prerelease",
StorybookSrcDir: "artifacts/storybook",
},
},
Enterprise2Mode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
},
PluginSignature: PluginSignature{
Sign: true,
AdminSign: true,
},
Docker: Docker{
ShouldSave: true,
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7,
},
Distribution: []Distribution{
Alpine,
Ubuntu,
},
PrereleaseBucket: "grafana-prerelease/artifacts/docker",
},
Buckets: Buckets{
Artifacts: "grafana-prerelease/artifacts/downloads",
ArtifactsEnterprise2: "grafana-prerelease/artifacts/downloads-enterprise2",
CDNAssets: "grafana-prerelease",
CDNAssetsDir: "artifacts/static-assets",
Storybook: "grafana-prerelease",
StorybookSrcDir: "artifacts/storybook",
},
},
}
pkg/build/cryptoutil/md5.go
+35
View File
@@ -0,0 +1,35 @@
package cryptoutil
import (
"crypto/md5"
"fmt"
"io"
"log"
"os"
)
func MD5File(fpath string) error {
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
fd, err := os.Open(fpath)
if err != nil {
return err
}
defer func() {
if err := fd.Close(); err != nil {
log.Printf("error closing file at '%s': %s", fpath, err.Error())
}
}()
h := md5.New() // nolint:gosec
if _, err = io.Copy(h, fd); err != nil {
return err
}
// nolint:gosec
if err := os.WriteFile(fpath+".md5", []byte(fmt.Sprintf("%x\n", h.Sum(nil))), 0664); err != nil {
return err
}
return nil
}
pkg/build/droneutil/docs.go
+2
View File
@@ -0,0 +1,2 @@
// Package droneutil provides utility functions for working with Drone.
package droneutil
pkg/build/droneutil/event.go
+34
View File
@@ -0,0 +1,34 @@
package droneutil
import (
"fmt"
"os"
"strings"
)
// Lookup is the equivalent of os.LookupEnv, but also accepts a list of strings rather than only checking os.Environ()
func Lookup(values []string, val string) (string, bool) {
for _, v := range values {
prefix := val + "="
if strings.HasPrefix(v, prefix) {
return strings.TrimPrefix(v, prefix), true
}
}
return "", false
}
// GetDroneEvent looks for the "DRONE_BUILD_EVENT" in the provided env list and returns the value.
// if it was not found, then an error is returned.
func GetDroneEvent(env []string) (string, error) {
event, ok := Lookup(env, "DRONE_BUILD_EVENT")
if !ok {
return "", fmt.Errorf("failed to get DRONE_BUILD_EVENT environmental variable")
}
return event, nil
}
// GetDroneEventFromEnv returns the value of DRONE_BUILD_EVENT from os.Environ()
func GetDroneEventFromEnv() (string, error) {
return GetDroneEvent(os.Environ())
}
pkg/build/droneutil/event_test.go
+36
View File
@@ -0,0 +1,36 @@
package droneutil_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/stretchr/testify/require"
)
func TestGetDroneEvent(t *testing.T) {
t.Run("Should return the Drone Event", func(t *testing.T) {
env := []string{"DRONE_BUILD_EVENT=pull_request"}
droneEvent, err := droneutil.GetDroneEvent(env)
require.NoError(t, err)
require.Equal(t, droneEvent, "pull_request")
})
t.Run("Should return error, Drone Event env var is missing", func(t *testing.T) {
droneEvent, err := droneutil.GetDroneEvent([]string{})
require.Error(t, err)
require.Empty(t, droneEvent)
})
}
func TestLookup(t *testing.T) {
env := []string{"", "EXAMPLE_KEY=value", "EXAMPLE_KEY"}
t.Run("A valid lookup should return a string and no error", func(t *testing.T) {
val, ok := droneutil.Lookup(env, "EXAMPLE_KEY")
require.True(t, ok)
require.Equal(t, val, "value")
})
t.Run("An invalid lookup should return an error", func(t *testing.T) {
_, ok := droneutil.Lookup(env, "EXAMPLE_KEY_DOES_NOT_EXIST")
require.False(t, ok)
})
}
pkg/build/errutil/group.go
+61
View File
@@ -0,0 +1,61 @@
package errutil
import (
"context"
"log"
"sync"
)
type Group struct {
cancel func()
wg sync.WaitGroup
errOnce sync.Once
err error
}
func GroupWithContext(ctx context.Context) (*Group, context.Context) {
ctx, cancel := context.WithCancel(ctx)
return &Group{cancel: cancel}, ctx
}
// Wait waits for any wrapped goroutines to finish and returns any error having occurred in one of them.
func (g *Group) Wait() error {
log.Println("Waiting on Group")
g.wg.Wait()
if g.cancel != nil {
log.Println("Group canceling its context after waiting")
g.cancel()
}
return g.err
}
// Cancel cancels the associated context.
func (g *Group) Cancel() {
log.Println("Group's Cancel method being called")
g.cancel()
}
// Wrap wraps a function to be executed in a goroutine.
func (g *Group) Wrap(f func() error) func() {
g.wg.Add(1)
return func() {
defer g.wg.Done()
if err := f(); err != nil {
g.errOnce.Do(func() {
log.Printf("An error occurred in Group: %s", err)
g.err = err
if g.cancel != nil {
log.Println("Group canceling its context due to error")
g.cancel()
}
})
}
}
}
// Go wraps the provided function and executes it in a goroutine.
func (g *Group) Go(f func() error) {
wrapped := g.Wrap(f)
go wrapped()
}
pkg/build/executil/exec.go
+46
View File
@@ -0,0 +1,46 @@
package executil
import (
"context"
"fmt"
"os/exec"
"strings"
)
func RunAt(ctx context.Context, dir, cmd string, args ...string) error {
// Ignore gosec G204 as this function is only used in the build process.
//nolint:gosec
c := exec.CommandContext(ctx, cmd, args...)
c.Dir = dir
b, err := c.CombinedOutput()
if err != nil {
return fmt.Errorf("%w. '%s %v': %s", err, cmd, args, string(b))
}
return nil
}
func Run(ctx context.Context, cmd string, args ...string) error {
return RunAt(ctx, ".", cmd, args...)
}
func OutputAt(ctx context.Context, dir, cmd string, args ...string) (string, error) {
// Ignore gosec G204 as this function is only used in the build process.
//nolint:gosec
c := exec.CommandContext(ctx, cmd, args...)
c.Dir = dir
b, err := c.CombinedOutput()
if err != nil {
return "", err
}
return strings.TrimSpace(string(b)), nil
}
func Output(ctx context.Context, cmd string, args ...string) (string, error) {
return OutputAt(ctx, ".", cmd, args...)
}
pkg/build/fsutil/copy_test.go
+87
View File
@@ -0,0 +1,87 @@
package fsutil_test
import (
"os"
"runtime"
"testing"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCopyFile(t *testing.T) {
src, err := os.CreateTemp("", "")
require.NoError(t, err)
defer func() {
if err := os.RemoveAll(src.Name()); err != nil {
t.Log(err)
}
}()
err = os.WriteFile(src.Name(), []byte("Contents"), 0600)
require.NoError(t, err)
dst, err := os.CreateTemp("", "")
require.NoError(t, err)
defer func() {
if err := os.RemoveAll(dst.Name()); err != nil {
t.Log(err)
}
}()
err = fsutil.CopyFile(src.Name(), dst.Name())
require.NoError(t, err)
}
func TestCopyFile_Permissions(t *testing.T) {
perms := os.FileMode(0700)
if runtime.GOOS == "windows" {
// Windows doesn't have file Unix style file permissions
// It seems you have either 0444 for read-only or 0666 for read-write
perms = os.FileMode(0666)
}
src, err := os.CreateTemp("", "")
require.NoError(t, err)
defer func() {
if err := os.RemoveAll(src.Name()); err != nil {
t.Log(err)
}
}()
err = os.WriteFile(src.Name(), []byte("Contents"), perms)
require.NoError(t, err)
err = os.Chmod(src.Name(), perms)
require.NoError(t, err)
dst, err := os.CreateTemp("", "")
require.NoError(t, err)
defer func() {
if err := os.RemoveAll(dst.Name()); err != nil {
t.Log(err)
}
}()
err = fsutil.CopyFile(src.Name(), dst.Name())
require.NoError(t, err)
fi, err := os.Stat(dst.Name())
require.NoError(t, err)
assert.Equal(t, perms, fi.Mode()&os.ModePerm)
}
// Test case where destination directory doesn't exist.
func TestCopyFile_NonExistentDestDir(t *testing.T) {
src, err := os.CreateTemp("", "")
require.NoError(t, err)
defer func() {
if err := os.RemoveAll(src.Name()); err != nil {
t.Log(err)
}
}()
err = fsutil.CopyFile(src.Name(), "non-existent/dest")
require.EqualError(t, err, "destination directory doesn't exist: \"non-existent\"")
}
pkg/build/fsutil/copyfile.go
+107
View File
@@ -0,0 +1,107 @@
package fsutil
import (
"fmt"
"io"
"log"
"os"
"path/filepath"
)
// CopyFile copies a file from src to dst.
//
// If src and dst files exist, and are the same, then return success. Otherwise, attempt to create a hard link
// between the two files. If that fails, copy the file contents from src to dst.
func CopyFile(src, dst string) (err error) {
absSrc, err := filepath.Abs(src)
if err != nil {
return fmt.Errorf("failed to get absolute path of source file %q: %w", src, err)
}
sfi, err := os.Stat(src)
if err != nil {
err = fmt.Errorf("couldn't stat source file %q: %w", absSrc, err)
return
}
if !sfi.Mode().IsRegular() {
// Cannot copy non-regular files (e.g., directories, symlinks, devices, etc.)
return fmt.Errorf("non-regular source file %s (%q)", absSrc, sfi.Mode().String())
}
dpath := filepath.Dir(dst)
exists, err := Exists(dpath)
if err != nil {
return err
}
if !exists {
err = fmt.Errorf("destination directory doesn't exist: %q", dpath)
return
}
var dfi os.FileInfo
dfi, err = os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
return
}
} else {
if !(dfi.Mode().IsRegular()) {
return fmt.Errorf("non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
}
if os.SameFile(sfi, dfi) {
return copyPermissions(sfi.Name(), dfi.Name())
}
}
err = copyFileContents(src, dst)
return err
}
// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
//nolint:gosec
in, err := os.Open(src)
if err != nil {
return
}
defer func() {
if err := in.Close(); err != nil {
log.Println("error closing file", err)
}
}()
//nolint:gosec
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
if cerr := out.Close(); cerr != nil && err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
if err := out.Sync(); err != nil {
return err
}
return copyPermissions(src, dst)
}
func copyPermissions(src, dst string) error {
sfi, err := os.Lstat(src)
if err != nil {
return err
}
if err := os.Chmod(dst, sfi.Mode()); err != nil {
return err
}
return nil
}
pkg/build/fsutil/createtemp.go
+43
View File
@@ -0,0 +1,43 @@
package fsutil
import (
"fmt"
"os"
)
// CreateTempFile generates a temp filepath, based on the provided suffix.
// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807.
func CreateTempFile(sfx string) (string, error) {
var suffix string
if sfx != "" {
suffix = fmt.Sprintf("*-%s", sfx)
} else {
suffix = sfx
}
f, err := os.CreateTemp("", suffix)
if err != nil {
return "", err
}
if err := f.Close(); err != nil {
return "", err
}
return f.Name(), nil
}
// CreateTempDir generates a temp directory, based on the provided suffix.
// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807/.
func CreateTempDir(sfx string) (string, error) {
var suffix string
if sfx != "" {
suffix = fmt.Sprintf("*-%s", sfx)
} else {
suffix = sfx
}
dir, err := os.MkdirTemp("", suffix)
if err != nil {
return "", err
}
return dir, nil
}
pkg/build/fsutil/createtemp_test.go
+48
View File
@@ -0,0 +1,48 @@
package fsutil
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestCreateTempFile(t *testing.T) {
t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807", func(t *testing.T) {
filePath, err := CreateTempFile("")
require.NoError(t, err)
pathParts := strings.Split(filePath, "/")
require.Greater(t, len(pathParts), 1)
require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1)
})
t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar", func(t *testing.T) {
filePath, err := CreateTempFile("foobar")
require.NoError(t, err)
pathParts := strings.Split(filePath, "/")
require.Greater(t, len(pathParts), 1)
require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2)
})
}
func TestCreateTempDir(t *testing.T) {
t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807/", func(t *testing.T) {
filePath, err := CreateTempFile("")
require.NoError(t, err)
pathParts := strings.Split(filePath, "/")
require.Greater(t, len(pathParts), 1)
require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1)
})
t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar/", func(t *testing.T) {
filePath, err := CreateTempFile("foobar")
require.NoError(t, err)
pathParts := strings.Split(filePath, "/")
require.Greater(t, len(pathParts), 1)
require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2)
})
}
pkg/build/fsutil/exists_test.go
+15
View File
@@ -0,0 +1,15 @@
package fsutil_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/stretchr/testify/require"
)
func TestExists_NonExistent(t *testing.T) {
exists, err := fsutil.Exists("non-existent")
require.NoError(t, err)
require.False(t, exists)
}
pkg/build/fsutil/exsits.go
+16
View File
@@ -0,0 +1,16 @@
package fsutil
import "os"
// Exists determines whether a file/directory exists or not.
func Exists(fpath string) (bool, error) {
_, err := os.Stat(fpath)
if err != nil {
if !os.IsNotExist(err) {
return false, err
}
return false, nil
}
return true, nil
}
pkg/build/gcloud/auth.go
+65
View File
@@ -0,0 +1,65 @@
package gcloud
import (
"encoding/base64"
"encoding/json"
"fmt"
"log"
"os"
"os/exec"
"strings"
)
func GetDecodedKey() ([]byte, error) {
gcpKey := strings.TrimSpace(os.Getenv("GCP_KEY"))
if gcpKey == "" {
return nil, fmt.Errorf("the environment variable GCP_KEY must be set")
}
gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKey)
if err != nil {
// key is not always base64 encoded
validKey := []byte(gcpKey)
if json.Valid(validKey) {
return validKey, nil
}
return nil, fmt.Errorf("error decoding the gcp_key, err: %q", err)
}
return gcpKeyB, nil
}
func ActivateServiceAccount() error {
byteKey, err := GetDecodedKey()
if err != nil {
return err
}
f, err := os.CreateTemp("", "*.json")
if err != nil {
return err
}
defer func() {
if err := os.Remove(f.Name()); err != nil {
log.Printf("error removing %s: %s", f.Name(), err)
}
}()
defer func() {
if err := f.Close(); err != nil {
log.Println("error closing file:", err)
}
}()
if _, err := f.Write(byteKey); err != nil {
return fmt.Errorf("failed to write GCP key file: %w", err)
}
keyArg := fmt.Sprintf("--key-file=%s", f.Name())
//nolint:gosec
cmd := exec.Command("gcloud", "auth", "activate-service-account", keyArg)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to sign into GCP: %w\n%s", err, output)
}
return nil
}
pkg/build/gcloud/storage/gsutil.go
+457
View File
@@ -0,0 +1,457 @@
package storage
import (
"context"
"errors"
"fmt"
"io"
"log"
"mime"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"cloud.google.com/go/storage"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/grafana/grafana/pkg/build/gcloud"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)
var (
// ErrorNilBucket is returned when a function is called where a bucket argument is expected and the bucket is nil.
ErrorNilBucket = errors.New("a bucket must be provided")
)
const (
// maxThreads specify the number of max threads that can run at the same time.
// Set to 1000, since the maximum number of simultaneous open files for the runners is 1024.
maxThreads = 1000
)
// Client wraps the gcloud storage Client with convenient helper functions.
// By using an embedded type we can still use the functions provided by storage.Client if we need to.
type Client struct {
storage.Client
}
// File represents a file in Google Cloud Storage.
type File struct {
FullPath string
PathTrimmed string
}
// New creates a new Client by checking for the Google Cloud SDK auth key and/or environment variable.
func New() (*Client, error) {
client, err := newClient()
if err != nil {
return nil, err
}
return &Client{
Client: *client,
}, nil
}
// newClient initializes the google-cloud-storage (GCS) client.
// It first checks for the application-default_credentials.json file then the GCP_KEY environment variable.
func newClient() (*storage.Client, error) {
ctx := context.Background()
byteKey, err := gcloud.GetDecodedKey()
if err != nil {
return nil, fmt.Errorf("failed to get gcp key, err: %w", err)
}
client, err := storage.NewClient(ctx, option.WithCredentialsJSON(byteKey))
if err != nil {
log.Println("failed to login with GCP_KEY, trying with default application credentials...")
client, err = storage.NewClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to open Google Cloud Storage client: %w", err)
}
}
return client, nil
}
// CopyLocalDir copies a local directory 'dir' to the bucket 'bucket' at the path 'bucketPath'.
func (client *Client) CopyLocalDir(ctx context.Context, dir string, bucket *storage.BucketHandle, bucketPath string, trim bool) error {
if bucket == nil {
return ErrorNilBucket
}
files, err := ListLocalFiles(dir)
if err != nil {
return err
}
log.Printf("Number or files to be copied over: %d\n", len(files))
for _, chunk := range asChunks(files, maxThreads) {
var wg sync.WaitGroup
for _, f := range chunk {
wg.Add(1)
go func(file File) {
defer wg.Done()
err = client.Copy(ctx, file, bucket, bucketPath, trim)
if err != nil {
log.Printf("failed to copy objects, err: %s\n", err.Error())
}
}(f)
}
wg.Wait()
}
return nil
}
// Copy copies a single local file into the bucket at the provided path.
// trim variable should be set to true if the full object path is needed - false otherwise.
func (client *Client) Copy(ctx context.Context, file File, bucket *storage.BucketHandle, remote string, trim bool) error {
if bucket == nil {
return ErrorNilBucket
}
localFile, err := os.Open(file.FullPath)
if err != nil {
return fmt.Errorf("failed to open file %s, err: %q", file.FullPath, err)
}
defer func() {
if err := localFile.Close(); err != nil {
log.Println("failed to close localfile", "err", err)
}
}()
extension := strings.ToLower(path.Ext(file.FullPath))
contentType := mime.TypeByExtension(extension)
filePath := file.FullPath
if trim {
filePath = file.PathTrimmed
}
objectPath := path.Join(remote, filePath)
wc := bucket.Object(objectPath).NewWriter(ctx)
wc.ContentType = contentType
defer func() {
if err := wc.Close(); err != nil {
log.Println("failed to close writer", "err", err)
}
}()
if _, err = io.Copy(wc, localFile); err != nil {
return fmt.Errorf("failed to copy to Cloud Storage: %w", err)
}
log.Printf("Successfully uploaded tarball to Google Cloud Storage, path: %s/%s\n", remote, file.FullPath)
return nil
}
// CopyRemoteDir copies an entire directory 'from' from the bucket 'fromBucket' into the 'toBucket' at the path 'to'.
func (client *Client) CopyRemoteDir(ctx context.Context, fromBucket *storage.BucketHandle, from string, toBucket *storage.BucketHandle, to string) error {
if toBucket == nil || fromBucket == nil {
return ErrorNilBucket
}
files, err := ListRemoteFiles(ctx, fromBucket, FilesFilter{Prefix: from})
if err != nil {
return err
}
var ch = make(chan File, len(files))
var wg sync.WaitGroup
wg.Add(maxThreads)
for i := 0; i < maxThreads; i++ {
go func() {
for {
file, ok := <-ch
if !ok {
wg.Done()
return
}
if err := client.RemoteCopy(ctx, file, fromBucket, toBucket, to); err != nil {
log.Printf("failed to copy files between buckets: err: %s\n", err.Error())
return
}
}
}()
}
for _, file := range files {
ch <- file
}
close(ch)
wg.Wait()
return nil
}
// RemoteCopy will copy the file 'file' from the 'fromBucket' to the 'toBucket' at the path 'path'.
func (client *Client) RemoteCopy(ctx context.Context, file File, fromBucket, toBucket *storage.BucketHandle, path string) error {
// Should this be path.Join instead of filepath.Join? filepath.Join on Windows will produce `\\` separators instead of `/`.
var (
src = fromBucket.Object(file.FullPath)
dstObject = filepath.Join(path, file.PathTrimmed)
dst = toBucket.Object(dstObject)
)
if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
return fmt.Errorf("failed to copy object %s, to %s, err: %w", file.FullPath, dstObject, err)
}
log.Printf("%s was successfully copied to %v bucket!.\n\n", file.FullPath, toBucket)
return nil
}
// DeleteDir deletes a directory at 'path' from the bucket.
func (client *Client) DeleteDir(ctx context.Context, bucket *storage.BucketHandle, path string) error {
if bucket == nil {
return ErrorNilBucket
}
files, err := ListRemoteFiles(ctx, bucket, FilesFilter{Prefix: path})
if err != nil {
return err
}
var ch = make(chan string, len(files))
var wg sync.WaitGroup
wg.Add(maxThreads)
for i := 0; i < maxThreads; i++ {
go func() {
for {
fullPath, ok := <-ch
if !ok {
wg.Done()
return
}
err := client.Delete(ctx, bucket, fullPath)
if err != nil && !errors.Is(err, storage.ErrObjectNotExist) {
log.Printf("failed to delete objects, err %s\n", err.Error())
panic(err)
}
}
}()
}
for _, file := range files {
ch <- file.FullPath
}
close(ch)
wg.Wait()
return nil
}
// Delete deletes single item from the bucket at 'path'.
func (client *Client) Delete(ctx context.Context, bucket *storage.BucketHandle, path string) error {
object := bucket.Object(path)
if err := object.Delete(ctx); err != nil {
return fmt.Errorf("cannot delete %s, err: %w", path, err)
}
log.Printf("Successfully deleted tarball to Google Cloud Storage, path: %s", path)
return nil
}
// ListLocalFiles lists files in a local filesystem.
func ListLocalFiles(dir string) ([]File, error) {
var files []File
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
files = append(files, File{
FullPath: path,
// Strip the dir name from the filepath
PathTrimmed: strings.ReplaceAll(path, dir, ""),
})
}
return nil
})
if err != nil {
return nil, fmt.Errorf("error walking path: %v", err)
}
return files, nil
}
type FilesFilter struct {
Prefix string
FileExts []string
}
// ListRemoteFiles lists all the files in the directory (filtering by FilesFilter) and returns a File struct for each one.
func ListRemoteFiles(ctx context.Context, bucket *storage.BucketHandle, filter FilesFilter) ([]File, error) {
if bucket == nil {
return []File{}, ErrorNilBucket
}
it := bucket.Objects(ctx, &storage.Query{
Prefix: filter.Prefix,
})
var files []File
for {
attrs, err := it.Next()
if err != nil {
if errors.Is(err, iterator.Done) {
break
}
}
if err != nil {
return nil, fmt.Errorf("failed to iterate through bucket, err: %w", err)
}
extMatch := len(filter.FileExts) == 0
for _, ext := range filter.FileExts {
if ext == filepath.Ext(attrs.Name) {
extMatch = true
break
}
}
if extMatch {
files = append(files, File{FullPath: attrs.Name, PathTrimmed: strings.TrimPrefix(attrs.Name, filter.Prefix)})
}
}
return files, nil
}
// DownloadDirectory downloads files from bucket (filtering by FilesFilter) to destPath on disk.
func (client *Client) DownloadDirectory(ctx context.Context, bucket *storage.BucketHandle, destPath string, filter FilesFilter) error {
if bucket == nil {
return ErrorNilBucket
}
files, err := ListRemoteFiles(ctx, bucket, filter)
if err != nil {
return err
}
// return err if dir already exists
exists, err := fsutil.Exists(destPath)
if err != nil {
return err
}
if exists {
return fmt.Errorf("destination path %q already exists", destPath)
}
err = os.MkdirAll(destPath, 0750)
if err != nil && !os.IsExist(err) {
return err
}
for _, file := range files {
err = client.downloadFile(ctx, bucket, file.FullPath, filepath.Join(destPath, file.PathTrimmed))
if err != nil {
return err
}
}
return nil
}
// GetLatestMainBuild gets the latest main build which is successfully uploaded to the gcs bucket.
func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path string) (string, error) {
if bucket == nil {
return "", ErrorNilBucket
}
it := bucket.Objects(ctx, &storage.Query{
Prefix: path,
})
var files []string
for {
attrs, err := it.Next()
if errors.Is(err, iterator.Done) {
break
}
if err != nil {
return "", fmt.Errorf("failed to iterate through bucket, err: %w", err)
}
files = append(files, attrs.Name)
}
var latestVersion string
for i := len(files) - 1; i >= 0; i-- {
captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`)
if captureVersion.MatchString(files[i]) {
latestVersion = captureVersion.FindString(files[i])
break
}
}
return latestVersion, nil
}
// downloadFile downloads an object to a file.
func (client *Client) downloadFile(ctx context.Context, bucket *storage.BucketHandle, objectName, destFileName string) error {
if bucket == nil {
return ErrorNilBucket
}
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
defer cancel()
// nolint:gosec
f, err := os.Create(destFileName)
if err != nil {
return fmt.Errorf("os.Create: %v", err)
}
rc, err := bucket.Object(objectName).NewReader(ctx)
if err != nil {
return fmt.Errorf("Object(%q).NewReader: %v", objectName, err)
}
defer func() {
if err := rc.Close(); err != nil {
log.Println("failed to close reader", "err", err)
}
}()
if _, err := io.Copy(f, rc); err != nil {
return fmt.Errorf("io.Copy: %v", err)
}
if err = f.Close(); err != nil {
return fmt.Errorf("f.Close: %v", err)
}
return nil
}
// asChunks will split the supplied []File into slices with a max size of `chunkSize`
// []string{"a", "b", "c"}, 1 => [][]string{[]string{"a"}, []string{"b"}, []string{"c"}}
// []string{"a", "b", "c"}, 2 => [][]string{[]string{"a", "b"}, []string{"c"}}.
func asChunks(files []File, chunkSize int) [][]File {
var fileChunks [][]File
if len(files) == 0 {
return [][]File{}
}
if len(files) > chunkSize && chunkSize > 0 {
for i := 0; i < len(files); i += chunkSize {
end := i + chunkSize
if end > len(files) {
end = len(files)
}
fileChunks = append(fileChunks, files[i:end])
}
} else {
fileChunks = [][]File{files}
}
return fileChunks
}
pkg/build/gcloud/storage/gsutil_test.go
+159
View File
@@ -0,0 +1,159 @@
package storage
import (
"testing"
"github.com/stretchr/testify/require"
)
func Test_asChunks(t *testing.T) {
type args struct {
files []File
chunkSize int
}
tcs := []struct {
name string
args args
expected [][]File
}{
{
name: "Happy path #1",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
{FullPath: "/1"},
{FullPath: "/2"},
{FullPath: "/3"},
},
chunkSize: 5,
},
expected: [][]File{
{{FullPath: "/a"}, {FullPath: "/b"}, {FullPath: "/c"}, {FullPath: "/1"}, {FullPath: "/2"}},
{{FullPath: "/3"}},
},
},
{
name: "Happy path #2",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
{FullPath: "/1"},
{FullPath: "/2"},
{FullPath: "/3"},
},
chunkSize: 2,
},
expected: [][]File{
{{FullPath: "/a"}, {FullPath: "/b"}},
{{FullPath: "/c"}, {FullPath: "/1"}},
{{FullPath: "/2"}, {FullPath: "/3"}},
},
},
{
name: "Happy path #3",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
chunkSize: 1,
},
expected: [][]File{
{{FullPath: "/a"}},
{{FullPath: "/b"}},
{{FullPath: "/c"}},
},
},
{
name: "A chunkSize with 0 value returns the input as a single chunk",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
chunkSize: 0,
},
expected: [][]File{
{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
},
},
{
name: "A chunkSize with negative value returns the input as a single chunk",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
chunkSize: -1,
},
expected: [][]File{
{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
},
},
{
name: "A chunkSize greater than the size on input returns the input as a single chunk",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
chunkSize: 5,
},
expected: [][]File{
{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
},
},
{
name: "A chunkSize equal the size on input returns the input as a single chunk",
args: args{
files: []File{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
chunkSize: 3,
},
expected: [][]File{
{
{FullPath: "/a"},
{FullPath: "/b"},
{FullPath: "/c"},
},
},
},
{
name: "An empty input returns empty chunks",
args: args{
files: []File{},
chunkSize: 3,
},
expected: [][]File{},
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
result := asChunks(tc.args.files, tc.args.chunkSize)
require.Equal(t, tc.expected, result)
})
}
}
pkg/build/git/git.go
+25
View File
@@ -0,0 +1,25 @@
package git
import (
"fmt"
"regexp"
)
const (
MainBranch = "main"
HomeDir = "."
RepoOwner = "grafana"
OSSRepo = "grafana"
EnterpriseRepo = "grafana-enterprise"
EnterpriseCheckName = "Grafana Enterprise"
EnterpriseCheckDescription = "Downstream tests to ensure that your changes are compatible with Grafana Enterprise"
)
func PRCheckRegexp() *regexp.Regexp {
reBranch, err := regexp.Compile(`^prc-([0-9]+)-([A-Za-z0-9]+)\/(.+)$`)
if err != nil {
panic(fmt.Sprintf("Failed to compile regexp: %s", err))
}
return reBranch
}
pkg/build/git/git_test.go
+56
View File
@@ -0,0 +1,56 @@
package git_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/git"
"github.com/stretchr/testify/assert"
)
func TestPRCheckRegexp(t *testing.T) {
type match struct {
String string
Commit string
Branch string
PR string
}
var (
shouldMatch = []match{
{
String: "prc-1-a1b2c3d4/branch-name",
Branch: "branch-name",
Commit: "a1b2c3d4",
PR: "1",
},
{
String: "prc-111-a1b2c3d4/branch/name",
Branch: "branch/name",
Commit: "a1b2c3d4",
PR: "111",
},
{
String: "prc-102930122-a1b2c3d4/branch-name",
Branch: "branch-name",
Commit: "a1b2c3d4",
PR: "102930122",
},
}
shouldNotMatch = []string{"prc-a/branch", "km/test", "test", "prc", "prc/test", "price"}
)
regex := git.PRCheckRegexp()
for _, v := range shouldMatch {
assert.Truef(t, regex.MatchString(v.String), "regex '%s' should match %s", regex.String(), v)
m := regex.FindStringSubmatch(v.String)
assert.Equal(t, m[1], v.PR)
assert.Equal(t, m[2], v.Commit)
assert.Equal(t, m[3], v.Branch)
}
for _, v := range shouldNotMatch {
assert.False(t, regex.MatchString(v), "regex '%s' should not match %s", regex.String(), v)
}
}
pkg/build/golangutils/build.go
+124
View File
@@ -0,0 +1,124 @@
package golangutils
import (
"context"
"fmt"
"io"
"os/exec"
"strings"
"github.com/grafana/grafana/pkg/build/config"
)
type BuildOpts struct {
// Package refers to the path to the `main` package containing `func main`
Package string
// Output is used as the -o argument in the go build command
Output string
// Workdir should define some place in the module where the package path resolves.
// Go commands need to be ran inside a the Go module directory.
Workdir string
GoOS config.OS
GoArch config.Architecture
GoArm string
Go386 string
CC string
LibC string
CGoEnabled bool
CGoCFlags string
// LdFlags are joined by a space character and provided to the -ldflags argument.
// A valid element here would be `-X 'main.version=1.0.0'`.
LdFlags []string
Stdout io.ReadWriter
Stderr io.ReadWriter
Stdin io.ReadWriter
// ExtraEnv allows consumers to provide extra env args that are not defined above.
// A single element should be formatted using like so: {NAME}={VALUE}. Example: GOOS=linux.
ExtraEnv []string
// ExtraArgs allows consumers to provide extra arguments that are not defined above.
// Flag names and values should be two separate elements.
// These flags will be appended to the command arguments before the package path in "go build".
ExtraArgs []string
}
// Env constructs a list of key/value pairs for setting a build command's environment.
// Should we consider using something to unmarshal the struct to env?
func (opts BuildOpts) Env() []string {
env := []string{}
if opts.CGoEnabled {
env = append(env, "CGO_ENABLED=1")
}
if opts.GoOS != "" {
env = append(env, fmt.Sprintf("GOOS=%s", opts.GoOS))
}
if opts.GoArch != "" {
env = append(env, fmt.Sprintf("GOARCH=%s", opts.GoArch))
}
if opts.CC != "" {
env = append(env, fmt.Sprintf("CC=%s", opts.CC))
}
if opts.CGoCFlags != "" {
env = append(env, fmt.Sprintf("CGO_CFLAGS=%s", opts.CGoCFlags))
}
if opts.GoArm != "" {
env = append(env, fmt.Sprintf("GOARM=%s", opts.GoArm))
}
if opts.ExtraEnv != nil {
return append(opts.ExtraEnv, env...)
}
return env
}
// Args constructs a list of flags and values for use with the exec.Command type when running "go build".
func (opts BuildOpts) Args() []string {
args := []string{}
if opts.LdFlags != nil {
args = append(args, "-ldflags", strings.Join(opts.LdFlags, " "))
}
if opts.Output != "" {
args = append(args, "-o", opts.Output)
}
if opts.ExtraArgs != nil {
args = append(args, opts.ExtraArgs...)
}
args = append(args, opts.Package)
return args
}
// Build runs the go build process in the current shell given the opts.
// This function will panic if no Stdout/Stderr/Stdin is provided in the opts.
func RunBuild(ctx context.Context, opts BuildOpts) error {
env := opts.Env()
args := append([]string{"build"}, opts.Args()...)
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
cmd := exec.CommandContext(ctx, "go", args...)
cmd.Env = env
cmd.Stdout = opts.Stdout
cmd.Stderr = opts.Stderr
cmd.Stdin = opts.Stdin
cmd.Dir = opts.Workdir
return cmd.Run()
}
pkg/build/golangutils/doc.go
+2
View File
@@ -0,0 +1,2 @@
// Package golangutils holds utility functions, wrappers, and types for building Go binaries for Grafana.
package golangutils
pkg/build/grafana/build.go
+123
View File
@@ -0,0 +1,123 @@
package grafana
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/cryptoutil"
"github.com/grafana/grafana/pkg/build/golangutils"
)
var binaries = []string{"grafana", "grafana-server", "grafana-cli"}
const (
SuffixEnterprise2 = "-enterprise2"
)
const (
ExtensionExe = ".exe"
)
func GrafanaLDFlags(version string, r config.Revision) []string {
return []string{
"-w",
fmt.Sprintf("-X main.version=%s", version),
fmt.Sprintf("-X main.commit=%s", r.SHA256),
fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp),
fmt.Sprintf("-X main.buildBranch=%s", r.Branch),
}
}
// BinaryFolder returns the path to where the Grafana binary is build given the provided arguments.
func BinaryFolder(edition config.Edition, args BuildArgs) string {
sfx := ""
if edition == config.EditionEnterprise2 {
sfx = SuffixEnterprise2
}
arch := string(args.GoArch)
if args.GoArch == config.ArchARM {
arch = string(args.GoArch) + "v" + args.GoArm
}
format := fmt.Sprintf("%s-%s", args.GoOS, arch)
if args.LibC != "" {
format += fmt.Sprintf("-%s", args.LibC)
}
format += sfx
if args.GoOS == config.OSWindows {
format += ExtensionExe
}
return format
}
func GrafanaDescriptor(opts golangutils.BuildOpts) string {
libcPart := ""
if opts.LibC != "" {
libcPart = fmt.Sprintf("/%s", opts.LibC)
}
arch := string(opts.GoArch)
if opts.GoArch == config.ArchARM {
arch = string(opts.GoArch) + "v" + opts.GoArm
}
return fmt.Sprintf("%s/%s%s", opts.GoOS, arch, libcPart)
}
// BuildGrafanaBinary builds a certain binary according to certain parameters.
func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArgs, edition config.Edition) error {
opts := args.BuildOpts
opts.ExtraEnv = os.Environ()
revision, err := config.GrafanaRevision(ctx, opts.Workdir)
if err != nil {
return err
}
folder := BinaryFolder(edition, args)
if opts.GoOS == config.OSWindows {
name += ExtensionExe
}
binary := filepath.Join(opts.Workdir, "bin", folder, name)
opts.Output = binary
if err := os.RemoveAll(binary); err != nil {
return fmt.Errorf("failed to remove %q: %w", binary, err)
}
if err := os.RemoveAll(binary + ".md5"); err != nil {
return fmt.Errorf("failed to remove %q: %w", binary+".md5", err)
}
descriptor := GrafanaDescriptor(opts)
log.Printf("Building %q for %s\nwith env: %v", binary, descriptor, opts.Env())
opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...)
if edition == config.EditionEnterprise2 {
opts.ExtraArgs = []string{"-tags=pro"}
}
log.Printf("Running command 'go %s'", opts.Args())
if err := golangutils.RunBuild(ctx, opts); err != nil {
return err
}
// Create an MD5 checksum of the binary, to be included in the archive for
// automatic upgrades.
if err := cryptoutil.MD5File(binary); err != nil {
return err
}
return nil
}
pkg/build/grafana/variant.go
+160
View File
@@ -0,0 +1,160 @@
package grafana
import (
"bytes"
"context"
"fmt"
"path/filepath"
"github.com/grafana/grafana/pkg/build/compilers"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/golangutils"
)
// BuildArgs represent the build parameters that define the "go build" behavior of a single variant.
// These arguments are applied as environment variables and arguments to the "go build" command.
type BuildArgs struct {
golangutils.BuildOpts
DebArch config.Architecture
RPMArch config.Architecture
}
type BuildVariantOpts struct {
Variant config.Variant
Edition config.Edition
Version string
GrafanaDir string
}
// BuildVariant builds a certain variant of the grafana-server and grafana-cli binaries sequentially.
func BuildVariant(ctx context.Context, opts BuildVariantOpts) error {
grafanaDir, err := filepath.Abs(opts.GrafanaDir)
if err != nil {
return err
}
var (
args = VariantBuildArgs(opts.Variant)
)
for _, binary := range binaries {
// Note that for Golang cmd paths we must use the relative path and the Linux file separators (/) even for Windows users.
var (
pkg = fmt.Sprintf("./pkg/cmd/%s", binary)
stdout = bytes.NewBuffer(nil)
stderr = bytes.NewBuffer(nil)
)
args.Workdir = grafanaDir
args.Stdout = stdout
args.Stderr = stderr
args.Package = pkg
if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil {
return fmt.Errorf("failed to build %s for %s: %w\nstdout: %s\nstderr: %s", pkg, opts.Variant, err, stdout.String(), stderr.String())
}
}
return nil
}
var ldFlagsStatic = []string{"-linkmode=external", "-extldflags=-static"}
var variantArgs = map[config.Variant]BuildArgs{
config.VariantArmV6: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
CGoEnabled: true,
GoArch: config.ArchARM,
GoArm: "6",
CC: compilers.ArmV6,
},
DebArch: config.ArchARMHF,
},
config.VariantArmV7: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
CGoEnabled: true,
GoArch: config.ArchARM,
GoArm: "7",
CC: compilers.Armv7,
},
DebArch: config.ArchARMHF,
RPMArch: config.ArchARMHFP,
},
config.VariantArmV7Musl: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
CGoEnabled: true,
GoArch: config.ArchARM,
GoArm: "7",
LibC: config.LibCMusl,
CC: compilers.Armv7Musl,
LdFlags: ldFlagsStatic,
},
},
config.VariantArm64: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
CGoEnabled: true,
GoArch: config.ArchARM64,
CC: compilers.Arm64,
},
DebArch: config.ArchARM64,
RPMArch: "aarch64",
},
config.VariantArm64Musl: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
GoArch: config.ArchARM64,
CGoEnabled: true,
CC: compilers.Arm64Musl,
LibC: config.LibCMusl,
LdFlags: ldFlagsStatic,
},
},
config.VariantDarwinAmd64: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSDarwin,
CGoEnabled: true,
GoArch: config.ArchAMD64,
CC: compilers.Osx64,
},
},
config.VariantWindowsAmd64: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSWindows,
GoArch: config.ArchAMD64,
CC: compilers.Win64,
CGoEnabled: true,
CGoCFlags: "-D_WIN32_WINNT=0x0601",
},
},
config.VariantLinuxAmd64: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
GoArch: config.ArchAMD64,
CC: compilers.LinuxX64,
},
DebArch: config.ArchAMD64,
RPMArch: config.ArchAMD64,
},
config.VariantLinuxAmd64Musl: {
BuildOpts: golangutils.BuildOpts{
GoOS: config.OSLinux,
GoArch: config.ArchAMD64,
CC: compilers.LinuxX64Musl,
LibC: config.LibCMusl,
LdFlags: ldFlagsStatic,
},
},
}
func VariantBuildArgs(v config.Variant) BuildArgs {
if val, ok := variantArgs[v]; ok {
return val
}
return BuildArgs{}
}
pkg/build/packaging/artifacts.go
+140
View File
@@ -0,0 +1,140 @@
package packaging
import (
"fmt"
"strings"
"github.com/grafana/grafana/pkg/build/config"
)
const ReleaseFolder = "release"
const MainFolder = "main"
const EnterpriseSfx = "-enterprise"
const CacheSettings = "Cache-Control:public, max-age="
type buildArtifact struct {
Os string
Arch string
urlPostfix string
packagePostfix string
}
type PublishConfig struct {
config.Config
Edition config.Edition
ReleaseMode config.ReleaseMode
GrafanaAPIKey string
WhatsNewURL string
ReleaseNotesURL string
DryRun bool
TTL string
SimulateRelease bool
}
const rhelOS = "rhel"
const debOS = "deb"
func (t buildArtifact) GetURL(baseArchiveURL string, cfg PublishConfig) string {
rev := ""
prefix := "-"
if t.Os == debOS {
prefix = "_"
} else if t.Os == rhelOS {
rev = "-1"
}
version := cfg.Version
verComponents := strings.Split(version, "-")
if len(verComponents) > 2 {
panic(fmt.Sprintf("Version string contains more than one hyphen: %q", version))
}
switch t.Os {
case debOS, rhelOS:
if len(verComponents) > 1 {
// With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this
// is considered of lower lexical value than the empty character, and this way pre-release versions are
// considered to be of a lower version than the final version (which lacks this suffix).
version = fmt.Sprintf("%s~%s", verComponents[0], verComponents[1])
}
}
// https://dl.grafana.com/oss/main/grafana_8.5.0~54094pre_armhf.deb: 404 Not Found
url := fmt.Sprintf("%s%s%s%s%s%s", baseArchiveURL, t.packagePostfix, prefix, version, rev, t.urlPostfix)
return url
}
var ArtifactConfigs = []buildArtifact{
{
Os: debOS,
Arch: "arm64",
urlPostfix: "_arm64.deb",
},
{
Os: rhelOS,
Arch: "arm64",
urlPostfix: ".aarch64.rpm",
},
{
Os: "linux",
Arch: "arm64",
urlPostfix: ".linux-arm64.tar.gz",
},
{
Os: debOS,
Arch: "armv7",
urlPostfix: "_armhf.deb",
},
{
Os: debOS,
Arch: "armv6",
packagePostfix: "-rpi",
urlPostfix: "_armhf.deb",
},
{
Os: rhelOS,
Arch: "armv7",
urlPostfix: ".armhfp.rpm",
},
{
Os: "linux",
Arch: "armv6",
urlPostfix: ".linux-armv6.tar.gz",
},
{
Os: "linux",
Arch: "armv7",
urlPostfix: ".linux-armv7.tar.gz",
},
{
Os: "darwin",
Arch: "amd64",
urlPostfix: ".darwin-amd64.tar.gz",
},
{
Os: "deb",
Arch: "amd64",
urlPostfix: "_amd64.deb",
},
{
Os: rhelOS,
Arch: "amd64",
urlPostfix: ".x86_64.rpm",
},
{
Os: "linux",
Arch: "amd64",
urlPostfix: ".linux-amd64.tar.gz",
},
{
Os: "win",
Arch: "amd64",
urlPostfix: ".windows-amd64.zip",
},
{
Os: "win-installer",
Arch: "amd64",
urlPostfix: ".windows-amd64.msi",
},
}
pkg/build/packaging/deb.go
+246
View File
@@ -0,0 +1,246 @@
package packaging
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/grafana/grafana/pkg/infra/fs"
)
func writeAptlyConf(dbDir, repoDir string) error {
aptlyConf := fmt.Sprintf(`{
"rootDir": "%s",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgProvider": "gpg2",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"skipContentsPublishing": false,
"FileSystemPublishEndpoints": {
"repo": {
"rootDir": "%s",
"linkMethod": "copy"
}
},
"S3PublishEndpoints": {},
"SwiftPublishEndpoints": {}
}
`, dbDir, repoDir)
home, err := os.UserHomeDir()
if err != nil {
return err
}
return os.WriteFile(filepath.Join(home, ".aptly.conf"), []byte(aptlyConf), 0600)
}
// downloadDebs downloads Deb packages.
func downloadDebs(cfg PublishConfig, workDir string) error {
if cfg.Bucket == "" {
panic("cfg.Bucket has to be set")
}
if !strings.HasSuffix(workDir, string(filepath.Separator)) {
workDir += string(filepath.Separator)
}
var version string
if cfg.ReleaseMode.Mode == config.TagMode {
if cfg.ReleaseMode.IsBeta {
version = strings.ReplaceAll(cfg.Version, "-", "~")
} else {
version = cfg.Version
}
}
if version == "" {
panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
}
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
u := fmt.Sprintf("gs://%s/%s/%s/grafana%s_%s_*.deb*", cfg.Bucket,
strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
log.Printf("Downloading Deb packages %q...\n", u)
args := []string{
"-m",
"cp",
u,
workDir,
}
//nolint:gosec
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download Deb packages %q: %w\n%s", u, err, output)
}
return nil
}
// UpdateDebRepo updates the Debian repository with the new release.
func UpdateDebRepo(cfg PublishConfig, workDir string) error {
if cfg.ReleaseMode.Mode != config.TagMode {
panic(fmt.Sprintf("Unsupported version mode: %s", cfg.ReleaseMode.Mode))
}
if cfg.ReleaseMode.IsTest {
if cfg.Config.DebDBBucket == DefaultDebDBBucket {
return fmt.Errorf("in test-release mode, the default Deb DB bucket shouldn't be used")
}
if cfg.Config.DebRepoBucket == DefaultDebRepoBucket {
return fmt.Errorf("in test-release mode, the default Deb repo bucket shouldn't be used")
}
}
if err := downloadDebs(cfg, workDir); err != nil {
return err
}
repoName := "grafana"
if cfg.ReleaseMode.IsBeta {
repoName = "beta"
}
repoRoot, err := fsutil.CreateTempDir("deb-repo")
if err != nil {
return err
}
defer func() {
if err := os.RemoveAll(repoRoot); err != nil {
log.Printf("Failed to remove temporary directory %q: %s\n", repoRoot, err.Error())
}
}()
dbDir := filepath.Join(repoRoot, "db")
repoDir := filepath.Join(repoRoot, "repo")
tmpDir := filepath.Join(repoRoot, "tmp")
for _, dpath := range []string{dbDir, repoDir, tmpDir} {
if err := os.MkdirAll(dpath, 0750); err != nil {
return err
}
}
if err := writeAptlyConf(dbDir, repoDir); err != nil {
return err
}
// Download the Debian repo database
u := fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
log.Printf("Downloading Debian repo database from %s...\n", u)
//nolint:gosec
cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", u, dbDir)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download Debian repo database: %w\n%s", err, output)
}
if err := addPkgsToRepo(cfg, workDir, tmpDir, repoName); err != nil {
return err
}
log.Println("Updating local Debian package repository...")
// Update published local repository. This assumes that there exists already a local, published repo.
for _, tp := range []string{"stable", "beta"} {
passArg := fmt.Sprintf("-passphrase-file=%s", cfg.GPGPassPath)
//nolint:gosec
cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp,
"filesystem:repo:grafana")
if output, err := cmd.CombinedOutput(); err != nil {
return cli.Exit(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1)
}
}
// Update database in GCS
u = fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
if cfg.DryRun {
log.Printf("Simulating upload of Debian repo database to GCS (%s)\n", u)
} else {
log.Printf("Uploading Debian repo database to GCS (%s)...\n", u)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.Exit(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1)
}
}
// Update metadata and binaries in repository bucket
u = fmt.Sprintf("gs://%s/%s/deb", cfg.DebRepoBucket, strings.ToLower(string(cfg.Edition)))
grafDir := filepath.Join(repoDir, "grafana")
if cfg.DryRun {
log.Printf("Simulating upload of Debian repo resources to GCS (%s)\n", u)
} else {
log.Printf("Uploading Debian repo resources to GCS (%s)...\n", u)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.Exit(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1)
}
allRepoResources := fmt.Sprintf("%s/**/*", u)
log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.Exit(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1)
}
}
return nil
}
func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unsupported edition %q", cfg.Edition)
}
log.Printf("Adding packages to Debian %q repo...\n", repoName)
// TODO: Be more specific about filename pattern
debs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s*.deb", sfx)))
if err != nil {
return err
}
for _, deb := range debs {
basename := filepath.Base(deb)
if strings.Contains(basename, "latest") {
continue
}
tgt := filepath.Join(tmpDir, basename)
if err := fs.CopyFile(deb, tgt); err != nil {
return err
}
}
// XXX: Adds too many packages in enterprise (Arve: What does this mean exactly?)
//nolint:gosec
cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.Exit(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1)
}
return nil
}
pkg/build/packaging/docs.go
+2
View File
@@ -0,0 +1,2 @@
// Package packaging holds functions and types for creating the tar.gz, deb, and rpm packages of Grafana.
package packaging
pkg/build/packaging/errors.go
+1
View File
@@ -0,0 +1 @@
package packaging
pkg/build/packaging/grafana.go
+1127
View File
File diff suppressed because it is too large Load Diff
pkg/build/packaging/grafana_test.go
+22
View File
@@ -0,0 +1,22 @@
package packaging_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/stretchr/testify/assert"
)
func TestPackageRegexp(t *testing.T) {
t.Run("It should match enterprise2 packages", func(t *testing.T) {
rgx := packaging.PackageRegexp(config.EditionEnterprise2)
matches := []string{
"grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz",
"grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz.sha256",
}
for _, v := range matches {
assert.Truef(t, rgx.MatchString(v), "'%s' should match regex '%s'", v, rgx.String())
}
})
}
pkg/build/packaging/rpm.go
+370
View File
@@ -0,0 +1,370 @@
package packaging
import (
"bytes"
"crypto"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
// Consider switching this over to a community fork unless there is
// an option to move us away from OpenPGP.
"golang.org/x/crypto/openpgp" //nolint:staticcheck
"golang.org/x/crypto/openpgp/armor" //nolint:staticcheck
"golang.org/x/crypto/openpgp/packet" //nolint:staticcheck
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/grafana/grafana/pkg/infra/fs"
)
// UpdateRPMRepo updates the RPM repository with the new release.
func UpdateRPMRepo(cfg PublishConfig, workDir string) error {
if cfg.ReleaseMode.Mode != config.TagMode {
panic(fmt.Sprintf("Unsupported version mode %s", cfg.ReleaseMode.Mode))
}
if cfg.ReleaseMode.IsTest && cfg.Config.RPMRepoBucket == DefaultRPMRepoBucket {
return fmt.Errorf("in test-release mode, the default RPM repo bucket shouldn't be used")
}
if err := downloadRPMs(cfg, workDir); err != nil {
return err
}
repoRoot, err := fsutil.CreateTempDir("rpm-repo")
if err != nil {
return err
}
defer func() {
if err := os.RemoveAll(repoRoot); err != nil {
log.Printf("Failed to remove %q: %s\n", repoRoot, err.Error())
}
}()
repoName := "rpm"
if cfg.ReleaseMode.IsBeta {
repoName = "rpm-beta"
}
folderURI := fmt.Sprintf("gs://%s/%s/%s", cfg.RPMRepoBucket, strings.ToLower(string(cfg.Edition)), repoName)
// Download the RPM database
log.Printf("Downloading RPM database from GCS (%s)...\n", folderURI)
//nolint:gosec
cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", folderURI, repoRoot)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download RPM database from GCS: %w\n%s", err, output)
}
// Add the new release to the repo
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unsupported edition %q", cfg.Edition)
}
allRPMs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s-*.rpm", sfx)))
if err != nil {
return fmt.Errorf("failed to list RPMs in %q: %w", workDir, err)
}
rpms := []string{}
for _, rpm := range allRPMs {
if strings.Contains(rpm, "-latest") {
continue
}
rpms = append(rpms, rpm)
}
// XXX: What does the following comment mean?
// adds to many files for enterprise
for _, rpm := range rpms {
if err := fs.CopyFile(rpm, filepath.Join(repoRoot, filepath.Base(rpm))); err != nil {
return err
}
}
//nolint:gosec
cmd = exec.Command("createrepo", repoRoot)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create repo at %q: %w\n%s", repoRoot, err, output)
}
if err := signRPMRepo(repoRoot, cfg); err != nil {
return err
}
// Update the repo in GCS
// Sync packages first to avoid cache misses
if cfg.DryRun {
log.Printf("Simulating upload of RPMs to GCS (%s)\n", folderURI)
} else {
log.Printf("Uploading RPMs to GCS (%s)...\n", folderURI)
args := []string{"-m", "cp"}
args = append(args, rpms...)
args = append(args, folderURI)
//nolint:gosec
cmd = exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload RPMs to GCS: %w\n%s", err, output)
}
}
if cfg.DryRun {
log.Printf("Simulating upload of RPM repo metadata to GCS (%s)\n", folderURI)
} else {
log.Printf("Uploading RPM repo metadata to GCS (%s)...\n", folderURI)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", repoRoot, folderURI)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload RPM repo metadata to GCS: %w\n%s", err, output)
}
allRepoResources := fmt.Sprintf("%s/**/*", folderURI)
log.Printf("Setting cache ttl for RPM repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to set cache ttl for RPM repo resources on GCS: %w\n%s", err, output)
}
}
return nil
}
// downloadRPMs downloads RPM packages.
func downloadRPMs(cfg PublishConfig, workDir string) error {
if !strings.HasSuffix(workDir, string(filepath.Separator)) {
workDir += string(filepath.Separator)
}
var version string
if cfg.ReleaseMode.Mode == config.TagMode {
if cfg.ReleaseMode.IsBeta {
version = strings.ReplaceAll(cfg.Version, "-", "~")
} else {
version = cfg.Version
}
}
if version == "" {
panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
}
var sfx string
switch cfg.Edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = EnterpriseSfx
default:
return fmt.Errorf("unrecognized edition %q", cfg.Edition)
}
u := fmt.Sprintf("gs://%s/%s/%s/grafana%s-%s-*.*.rpm*", cfg.Bucket,
strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
log.Printf("Downloading RPM packages %q...\n", u)
args := []string{
"-m",
"cp",
u,
workDir,
}
//nolint:gosec
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to download RPM packages %q: %w\n%s", u, err, output)
}
return nil
}
func getPublicKey(cfg PublishConfig) (*packet.PublicKey, error) {
f, err := os.Open(cfg.GPGPublicKey)
if err != nil {
return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPublicKey, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
return
}
}(f)
block, err := armor.Decode(f)
if err != nil {
return nil, err
}
if block.Type != openpgp.PublicKeyType {
return nil, fmt.Errorf("invalid public key block type: %q", block.Type)
}
packetReader := packet.NewReader(block.Body)
pkt, err := packetReader.Next()
if err != nil {
return nil, err
}
key, ok := pkt.(*packet.PublicKey)
if !ok {
return nil, fmt.Errorf("got non-public key from packet reader: %T", pkt)
}
return key, nil
}
func getPrivateKey(cfg PublishConfig) (*packet.PrivateKey, error) {
f, err := os.Open(cfg.GPGPrivateKey)
if err != nil {
return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPrivateKey, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
return
}
}(f)
passphraseB, err := os.ReadFile(cfg.GPGPassPath)
if err != nil {
return nil, fmt.Errorf("failed to read %q: %w", cfg.GPGPrivateKey, err)
}
passphraseB = bytes.TrimSuffix(passphraseB, []byte("\n"))
block, err := armor.Decode(f)
if err != nil {
return nil, err
}
if block.Type != openpgp.PrivateKeyType {
return nil, fmt.Errorf("invalid private key block type: %q", block.Type)
}
packetReader := packet.NewReader(block.Body)
pkt, err := packetReader.Next()
if err != nil {
return nil, err
}
key, ok := pkt.(*packet.PrivateKey)
if !ok {
return nil, fmt.Errorf("got non-private key from packet reader: %T", pkt)
}
if err := key.Decrypt(passphraseB); err != nil {
return nil, fmt.Errorf("failed to decrypt private key: %w", err)
}
return key, nil
}
// signRPMRepo signs an RPM repository using PGP.
// The signature gets written to the file repodata/repomd.xml.asc.
func signRPMRepo(repoRoot string, cfg PublishConfig) error {
if cfg.GPGPublicKey == "" || cfg.GPGPrivateKey == "" {
return fmt.Errorf("private or public key is empty")
}
log.Printf("Signing RPM repo")
pubKey, err := getPublicKey(cfg)
if err != nil {
return err
}
privKey, err := getPrivateKey(cfg)
if err != nil {
return err
}
pcfg := packet.Config{
DefaultHash: crypto.SHA256,
DefaultCipher: packet.CipherAES256,
DefaultCompressionAlgo: packet.CompressionZLIB,
CompressionConfig: &packet.CompressionConfig{
Level: 9,
},
RSABits: 4096,
}
currentTime := pcfg.Now()
uid := packet.NewUserId("", "", "")
isPrimaryID := false
keyLifetimeSecs := uint32(86400 * 365)
signer := openpgp.Entity{
PrimaryKey: pubKey,
PrivateKey: privKey,
Identities: map[string]*openpgp.Identity{
uid.Id: {
Name: uid.Name,
UserId: uid,
SelfSignature: &packet.Signature{
CreationTime: currentTime,
SigType: packet.SigTypePositiveCert,
PubKeyAlgo: packet.PubKeyAlgoRSA,
Hash: pcfg.Hash(),
IsPrimaryId: &isPrimaryID,
FlagsValid: true,
FlagSign: true,
FlagCertify: true,
IssuerKeyId: &pubKey.KeyId,
},
},
},
Subkeys: []openpgp.Subkey{
{
PublicKey: pubKey,
PrivateKey: privKey,
Sig: &packet.Signature{
CreationTime: currentTime,
SigType: packet.SigTypeSubkeyBinding,
PubKeyAlgo: packet.PubKeyAlgoRSA,
Hash: pcfg.Hash(),
PreferredHash: []uint8{8}, // SHA-256
FlagsValid: true,
FlagEncryptStorage: true,
FlagEncryptCommunications: true,
IssuerKeyId: &pubKey.KeyId,
KeyLifetimeSecs: &keyLifetimeSecs,
},
},
},
}
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
freader, err := os.Open(filepath.Join(repoRoot, "repodata", "repomd.xml"))
if err != nil {
return err
}
defer func(freader *os.File) {
err := freader.Close()
if err != nil {
return
}
}(freader)
// Ignore gosec G304 as this function is only used in the build process.
//nolint:gosec
sigwriter, err := os.Create(filepath.Join(repoRoot, "repodata", "repomd.xml.asc"))
if err != nil {
return err
}
defer func(sigwriter *os.File) {
err := sigwriter.Close()
if err != nil {
return
}
}(sigwriter)
if err := openpgp.ArmoredDetachSignText(sigwriter, &signer, freader, nil); err != nil {
return fmt.Errorf("failed to write PGP signature: %w", err)
}
if err := sigwriter.Close(); err != nil {
return fmt.Errorf("failed to write PGP signature: %w", err)
}
return nil
}
pkg/build/packaging/rpm_test.go
+146
View File
@@ -0,0 +1,146 @@
package packaging
import (
"os"
"path/filepath"
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/stretchr/testify/require"
)
const pubKey = `-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: OpenPGP.js v4.10.10
Comment: https://openpgpjs.org
xsBNBGM1b9wBCADZM49X7vwOS93KbgA6yhpwrYf8ZlzksGcDaYgp1IzvqHbs
xeU1mmBYVH/bSKRDG0tt3Qdky4Nvl4Oqd+g0e2ZGjmlEy9zUiPTTK/BtXT+5
s8oqih2NIAkyF91BNZABAgvh/vJdYImhYeUQBDqMJgqZ/Y/Ha31N7rSW+jUt
LHspbN0ztJYjuEd/bg2NKH7Gs/AyNvX9IQTC4k7iRRafx7q/PBCVtsk+NCwz
BEkL93xpAdcdYiMNrRP2eIHQjBmNZ/oUCkcDsLCBvcSq6P2lGpNnpPzVoTJf
v2qrWkVn5txJJsOkmBGpEDbECPunVilrWO6RPomP0yYkr6NE4XeCJ3QhABEB
AAHNGWR1bW15IDxkdW1teUBob3RtYWlsLmNvbT7CwI0EEAEIACAFAmM1b9wG
CwkHCAMCBBUICgIEFgIBAAIZAQIbAwIeAQAhCRAoJ1i5w6kkAxYhBCQv+iwt
IFn7vj9PLygnWLnDqSQDPxkH/0Ju2Cah+bOxl09uv2Ft2BVlQh0u+wJyRVgs
KxTxldAXFZwMrN4wK/GUoGWDiy2tzNtoVE6GpxWUj+LvSGFaVLNVjW+Le77I
BP/sl1wKHJbQhseKc7Mz5Zj3i0F1FPM+rLik7tNk6kiEBqYVyyXahyT98Hu1
1OKEV+8NiRG47iNgd/dpgEdVSS4DN/dL6m5q+CVy9YnlR+wXxF/2xcMmWBzR
V2cPVw0JzunpUV8lDDQ/n1sPw61D3oL1aH0bkn8aA8pEceKOVIYOaja7LkLX
uSlROlALA/M2fuubradW9I3FcrJNn+/xA52el2l/Hn/Syf9GQV/Ll/R+qKIo
Z57xWd7OwE0EYzVv3AEIAJl/PNYOF2prNKY58BfZ74XurDb9mNlZ1wsIqrOu
J/euzHEnzkCAjMUuXV7wcugjQlmpcZn6Y0QmQ2uX7SwPCMovDvngbXeAfbdd
6FUKecQ0sG54Plm8HSMNdjetdUVl7ACxjJO8Rdc/Asx7ua7gMm42CVfqMj4L
qN5foUBlaKJ1iGKUpQ+673UQWMYeOBuu9G8awbSzGaphN97CIX7xEMGzGeff
yHLHK+MsfX935uDgDwJQzxJKEugIJDMKgWOLgVz1jRCsJKHlywHTWpVuMiKY
Wnuq4tDNLBUQtaRL7uclG7Wejw/XNN0uD/zNHPgF5rmlYHVhrtDbBCP2XqTn
WU8AEQEAAcLAdgQYAQgACQUCYzVv3AIbDAAhCRAoJ1i5w6kkAxYhBCQv+iwt
IFn7vj9PLygnWLnDqSQDFqYH/AkdNaPUQlE7RQBigNRGOFBuqjhbLsV/rZf+
/4K6wDHojM606lgLm57T4NUXnk53VIF3KO8+v8N11mCtPb+zBngfvVU14COC
HEDNdOK19TlR+tH25cftfUiF+OJsgMQysErGuFEtwLE6TNzpQIcnw7SbjxMr
EGacF9xCBKexB6MlR3GwJ2LBUJm3Lq/fvqImztoTlKDsrpk4JOH5FfYG+G2f
1zU73fVsCCElX4qA/49rRQf0RNfhjRjmHULP8hSvCXUEhfiBggEgxof/vKlC
qauHC55luuIeabju8HaXTjpz019cq+3IUgewX/ky0PhQXEW9SoODKabPY2yS
yUbHFm4=
=OCSx
-----END PGP PUBLIC KEY BLOCK-----
`
const privKey = `-----BEGIN PGP PRIVATE KEY BLOCK-----
Version: OpenPGP.js v4.10.10
Comment: https://openpgpjs.org
xcMGBGM1b9wBCADZM49X7vwOS93KbgA6yhpwrYf8ZlzksGcDaYgp1IzvqHbs
xeU1mmBYVH/bSKRDG0tt3Qdky4Nvl4Oqd+g0e2ZGjmlEy9zUiPTTK/BtXT+5
s8oqih2NIAkyF91BNZABAgvh/vJdYImhYeUQBDqMJgqZ/Y/Ha31N7rSW+jUt
LHspbN0ztJYjuEd/bg2NKH7Gs/AyNvX9IQTC4k7iRRafx7q/PBCVtsk+NCwz
BEkL93xpAdcdYiMNrRP2eIHQjBmNZ/oUCkcDsLCBvcSq6P2lGpNnpPzVoTJf
v2qrWkVn5txJJsOkmBGpEDbECPunVilrWO6RPomP0yYkr6NE4XeCJ3QhABEB
AAH+CQMIuDEg1p2Y6zbg0EQ3JvsP7VQBGsuXg9khTjktoxhwici/d+rcIW7Q
SuKWJGqs83LTeeGmS+9etNtf3LqRdPnI7f0qbT47mAqvp2gn7Rvbrabk+5Jj
AQS/DDLlWNiWsPrMBMZ7TZpiQ+g7gnIZaV10taFupYJr69AjtED+NPu8LOvZ
2ItK9xBqOwl5mkNe7ps/uTT6jwYSWxeObp4ymnLDLONY3eHuaYP9QB/NSlw0
80Wo5qBPljlU8JdbEoLFU4gY6wkEbLa/DVbEVXSHfWVtr8jZbzHW39TBxpG2
Dxk52EVyu8Gf9XIQN2ZjDP3CzBGmlxJjLxLUD4GmRSPaDGK7LCN9ZztaXy3Y
WtF6RJfNzEoDdCaV0kkM3AskQDsQ+CWsDVsbbQyDtfncVG6cDzqmoDrBCSq1
Bsoz07k2hj9VP0aP2xU78qcpJWO2rmhAHy9W2NqjXSBJriy1JXrK5o2/lUUr
94R8NLvqeVbInUw/zovVctaujHIBhNKL9wn2T0LWrA2OEJUz0HWo6ZQSaNzl
Obtz0M8gCj/4sDYjRAiDk50FzOcZp8ijYQFVypQTVzHki5T/JfvBnMpo+4Uc
93QB1woyiZuJCIj7DpY3MkZ5fTDtgJPa+0k8r+lPnAmE6auGUaH7JRKhbBu0
8faDwaiSv3kD3EEDffoWX/axLLYta9jTDnitTXbf1jY03pdJeiU/ZX0BQTZi
pehZ/6yi/qXM/F8HDVEWriSLqVsMLrXXeFIojAc3fJ/QPpAZSx6E/Fe2xh8c
yURov5krU1zNJDwqC3SjHsHQ/UlLtamDDmmuXX+xb1CwIDd6WksGsCbe/LoN
TxViV4hOjIeh5TwRP5jQaqsVKCT8fzoDrRXy76taT+Zaaen+J6rC51HQwyEq
Qgf1e7WodzN3r10UV6/L/wNkfdWJgf5MzRlkdW1teSA8ZHVtbXlAaG90bWFp
bC5jb20+wsCNBBABCAAgBQJjNW/cBgsJBwgDAgQVCAoCBBYCAQACGQECGwMC
HgEAIQkQKCdYucOpJAMWIQQkL/osLSBZ+74/Ty8oJ1i5w6kkAz8ZB/9Cbtgm
ofmzsZdPbr9hbdgVZUIdLvsCckVYLCsU8ZXQFxWcDKzeMCvxlKBlg4strczb
aFROhqcVlI/i70hhWlSzVY1vi3u+yAT/7JdcChyW0IbHinOzM+WY94tBdRTz
Pqy4pO7TZOpIhAamFcsl2ock/fB7tdTihFfvDYkRuO4jYHf3aYBHVUkuAzf3
S+puavglcvWJ5UfsF8Rf9sXDJlgc0VdnD1cNCc7p6VFfJQw0P59bD8OtQ96C
9Wh9G5J/GgPKRHHijlSGDmo2uy5C17kpUTpQCwPzNn7rm62nVvSNxXKyTZ/v
8QOdnpdpfx5/0sn/RkFfy5f0fqiiKGee8Vnex8MGBGM1b9wBCACZfzzWDhdq
azSmOfAX2e+F7qw2/ZjZWdcLCKqzrif3rsxxJ85AgIzFLl1e8HLoI0JZqXGZ
+mNEJkNrl+0sDwjKLw754G13gH23XehVCnnENLBueD5ZvB0jDXY3rXVFZewA
sYyTvEXXPwLMe7mu4DJuNglX6jI+C6jeX6FAZWiidYhilKUPuu91EFjGHjgb
rvRvGsG0sxmqYTfewiF+8RDBsxnn38hyxyvjLH1/d+bg4A8CUM8SShLoCCQz
CoFji4Fc9Y0QrCSh5csB01qVbjIimFp7quLQzSwVELWkS+7nJRu1no8P1zTd
Lg/8zRz4Bea5pWB1Ya7Q2wQj9l6k51lPABEBAAH+CQMIwr3YSD15lYrgItoy
MDsrWqMMHJsSxusbQiK0KLgjFBuDuTolsu9zqQCHEm2dxChqT+yQ6AeeynRD
pDMVkHEvhShvGUhB6Bu5wClHj8+xFpyprShE/KbEuppNdfIRgWVYc7UX+TYz
6BymqhzKyIw2Q33ocrXgTRZ02HM7urKVvAhsJCEff0paByOzCspiv/TPRihi
7GAZY0wFLDPe9qr+07ExT2ndMDX8Xb1mlg8IeaSWUaNilm7M8oW3xnUBnXeD
XglTkObGeRVXAINim9uL4soT3lamN4QwgBus9WzFqOOCMk11fjatY8kY1zX9
epO27igGtMwTFl11XcQLlFyvlgPBeWtFam7RiDPa3VF0XubmBYZBmqWpccWs
xl0xHCtUK7Pd0O4kSqxsL9cB0MX9iR1yPkM8wA++Mp6pEfNcXUrGIdlie0H5
aCq8rguYG5VuFosSUatdCbpRVGBxGnhxHes0mNTPgwAoAVNYBWXH5iq5HxKy
i3Zy5V7ZKSyDrfg/0AajtDW5h3g+wglUI9UCdT4tNLFwYbhHqGH2xdBztYI0
iSJ7COLmo26smkA8UXxsrlw8PWPzpbhQOG06EbMjncJimJDMI1YDC6ag7M5l
OcG9uXZQ22ipAz5CSPtyL0/0WAp4yyn1VQRBK42n/y9ld+dMbuq6majazb15
6sEgHUKERcwGs0Ftfj5Zamwhm7ZoIe26XEqvcshpQpv1Q9hktluVeSbiVaBe
Nl8zUZHlo/0zUc5j7G5Up58t+ChSsyOFJGM7CGkKHHawBZYCs0EcpsdAPr3T
1C8A0Wt9POTETYM4pZFOoLds6VTolZZcxeBN5YPoN2kbwFpOgPJN09Zz8z8S
4psQRV4KQ92XDPZ/6q2BH5i2+F2ZwUsvCR4DwgzbVGZSRV6mM7lkjZSmnWfC
AE7DUl7XwsB2BBgBCAAJBQJjNW/cAhsMACEJECgnWLnDqSQDFiEEJC/6LC0g
Wfu+P08vKCdYucOpJAMWpgf8CR01o9RCUTtFAGKA1EY4UG6qOFsuxX+tl/7/
grrAMeiMzrTqWAubntPg1ReeTndUgXco7z6/w3XWYK09v7MGeB+9VTXgI4Ic
QM104rX1OVH60fblx+19SIX44myAxDKwSsa4US3AsTpM3OlAhyfDtJuPEysQ
ZpwX3EIEp7EHoyVHcbAnYsFQmbcur9++oibO2hOUoOyumTgk4fkV9gb4bZ/X
NTvd9WwIISVfioD/j2tFB/RE1+GNGOYdQs/yFK8JdQSF+IGCASDGh/+8qUKp
q4cLnmW64h5puO7wdpdOOnPTX1yr7chSB7Bf+TLQ+FBcRb1Kg4Mpps9jbJLJ
RscWbg==
=KJNy
-----END PGP PRIVATE KEY BLOCK-----
`
// Dummy GPG keys, used only for testing
// nolint:gosec
const passPhrase = `MkDgjkrgdGxt`
func TestSignRPMRepo(t *testing.T) {
repoDir := t.TempDir()
workDir := t.TempDir()
pubKeyPath := filepath.Join(workDir, "pub.key")
err := os.WriteFile(pubKeyPath, []byte(pubKey), 0600)
require.NoError(t, err)
privKeyPath := filepath.Join(workDir, "priv.key")
err = os.WriteFile(privKeyPath, []byte(privKey), 0600)
require.NoError(t, err)
passPhrasePath := filepath.Join(workDir, "passphrase")
err = os.WriteFile(passPhrasePath, []byte(passPhrase), 0600)
require.NoError(t, err)
err = os.Mkdir(filepath.Join(repoDir, "repodata"), 0700)
require.NoError(t, err)
err = os.WriteFile(filepath.Join(repoDir, "repodata", "repomd.xml"), []byte("<xml></xml>"), 0600)
require.NoError(t, err)
cfg := PublishConfig{
Config: config.Config{
GPGPrivateKey: privKeyPath,
GPGPublicKey: pubKeyPath,
GPGPassPath: passPhrasePath,
},
}
err = signRPMRepo(repoDir, cfg)
require.NoError(t, err)
}
pkg/build/plugins/build.go
+66
View File
@@ -0,0 +1,66 @@
package plugins
import (
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/errutil"
"github.com/grafana/grafana/pkg/build/syncutil"
"github.com/grafana/grafana/pkg/infra/fs"
)
type PluginSigningMode = int
// BuildPlugins builds internal plugins.
// The built plugins are placed in plugins-bundled/dist/.
func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.BuildConfig) error {
log.Printf("Building plugins in %q...", grafanaDir)
root := filepath.Join(grafanaDir, "plugins-bundled", "internal")
fis, err := os.ReadDir(root)
if err != nil {
return err
}
for i := range fis {
fi := fis[i]
if !fi.IsDir() {
continue
}
dpath := filepath.Join(root, fi.Name())
p.Schedule(g.Wrap(func() error {
log.Printf("Building plugin %q...", dpath)
cmd := exec.Command("yarn", "build")
cmd.Dir = dpath
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("yarn build failed: %s", output)
}
dstPath := filepath.Join("plugins-bundled", "dist", fi.Name())
if err := fs.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil {
return err
}
if !verMode.PluginSignature.Sign {
return nil
}
return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign)
}))
}
if err := g.Wait(); err != nil {
return err
}
log.Printf("Built all plug-ins successfully!")
return nil
}
pkg/build/plugins/download.go
+118
View File
@@ -0,0 +1,118 @@
package plugins
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/build/errutil"
"github.com/grafana/grafana/pkg/build/syncutil"
)
// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
func logCloseError(closeFunc func() error) {
if err := closeFunc(); err != nil {
log.Println(err)
}
}
// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
func logError(err error) {
if err != nil {
log.Println(err)
}
}
// pluginManifest has details of an external plugin package.
type pluginManifest struct {
Name string `json:"name"`
Version string `json:"version"`
Checksum string `json:"checksum"`
}
// pluginsManifest represents a manifest of Grafana's external plugins.
type pluginsManifest struct {
Plugins []pluginManifest `json:"plugins"`
}
// downloadPlugins downloads Grafana plugins that should be bundled into packages.
//
// The plugin archives are downloaded into <grafanaDir>/plugins-bundled.
func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error {
g, _ := errutil.GroupWithContext(ctx)
log.Println("Downloading external plugins...")
var m pluginsManifest
manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json")
//nolint:gosec
manifestB, err := os.ReadFile(manifestPath)
if err != nil {
return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err)
}
if err := json.Unmarshal(manifestB, &m); err != nil {
return err
}
for i := range m.Plugins {
pm := m.Plugins[i]
p.Schedule(g.Wrap(func() error {
tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version))
//nolint:gosec
out, err := os.Create(tgt)
if err != nil {
return err
}
defer logCloseError(out.Close)
u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name,
pm.Version)
log.Printf("Downloading plugin %q to %q...", u, tgt)
// nolint:gosec
resp, err := http.Get(u)
if err != nil {
return fmt.Errorf("downloading %q failed: %w", u, err)
}
defer logError(resp.Body.Close())
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode)
}
if _, err := io.Copy(out, resp.Body); err != nil {
return fmt.Errorf("downloading %q failed: %w", u, err)
}
if err := out.Close(); err != nil {
return fmt.Errorf("downloading %q failed: %w", u, err)
}
//nolint:gosec
fd, err := os.Open(tgt)
if err != nil {
return err
}
defer logCloseError(fd.Close)
h := sha256.New()
if _, err := io.Copy(h, fd); err != nil {
return err
}
chksum := hex.EncodeToString(h.Sum(nil))
if chksum != pm.Checksum {
return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum)
}
return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled"))
}))
}
return g.Wait()
}
pkg/build/plugins/manifest.go
+204
View File
@@ -0,0 +1,204 @@
package plugins
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
)
type manifest struct {
Plugin string `json:"plugin"`
Version string `json:"version"`
Files map[string]string `json:"files"`
}
func getManifest(dpath string, chksums map[string]string) (manifest, error) {
m := manifest{}
type pluginInfo struct {
Version string `json:"version"`
}
type plugin struct {
ID string `json:"id"`
Info pluginInfo `json:"info"`
}
//nolint:gosec
f, err := os.Open(filepath.Join(dpath, "plugin.json"))
if err != nil {
return m, err
}
decoder := json.NewDecoder(f)
var p plugin
if err := decoder.Decode(&p); err != nil {
return m, err
}
if p.ID == "" {
return m, fmt.Errorf("plugin.json doesn't define id")
}
if p.Info.Version == "" {
return m, fmt.Errorf("plugin.json doesn't define info.version")
}
return manifest{
Plugin: p.ID,
Version: p.Info.Version,
Files: chksums,
}, nil
}
// BuildManifest requests a plugin's signed manifest file fromt he Grafana API.
// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires
// an admin API key.
func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error {
log.Printf("Building manifest for plug-in at %q", dpath)
apiKey := os.Getenv("GRAFANA_API_KEY")
if apiKey == "" {
return fmt.Errorf("GRAFANA_API_KEY must be set")
}
manifestPath := filepath.Join(dpath, "MANIFEST.txt")
chksums, err := getChksums(dpath, manifestPath)
if err != nil {
return err
}
m, err := getManifest(dpath, chksums)
if err != nil {
return err
}
b := bytes.NewBuffer(nil)
encoder := json.NewEncoder(b)
if err := encoder.Encode(&m); err != nil {
return err
}
jsonB := b.Bytes()
u := "https://grafana.com/api/plugins/ci/sign"
if !signingAdmin {
u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin)
}
log.Printf("Requesting signed manifest from Grafana API...")
req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB))
if err != nil {
return err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey))
req.Header.Add("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println("failed to close response body, err: %w", err)
}
}()
if resp.StatusCode != 200 {
msg, err := io.ReadAll(resp.Body)
if err != nil {
log.Printf("Failed to read response body: %s", err)
msg = []byte("")
}
return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg))
}
log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath)
//nolint:gosec
f, err := os.Create(manifestPath)
if err != nil {
return fmt.Errorf("failed to create %s: %w", manifestPath, err)
}
defer func() {
if err := f.Close(); err != nil {
log.Println("failed to close file, err: %w", err)
}
}()
if _, err := io.Copy(f, resp.Body); err != nil {
return fmt.Errorf("failed to write %s: %w", manifestPath, err)
}
if err := f.Close(); err != nil {
return fmt.Errorf("failed to write %s: %w", manifestPath, err)
}
return nil
}
func getChksums(dpath, manifestPath string) (map[string]string, error) {
manifestPath = filepath.Clean(manifestPath)
chksums := map[string]string{}
if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if fi.IsDir() {
return nil
}
path = filepath.Clean(path)
// Handle symbolic links
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
finalPath, err := filepath.EvalSymlinks(path)
if err != nil {
return err
}
log.Printf("Handling symlink %q, pointing to %q", path, finalPath)
info, err := os.Stat(finalPath)
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if _, err := filepath.Rel(dpath, finalPath); err != nil {
return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath)
}
if finalPath == manifestPath {
return nil
}
}
if path == manifestPath {
return nil
}
h := sha256.New()
//nolint:gosec
f, err := os.Open(path)
if err != nil {
return err
}
defer logCloseError(f.Close)
if _, err := io.Copy(h, f); err != nil {
return err
}
relPath, err := filepath.Rel(dpath, path)
if err != nil {
return err
}
chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil))
return nil
}); err != nil {
return nil, err
}
return chksums, nil
}
pkg/build/plugins/zip.go
+64
View File
@@ -0,0 +1,64 @@
package plugins
import (
"archive/zip"
"io"
"log"
"os"
"path/filepath"
)
// Unzip unzips a plugin.
func Unzip(fpath, tgtDir string) error {
log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir)
r, err := zip.OpenReader(fpath)
if err != nil {
return err
}
defer logCloseError(r.Close)
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
log.Printf("Extracting zip member %q...", f.Name)
rc, err := f.Open()
if err != nil {
return err
}
defer logCloseError(rc.Close)
//nolint:gosec
dstPath := filepath.Join(tgtDir, f.Name)
if f.FileInfo().IsDir() {
return os.MkdirAll(dstPath, f.Mode())
}
if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil {
return err
}
//nolint:gosec
fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer logCloseError(fd.Close)
// nolint:gosec
if _, err := io.Copy(fd, rc); err != nil {
return err
}
return fd.Close()
}
for _, f := range r.File {
if err := extractAndWriteFile(f); err != nil {
return err
}
}
return nil
}
pkg/build/stringutil/contains.go
+10
View File
@@ -0,0 +1,10 @@
package stringutil
func Contains(arr []string, s string) bool {
for _, e := range arr {
if e == s {
return true
}
}
return false
}
pkg/build/syncutil/pool.go
+43
View File
@@ -0,0 +1,43 @@
package syncutil
import (
"log"
"runtime"
)
func worker(jobs chan func()) {
for j := range jobs {
j()
}
}
// WorkerPool represents a concurrent worker pool.
type WorkerPool struct {
NumWorkers int
jobs chan func()
}
// NewWorkerPool constructs a new WorkerPool.
func NewWorkerPool(numWorkers int) WorkerPool {
if numWorkers <= 0 {
numWorkers = runtime.NumCPU()
}
log.Printf("Creating worker pool with %d workers", numWorkers)
jobs := make(chan func(), 100)
for i := 0; i < numWorkers; i++ {
go worker(jobs)
}
return WorkerPool{
NumWorkers: numWorkers,
jobs: jobs,
}
}
// Schedule schedules a job to be executed by a worker in the pool.
func (p WorkerPool) Schedule(job func()) {
p.jobs <- job
}
func (p WorkerPool) Close() {
close(p.jobs)
}
scripts/drone/steps/lib.star
+1
View File
@@ -1041,6 +1041,7 @@ def publish_grafanacom_step(edition, ver_mode):
],
'environment': {
'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
'GCP_KEY': from_secret('gcp_key'),
},
'commands': [
cmd,