diff --git a/.drone.yml b/.drone.yml
index 6549e720980..afc1eec0b05 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -3233,6 +3233,8 @@ steps:
- publish-linux-packages-deb
- publish-linux-packages-rpm
environment:
+ GCP_KEY:
+ from_secret: gcp_key
GRAFANA_COM_API_KEY:
from_secret: grafana_api_key
image: grafana/grafana-ci-deploy:1.3.1
@@ -3326,6 +3328,8 @@ steps:
- publish-linux-packages-deb
- publish-linux-packages-rpm
environment:
+ GCP_KEY:
+ from_secret: gcp_key
GRAFANA_COM_API_KEY:
from_secret: grafana_api_key
image: grafana/grafana-ci-deploy:1.3.1
@@ -4792,6 +4796,6 @@ kind: secret
name: aws_secret_access_key
---
kind: signature
-hmac: bf3d2885e476653b0f3bcecb4e0335b2d243285d90e3d74492bf794a90615e06
+hmac: 702107d1f512b722de5103aa8c80ff0f0048b3e42acbc3af17064301414def49
...
diff --git a/pkg/build/cmd/argcount_wrapper.go b/pkg/build/cmd/argcount_wrapper.go
new file mode 100644
index 00000000000..690695cd350
--- /dev/null
+++ b/pkg/build/cmd/argcount_wrapper.go
@@ -0,0 +1,31 @@
+package main
+
+import "github.com/urfave/cli/v2"
+
+// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided.
+func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc {
+ return func(ctx *cli.Context) error {
+ if ctx.NArg() != num {
+ if err := cli.ShowSubcommandHelp(ctx); err != nil {
+ return cli.Exit(err.Error(), 1)
+ }
+ return cli.Exit("", 1)
+ }
+
+ return action(ctx)
+ }
+}
+
+// ArgCountWrapper will cause the action to fail if there were more than `num` args provided.
+func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc {
+ return func(ctx *cli.Context) error {
+ if ctx.NArg() > max {
+ if err := cli.ShowSubcommandHelp(ctx); err != nil {
+ return cli.Exit(err.Error(), 1)
+ }
+ return cli.Exit("", 1)
+ }
+
+ return action(ctx)
+ }
+}
diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go
new file mode 100644
index 00000000000..413ad66efa1
--- /dev/null
+++ b/pkg/build/cmd/flags.go
@@ -0,0 +1,59 @@
+package main
+
+import "github.com/urfave/cli/v2"
+
+var (
+ jobsFlag = cli.IntFlag{
+ Name: "jobs",
+ Usage: "Number of parallel jobs",
+ }
+ buildIDFlag = cli.StringFlag{
+ Name: "build-id",
+ Usage: "Optionally supply a build ID to be part of the version",
+ }
+ editionFlag = cli.StringFlag{
+ Name: "edition",
+ Usage: "The edition of Grafana to build (oss or enterprise)",
+ Value: "oss",
+ }
+ variantsFlag = cli.StringFlag{
+ Name: "variants",
+ Usage: "Comma-separated list of variants to build",
+ }
+ triesFlag = cli.IntFlag{
+ Name: "tries",
+ Usage: "Specify number of tries before failing",
+ Value: 1,
+ }
+ noInstallDepsFlag = cli.BoolFlag{
+ Name: "no-install-deps",
+ Usage: "Don't install dependencies",
+ }
+ signingAdminFlag = cli.BoolFlag{
+ Name: "signing-admin",
+ Usage: "Use manifest signing admin API endpoint?",
+ }
+ signFlag = cli.BoolFlag{
+ Name: "sign",
+ Usage: "Enable plug-in signing (you must set GRAFANA_API_KEY)",
+ }
+ dryRunFlag = cli.BoolFlag{
+ Name: "dry-run",
+ Usage: "Only simulate actions",
+ }
+ gcpKeyFlag = cli.StringFlag{
+ Name: "gcp-key",
+ Usage: "Google Cloud Platform key file",
+ Required: true,
+ }
+ gitHubTokenFlag = cli.StringFlag{
+ Name: "github-token",
+ Value: "",
+ EnvVars: []string{"GITHUB_TOKEN"},
+ Usage: "GitHub token",
+ }
+ tagFlag = cli.StringFlag{
+ Name: "tag",
+ Usage: "Grafana version tag",
+ }
+)
diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go
new file mode 100644
index 00000000000..43f5bc9eb0f
--- /dev/null
+++ b/pkg/build/cmd/grafanacom.go
@@ -0,0 +1,323 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "net/url"
+ "os"
+
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/urfave/cli/v2"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/gcloud"
+ "github.com/grafana/grafana/pkg/build/gcloud/storage"
+ "github.com/grafana/grafana/pkg/build/packaging"
+)
+
+const grafanaAPI = "https://grafana.com/api"
+
+// GrafanaCom implements the sub-command "grafana-com".
+func GrafanaCom(c *cli.Context) error {
+ bucketStr := c.String("src-bucket")
+ edition := config.Edition(c.String("edition"))
+
+ if err := gcloud.ActivateServiceAccount(); err != nil {
+ return fmt.Errorf("couldn't activate service account, err: %w", err)
+ }
+
+ metadata, err := config.GenerateMetadata(c)
+ if err != nil {
+ return err
+ }
+
+ releaseMode, err := metadata.GetReleaseMode()
+ if err != nil {
+ return err
+ }
+
+ version := metadata.GrafanaVersion
+ if releaseMode.Mode == config.Cronjob {
+ gcs, err := storage.New()
+ if err != nil {
+ return err
+ }
+ bucket := gcs.Bucket(bucketStr)
+ latestMainVersion, err := storage.GetLatestMainBuild(c.Context, bucket, filepath.Join(string(edition), "main"))
+ if err != nil {
+ return err
+ }
+ version = latestMainVersion
+ }
+
+ dryRun := c.Bool("dry-run")
+ simulateRelease := c.Bool("simulate-release")
+ // Test release mode and dryRun imply simulateRelease
+ if releaseMode.IsTest || dryRun {
+ simulateRelease = true
+ }
+
+ grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY"))
+ if grafanaAPIKey == "" {
+ return cli.Exit("the environment variable GRAFANA_COM_API_KEY must be set", 1)
+ }
+ whatsNewURL, releaseNotesURL, err := getReleaseURLs()
+ if err != nil {
+ return cli.Exit(err.Error(), 1)
+ }
+
+ // TODO: Verify config values
+ cfg := packaging.PublishConfig{
+ Config: config.Config{
+ Version: version,
+ },
+ Edition: edition,
+ ReleaseMode: releaseMode,
+ GrafanaAPIKey: grafanaAPIKey,
+ WhatsNewURL: whatsNewURL,
+ ReleaseNotesURL: releaseNotesURL,
+ DryRun: dryRun,
+ TTL: c.String("ttl"),
+ SimulateRelease: simulateRelease,
+ }
+
+ if err := publishPackages(cfg); err != nil {
+ return cli.Exit(err.Error(), 1)
+ }
+
+ log.Println("Successfully published packages to grafana.com!")
+ return nil
+}
+
+func getReleaseURLs() (string, string, error) {
+ type grafanaConf struct {
+ WhatsNewURL string `json:"whatsNewUrl"`
+ ReleaseNotesURL string `json:"releaseNotesUrl"`
+ }
+ type packageConf struct {
+ Grafana grafanaConf `json:"grafana"`
+ }
+
+ pkgB, err := os.ReadFile("package.json")
+ if err != nil {
+ return "", "", fmt.Errorf("failed to read package.json: %w", err)
+ }
+
+ var pconf packageConf
+ if err := json.Unmarshal(pkgB, &pconf); err != nil {
+ return "", "", fmt.Errorf("failed to decode package.json: %w", err)
+ }
+ if _, err := url.ParseRequestURI(pconf.Grafana.WhatsNewURL); err != nil {
+ return "", "", fmt.Errorf("grafana.whatsNewUrl is invalid in package.json: %q", pconf.Grafana.WhatsNewURL)
+ }
+ if _, err := url.ParseRequestURI(pconf.Grafana.ReleaseNotesURL); err != nil {
+ return "", "", fmt.Errorf("grafana.releaseNotesUrl is invalid in package.json: %q",
+ pconf.Grafana.ReleaseNotesURL)
+ }
+
+ return pconf.Grafana.WhatsNewURL, pconf.Grafana.ReleaseNotesURL, nil
+}
+
+// publishPackages publishes packages to grafana.com.
+func publishPackages(cfg packaging.PublishConfig) error {
+ log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n",
+ cfg.Version, cfg.Edition, cfg.ReleaseMode.Mode, cfg.DryRun, cfg.SimulateRelease)
+
+ versionStr := fmt.Sprintf("v%s", cfg.Version)
+ log.Printf("Creating release %s at grafana.com...\n", versionStr)
+
+ var sfx string
+ var pth string
+ switch cfg.Edition {
+ case config.EditionOSS:
+ pth = "oss"
+ case config.EditionEnterprise:
+ pth = "enterprise"
+ sfx = packaging.EnterpriseSfx
+ default:
+ return fmt.Errorf("unrecognized edition %q", cfg.Edition)
+ }
+
+ switch cfg.ReleaseMode.Mode {
+ case config.MainMode, config.DownstreamMode, config.CronjobMode:
+ pth = path.Join(pth, packaging.MainFolder)
+ default:
+ pth = path.Join(pth, packaging.ReleaseFolder)
+ }
+
+ product := fmt.Sprintf("grafana%s", sfx)
+ pth = path.Join(pth, product)
+ baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth)
+
+ var builds []buildRepr
+ for _, ba := range packaging.ArtifactConfigs {
+ u := ba.GetURL(baseArchiveURL, cfg)
+
+ sha256, err := getSHA256(u)
+ if err != nil {
+ return err
+ }
+
+ builds = append(builds, buildRepr{
+ OS: ba.Os,
+ URL: u,
+ SHA256: string(sha256),
+ Arch: ba.Arch,
+ })
+ }
+
+ r := releaseRepr{
+ Version: cfg.Version,
+ ReleaseDate: time.Now().UTC(),
+ Builds: builds,
+ Stable: cfg.ReleaseMode.Mode == config.TagMode && !cfg.ReleaseMode.IsBeta && !cfg.ReleaseMode.IsTest,
+ Beta: cfg.ReleaseMode.IsBeta,
+ Nightly: cfg.ReleaseMode.Mode == config.CronjobMode,
+ }
+ if cfg.ReleaseMode.Mode == config.TagMode || r.Beta {
+ r.WhatsNewURL = cfg.WhatsNewURL
+ r.ReleaseNotesURL = cfg.ReleaseNotesURL
+ }
+
+ if err := postRequest(cfg, "versions", r, fmt.Sprintf("create release %s", r.Version)); err != nil {
+ return err
+ }
+
+ if err := postRequest(cfg, fmt.Sprintf("versions/%s", cfg.Version), r,
+ fmt.Sprintf("update release %s", cfg.Version)); err != nil {
+ return err
+ }
+
+ for _, b := range r.Builds {
+ if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages", cfg.Version), b,
+ fmt.Sprintf("create build %s %s", b.OS, b.Arch)); err != nil {
+ return err
+ }
+ if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages/%s/%s", cfg.Version, b.Arch, b.OS), b,
+ fmt.Sprintf("update build %s %s", b.OS, b.Arch)); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func getSHA256(u string) ([]byte, error) {
+ shaURL := fmt.Sprintf("%s.sha256", u)
+ // nolint:gosec
+ resp, err := http.Get(shaURL)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if err := resp.Body.Close(); err != nil {
+ log.Println("failed to close response body, err: %w", err)
+ }
+ }()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("failed downloading %s: %s", u, resp.Status)
+ }
+
+ sha256, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ return sha256, nil
+}
+
+func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error {
+ var sfx string
+ switch cfg.Edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = packaging.EnterpriseSfx
+ default:
+ return fmt.Errorf("unrecognized edition %q", cfg.Edition)
+ }
+ product := fmt.Sprintf("grafana%s", sfx)
+
+ jsonB, err := json.Marshal(obj)
+ if err != nil {
+ return fmt.Errorf("failed to JSON encode release: %w", err)
+ }
+
+ u, err := constructURL(product, pth)
+ if err != nil {
+ return err
+ }
+ req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(jsonB))
+ if err != nil {
+ return err
+ }
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cfg.GrafanaAPIKey))
+ req.Header.Add("Content-Type", "application/json")
+
+ log.Printf("Posting to grafana.com API, %s - JSON: %s\n", u, string(jsonB))
+ if cfg.SimulateRelease {
+ log.Println("Only simulating request")
+ return nil
+ }
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed posting to %s (%s): %s", u, descr, err)
+ }
+ defer func() {
+ if err := resp.Body.Close(); err != nil {
+ log.Println("failed to close response body, err: %w", err)
+ }
+ }()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return err
+ }
+
+ if strings.Contains(string(body), "already exists") || strings.Contains(string(body), "Nothing to update") {
+ log.Printf("Already exists: %s\n", descr)
+ return nil
+ }
+
+ return fmt.Errorf("failed posting to %s (%s): %s", u, descr, resp.Status)
+ }
+
+ log.Printf("Successfully posted to grafana.com API, %s\n", u)
+
+ return nil
+}
+
+func constructURL(product string, pth string) (string, error) {
+ productPath := filepath.Clean(filepath.Join("/", product, pth))
+ u, err := url.Parse(grafanaAPI)
+ if err != nil {
+ return "", err
+ }
+ u.Path = path.Join(u.Path, productPath)
+ return u.String(), err
+}
+
+type buildRepr struct {
+ OS string `json:"os"`
+ URL string `json:"url"`
+ SHA256 string `json:"sha256"`
+ Arch string `json:"arch"`
+}
+
+type releaseRepr struct {
+ Version string `json:"version"`
+ ReleaseDate time.Time `json:"releaseDate"`
+ Stable bool `json:"stable"`
+ Beta bool `json:"beta"`
+ Nightly bool `json:"nightly"`
+ WhatsNewURL string `json:"whatsNewUrl"`
+ ReleaseNotesURL string `json:"releaseNotesUrl"`
+ Builds []buildRepr `json:"-"`
+}
diff --git a/pkg/build/cmd/grafanacom_test.go b/pkg/build/cmd/grafanacom_test.go
new file mode 100644
index 00000000000..bf90874b1c9
--- /dev/null
+++ b/pkg/build/cmd/grafanacom_test.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+ "testing"
+)
+
+func Test_constructURL(t *testing.T) {
+ type args struct {
+ product string
+ pth string
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ wantErr bool
+ }{
+ {name: "cleans .. sequence", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false},
+ {name: "doesn't clean anything - non malicious url", args: args{"foo", "bar"}, want: "https://grafana.com/api/foo/bar", wantErr: false},
+ {name: "doesn't clean anything - three dots", args: args{"...", "..."}, want: "https://grafana.com/api/.../...", wantErr: false},
+ {name: "cleans .", args: args{"..", ".."}, want: "https://grafana.com/api", wantErr: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := constructURL(tt.args.product, tt.args.pth)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("constructURL() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("constructURL() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go
new file mode 100644
index 00000000000..23f16ed0ea2
--- /dev/null
+++ b/pkg/build/cmd/main.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+ "log"
+ "os"
+
+ "github.com/urfave/cli/v2"
+)
+
+var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5)
+
+//nolint:unused
+func registerAppCommand(c *cli.Command) {
+ additionalCommands = append(additionalCommands, c)
+}
+
+func main() {
+ app := cli.NewApp()
+ app.Commands = cli.Commands{
+ {
+ Name: "publish",
+ Usage: "Publish packages to Grafana com and repositories",
+ Subcommands: cli.Commands{
+ {
+ Name: "grafana-com",
+ Usage: "Publish packages to grafana.com",
+ Action: GrafanaCom,
+ Flags: []cli.Flag{
+ &editionFlag,
+ &buildIDFlag,
+ &dryRunFlag,
+ &cli.StringFlag{
+ Name: "src-bucket",
+ Value: "grafana-downloads",
+ Usage: "Google Cloud Storage bucket",
+ },
+ },
+ },
+ },
+ },
+ }
+
+ app.Commands = append(app.Commands, additionalCommands...)
+
+ if err := app.Run(os.Args); err != nil {
+ log.Fatalln(err)
+ }
+}
diff --git a/pkg/build/compilers/install.go b/pkg/build/compilers/install.go
new file mode 100644
index 00000000000..9a45ba48c0b
--- /dev/null
+++ b/pkg/build/compilers/install.go
@@ -0,0 +1,50 @@
+package compilers
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+)
+
+const (
+ ArmV6 = "/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc"
+ Armv7 = "arm-linux-gnueabihf-gcc"
+ Armv7Musl = "/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc"
+ Arm64 = "aarch64-linux-gnu-gcc"
+ Arm64Musl = "/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc"
+ Osx64 = "/tmp/osxcross/target/bin/o64-clang"
+ Win64 = "x86_64-w64-mingw32-gcc"
+ LinuxX64 = "/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc"
+ LinuxX64Musl = "/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc"
+)
+
+func Install() error {
+ // From the os.TempDir documentation:
+ // On Unix systems, it returns $TMPDIR if non-empty,
+ // else /tmp. On Windows, it uses GetTempPath,
+ // returning the first non-empty value from %TMP%, %TEMP%, %USERPROFILE%,
+ // or the Windows directory. On Plan 9, it returns /tmp.
+ tmp := os.TempDir()
+
+ var (
+ centosArchive = "x86_64-centos6-linux-gnu.tar.xz"
+ osxArchive = "osxcross.tar.xz"
+ )
+
+ for _, fname := range []string{centosArchive, osxArchive} {
+ path := filepath.Join(tmp, fname)
+ if _, err := os.Stat(path); err != nil {
+ return fmt.Errorf("stat error: %w", err)
+ }
+ // Ignore gosec G204 as this function is only used in the build process.
+ //nolint:gosec
+ cmd := exec.Command("tar", "xfJ", fname)
+ cmd.Dir = tmp
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to unpack %q: %q, %w", fname, output, err)
+ }
+ }
+
+ return nil
+}
diff --git a/pkg/build/config/config.go b/pkg/build/config/config.go
new file mode 100644
index 00000000000..7aa4176e13a
--- /dev/null
+++ b/pkg/build/config/config.go
@@ -0,0 +1,18 @@
+package config
+
+type Config struct {
+ Version string
+ Bucket string
+ DebRepoBucket string
+ DebDBBucket string
+ RPMRepoBucket string
+ GPGPassPath string
+ GPGPrivateKey string
+ GPGPublicKey string
+ NumWorkers int
+ GitHubUser string
+ GitHubToken string
+ PullEnterprise bool
+ PackageVersion string
+ SignPackages bool
+}
diff --git a/pkg/build/config/edition.go b/pkg/build/config/edition.go
new file mode 100644
index 00000000000..48d7375f1cd
--- /dev/null
+++ b/pkg/build/config/edition.go
@@ -0,0 +1,9 @@
+package config
+
+type Edition string
+
+const (
+ EditionOSS Edition = "oss"
+ EditionEnterprise Edition = "enterprise"
+ EditionEnterprise2 Edition = "enterprise2"
+)
diff --git a/pkg/build/config/genmetadata.go b/pkg/build/config/genmetadata.go
new file mode 100644
index 00000000000..c201d77dd39
--- /dev/null
+++ b/pkg/build/config/genmetadata.go
@@ -0,0 +1,102 @@
+package config
+
+import (
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/grafana/grafana/pkg/build/droneutil"
+ "github.com/urfave/cli/v2"
+)
+
+func GenerateMetadata(c *cli.Context) (Metadata, error) {
+ var metadata Metadata
+ version := ""
+
+ event, err := droneutil.GetDroneEventFromEnv()
+ if err != nil {
+ return Metadata{}, err
+ }
+
+ tag, ok := os.LookupEnv("DRONE_TAG")
+ if !ok {
+ fmt.Println("DRONE_TAG envvar not present, %w", err)
+ }
+
+ var releaseMode ReleaseMode
+ switch event {
+ case string(PullRequestMode):
+ releaseMode = ReleaseMode{Mode: PullRequestMode}
+ case Push:
+ mode, err := CheckDroneTargetBranch()
+ if err != nil {
+ return Metadata{}, err
+ }
+ releaseMode = ReleaseMode{Mode: mode}
+ case Custom:
+ if edition, _ := os.LookupEnv("EDITION"); edition == string(EditionEnterprise2) {
+ releaseMode = ReleaseMode{Mode: Enterprise2Mode}
+ if tag != "" {
+ version = strings.TrimPrefix(tag, "v")
+ }
+ break
+ }
+ mode, err := CheckDroneTargetBranch()
+ if err != nil {
+ return Metadata{}, err
+ }
+ // if there is a custom event targeting the main branch, that's an enterprise downstream build
+ if mode == MainBranch {
+ releaseMode = ReleaseMode{Mode: DownstreamMode}
+ } else {
+ releaseMode = ReleaseMode{Mode: mode}
+ }
+ case Tag, Promote:
+ if tag == "" {
+ return Metadata{}, fmt.Errorf("DRONE_TAG envvar not present for a tag/promotion event, %w", err)
+ }
+ version = strings.TrimPrefix(tag, "v")
+ mode, err := CheckSemverSuffix()
+ if err != nil {
+ return Metadata{}, err
+ }
+ releaseMode = mode
+ case Cronjob:
+ releaseMode = ReleaseMode{Mode: CronjobMode}
+ }
+
+ if version == "" {
+ version, err = generateVersionFromBuildID()
+ if err != nil {
+ return Metadata{}, err
+ }
+ }
+
+ currentCommit, err := GetDroneCommit()
+ if err != nil {
+ return Metadata{}, err
+ }
+ metadata = Metadata{
+ GrafanaVersion: version,
+ ReleaseMode: releaseMode,
+ GrabplVersion: c.App.Version,
+ CurrentCommit: currentCommit,
+ }
+
+ fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode)
+
+ return metadata, nil
+}
+
+func generateVersionFromBuildID() (string, error) {
+ buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER")
+ if !ok {
+ return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable")
+ }
+ var err error
+ version, err := GetGrafanaVersion(buildID, ".")
+ if err != nil {
+ return "", err
+ }
+ return version, nil
+}
diff --git a/pkg/build/config/genmetadata_test.go b/pkg/build/config/genmetadata_test.go
new file mode 100644
index 00000000000..605b68496c7
--- /dev/null
+++ b/pkg/build/config/genmetadata_test.go
@@ -0,0 +1,81 @@
+package config
+
+import (
+ "flag"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "github.com/urfave/cli/v2"
+)
+
+const (
+ DroneBuildEvent = "DRONE_BUILD_EVENT"
+ DroneTargetBranch = "DRONE_TARGET_BRANCH"
+ DroneTag = "DRONE_TAG"
+ DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE"
+ DroneBuildNumber = "DRONE_BUILD_NUMBER"
+)
+
+const (
+ hashedGrafanaVersion = "9.2.0-12345pre"
+ versionedBranch = "v9.2.x"
+)
+
+func TestGetMetadata(t *testing.T) {
+ tcs := []struct {
+ envMap map[string]string
+ expVersion string
+ mode ReleaseMode
+ }{
+ {map[string]string{DroneBuildEvent: PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: PullRequestMode}},
+ {map[string]string{DroneBuildEvent: Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}},
+ {map[string]string{DroneBuildEvent: Push, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: MainMode}},
+ {map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}},
+ {map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: DownstreamMode}},
+ {map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345", "EDITION": string(EditionEnterprise2)}, hashedGrafanaVersion, ReleaseMode{Mode: Enterprise2Mode}},
+ {map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}},
+ {map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}},
+ {map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}},
+ {map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}},
+ {map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}},
+ {map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}},
+ }
+
+ ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil)
+ for _, tc := range tcs {
+ t.Run("Should return valid metadata, ", func(t *testing.T) {
+ setUpEnv(t, tc.envMap)
+ testMetadata(t, ctx, tc.expVersion, tc.mode)
+ })
+ }
+}
+
+func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode ReleaseMode) {
+ t.Helper()
+
+ metadata, err := GenerateMetadata(ctx)
+ require.NoError(t, err)
+ t.Run("with a valid version", func(t *testing.T) {
+ expVersion := metadata.GrafanaVersion
+ require.Equal(t, expVersion, version)
+ })
+
+ t.Run("with a valid release mode from the built-in list", func(t *testing.T) {
+ expMode := metadata.ReleaseMode
+ require.NoError(t, err)
+ require.Equal(t, expMode, releaseMode)
+ })
+}
+
+func setUpEnv(t *testing.T, envMap map[string]string) {
+ t.Helper()
+
+ os.Clearenv()
+ err := os.Setenv("DRONE_COMMIT", "abcd12345")
+ require.NoError(t, err)
+ for k, v := range envMap {
+ err := os.Setenv(k, v)
+ require.NoError(t, err)
+ }
+}
diff --git a/pkg/build/config/package.json b/pkg/build/config/package.json
new file mode 100644
index 00000000000..1a5184b05c7
--- /dev/null
+++ b/pkg/build/config/package.json
@@ -0,0 +1,3 @@
+{
+ "version": "9.2.0-pre"
+}
diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go
new file mode 100644
index 00000000000..aa104916652
--- /dev/null
+++ b/pkg/build/config/revision.go
@@ -0,0 +1,55 @@
+package config
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+ "time"
+
+ "github.com/grafana/grafana/pkg/build/executil"
+)
+
+type Revision struct {
+ Timestamp int64
+ SHA256 string
+ Branch string
+}
+
+func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) {
+ out, err := executil.OutputAt(ctx, dir, "git", "show", "-s", "--format=%ct")
+ if err != nil {
+ return time.Now().Unix(), nil
+ }
+
+ stamp, err := strconv.ParseInt(out, 10, 64)
+ if err != nil {
+ return 0, fmt.Errorf("failed to parse output from git show: %q", out)
+ }
+
+ return stamp, nil
+}
+
+// GrafanaRevision uses git commands to get information about the checked out Grafana code located at 'grafanaDir'.
+// This could maybe be a more generic "Describe" function in the "git" package.
+func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) {
+ stamp, err := GrafanaTimestamp(ctx, grafanaDir)
+ if err != nil {
+ return Revision{}, err
+ }
+
+ sha, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--short", "HEAD")
+ if err != nil {
+ return Revision{}, err
+ }
+
+ branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD")
+ if err != nil {
+ return Revision{}, err
+ }
+
+ return Revision{
+ SHA256: sha,
+ Branch: branch,
+ Timestamp: stamp,
+ }, nil
+}
diff --git a/pkg/build/config/variant.go b/pkg/build/config/variant.go
new file mode 100644
index 00000000000..4fde6a89a74
--- /dev/null
+++ b/pkg/build/config/variant.go
@@ -0,0 +1,63 @@
+package config
+
+// Variant is the OS / Architecture combination that Grafana can be compiled for.
+type Variant string
+
+const (
+ VariantLinuxAmd64 Variant = "linux-amd64"
+ VariantLinuxAmd64Musl Variant = "linux-amd64-musl"
+ VariantArmV6 Variant = "linux-armv6"
+ VariantArmV7 Variant = "linux-armv7"
+ VariantArmV7Musl Variant = "linux-armv7-musl"
+ VariantArm64 Variant = "linux-arm64"
+ VariantArm64Musl Variant = "linux-arm64-musl"
+ VariantDarwinAmd64 Variant = "darwin-amd64"
+ VariantWindowsAmd64 Variant = "windows-amd64"
+)
+
+var AllVariants = []Variant{
+ VariantArmV6,
+ VariantArmV7,
+ VariantArmV7Musl,
+ VariantArm64,
+ VariantArm64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+}
+
+// Architecture is an allowed value in the GOARCH environment variable.
+type Architecture string
+
+const (
+ ArchAMD64 Architecture = "amd64"
+ ArchARMv6 Architecture = "armv6"
+ ArchARMv7 Architecture = "armv7"
+ ArchARM64 Architecture = "arm64"
+ ArchARMHF Architecture = "armhf"
+ ArchARMHFP Architecture = "armhfp"
+ ArchARM Architecture = "arm"
+)
+
+type OS string
+
+const (
+ OSWindows OS = "windows"
+ OSDarwin OS = "darwin"
+ OSLinux OS = "linux"
+)
+
+type LibC string
+
+const (
+ LibCMusl = "musl"
+)
+
+// Distribution is the base os image where the Grafana image is built on.
+type Distribution string
+
+const (
+ Ubuntu Distribution = "ubuntu"
+ Alpine Distribution = "alpine"
+)
diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go
new file mode 100644
index 00000000000..aaeeaf56652
--- /dev/null
+++ b/pkg/build/config/version.go
@@ -0,0 +1,157 @@
+package config
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/grafana/grafana/pkg/build/git"
+)
+
+type Metadata struct {
+ GrafanaVersion string `json:"version,omitempty"`
+ ReleaseMode ReleaseMode `json:"releaseMode,omitempty"`
+ GrabplVersion string `json:"grabplVersion,omitempty"`
+ CurrentCommit string `json:"currentCommit,omitempty"`
+}
+
+type ReleaseMode struct {
+ Mode VersionMode `json:"mode,omitempty"`
+ IsBeta bool `json:"isBeta,omitempty"`
+ IsTest bool `json:"isTest,omitempty"`
+}
+
+type PluginSignature struct {
+ Sign bool `json:"sign,omitempty"`
+ AdminSign bool `json:"adminSign,omitempty"`
+}
+
+type Docker struct {
+ ShouldSave bool `json:"shouldSave,omitempty"`
+ Distribution []Distribution `json:"distribution,omitempty"`
+ Architectures []Architecture `json:"archs,omitempty"`
+ PrereleaseBucket string `json:"prereleaseBucket,omitempty"`
+}
+
+type Buckets struct {
+ Artifacts string `json:"artifacts,omitempty"`
+ ArtifactsEnterprise2 string `json:"artifactsEnterprise2,omitempty"`
+ CDNAssets string `json:"CDNAssets,omitempty"`
+ CDNAssetsDir string `json:"CDNAssetsDir,omitempty"`
+ Storybook string `json:"storybook,omitempty"`
+ StorybookSrcDir string `json:"storybookSrcDir,omitempty"`
+}
+
+// BuildConfig represents the struct that defines all of the different variables used to build Grafana
+type BuildConfig struct {
+ Variants []Variant `json:"variants,omitempty"`
+ PluginSignature PluginSignature `json:"pluginSignature,omitempty"`
+ Docker Docker `json:"docker,omitempty"`
+ Buckets Buckets `json:"buckets,omitempty"`
+}
+
+func (md *Metadata) GetReleaseMode() (ReleaseMode, error) {
+ return md.ReleaseMode, nil
+}
+
+// VersionMap is a map of versions. Each key of the Versions map is an event that uses the the config as the value for that key.
+// For example, the 'pull_request' key will have data in it that might cause Grafana to be built differently in a pull request,
+// than the way it will be built in 'main'
+type VersionMap map[VersionMode]BuildConfig
+
+// GetBuildConfig reads the embedded config.json and decodes it.
+func GetBuildConfig(mode VersionMode) (*BuildConfig, error) {
+ if v, ok := Versions[mode]; ok {
+ return &v, nil
+ }
+
+ return nil, fmt.Errorf("mode '%s' not found in version list", mode)
+}
+
+// GetGrafanaVersion gets the Grafana version from the package.json
+func GetGrafanaVersion(buildID, grafanaDir string) (string, error) {
+ pkgJSONPath := filepath.Join(grafanaDir, "package.json")
+ //nolint:gosec
+ pkgJSONB, err := os.ReadFile(pkgJSONPath)
+ if err != nil {
+ return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err)
+ }
+ pkgObj := map[string]interface{}{}
+ if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil {
+ return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err)
+ }
+
+ version := pkgObj["version"].(string)
+ if version == "" {
+ return "", fmt.Errorf("failed to read version from %q", pkgJSONPath)
+ }
+ if buildID != "" {
+ buildID = shortenBuildID(buildID)
+ verComponents := strings.Split(version, "-")
+ version = verComponents[0]
+ if len(verComponents) > 1 {
+ buildID = fmt.Sprintf("%s%s", buildID, verComponents[1])
+ }
+ version = fmt.Sprintf("%s-%s", version, buildID)
+ }
+
+ return version, nil
+}
+
+func CheckDroneTargetBranch() (VersionMode, error) {
+ rePRCheckBranch := git.PRCheckRegexp()
+ reRlsBranch := regexp.MustCompile(`^v\d+\.\d+\.x$`)
+ target := os.Getenv("DRONE_TARGET_BRANCH")
+ if target == "" {
+ return "", fmt.Errorf("failed to get DRONE_TARGET_BRANCH environmental variable")
+ } else if target == string(MainMode) {
+ return MainMode, nil
+ }
+ if reRlsBranch.MatchString(target) {
+ return ReleaseBranchMode, nil
+ }
+ if rePRCheckBranch.MatchString(target) {
+ return PullRequestMode, nil
+ }
+ fmt.Printf("unrecognized target branch: %s, defaulting to %s", target, PullRequestMode)
+ return PullRequestMode, nil
+}
+
+func CheckSemverSuffix() (ReleaseMode, error) {
+ reBetaRls := regexp.MustCompile(`beta.*`)
+ reTestRls := regexp.MustCompile(`test.*`)
+ tagSuffix, ok := os.LookupEnv("DRONE_SEMVER_PRERELEASE")
+ if !ok || tagSuffix == "" {
+ fmt.Println("DRONE_SEMVER_PRERELEASE doesn't exist for a tag, this is a release event...")
+ return ReleaseMode{Mode: TagMode}, nil
+ }
+ switch {
+ case reBetaRls.MatchString(tagSuffix):
+ return ReleaseMode{Mode: TagMode, IsBeta: true}, nil
+ case reTestRls.MatchString(tagSuffix):
+ return ReleaseMode{Mode: TagMode, IsTest: true}, nil
+ default:
+ fmt.Printf("DRONE_SEMVER_PRERELEASE is custom string, release event with %s suffix\n", tagSuffix)
+ return ReleaseMode{Mode: TagMode}, nil
+ }
+}
+
+func GetDroneCommit() (string, error) {
+ commit := strings.TrimSpace(os.Getenv("DRONE_COMMIT"))
+ if commit == "" {
+ return "", fmt.Errorf("the environment variable DRONE_COMMIT is missing")
+ }
+ return commit, nil
+}
+
+func shortenBuildID(buildID string) string {
+ buildID = strings.ReplaceAll(buildID, "-", "")
+ if len(buildID) < 9 {
+ return buildID
+ }
+
+ return buildID[0:8]
+}
diff --git a/pkg/build/config/version_mode.go b/pkg/build/config/version_mode.go
new file mode 100644
index 00000000000..a74b5b5f49e
--- /dev/null
+++ b/pkg/build/config/version_mode.go
@@ -0,0 +1,27 @@
+package config
+
+// VersionMode defines the source event that created a release or published version
+type VersionMode string
+
+const (
+ MainMode VersionMode = "main"
+ TagMode VersionMode = "release"
+ ReleaseBranchMode VersionMode = "branch"
+ PullRequestMode VersionMode = "pull_request"
+ DownstreamMode VersionMode = "downstream"
+ Enterprise2Mode VersionMode = "enterprise2"
+ CronjobMode VersionMode = "cron"
+)
+
+const (
+ Tag = "tag"
+ PullRequest = "pull_request"
+ Push = "push"
+ Custom = "custom"
+ Promote = "promote"
+ Cronjob = "cron"
+)
+
+const (
+ MainBranch = "main"
+)
diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go
new file mode 100644
index 00000000000..874cedef5f5
--- /dev/null
+++ b/pkg/build/config/versions.go
@@ -0,0 +1,206 @@
+package config
+
+const PublicBucket = "grafana-downloads"
+
+var Versions = VersionMap{
+ PullRequestMode: {
+ Variants: []Variant{
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ },
+ PluginSignature: PluginSignature{
+ Sign: false,
+ AdminSign: false,
+ },
+ Docker: Docker{
+ ShouldSave: false,
+ Architectures: []Architecture{
+ ArchAMD64,
+ },
+ Distribution: []Distribution{
+ Alpine,
+ },
+ },
+ },
+ MainMode: {
+ Variants: []Variant{
+ VariantArmV6,
+ VariantArmV7,
+ VariantArmV7Musl,
+ VariantArm64,
+ VariantArm64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+ },
+ PluginSignature: PluginSignature{
+ Sign: true,
+ AdminSign: true,
+ },
+ Docker: Docker{
+ ShouldSave: false,
+ Architectures: []Architecture{
+ ArchAMD64,
+ ArchARM64,
+ ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
+ },
+ Distribution: []Distribution{
+ Alpine,
+ Ubuntu,
+ },
+ },
+ Buckets: Buckets{
+ Artifacts: "grafana-downloads",
+ ArtifactsEnterprise2: "grafana-downloads-enterprise2",
+ CDNAssets: "grafana-static-assets",
+ Storybook: "grafana-storybook",
+ },
+ },
+ DownstreamMode: {
+ Variants: []Variant{
+ VariantArmV6,
+ VariantArmV7,
+ VariantArmV7Musl,
+ VariantArm64,
+ VariantArm64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+ },
+ PluginSignature: PluginSignature{
+ Sign: true,
+ AdminSign: true,
+ },
+ Docker: Docker{
+ ShouldSave: true,
+ Architectures: []Architecture{
+ ArchAMD64,
+ ArchARM64,
+ ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
+ },
+ Distribution: []Distribution{
+ Alpine,
+ Ubuntu,
+ },
+ },
+ Buckets: Buckets{
+ Artifacts: "grafana-downloads",
+ ArtifactsEnterprise2: "grafana-downloads-enterprise2",
+ CDNAssets: "grafana-static-assets",
+ },
+ },
+ ReleaseBranchMode: {
+ Variants: []Variant{
+ VariantArmV6,
+ VariantArmV7,
+ VariantArmV7Musl,
+ VariantArm64,
+ VariantArm64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+ },
+ PluginSignature: PluginSignature{
+ Sign: true,
+ AdminSign: true,
+ },
+ Docker: Docker{
+ ShouldSave: true,
+ Architectures: []Architecture{
+ ArchAMD64,
+ ArchARM64,
+ ArchARMv7,
+ },
+ Distribution: []Distribution{
+ Alpine,
+ Ubuntu,
+ },
+ },
+ Buckets: Buckets{
+ Artifacts: "grafana-downloads",
+ ArtifactsEnterprise2: "grafana-downloads-enterprise2",
+ CDNAssets: "grafana-static-assets",
+ },
+ },
+ TagMode: {
+ Variants: []Variant{
+ VariantArmV6,
+ VariantArmV7,
+ VariantArmV7Musl,
+ VariantArm64,
+ VariantArm64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+ },
+ PluginSignature: PluginSignature{
+ Sign: true,
+ AdminSign: true,
+ },
+ Docker: Docker{
+ ShouldSave: true,
+ Architectures: []Architecture{
+ ArchAMD64,
+ ArchARM64,
+ ArchARMv7,
+ },
+ Distribution: []Distribution{
+ Alpine,
+ Ubuntu,
+ },
+ PrereleaseBucket: "grafana-prerelease/artifacts/docker",
+ },
+ Buckets: Buckets{
+ Artifacts: "grafana-prerelease/artifacts/downloads",
+ ArtifactsEnterprise2: "grafana-prerelease/artifacts/downloads-enterprise2",
+ CDNAssets: "grafana-prerelease",
+ CDNAssetsDir: "artifacts/static-assets",
+ Storybook: "grafana-prerelease",
+ StorybookSrcDir: "artifacts/storybook",
+ },
+ },
+ Enterprise2Mode: {
+ Variants: []Variant{
+ VariantArmV6,
+ VariantArmV7,
+ VariantArmV7Musl,
+ VariantArm64,
+ VariantArm64Musl,
+ VariantDarwinAmd64,
+ VariantWindowsAmd64,
+ VariantLinuxAmd64,
+ VariantLinuxAmd64Musl,
+ },
+ PluginSignature: PluginSignature{
+ Sign: true,
+ AdminSign: true,
+ },
+ Docker: Docker{
+ ShouldSave: true,
+ Architectures: []Architecture{
+ ArchAMD64,
+ ArchARM64,
+ ArchARMv7,
+ },
+ Distribution: []Distribution{
+ Alpine,
+ Ubuntu,
+ },
+ PrereleaseBucket: "grafana-prerelease/artifacts/docker",
+ },
+ Buckets: Buckets{
+ Artifacts: "grafana-prerelease/artifacts/downloads",
+ ArtifactsEnterprise2: "grafana-prerelease/artifacts/downloads-enterprise2",
+ CDNAssets: "grafana-prerelease",
+ CDNAssetsDir: "artifacts/static-assets",
+ Storybook: "grafana-prerelease",
+ StorybookSrcDir: "artifacts/storybook",
+ },
+ },
+}
diff --git a/pkg/build/cryptoutil/md5.go b/pkg/build/cryptoutil/md5.go
new file mode 100644
index 00000000000..fa4b4fcc5ec
--- /dev/null
+++ b/pkg/build/cryptoutil/md5.go
@@ -0,0 +1,35 @@
+package cryptoutil
+
+import (
+ "crypto/md5"
+ "fmt"
+ "io"
+ "log"
+ "os"
+)
+
+func MD5File(fpath string) error {
+ // Ignore gosec G304 as this function is only used in the build process.
+ //nolint:gosec
+ fd, err := os.Open(fpath)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if err := fd.Close(); err != nil {
+ log.Printf("error closing file at '%s': %s", fpath, err.Error())
+ }
+ }()
+
+ h := md5.New() // nolint:gosec
+ if _, err = io.Copy(h, fd); err != nil {
+ return err
+ }
+
+ // nolint:gosec
+ if err := os.WriteFile(fpath+".md5", []byte(fmt.Sprintf("%x\n", h.Sum(nil))), 0664); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/pkg/build/droneutil/docs.go b/pkg/build/droneutil/docs.go
new file mode 100644
index 00000000000..8bf3753936e
--- /dev/null
+++ b/pkg/build/droneutil/docs.go
@@ -0,0 +1,2 @@
+// Package droneutil provides utility functions for working with Drone.
+package droneutil
diff --git a/pkg/build/droneutil/event.go b/pkg/build/droneutil/event.go
new file mode 100644
index 00000000000..707b87383c2
--- /dev/null
+++ b/pkg/build/droneutil/event.go
@@ -0,0 +1,34 @@
+package droneutil
+
+import (
+ "fmt"
+ "os"
+ "strings"
+)
+
+// Lookup is the equivalent of os.LookupEnv, but also accepts a list of strings rather than only checking os.Environ()
+func Lookup(values []string, val string) (string, bool) {
+ for _, v := range values {
+ prefix := val + "="
+ if strings.HasPrefix(v, prefix) {
+ return strings.TrimPrefix(v, prefix), true
+ }
+ }
+
+ return "", false
+}
+
+// GetDroneEvent looks for the "DRONE_BUILD_EVENT" in the provided env list and returns the value.
+// if it was not found, then an error is returned.
+func GetDroneEvent(env []string) (string, error) {
+ event, ok := Lookup(env, "DRONE_BUILD_EVENT")
+ if !ok {
+ return "", fmt.Errorf("failed to get DRONE_BUILD_EVENT environmental variable")
+ }
+ return event, nil
+}
+
+// GetDroneEventFromEnv returns the value of DRONE_BUILD_EVENT from os.Environ()
+func GetDroneEventFromEnv() (string, error) {
+ return GetDroneEvent(os.Environ())
+}
diff --git a/pkg/build/droneutil/event_test.go b/pkg/build/droneutil/event_test.go
new file mode 100644
index 00000000000..f0efcfa284e
--- /dev/null
+++ b/pkg/build/droneutil/event_test.go
@@ -0,0 +1,36 @@
+package droneutil_test
+
+import (
+ "testing"
+
+ "github.com/grafana/grafana/pkg/build/droneutil"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetDroneEvent(t *testing.T) {
+ t.Run("Should return the Drone Event", func(t *testing.T) {
+ env := []string{"DRONE_BUILD_EVENT=pull_request"}
+ droneEvent, err := droneutil.GetDroneEvent(env)
+ require.NoError(t, err)
+ require.Equal(t, droneEvent, "pull_request")
+ })
+ t.Run("Should return error, Drone Event env var is missing", func(t *testing.T) {
+ droneEvent, err := droneutil.GetDroneEvent([]string{})
+ require.Error(t, err)
+ require.Empty(t, droneEvent)
+ })
+}
+
+func TestLookup(t *testing.T) {
+ env := []string{"", "EXAMPLE_KEY=value", "EXAMPLE_KEY"}
+ t.Run("A valid lookup should return a string and no error", func(t *testing.T) {
+ val, ok := droneutil.Lookup(env, "EXAMPLE_KEY")
+ require.True(t, ok)
+ require.Equal(t, val, "value")
+ })
+
+ t.Run("An invalid lookup should return an error", func(t *testing.T) {
+ _, ok := droneutil.Lookup(env, "EXAMPLE_KEY_DOES_NOT_EXIST")
+ require.False(t, ok)
+ })
+}
diff --git a/pkg/build/errutil/group.go b/pkg/build/errutil/group.go
new file mode 100644
index 00000000000..0844616e579
--- /dev/null
+++ b/pkg/build/errutil/group.go
@@ -0,0 +1,61 @@
+package errutil
+
+import (
+ "context"
+ "log"
+ "sync"
+)
+
+type Group struct {
+ cancel func()
+ wg sync.WaitGroup
+ errOnce sync.Once
+ err error
+}
+
+func GroupWithContext(ctx context.Context) (*Group, context.Context) {
+ ctx, cancel := context.WithCancel(ctx)
+ return &Group{cancel: cancel}, ctx
+}
+
+// Wait waits for any wrapped goroutines to finish and returns any error having occurred in one of them.
+func (g *Group) Wait() error {
+ log.Println("Waiting on Group")
+ g.wg.Wait()
+ if g.cancel != nil {
+ log.Println("Group canceling its context after waiting")
+ g.cancel()
+ }
+ return g.err
+}
+
+// Cancel cancels the associated context.
+func (g *Group) Cancel() {
+ log.Println("Group's Cancel method being called")
+ g.cancel()
+}
+
+// Wrap wraps a function to be executed in a goroutine.
+func (g *Group) Wrap(f func() error) func() {
+ g.wg.Add(1)
+ return func() {
+ defer g.wg.Done()
+
+ if err := f(); err != nil {
+ g.errOnce.Do(func() {
+ log.Printf("An error occurred in Group: %s", err)
+ g.err = err
+ if g.cancel != nil {
+ log.Println("Group canceling its context due to error")
+ g.cancel()
+ }
+ })
+ }
+ }
+}
+
+// Go wraps the provided function and executes it in a goroutine.
+func (g *Group) Go(f func() error) {
+ wrapped := g.Wrap(f)
+ go wrapped()
+}
diff --git a/pkg/build/executil/exec.go b/pkg/build/executil/exec.go
new file mode 100644
index 00000000000..e46f568a997
--- /dev/null
+++ b/pkg/build/executil/exec.go
@@ -0,0 +1,46 @@
+package executil
+
+import (
+ "context"
+ "fmt"
+ "os/exec"
+ "strings"
+)
+
+func RunAt(ctx context.Context, dir, cmd string, args ...string) error {
+ // Ignore gosec G204 as this function is only used in the build process.
+ //nolint:gosec
+ c := exec.CommandContext(ctx, cmd, args...)
+ c.Dir = dir
+
+ b, err := c.CombinedOutput()
+
+ if err != nil {
+ return fmt.Errorf("%w. '%s %v': %s", err, cmd, args, string(b))
+ }
+
+ return nil
+}
+
+func Run(ctx context.Context, cmd string, args ...string) error {
+ return RunAt(ctx, ".", cmd, args...)
+}
+
+func OutputAt(ctx context.Context, dir, cmd string, args ...string) (string, error) {
+ // Ignore gosec G204 as this function is only used in the build process.
+ //nolint:gosec
+ c := exec.CommandContext(ctx, cmd, args...)
+ c.Dir = dir
+
+ b, err := c.CombinedOutput()
+
+ if err != nil {
+ return "", err
+ }
+
+ return strings.TrimSpace(string(b)), nil
+}
+
+func Output(ctx context.Context, cmd string, args ...string) (string, error) {
+ return OutputAt(ctx, ".", cmd, args...)
+}
diff --git a/pkg/build/fsutil/copy_test.go b/pkg/build/fsutil/copy_test.go
new file mode 100644
index 00000000000..6c243ee69f7
--- /dev/null
+++ b/pkg/build/fsutil/copy_test.go
@@ -0,0 +1,87 @@
+package fsutil_test
+
+import (
+ "os"
+ "runtime"
+ "testing"
+
+ "github.com/grafana/grafana/pkg/build/fsutil"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCopyFile(t *testing.T) {
+ src, err := os.CreateTemp("", "")
+ require.NoError(t, err)
+ defer func() {
+ if err := os.RemoveAll(src.Name()); err != nil {
+ t.Log(err)
+ }
+ }()
+
+ err = os.WriteFile(src.Name(), []byte("Contents"), 0600)
+ require.NoError(t, err)
+
+ dst, err := os.CreateTemp("", "")
+ require.NoError(t, err)
+ defer func() {
+ if err := os.RemoveAll(dst.Name()); err != nil {
+ t.Log(err)
+ }
+ }()
+
+ err = fsutil.CopyFile(src.Name(), dst.Name())
+ require.NoError(t, err)
+}
+
+func TestCopyFile_Permissions(t *testing.T) {
+ perms := os.FileMode(0700)
+ if runtime.GOOS == "windows" {
+ // Windows doesn't have file Unix style file permissions
+ // It seems you have either 0444 for read-only or 0666 for read-write
+ perms = os.FileMode(0666)
+ }
+
+ src, err := os.CreateTemp("", "")
+ require.NoError(t, err)
+
+ defer func() {
+ if err := os.RemoveAll(src.Name()); err != nil {
+ t.Log(err)
+ }
+ }()
+
+ err = os.WriteFile(src.Name(), []byte("Contents"), perms)
+ require.NoError(t, err)
+ err = os.Chmod(src.Name(), perms)
+ require.NoError(t, err)
+
+ dst, err := os.CreateTemp("", "")
+ require.NoError(t, err)
+ defer func() {
+ if err := os.RemoveAll(dst.Name()); err != nil {
+ t.Log(err)
+ }
+ }()
+
+ err = fsutil.CopyFile(src.Name(), dst.Name())
+ require.NoError(t, err)
+
+ fi, err := os.Stat(dst.Name())
+ require.NoError(t, err)
+ assert.Equal(t, perms, fi.Mode()&os.ModePerm)
+}
+
+// Test case where destination directory doesn't exist.
+func TestCopyFile_NonExistentDestDir(t *testing.T) {
+ src, err := os.CreateTemp("", "")
+ require.NoError(t, err)
+ defer func() {
+ if err := os.RemoveAll(src.Name()); err != nil {
+ t.Log(err)
+ }
+ }()
+
+ err = fsutil.CopyFile(src.Name(), "non-existent/dest")
+ require.EqualError(t, err, "destination directory doesn't exist: \"non-existent\"")
+}
diff --git a/pkg/build/fsutil/copyfile.go b/pkg/build/fsutil/copyfile.go
new file mode 100644
index 00000000000..eb7edbdf263
--- /dev/null
+++ b/pkg/build/fsutil/copyfile.go
@@ -0,0 +1,107 @@
+package fsutil
+
+import (
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+)
+
+// CopyFile copies a file from src to dst.
+//
+// If src and dst files exist, and are the same, then return success. Otherwise, attempt to create a hard link
+// between the two files. If that fails, copy the file contents from src to dst.
+func CopyFile(src, dst string) (err error) {
+ absSrc, err := filepath.Abs(src)
+ if err != nil {
+ return fmt.Errorf("failed to get absolute path of source file %q: %w", src, err)
+ }
+ sfi, err := os.Stat(src)
+ if err != nil {
+ err = fmt.Errorf("couldn't stat source file %q: %w", absSrc, err)
+ return
+ }
+ if !sfi.Mode().IsRegular() {
+ // Cannot copy non-regular files (e.g., directories, symlinks, devices, etc.)
+ return fmt.Errorf("non-regular source file %s (%q)", absSrc, sfi.Mode().String())
+ }
+ dpath := filepath.Dir(dst)
+ exists, err := Exists(dpath)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ err = fmt.Errorf("destination directory doesn't exist: %q", dpath)
+ return
+ }
+
+ var dfi os.FileInfo
+ dfi, err = os.Stat(dst)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return
+ }
+ } else {
+ if !(dfi.Mode().IsRegular()) {
+ return fmt.Errorf("non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
+ }
+ if os.SameFile(sfi, dfi) {
+ return copyPermissions(sfi.Name(), dfi.Name())
+ }
+ }
+
+ err = copyFileContents(src, dst)
+ return err
+}
+
+// copyFileContents copies the contents of the file named src to the file named
+// by dst. The file will be created if it does not already exist. If the
+// destination file exists, all it's contents will be replaced by the contents
+// of the source file.
+func copyFileContents(src, dst string) (err error) {
+ //nolint:gosec
+ in, err := os.Open(src)
+ if err != nil {
+ return
+ }
+ defer func() {
+ if err := in.Close(); err != nil {
+ log.Println("error closing file", err)
+ }
+ }()
+
+ //nolint:gosec
+ out, err := os.Create(dst)
+ if err != nil {
+ return
+ }
+ defer func() {
+ if cerr := out.Close(); cerr != nil && err == nil {
+ err = cerr
+ }
+ }()
+
+ if _, err = io.Copy(out, in); err != nil {
+ return
+ }
+
+ if err := out.Sync(); err != nil {
+ return err
+ }
+
+ return copyPermissions(src, dst)
+}
+
+func copyPermissions(src, dst string) error {
+ sfi, err := os.Lstat(src)
+ if err != nil {
+ return err
+ }
+
+ if err := os.Chmod(dst, sfi.Mode()); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/pkg/build/fsutil/createtemp.go b/pkg/build/fsutil/createtemp.go
new file mode 100644
index 00000000000..21720a9a3d8
--- /dev/null
+++ b/pkg/build/fsutil/createtemp.go
@@ -0,0 +1,43 @@
+package fsutil
+
+import (
+ "fmt"
+ "os"
+)
+
+// CreateTempFile generates a temp filepath, based on the provided suffix.
+// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807.
+func CreateTempFile(sfx string) (string, error) {
+ var suffix string
+ if sfx != "" {
+ suffix = fmt.Sprintf("*-%s", sfx)
+ } else {
+ suffix = sfx
+ }
+ f, err := os.CreateTemp("", suffix)
+ if err != nil {
+ return "", err
+ }
+ if err := f.Close(); err != nil {
+ return "", err
+ }
+
+ return f.Name(), nil
+}
+
+// CreateTempDir generates a temp directory, based on the provided suffix.
+// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807/.
+func CreateTempDir(sfx string) (string, error) {
+ var suffix string
+ if sfx != "" {
+ suffix = fmt.Sprintf("*-%s", sfx)
+ } else {
+ suffix = sfx
+ }
+ dir, err := os.MkdirTemp("", suffix)
+ if err != nil {
+ return "", err
+ }
+
+ return dir, nil
+}
diff --git a/pkg/build/fsutil/createtemp_test.go b/pkg/build/fsutil/createtemp_test.go
new file mode 100644
index 00000000000..640585f4ea5
--- /dev/null
+++ b/pkg/build/fsutil/createtemp_test.go
@@ -0,0 +1,48 @@
+package fsutil
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestCreateTempFile(t *testing.T) {
+ t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807", func(t *testing.T) {
+ filePath, err := CreateTempFile("")
+ require.NoError(t, err)
+
+ pathParts := strings.Split(filePath, "/")
+ require.Greater(t, len(pathParts), 1)
+ require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1)
+ })
+
+ t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar", func(t *testing.T) {
+ filePath, err := CreateTempFile("foobar")
+ require.NoError(t, err)
+
+ pathParts := strings.Split(filePath, "/")
+ require.Greater(t, len(pathParts), 1)
+ require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2)
+ })
+}
+
+func TestCreateTempDir(t *testing.T) {
+ t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807/", func(t *testing.T) {
+ filePath, err := CreateTempFile("")
+ require.NoError(t, err)
+
+ pathParts := strings.Split(filePath, "/")
+ require.Greater(t, len(pathParts), 1)
+ require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1)
+ })
+
+ t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar/", func(t *testing.T) {
+ filePath, err := CreateTempFile("foobar")
+ require.NoError(t, err)
+
+ pathParts := strings.Split(filePath, "/")
+ require.Greater(t, len(pathParts), 1)
+ require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2)
+ })
+}
diff --git a/pkg/build/fsutil/exists_test.go b/pkg/build/fsutil/exists_test.go
new file mode 100644
index 00000000000..1428654b9b8
--- /dev/null
+++ b/pkg/build/fsutil/exists_test.go
@@ -0,0 +1,15 @@
+package fsutil_test
+
+import (
+ "testing"
+
+ "github.com/grafana/grafana/pkg/build/fsutil"
+ "github.com/stretchr/testify/require"
+)
+
+func TestExists_NonExistent(t *testing.T) {
+ exists, err := fsutil.Exists("non-existent")
+ require.NoError(t, err)
+
+ require.False(t, exists)
+}
diff --git a/pkg/build/fsutil/exsits.go b/pkg/build/fsutil/exsits.go
new file mode 100644
index 00000000000..23e09d20133
--- /dev/null
+++ b/pkg/build/fsutil/exsits.go
@@ -0,0 +1,16 @@
+package fsutil
+
+import "os"
+
+// Exists determines whether a file/directory exists or not.
+func Exists(fpath string) (bool, error) {
+ _, err := os.Stat(fpath)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return false, err
+ }
+ return false, nil
+ }
+
+ return true, nil
+}
diff --git a/pkg/build/gcloud/auth.go b/pkg/build/gcloud/auth.go
new file mode 100644
index 00000000000..ad68fbc5d17
--- /dev/null
+++ b/pkg/build/gcloud/auth.go
@@ -0,0 +1,65 @@
+package gcloud
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "strings"
+)
+
+func GetDecodedKey() ([]byte, error) {
+ gcpKey := strings.TrimSpace(os.Getenv("GCP_KEY"))
+ if gcpKey == "" {
+ return nil, fmt.Errorf("the environment variable GCP_KEY must be set")
+ }
+
+ gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKey)
+ if err != nil {
+ // key is not always base64 encoded
+ validKey := []byte(gcpKey)
+ if json.Valid(validKey) {
+ return validKey, nil
+ }
+ return nil, fmt.Errorf("error decoding the gcp_key, err: %q", err)
+ }
+
+ return gcpKeyB, nil
+}
+
+func ActivateServiceAccount() error {
+ byteKey, err := GetDecodedKey()
+ if err != nil {
+ return err
+ }
+
+ f, err := os.CreateTemp("", "*.json")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if err := os.Remove(f.Name()); err != nil {
+ log.Printf("error removing %s: %s", f.Name(), err)
+ }
+ }()
+
+ defer func() {
+ if err := f.Close(); err != nil {
+ log.Println("error closing file:", err)
+ }
+ }()
+
+ if _, err := f.Write(byteKey); err != nil {
+ return fmt.Errorf("failed to write GCP key file: %w", err)
+ }
+ keyArg := fmt.Sprintf("--key-file=%s", f.Name())
+ //nolint:gosec
+ cmd := exec.Command("gcloud", "auth", "activate-service-account", keyArg)
+
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to sign into GCP: %w\n%s", err, output)
+ }
+ return nil
+}
diff --git a/pkg/build/gcloud/storage/gsutil.go b/pkg/build/gcloud/storage/gsutil.go
new file mode 100644
index 00000000000..461fd58c98a
--- /dev/null
+++ b/pkg/build/gcloud/storage/gsutil.go
@@ -0,0 +1,457 @@
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "mime"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "github.com/grafana/grafana/pkg/build/fsutil"
+ "github.com/grafana/grafana/pkg/build/gcloud"
+ "google.golang.org/api/iterator"
+ "google.golang.org/api/option"
+)
+
+var (
+ // ErrorNilBucket is returned when a function is called where a bucket argument is expected and the bucket is nil.
+ ErrorNilBucket = errors.New("a bucket must be provided")
+)
+
+const (
+ // maxThreads specify the number of max threads that can run at the same time.
+ // Set to 1000, since the maximum number of simultaneous open files for the runners is 1024.
+ maxThreads = 1000
+)
+
+// Client wraps the gcloud storage Client with convenient helper functions.
+// By using an embedded type we can still use the functions provided by storage.Client if we need to.
+type Client struct {
+ storage.Client
+}
+
+// File represents a file in Google Cloud Storage.
+type File struct {
+ FullPath string
+ PathTrimmed string
+}
+
+// New creates a new Client by checking for the Google Cloud SDK auth key and/or environment variable.
+func New() (*Client, error) {
+ client, err := newClient()
+ if err != nil {
+ return nil, err
+ }
+
+ return &Client{
+ Client: *client,
+ }, nil
+}
+
+// newClient initializes the google-cloud-storage (GCS) client.
+// It first checks for the application-default_credentials.json file then the GCP_KEY environment variable.
+func newClient() (*storage.Client, error) {
+ ctx := context.Background()
+
+ byteKey, err := gcloud.GetDecodedKey()
+ if err != nil {
+ return nil, fmt.Errorf("failed to get gcp key, err: %w", err)
+ }
+ client, err := storage.NewClient(ctx, option.WithCredentialsJSON(byteKey))
+ if err != nil {
+ log.Println("failed to login with GCP_KEY, trying with default application credentials...")
+ client, err = storage.NewClient(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open Google Cloud Storage client: %w", err)
+ }
+ }
+
+ return client, nil
+}
+
+// CopyLocalDir copies a local directory 'dir' to the bucket 'bucket' at the path 'bucketPath'.
+func (client *Client) CopyLocalDir(ctx context.Context, dir string, bucket *storage.BucketHandle, bucketPath string, trim bool) error {
+ if bucket == nil {
+ return ErrorNilBucket
+ }
+
+ files, err := ListLocalFiles(dir)
+ if err != nil {
+ return err
+ }
+ log.Printf("Number or files to be copied over: %d\n", len(files))
+
+ for _, chunk := range asChunks(files, maxThreads) {
+ var wg sync.WaitGroup
+ for _, f := range chunk {
+ wg.Add(1)
+ go func(file File) {
+ defer wg.Done()
+ err = client.Copy(ctx, file, bucket, bucketPath, trim)
+ if err != nil {
+ log.Printf("failed to copy objects, err: %s\n", err.Error())
+ }
+ }(f)
+ }
+ wg.Wait()
+ }
+
+ return nil
+}
+
+// Copy copies a single local file into the bucket at the provided path.
+// trim variable should be set to true if the full object path is needed - false otherwise.
+func (client *Client) Copy(ctx context.Context, file File, bucket *storage.BucketHandle, remote string, trim bool) error {
+ if bucket == nil {
+ return ErrorNilBucket
+ }
+
+ localFile, err := os.Open(file.FullPath)
+ if err != nil {
+ return fmt.Errorf("failed to open file %s, err: %q", file.FullPath, err)
+ }
+ defer func() {
+ if err := localFile.Close(); err != nil {
+ log.Println("failed to close localfile", "err", err)
+ }
+ }()
+
+ extension := strings.ToLower(path.Ext(file.FullPath))
+ contentType := mime.TypeByExtension(extension)
+
+ filePath := file.FullPath
+ if trim {
+ filePath = file.PathTrimmed
+ }
+
+ objectPath := path.Join(remote, filePath)
+
+ wc := bucket.Object(objectPath).NewWriter(ctx)
+ wc.ContentType = contentType
+ defer func() {
+ if err := wc.Close(); err != nil {
+ log.Println("failed to close writer", "err", err)
+ }
+ }()
+
+ if _, err = io.Copy(wc, localFile); err != nil {
+ return fmt.Errorf("failed to copy to Cloud Storage: %w", err)
+ }
+
+ log.Printf("Successfully uploaded tarball to Google Cloud Storage, path: %s/%s\n", remote, file.FullPath)
+
+ return nil
+}
+
+// CopyRemoteDir copies an entire directory 'from' from the bucket 'fromBucket' into the 'toBucket' at the path 'to'.
+func (client *Client) CopyRemoteDir(ctx context.Context, fromBucket *storage.BucketHandle, from string, toBucket *storage.BucketHandle, to string) error {
+ if toBucket == nil || fromBucket == nil {
+ return ErrorNilBucket
+ }
+
+ files, err := ListRemoteFiles(ctx, fromBucket, FilesFilter{Prefix: from})
+ if err != nil {
+ return err
+ }
+
+ var ch = make(chan File, len(files))
+ var wg sync.WaitGroup
+ wg.Add(maxThreads)
+
+ for i := 0; i < maxThreads; i++ {
+ go func() {
+ for {
+ file, ok := <-ch
+ if !ok {
+ wg.Done()
+ return
+ }
+ if err := client.RemoteCopy(ctx, file, fromBucket, toBucket, to); err != nil {
+ log.Printf("failed to copy files between buckets: err: %s\n", err.Error())
+ return
+ }
+ }
+ }()
+ }
+
+ for _, file := range files {
+ ch <- file
+ }
+
+ close(ch)
+ wg.Wait()
+
+ return nil
+}
+
+// RemoteCopy will copy the file 'file' from the 'fromBucket' to the 'toBucket' at the path 'path'.
+func (client *Client) RemoteCopy(ctx context.Context, file File, fromBucket, toBucket *storage.BucketHandle, path string) error {
+ // Should this be path.Join instead of filepath.Join? filepath.Join on Windows will produce `\\` separators instead of `/`.
+ var (
+ src = fromBucket.Object(file.FullPath)
+ dstObject = filepath.Join(path, file.PathTrimmed)
+ dst = toBucket.Object(dstObject)
+ )
+
+ if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
+ return fmt.Errorf("failed to copy object %s, to %s, err: %w", file.FullPath, dstObject, err)
+ }
+
+ log.Printf("%s was successfully copied to %v bucket!.\n\n", file.FullPath, toBucket)
+ return nil
+}
+
+// DeleteDir deletes a directory at 'path' from the bucket.
+func (client *Client) DeleteDir(ctx context.Context, bucket *storage.BucketHandle, path string) error {
+ if bucket == nil {
+ return ErrorNilBucket
+ }
+
+ files, err := ListRemoteFiles(ctx, bucket, FilesFilter{Prefix: path})
+ if err != nil {
+ return err
+ }
+
+ var ch = make(chan string, len(files))
+ var wg sync.WaitGroup
+ wg.Add(maxThreads)
+
+ for i := 0; i < maxThreads; i++ {
+ go func() {
+ for {
+ fullPath, ok := <-ch
+ if !ok {
+ wg.Done()
+ return
+ }
+ err := client.Delete(ctx, bucket, fullPath)
+ if err != nil && !errors.Is(err, storage.ErrObjectNotExist) {
+ log.Printf("failed to delete objects, err %s\n", err.Error())
+ panic(err)
+ }
+ }
+ }()
+ }
+
+ for _, file := range files {
+ ch <- file.FullPath
+ }
+
+ close(ch)
+ wg.Wait()
+
+ return nil
+}
+
+// Delete deletes single item from the bucket at 'path'.
+func (client *Client) Delete(ctx context.Context, bucket *storage.BucketHandle, path string) error {
+ object := bucket.Object(path)
+ if err := object.Delete(ctx); err != nil {
+ return fmt.Errorf("cannot delete %s, err: %w", path, err)
+ }
+ log.Printf("Successfully deleted tarball to Google Cloud Storage, path: %s", path)
+ return nil
+}
+
+// ListLocalFiles lists files in a local filesystem.
+func ListLocalFiles(dir string) ([]File, error) {
+ var files []File
+ err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if !info.IsDir() {
+ files = append(files, File{
+ FullPath: path,
+ // Strip the dir name from the filepath
+ PathTrimmed: strings.ReplaceAll(path, dir, ""),
+ })
+ }
+ return nil
+ })
+
+ if err != nil {
+ return nil, fmt.Errorf("error walking path: %v", err)
+ }
+
+ return files, nil
+}
+
+type FilesFilter struct {
+ Prefix string
+ FileExts []string
+}
+
+// ListRemoteFiles lists all the files in the directory (filtering by FilesFilter) and returns a File struct for each one.
+func ListRemoteFiles(ctx context.Context, bucket *storage.BucketHandle, filter FilesFilter) ([]File, error) {
+ if bucket == nil {
+ return []File{}, ErrorNilBucket
+ }
+
+ it := bucket.Objects(ctx, &storage.Query{
+ Prefix: filter.Prefix,
+ })
+
+ var files []File
+ for {
+ attrs, err := it.Next()
+ if err != nil {
+ if errors.Is(err, iterator.Done) {
+ break
+ }
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to iterate through bucket, err: %w", err)
+ }
+
+ extMatch := len(filter.FileExts) == 0
+ for _, ext := range filter.FileExts {
+ if ext == filepath.Ext(attrs.Name) {
+ extMatch = true
+ break
+ }
+ }
+
+ if extMatch {
+ files = append(files, File{FullPath: attrs.Name, PathTrimmed: strings.TrimPrefix(attrs.Name, filter.Prefix)})
+ }
+ }
+
+ return files, nil
+}
+
+// DownloadDirectory downloads files from bucket (filtering by FilesFilter) to destPath on disk.
+func (client *Client) DownloadDirectory(ctx context.Context, bucket *storage.BucketHandle, destPath string, filter FilesFilter) error {
+ if bucket == nil {
+ return ErrorNilBucket
+ }
+
+ files, err := ListRemoteFiles(ctx, bucket, filter)
+ if err != nil {
+ return err
+ }
+
+ // return err if dir already exists
+ exists, err := fsutil.Exists(destPath)
+ if err != nil {
+ return err
+ }
+ if exists {
+ return fmt.Errorf("destination path %q already exists", destPath)
+ }
+
+ err = os.MkdirAll(destPath, 0750)
+ if err != nil && !os.IsExist(err) {
+ return err
+ }
+
+ for _, file := range files {
+ err = client.downloadFile(ctx, bucket, file.FullPath, filepath.Join(destPath, file.PathTrimmed))
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// GetLatestMainBuild gets the latest main build which is successfully uploaded to the gcs bucket.
+func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path string) (string, error) {
+ if bucket == nil {
+ return "", ErrorNilBucket
+ }
+
+ it := bucket.Objects(ctx, &storage.Query{
+ Prefix: path,
+ })
+
+ var files []string
+ for {
+ attrs, err := it.Next()
+ if errors.Is(err, iterator.Done) {
+ break
+ }
+ if err != nil {
+ return "", fmt.Errorf("failed to iterate through bucket, err: %w", err)
+ }
+
+ files = append(files, attrs.Name)
+ }
+
+ var latestVersion string
+ for i := len(files) - 1; i >= 0; i-- {
+ captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`)
+ if captureVersion.MatchString(files[i]) {
+ latestVersion = captureVersion.FindString(files[i])
+ break
+ }
+ }
+
+ return latestVersion, nil
+}
+
+// downloadFile downloads an object to a file.
+func (client *Client) downloadFile(ctx context.Context, bucket *storage.BucketHandle, objectName, destFileName string) error {
+ if bucket == nil {
+ return ErrorNilBucket
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, time.Second*10)
+ defer cancel()
+
+ // nolint:gosec
+ f, err := os.Create(destFileName)
+ if err != nil {
+ return fmt.Errorf("os.Create: %v", err)
+ }
+
+ rc, err := bucket.Object(objectName).NewReader(ctx)
+ if err != nil {
+ return fmt.Errorf("Object(%q).NewReader: %v", objectName, err)
+ }
+ defer func() {
+ if err := rc.Close(); err != nil {
+ log.Println("failed to close reader", "err", err)
+ }
+ }()
+
+ if _, err := io.Copy(f, rc); err != nil {
+ return fmt.Errorf("io.Copy: %v", err)
+ }
+
+ if err = f.Close(); err != nil {
+ return fmt.Errorf("f.Close: %v", err)
+ }
+
+ return nil
+}
+
+// asChunks will split the supplied []File into slices with a max size of `chunkSize`
+// []string{"a", "b", "c"}, 1 => [][]string{[]string{"a"}, []string{"b"}, []string{"c"}}
+// []string{"a", "b", "c"}, 2 => [][]string{[]string{"a", "b"}, []string{"c"}}.
+func asChunks(files []File, chunkSize int) [][]File {
+ var fileChunks [][]File
+
+ if len(files) == 0 {
+ return [][]File{}
+ }
+
+ if len(files) > chunkSize && chunkSize > 0 {
+ for i := 0; i < len(files); i += chunkSize {
+ end := i + chunkSize
+
+ if end > len(files) {
+ end = len(files)
+ }
+ fileChunks = append(fileChunks, files[i:end])
+ }
+ } else {
+ fileChunks = [][]File{files}
+ }
+ return fileChunks
+}
diff --git a/pkg/build/gcloud/storage/gsutil_test.go b/pkg/build/gcloud/storage/gsutil_test.go
new file mode 100644
index 00000000000..1e2ecb7c06b
--- /dev/null
+++ b/pkg/build/gcloud/storage/gsutil_test.go
@@ -0,0 +1,159 @@
+package storage
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func Test_asChunks(t *testing.T) {
+ type args struct {
+ files []File
+ chunkSize int
+ }
+ tcs := []struct {
+ name string
+ args args
+ expected [][]File
+ }{
+ {
+ name: "Happy path #1",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ {FullPath: "/1"},
+ {FullPath: "/2"},
+ {FullPath: "/3"},
+ },
+ chunkSize: 5,
+ },
+ expected: [][]File{
+ {{FullPath: "/a"}, {FullPath: "/b"}, {FullPath: "/c"}, {FullPath: "/1"}, {FullPath: "/2"}},
+ {{FullPath: "/3"}},
+ },
+ },
+ {
+ name: "Happy path #2",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ {FullPath: "/1"},
+ {FullPath: "/2"},
+ {FullPath: "/3"},
+ },
+ chunkSize: 2,
+ },
+ expected: [][]File{
+ {{FullPath: "/a"}, {FullPath: "/b"}},
+ {{FullPath: "/c"}, {FullPath: "/1"}},
+ {{FullPath: "/2"}, {FullPath: "/3"}},
+ },
+ },
+ {
+ name: "Happy path #3",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ chunkSize: 1,
+ },
+ expected: [][]File{
+ {{FullPath: "/a"}},
+ {{FullPath: "/b"}},
+ {{FullPath: "/c"}},
+ },
+ },
+ {
+ name: "A chunkSize with 0 value returns the input as a single chunk",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ chunkSize: 0,
+ },
+ expected: [][]File{
+ {
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ },
+ },
+ {
+ name: "A chunkSize with negative value returns the input as a single chunk",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ chunkSize: -1,
+ },
+ expected: [][]File{
+ {
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ },
+ },
+ {
+ name: "A chunkSize greater than the size on input returns the input as a single chunk",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ chunkSize: 5,
+ },
+ expected: [][]File{
+ {
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ },
+ },
+ {
+ name: "A chunkSize equal the size on input returns the input as a single chunk",
+ args: args{
+ files: []File{
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ chunkSize: 3,
+ },
+ expected: [][]File{
+ {
+ {FullPath: "/a"},
+ {FullPath: "/b"},
+ {FullPath: "/c"},
+ },
+ },
+ },
+ {
+ name: "An empty input returns empty chunks",
+ args: args{
+ files: []File{},
+ chunkSize: 3,
+ },
+ expected: [][]File{},
+ },
+ }
+ for _, tc := range tcs {
+ t.Run(tc.name, func(t *testing.T) {
+ result := asChunks(tc.args.files, tc.args.chunkSize)
+ require.Equal(t, tc.expected, result)
+ })
+ }
+}
diff --git a/pkg/build/git/git.go b/pkg/build/git/git.go
new file mode 100644
index 00000000000..bb6c610cdcd
--- /dev/null
+++ b/pkg/build/git/git.go
@@ -0,0 +1,25 @@
+package git
+
+import (
+ "fmt"
+ "regexp"
+)
+
+const (
+ MainBranch = "main"
+ HomeDir = "."
+ RepoOwner = "grafana"
+ OSSRepo = "grafana"
+ EnterpriseRepo = "grafana-enterprise"
+ EnterpriseCheckName = "Grafana Enterprise"
+ EnterpriseCheckDescription = "Downstream tests to ensure that your changes are compatible with Grafana Enterprise"
+)
+
+func PRCheckRegexp() *regexp.Regexp {
+ reBranch, err := regexp.Compile(`^prc-([0-9]+)-([A-Za-z0-9]+)\/(.+)$`)
+ if err != nil {
+ panic(fmt.Sprintf("Failed to compile regexp: %s", err))
+ }
+
+ return reBranch
+}
diff --git a/pkg/build/git/git_test.go b/pkg/build/git/git_test.go
new file mode 100644
index 00000000000..c0cbd4c8cd8
--- /dev/null
+++ b/pkg/build/git/git_test.go
@@ -0,0 +1,56 @@
+package git_test
+
+import (
+ "testing"
+
+ "github.com/grafana/grafana/pkg/build/git"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestPRCheckRegexp(t *testing.T) {
+ type match struct {
+ String string
+ Commit string
+ Branch string
+ PR string
+ }
+
+ var (
+ shouldMatch = []match{
+ {
+ String: "prc-1-a1b2c3d4/branch-name",
+ Branch: "branch-name",
+ Commit: "a1b2c3d4",
+ PR: "1",
+ },
+ {
+ String: "prc-111-a1b2c3d4/branch/name",
+ Branch: "branch/name",
+ Commit: "a1b2c3d4",
+ PR: "111",
+ },
+ {
+ String: "prc-102930122-a1b2c3d4/branch-name",
+ Branch: "branch-name",
+ Commit: "a1b2c3d4",
+ PR: "102930122",
+ },
+ }
+
+ shouldNotMatch = []string{"prc-a/branch", "km/test", "test", "prc", "prc/test", "price"}
+ )
+
+ regex := git.PRCheckRegexp()
+
+ for _, v := range shouldMatch {
+ assert.Truef(t, regex.MatchString(v.String), "regex '%s' should match %s", regex.String(), v)
+ m := regex.FindStringSubmatch(v.String)
+ assert.Equal(t, m[1], v.PR)
+ assert.Equal(t, m[2], v.Commit)
+ assert.Equal(t, m[3], v.Branch)
+ }
+
+ for _, v := range shouldNotMatch {
+ assert.False(t, regex.MatchString(v), "regex '%s' should not match %s", regex.String(), v)
+ }
+}
diff --git a/pkg/build/golangutils/build.go b/pkg/build/golangutils/build.go
new file mode 100644
index 00000000000..14da56a7765
--- /dev/null
+++ b/pkg/build/golangutils/build.go
@@ -0,0 +1,124 @@
+package golangutils
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os/exec"
+ "strings"
+
+ "github.com/grafana/grafana/pkg/build/config"
+)
+
+type BuildOpts struct {
+ // Package refers to the path to the `main` package containing `func main`
+ Package string
+
+ // Output is used as the -o argument in the go build command
+ Output string
+
+ // Workdir should define some place in the module where the package path resolves.
+ // Go commands need to be ran inside a the Go module directory.
+ Workdir string
+
+ GoOS config.OS
+ GoArch config.Architecture
+ GoArm string
+ Go386 string
+ CC string
+ LibC string
+
+ CGoEnabled bool
+ CGoCFlags string
+
+ // LdFlags are joined by a space character and provided to the -ldflags argument.
+ // A valid element here would be `-X 'main.version=1.0.0'`.
+ LdFlags []string
+
+ Stdout io.ReadWriter
+ Stderr io.ReadWriter
+ Stdin io.ReadWriter
+
+ // ExtraEnv allows consumers to provide extra env args that are not defined above.
+ // A single element should be formatted using like so: {NAME}={VALUE}. Example: GOOS=linux.
+ ExtraEnv []string
+
+ // ExtraArgs allows consumers to provide extra arguments that are not defined above.
+ // Flag names and values should be two separate elements.
+ // These flags will be appended to the command arguments before the package path in "go build".
+ ExtraArgs []string
+}
+
+// Env constructs a list of key/value pairs for setting a build command's environment.
+// Should we consider using something to unmarshal the struct to env?
+func (opts BuildOpts) Env() []string {
+ env := []string{}
+ if opts.CGoEnabled {
+ env = append(env, "CGO_ENABLED=1")
+ }
+
+ if opts.GoOS != "" {
+ env = append(env, fmt.Sprintf("GOOS=%s", opts.GoOS))
+ }
+
+ if opts.GoArch != "" {
+ env = append(env, fmt.Sprintf("GOARCH=%s", opts.GoArch))
+ }
+
+ if opts.CC != "" {
+ env = append(env, fmt.Sprintf("CC=%s", opts.CC))
+ }
+
+ if opts.CGoCFlags != "" {
+ env = append(env, fmt.Sprintf("CGO_CFLAGS=%s", opts.CGoCFlags))
+ }
+
+ if opts.GoArm != "" {
+ env = append(env, fmt.Sprintf("GOARM=%s", opts.GoArm))
+ }
+
+ if opts.ExtraEnv != nil {
+ return append(opts.ExtraEnv, env...)
+ }
+
+ return env
+}
+
+// Args constructs a list of flags and values for use with the exec.Command type when running "go build".
+func (opts BuildOpts) Args() []string {
+ args := []string{}
+
+ if opts.LdFlags != nil {
+ args = append(args, "-ldflags", strings.Join(opts.LdFlags, " "))
+ }
+
+ if opts.Output != "" {
+ args = append(args, "-o", opts.Output)
+ }
+
+ if opts.ExtraArgs != nil {
+ args = append(args, opts.ExtraArgs...)
+ }
+
+ args = append(args, opts.Package)
+
+ return args
+}
+
+// Build runs the go build process in the current shell given the opts.
+// This function will panic if no Stdout/Stderr/Stdin is provided in the opts.
+func RunBuild(ctx context.Context, opts BuildOpts) error {
+ env := opts.Env()
+ args := append([]string{"build"}, opts.Args()...)
+ // Ignore gosec G304 as this function is only used in the build process.
+ //nolint:gosec
+ cmd := exec.CommandContext(ctx, "go", args...)
+ cmd.Env = env
+
+ cmd.Stdout = opts.Stdout
+ cmd.Stderr = opts.Stderr
+ cmd.Stdin = opts.Stdin
+ cmd.Dir = opts.Workdir
+
+ return cmd.Run()
+}
diff --git a/pkg/build/golangutils/doc.go b/pkg/build/golangutils/doc.go
new file mode 100644
index 00000000000..2cb33af05e1
--- /dev/null
+++ b/pkg/build/golangutils/doc.go
@@ -0,0 +1,2 @@
+// Package golangutils holds utility functions, wrappers, and types for building Go binaries for Grafana.
+package golangutils
diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go
new file mode 100644
index 00000000000..4d661e8d851
--- /dev/null
+++ b/pkg/build/grafana/build.go
@@ -0,0 +1,123 @@
+package grafana
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/cryptoutil"
+ "github.com/grafana/grafana/pkg/build/golangutils"
+)
+
+var binaries = []string{"grafana", "grafana-server", "grafana-cli"}
+
+const (
+ SuffixEnterprise2 = "-enterprise2"
+)
+
+const (
+ ExtensionExe = ".exe"
+)
+
+func GrafanaLDFlags(version string, r config.Revision) []string {
+ return []string{
+ "-w",
+ fmt.Sprintf("-X main.version=%s", version),
+ fmt.Sprintf("-X main.commit=%s", r.SHA256),
+ fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp),
+ fmt.Sprintf("-X main.buildBranch=%s", r.Branch),
+ }
+}
+
+// BinaryFolder returns the path to where the Grafana binary is build given the provided arguments.
+func BinaryFolder(edition config.Edition, args BuildArgs) string {
+ sfx := ""
+ if edition == config.EditionEnterprise2 {
+ sfx = SuffixEnterprise2
+ }
+
+ arch := string(args.GoArch)
+ if args.GoArch == config.ArchARM {
+ arch = string(args.GoArch) + "v" + args.GoArm
+ }
+
+ format := fmt.Sprintf("%s-%s", args.GoOS, arch)
+ if args.LibC != "" {
+ format += fmt.Sprintf("-%s", args.LibC)
+ }
+ format += sfx
+
+ if args.GoOS == config.OSWindows {
+ format += ExtensionExe
+ }
+
+ return format
+}
+
+func GrafanaDescriptor(opts golangutils.BuildOpts) string {
+ libcPart := ""
+ if opts.LibC != "" {
+ libcPart = fmt.Sprintf("/%s", opts.LibC)
+ }
+ arch := string(opts.GoArch)
+ if opts.GoArch == config.ArchARM {
+ arch = string(opts.GoArch) + "v" + opts.GoArm
+ }
+
+ return fmt.Sprintf("%s/%s%s", opts.GoOS, arch, libcPart)
+}
+
+// BuildGrafanaBinary builds a certain binary according to certain parameters.
+func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArgs, edition config.Edition) error {
+ opts := args.BuildOpts
+ opts.ExtraEnv = os.Environ()
+
+ revision, err := config.GrafanaRevision(ctx, opts.Workdir)
+ if err != nil {
+ return err
+ }
+
+ folder := BinaryFolder(edition, args)
+
+ if opts.GoOS == config.OSWindows {
+ name += ExtensionExe
+ }
+
+ binary := filepath.Join(opts.Workdir, "bin", folder, name)
+ opts.Output = binary
+
+ if err := os.RemoveAll(binary); err != nil {
+ return fmt.Errorf("failed to remove %q: %w", binary, err)
+ }
+
+ if err := os.RemoveAll(binary + ".md5"); err != nil {
+ return fmt.Errorf("failed to remove %q: %w", binary+".md5", err)
+ }
+
+ descriptor := GrafanaDescriptor(opts)
+
+ log.Printf("Building %q for %s\nwith env: %v", binary, descriptor, opts.Env())
+
+ opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...)
+
+ if edition == config.EditionEnterprise2 {
+ opts.ExtraArgs = []string{"-tags=pro"}
+ }
+
+ log.Printf("Running command 'go %s'", opts.Args())
+
+ if err := golangutils.RunBuild(ctx, opts); err != nil {
+ return err
+ }
+
+ // Create an MD5 checksum of the binary, to be included in the archive for
+ // automatic upgrades.
+ if err := cryptoutil.MD5File(binary); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/pkg/build/grafana/variant.go b/pkg/build/grafana/variant.go
new file mode 100644
index 00000000000..6ccd4abb8a4
--- /dev/null
+++ b/pkg/build/grafana/variant.go
@@ -0,0 +1,160 @@
+package grafana
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "path/filepath"
+
+ "github.com/grafana/grafana/pkg/build/compilers"
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/golangutils"
+)
+
+// BuildArgs represent the build parameters that define the "go build" behavior of a single variant.
+// These arguments are applied as environment variables and arguments to the "go build" command.
+type BuildArgs struct {
+ golangutils.BuildOpts
+ DebArch config.Architecture
+ RPMArch config.Architecture
+}
+
+type BuildVariantOpts struct {
+ Variant config.Variant
+ Edition config.Edition
+
+ Version string
+ GrafanaDir string
+}
+
+// BuildVariant builds a certain variant of the grafana-server and grafana-cli binaries sequentially.
+func BuildVariant(ctx context.Context, opts BuildVariantOpts) error {
+ grafanaDir, err := filepath.Abs(opts.GrafanaDir)
+ if err != nil {
+ return err
+ }
+
+ var (
+ args = VariantBuildArgs(opts.Variant)
+ )
+
+ for _, binary := range binaries {
+ // Note that for Golang cmd paths we must use the relative path and the Linux file separators (/) even for Windows users.
+ var (
+ pkg = fmt.Sprintf("./pkg/cmd/%s", binary)
+ stdout = bytes.NewBuffer(nil)
+ stderr = bytes.NewBuffer(nil)
+ )
+
+ args.Workdir = grafanaDir
+ args.Stdout = stdout
+ args.Stderr = stderr
+ args.Package = pkg
+
+ if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil {
+ return fmt.Errorf("failed to build %s for %s: %w\nstdout: %s\nstderr: %s", pkg, opts.Variant, err, stdout.String(), stderr.String())
+ }
+ }
+
+ return nil
+}
+
+var ldFlagsStatic = []string{"-linkmode=external", "-extldflags=-static"}
+
+var variantArgs = map[config.Variant]BuildArgs{
+ config.VariantArmV6: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ CGoEnabled: true,
+ GoArch: config.ArchARM,
+ GoArm: "6",
+ CC: compilers.ArmV6,
+ },
+ DebArch: config.ArchARMHF,
+ },
+ config.VariantArmV7: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ CGoEnabled: true,
+ GoArch: config.ArchARM,
+ GoArm: "7",
+ CC: compilers.Armv7,
+ },
+ DebArch: config.ArchARMHF,
+ RPMArch: config.ArchARMHFP,
+ },
+ config.VariantArmV7Musl: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ CGoEnabled: true,
+ GoArch: config.ArchARM,
+ GoArm: "7",
+ LibC: config.LibCMusl,
+ CC: compilers.Armv7Musl,
+ LdFlags: ldFlagsStatic,
+ },
+ },
+ config.VariantArm64: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ CGoEnabled: true,
+ GoArch: config.ArchARM64,
+ CC: compilers.Arm64,
+ },
+ DebArch: config.ArchARM64,
+ RPMArch: "aarch64",
+ },
+ config.VariantArm64Musl: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ GoArch: config.ArchARM64,
+ CGoEnabled: true,
+ CC: compilers.Arm64Musl,
+ LibC: config.LibCMusl,
+ LdFlags: ldFlagsStatic,
+ },
+ },
+ config.VariantDarwinAmd64: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSDarwin,
+ CGoEnabled: true,
+ GoArch: config.ArchAMD64,
+ CC: compilers.Osx64,
+ },
+ },
+ config.VariantWindowsAmd64: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSWindows,
+ GoArch: config.ArchAMD64,
+ CC: compilers.Win64,
+ CGoEnabled: true,
+ CGoCFlags: "-D_WIN32_WINNT=0x0601",
+ },
+ },
+ config.VariantLinuxAmd64: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ GoArch: config.ArchAMD64,
+ CC: compilers.LinuxX64,
+ },
+ DebArch: config.ArchAMD64,
+ RPMArch: config.ArchAMD64,
+ },
+ config.VariantLinuxAmd64Musl: {
+ BuildOpts: golangutils.BuildOpts{
+ GoOS: config.OSLinux,
+ GoArch: config.ArchAMD64,
+ CC: compilers.LinuxX64Musl,
+ LibC: config.LibCMusl,
+ LdFlags: ldFlagsStatic,
+ },
+ },
+}
+
+func VariantBuildArgs(v config.Variant) BuildArgs {
+ if val, ok := variantArgs[v]; ok {
+ return val
+ }
+
+ return BuildArgs{}
+}
diff --git a/pkg/build/packaging/artifacts.go b/pkg/build/packaging/artifacts.go
new file mode 100644
index 00000000000..bdce67170e1
--- /dev/null
+++ b/pkg/build/packaging/artifacts.go
@@ -0,0 +1,140 @@
+package packaging
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/grafana/grafana/pkg/build/config"
+)
+
+const ReleaseFolder = "release"
+const MainFolder = "main"
+const EnterpriseSfx = "-enterprise"
+const CacheSettings = "Cache-Control:public, max-age="
+
+type buildArtifact struct {
+ Os string
+ Arch string
+ urlPostfix string
+ packagePostfix string
+}
+
+type PublishConfig struct {
+ config.Config
+
+ Edition config.Edition
+ ReleaseMode config.ReleaseMode
+ GrafanaAPIKey string
+ WhatsNewURL string
+ ReleaseNotesURL string
+ DryRun bool
+ TTL string
+ SimulateRelease bool
+}
+
+const rhelOS = "rhel"
+const debOS = "deb"
+
+func (t buildArtifact) GetURL(baseArchiveURL string, cfg PublishConfig) string {
+ rev := ""
+ prefix := "-"
+ if t.Os == debOS {
+ prefix = "_"
+ } else if t.Os == rhelOS {
+ rev = "-1"
+ }
+
+ version := cfg.Version
+ verComponents := strings.Split(version, "-")
+ if len(verComponents) > 2 {
+ panic(fmt.Sprintf("Version string contains more than one hyphen: %q", version))
+ }
+
+ switch t.Os {
+ case debOS, rhelOS:
+ if len(verComponents) > 1 {
+ // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this
+ // is considered of lower lexical value than the empty character, and this way pre-release versions are
+ // considered to be of a lower version than the final version (which lacks this suffix).
+ version = fmt.Sprintf("%s~%s", verComponents[0], verComponents[1])
+ }
+ }
+
+ // https://dl.grafana.com/oss/main/grafana_8.5.0~54094pre_armhf.deb: 404 Not Found
+ url := fmt.Sprintf("%s%s%s%s%s%s", baseArchiveURL, t.packagePostfix, prefix, version, rev, t.urlPostfix)
+ return url
+}
+
+var ArtifactConfigs = []buildArtifact{
+ {
+ Os: debOS,
+ Arch: "arm64",
+ urlPostfix: "_arm64.deb",
+ },
+ {
+ Os: rhelOS,
+ Arch: "arm64",
+ urlPostfix: ".aarch64.rpm",
+ },
+ {
+ Os: "linux",
+ Arch: "arm64",
+ urlPostfix: ".linux-arm64.tar.gz",
+ },
+ {
+ Os: debOS,
+ Arch: "armv7",
+ urlPostfix: "_armhf.deb",
+ },
+ {
+ Os: debOS,
+ Arch: "armv6",
+ packagePostfix: "-rpi",
+ urlPostfix: "_armhf.deb",
+ },
+ {
+ Os: rhelOS,
+ Arch: "armv7",
+ urlPostfix: ".armhfp.rpm",
+ },
+ {
+ Os: "linux",
+ Arch: "armv6",
+ urlPostfix: ".linux-armv6.tar.gz",
+ },
+ {
+ Os: "linux",
+ Arch: "armv7",
+ urlPostfix: ".linux-armv7.tar.gz",
+ },
+ {
+ Os: "darwin",
+ Arch: "amd64",
+ urlPostfix: ".darwin-amd64.tar.gz",
+ },
+ {
+ Os: "deb",
+ Arch: "amd64",
+ urlPostfix: "_amd64.deb",
+ },
+ {
+ Os: rhelOS,
+ Arch: "amd64",
+ urlPostfix: ".x86_64.rpm",
+ },
+ {
+ Os: "linux",
+ Arch: "amd64",
+ urlPostfix: ".linux-amd64.tar.gz",
+ },
+ {
+ Os: "win",
+ Arch: "amd64",
+ urlPostfix: ".windows-amd64.zip",
+ },
+ {
+ Os: "win-installer",
+ Arch: "amd64",
+ urlPostfix: ".windows-amd64.msi",
+ },
+}
diff --git a/pkg/build/packaging/deb.go b/pkg/build/packaging/deb.go
new file mode 100644
index 00000000000..094207b0270
--- /dev/null
+++ b/pkg/build/packaging/deb.go
@@ -0,0 +1,246 @@
+package packaging
+
+import (
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "github.com/urfave/cli/v2"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/fsutil"
+ "github.com/grafana/grafana/pkg/infra/fs"
+)
+
+func writeAptlyConf(dbDir, repoDir string) error {
+ aptlyConf := fmt.Sprintf(`{
+ "rootDir": "%s",
+ "downloadConcurrency": 4,
+ "downloadSpeedLimit": 0,
+ "architectures": [],
+ "dependencyFollowSuggests": false,
+ "dependencyFollowRecommends": false,
+ "dependencyFollowAllVariants": false,
+ "dependencyFollowSource": false,
+ "dependencyVerboseResolve": false,
+ "gpgDisableSign": false,
+ "gpgDisableVerify": false,
+ "gpgProvider": "gpg2",
+ "downloadSourcePackages": false,
+ "skipLegacyPool": true,
+ "ppaDistributorID": "ubuntu",
+ "ppaCodename": "",
+ "skipContentsPublishing": false,
+ "FileSystemPublishEndpoints": {
+ "repo": {
+ "rootDir": "%s",
+ "linkMethod": "copy"
+ }
+ },
+ "S3PublishEndpoints": {},
+ "SwiftPublishEndpoints": {}
+}
+`, dbDir, repoDir)
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(filepath.Join(home, ".aptly.conf"), []byte(aptlyConf), 0600)
+}
+
+// downloadDebs downloads Deb packages.
+func downloadDebs(cfg PublishConfig, workDir string) error {
+ if cfg.Bucket == "" {
+ panic("cfg.Bucket has to be set")
+ }
+ if !strings.HasSuffix(workDir, string(filepath.Separator)) {
+ workDir += string(filepath.Separator)
+ }
+
+ var version string
+ if cfg.ReleaseMode.Mode == config.TagMode {
+ if cfg.ReleaseMode.IsBeta {
+ version = strings.ReplaceAll(cfg.Version, "-", "~")
+ } else {
+ version = cfg.Version
+ }
+ }
+ if version == "" {
+ panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
+ }
+
+ var sfx string
+ switch cfg.Edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = EnterpriseSfx
+ default:
+ return fmt.Errorf("unrecognized edition %q", cfg.Edition)
+ }
+
+ u := fmt.Sprintf("gs://%s/%s/%s/grafana%s_%s_*.deb*", cfg.Bucket,
+ strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
+ log.Printf("Downloading Deb packages %q...\n", u)
+ args := []string{
+ "-m",
+ "cp",
+ u,
+ workDir,
+ }
+ //nolint:gosec
+ cmd := exec.Command("gsutil", args...)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to download Deb packages %q: %w\n%s", u, err, output)
+ }
+
+ return nil
+}
+
+// UpdateDebRepo updates the Debian repository with the new release.
+func UpdateDebRepo(cfg PublishConfig, workDir string) error {
+ if cfg.ReleaseMode.Mode != config.TagMode {
+ panic(fmt.Sprintf("Unsupported version mode: %s", cfg.ReleaseMode.Mode))
+ }
+
+ if cfg.ReleaseMode.IsTest {
+ if cfg.Config.DebDBBucket == DefaultDebDBBucket {
+ return fmt.Errorf("in test-release mode, the default Deb DB bucket shouldn't be used")
+ }
+ if cfg.Config.DebRepoBucket == DefaultDebRepoBucket {
+ return fmt.Errorf("in test-release mode, the default Deb repo bucket shouldn't be used")
+ }
+ }
+
+ if err := downloadDebs(cfg, workDir); err != nil {
+ return err
+ }
+
+ repoName := "grafana"
+ if cfg.ReleaseMode.IsBeta {
+ repoName = "beta"
+ }
+
+ repoRoot, err := fsutil.CreateTempDir("deb-repo")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if err := os.RemoveAll(repoRoot); err != nil {
+ log.Printf("Failed to remove temporary directory %q: %s\n", repoRoot, err.Error())
+ }
+ }()
+
+ dbDir := filepath.Join(repoRoot, "db")
+ repoDir := filepath.Join(repoRoot, "repo")
+ tmpDir := filepath.Join(repoRoot, "tmp")
+ for _, dpath := range []string{dbDir, repoDir, tmpDir} {
+ if err := os.MkdirAll(dpath, 0750); err != nil {
+ return err
+ }
+ }
+
+ if err := writeAptlyConf(dbDir, repoDir); err != nil {
+ return err
+ }
+
+ // Download the Debian repo database
+ u := fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
+ log.Printf("Downloading Debian repo database from %s...\n", u)
+ //nolint:gosec
+ cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", u, dbDir)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to download Debian repo database: %w\n%s", err, output)
+ }
+
+ if err := addPkgsToRepo(cfg, workDir, tmpDir, repoName); err != nil {
+ return err
+ }
+
+ log.Println("Updating local Debian package repository...")
+ // Update published local repository. This assumes that there exists already a local, published repo.
+ for _, tp := range []string{"stable", "beta"} {
+ passArg := fmt.Sprintf("-passphrase-file=%s", cfg.GPGPassPath)
+ //nolint:gosec
+ cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp,
+ "filesystem:repo:grafana")
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return cli.Exit(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1)
+ }
+ }
+
+ // Update database in GCS
+ u = fmt.Sprintf("gs://%s/%s", cfg.DebDBBucket, strings.ToLower(string(cfg.Edition)))
+ if cfg.DryRun {
+ log.Printf("Simulating upload of Debian repo database to GCS (%s)\n", u)
+ } else {
+ log.Printf("Uploading Debian repo database to GCS (%s)...\n", u)
+ //nolint:gosec
+ cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return cli.Exit(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1)
+ }
+ }
+
+ // Update metadata and binaries in repository bucket
+ u = fmt.Sprintf("gs://%s/%s/deb", cfg.DebRepoBucket, strings.ToLower(string(cfg.Edition)))
+ grafDir := filepath.Join(repoDir, "grafana")
+ if cfg.DryRun {
+ log.Printf("Simulating upload of Debian repo resources to GCS (%s)\n", u)
+ } else {
+ log.Printf("Uploading Debian repo resources to GCS (%s)...\n", u)
+ //nolint:gosec
+ cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return cli.Exit(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1)
+ }
+ allRepoResources := fmt.Sprintf("%s/**/*", u)
+ log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources)
+ //nolint:gosec
+ cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return cli.Exit(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1)
+ }
+ }
+
+ return nil
+}
+
+func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error {
+ var sfx string
+ switch cfg.Edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = EnterpriseSfx
+ default:
+ return fmt.Errorf("unsupported edition %q", cfg.Edition)
+ }
+
+ log.Printf("Adding packages to Debian %q repo...\n", repoName)
+ // TODO: Be more specific about filename pattern
+ debs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s*.deb", sfx)))
+ if err != nil {
+ return err
+ }
+ for _, deb := range debs {
+ basename := filepath.Base(deb)
+ if strings.Contains(basename, "latest") {
+ continue
+ }
+
+ tgt := filepath.Join(tmpDir, basename)
+ if err := fs.CopyFile(deb, tgt); err != nil {
+ return err
+ }
+ }
+ // XXX: Adds too many packages in enterprise (Arve: What does this mean exactly?)
+ //nolint:gosec
+ cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return cli.Exit(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1)
+ }
+
+ return nil
+}
diff --git a/pkg/build/packaging/docs.go b/pkg/build/packaging/docs.go
new file mode 100644
index 00000000000..a723cb3d346
--- /dev/null
+++ b/pkg/build/packaging/docs.go
@@ -0,0 +1,2 @@
+// Package packaging holds functions and types for creating the tar.gz, deb, and rpm packages of Grafana.
+package packaging
diff --git a/pkg/build/packaging/errors.go b/pkg/build/packaging/errors.go
new file mode 100644
index 00000000000..c20b9edbfae
--- /dev/null
+++ b/pkg/build/packaging/errors.go
@@ -0,0 +1 @@
+package packaging
diff --git a/pkg/build/packaging/grafana.go b/pkg/build/packaging/grafana.go
new file mode 100644
index 00000000000..4500f72f569
--- /dev/null
+++ b/pkg/build/packaging/grafana.go
@@ -0,0 +1,1127 @@
+package packaging
+
+import (
+ "archive/tar"
+ "archive/zip"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/errutil"
+ "github.com/grafana/grafana/pkg/build/grafana"
+ "github.com/grafana/grafana/pkg/build/plugins"
+ "github.com/grafana/grafana/pkg/build/syncutil"
+ "github.com/grafana/grafana/pkg/infra/fs"
+)
+
+var (
+ ErrorNoBinaries = errors.New("no binaries found")
+ ErrorNoDebArch = errors.New("deb architecture not defined")
+ ErrorNoRPMArch = errors.New("rpm architecture not defined")
+)
+
+const (
+ maxAttempts = 3
+ enterpriseSfx = "-enterprise"
+ enterprise2Sfx = "-enterprise2"
+ DefaultDebDBBucket = "grafana-aptly-db"
+ DefaultDebRepoBucket = "grafana-repo"
+ DefaultRPMRepoBucket = "grafana-repo"
+ DefaultTTLSeconds = "300"
+)
+
+// PackageRegexp returns a regexp for matching packages corresponding to a certain Grafana edition.
+func PackageRegexp(edition config.Edition) *regexp.Regexp {
+ var sfx string
+ switch edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = "-enterprise"
+ case config.EditionEnterprise2:
+ sfx = "-enterprise2"
+ default:
+ panic(fmt.Sprintf("unrecognized edition %q", edition))
+ }
+ rePkg, err := regexp.Compile(fmt.Sprintf(`^grafana%s(?:-rpi)?[-_][^-_]+.*$`, sfx))
+ if err != nil {
+ panic(fmt.Sprintf("Failed to compile regexp: %s", err))
+ }
+
+ return rePkg
+}
+
+// PackageGrafana packages Grafana for various variants.
+func PackageGrafana(
+ ctx context.Context,
+ version string,
+ grafanaDir string,
+ cfg config.Config,
+ edition config.Edition,
+ variants []config.Variant,
+ shouldSign bool,
+ p syncutil.WorkerPool,
+) error {
+ if err := packageGrafana(ctx, edition, version, grafanaDir, variants, shouldSign, p); err != nil {
+ return err
+ }
+
+ if cfg.SignPackages {
+ if err := signRPMPackages(edition, cfg, grafanaDir); err != nil {
+ return err
+ }
+ }
+
+ if err := checksumPackages(grafanaDir, edition); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func packageGrafana(
+ ctx context.Context,
+ edition config.Edition,
+ version string,
+ grafanaDir string,
+ variants []config.Variant,
+ shouldSign bool,
+ p syncutil.WorkerPool,
+) error {
+ distDir := filepath.Join(grafanaDir, "dist")
+ exists, err := fs.Exists(distDir)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ log.Printf("directory %s doesn't exist - creating...", distDir)
+ //nolint
+ if err := os.MkdirAll(distDir, 0o755); err != nil {
+ return fmt.Errorf("couldn't create dist: %w", err)
+ }
+ }
+
+ var m pluginsManifest
+ manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json")
+ //nolint:gosec
+ manifestB, err := os.ReadFile(manifestPath)
+ if err != nil {
+ return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err)
+ }
+ if err := json.Unmarshal(manifestB, &m); err != nil {
+ return err
+ }
+
+ g, ctx := errutil.GroupWithContext(ctx)
+ for _, v := range variants {
+ packageVariant(ctx, v, edition, version, grafanaDir, shouldSign, g, p, m)
+ }
+ if err := g.Wait(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// packageVariant packages Grafana for a certain variant.
+func packageVariant(
+ ctx context.Context,
+ v config.Variant,
+ edition config.Edition,
+ version string,
+ grafanaDir string,
+ shouldSign bool,
+ g *errutil.Group,
+ p syncutil.WorkerPool,
+ m pluginsManifest,
+) {
+ p.Schedule(g.Wrap(func() error {
+ // We've experienced spurious packaging failures, so retry on failure.
+ i := 0
+ for {
+ if err := realPackageVariant(ctx, v, edition, version, grafanaDir, m, shouldSign); err != nil {
+ i++
+ if i < maxAttempts {
+ log.Printf("Packaging for variant %s, %s edition failed: %s, trying again", v, edition, err)
+ continue
+ }
+
+ return err
+ }
+
+ break
+ }
+
+ return nil
+ }))
+}
+
+// signRPMPackages signs the RPM packages.
+func signRPMPackages(edition config.Edition, cfg config.Config, grafanaDir string) error {
+ log.Printf("Signing %s RPM packages...", edition)
+ var sfx string
+ switch edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = enterpriseSfx
+ case config.EditionEnterprise2:
+ sfx = enterprise2Sfx
+ default:
+ panic(fmt.Sprintf("Unrecognized edition %s", edition))
+ }
+ rpms, err := filepath.Glob(filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-*.rpm", sfx)))
+ if err != nil {
+ return err
+ }
+
+ rpmArgs := append([]string{"--addsign"}, rpms...)
+ log.Printf("Invoking rpm with args: %+v", rpmArgs)
+ //nolint:gosec
+ cmd := exec.Command("rpm", rpmArgs...)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to sign RPM packages: %s", output)
+ }
+ if err := os.Remove(cfg.GPGPassPath); err != nil {
+ return fmt.Errorf("failed to remove %q: %w", cfg.GPGPassPath, err)
+ }
+
+ log.Printf("Verifying %s RPM packages...", edition)
+ // The output changed between rpm versions
+ reOutput := regexp.MustCompile("(?:digests signatures OK)|(?:pgp.+OK)")
+ for _, p := range rpms {
+ //nolint:gosec
+ cmd := exec.Command("rpm", "-K", p)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("failed to verify RPM signature: %w", err)
+ }
+
+ if !reOutput.Match(output) {
+ return fmt.Errorf("RPM package %q not verified: %s", p, output)
+ }
+ }
+
+ return nil
+}
+
+// checksumPackages generates package checksums with SHA-256.
+func checksumPackages(grafanaDir string, edition config.Edition) error {
+ log.Printf("Checksumming %s packages...", edition)
+ distDir := filepath.Join(grafanaDir, "dist")
+ rePkg := PackageRegexp(edition)
+ if err := filepath.Walk(distDir, func(fpath string, info os.FileInfo, err error) error {
+ if err != nil {
+ var pathErr *os.PathError
+ if errors.As(err, &pathErr) {
+ log.Printf("path error in walk function for file %q: %s", pathErr.Path, pathErr.Err.Error())
+ return nil
+ }
+ return fmt.Errorf("walking through dist folder failed: %w", err)
+ }
+
+ if info.IsDir() {
+ return nil
+ }
+
+ fname := filepath.Base(fpath)
+ if strings.HasSuffix(fname, ".sha256") || strings.HasSuffix(fname, ".version") || !rePkg.MatchString(fname) {
+ log.Printf("Ignoring non-package %q", fpath)
+ return nil
+ }
+
+ return shaFile(fpath)
+ }); err != nil {
+ return fmt.Errorf("checksumming packages in %q failed: %w", distDir, err)
+ }
+
+ log.Printf("Successfully checksummed %s packages", edition)
+ return nil
+}
+
+func shaFile(fpath string) error {
+ //nolint:gosec
+ fd, err := os.Open(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to open %q: %w", fpath, err)
+ }
+ defer func() {
+ if err := fd.Close(); err != nil {
+ log.Println(err)
+ }
+ }()
+
+ h := sha256.New()
+ _, err = io.Copy(h, fd)
+ if err != nil {
+ return fmt.Errorf("failed to read %q: %w", fpath, err)
+ }
+
+ //nolint:gosec
+ out, err := os.Create(fpath + ".sha256")
+ if err != nil {
+ return fmt.Errorf("failed to create %q: %w", fpath+".sha256", err)
+ }
+ defer func() {
+ if err := out.Close(); err != nil {
+ log.Println("failed to close file", out.Name())
+ }
+ }()
+
+ if _, err = fmt.Fprintf(out, "%x\n", h.Sum(nil)); err != nil {
+ return fmt.Errorf("failed to write %q: %w", out.Name(), err)
+ }
+
+ return nil
+}
+
+// createPackage creates a Linux package.
+func createPackage(srcDir string, options linuxPackageOptions) error {
+ binary := "grafana"
+ cliBinary := "grafana-cli"
+ serverBinary := "grafana-server"
+
+ packageRoot, err := os.MkdirTemp("", "grafana-linux-pack")
+ if err != nil {
+ return fmt.Errorf("failed to create temporary directory: %w", err)
+ }
+ defer func() {
+ if err := os.RemoveAll(packageRoot); err != nil {
+ log.Println(err)
+ }
+ }()
+
+ for _, dname := range []string{
+ options.homeDir,
+ options.configDir,
+ "etc/init.d",
+ options.etcDefaultPath,
+ "usr/lib/systemd/system",
+ "usr/sbin",
+ } {
+ dpath := filepath.Join(packageRoot, dname)
+ //nolint
+ if err := os.MkdirAll(dpath, 0o755); err != nil {
+ return fmt.Errorf("failed to make directory %q: %w", dpath, err)
+ }
+ }
+
+ if err := fs.CopyFile(filepath.Join(options.wrapperFilePath, binary),
+ filepath.Join(packageRoot, "usr", "sbin", binary)); err != nil {
+ return err
+ }
+ if err := fs.CopyFile(filepath.Join(options.wrapperFilePath, cliBinary),
+ filepath.Join(packageRoot, "usr", "sbin", cliBinary)); err != nil {
+ return err
+ }
+ if err := fs.CopyFile(filepath.Join(options.wrapperFilePath, serverBinary),
+ filepath.Join(packageRoot, "usr", "sbin", serverBinary)); err != nil {
+ return err
+ }
+ if err := fs.CopyFile(options.initdScriptSrc, filepath.Join(packageRoot, options.initdScriptFilePath)); err != nil {
+ return err
+ }
+ if err := fs.CopyFile(options.defaultFileSrc, filepath.Join(packageRoot, options.etcDefaultFilePath)); err != nil {
+ return err
+ }
+ if err := fs.CopyFile(options.systemdFileSrc, filepath.Join(packageRoot, options.systemdServiceFilePath)); err != nil {
+ return err
+ }
+ if err := fs.CopyRecursive(srcDir, filepath.Join(packageRoot, options.homeDir)); err != nil {
+ return err
+ }
+
+ if err := executeFPM(options, packageRoot, srcDir); err != nil {
+ return err
+ }
+
+ return nil
+}
+func executeFPM(options linuxPackageOptions, packageRoot, srcDir string) error {
+ name := "grafana"
+ vendor := "Grafana"
+ if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 {
+ vendor += " Enterprise"
+ if options.edition == config.EditionEnterprise2 {
+ name += enterprise2Sfx
+ } else if options.edition == config.EditionEnterprise {
+ name += enterpriseSfx
+ }
+ }
+
+ if options.goArch == config.ArchARM && options.goArm == "6" {
+ name += "-rpi"
+ }
+
+ pkgVersion := packageVersion(options)
+ args := []string{
+ "-s", "dir",
+ "--description", "Grafana",
+ "-C", packageRoot,
+ "--url", "https://grafana.com",
+ "--maintainer", "contact@grafana.com",
+ "--config-files", options.initdScriptFilePath,
+ "--config-files", options.etcDefaultFilePath,
+ "--config-files", options.systemdServiceFilePath,
+ "--after-install", options.postinstSrc,
+ "--version", pkgVersion,
+ "-p", "dist/",
+ "--name", name,
+ "--vendor", vendor,
+ "-a", string(options.packageArch),
+ }
+ if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 || options.goArch == config.ArchARMv6 {
+ args = append(args, "--conflicts", "grafana")
+ }
+ if options.edition == config.EditionOSS {
+ args = append(args, "--license", "\"AGPLv3\"")
+ }
+ switch options.packageType {
+ case packageTypeRpm:
+ args = append(args, "-t", "rpm", "--rpm-posttrans", "packaging/rpm/control/posttrans")
+ args = append(args, "--rpm-digest", "sha256")
+ case packageTypeDeb:
+ args = append(args, "-t", "deb", "--deb-no-default-config-files")
+ default:
+ panic(fmt.Sprintf("Unrecognized package type %d", options.packageType))
+ }
+ for _, dep := range options.depends {
+ args = append(args, "--depends", dep)
+ }
+ args = append(args, ".")
+
+ distDir := filepath.Join(options.grafanaDir, "dist")
+ log.Printf("Generating package in %q (source directory %q)", distDir, srcDir)
+
+ cmdStr := "fpm"
+ for _, arg := range args {
+ if strings.Contains(arg, " ") {
+ arg = fmt.Sprintf("'%s'", arg)
+ }
+ cmdStr += fmt.Sprintf(" %s", arg)
+ }
+ log.Printf("Creating %s package: %s...", options.packageType, cmdStr)
+ const rvmPath = "/etc/profile.d/rvm.sh"
+ exists, err := fs.Exists(rvmPath)
+ if err != nil {
+ return err
+ }
+ if exists {
+ cmdStr = fmt.Sprintf("source %q && %s", rvmPath, cmdStr)
+ log.Printf("Sourcing %q before running fpm", rvmPath)
+ }
+ //nolint:gosec
+ cmd := exec.Command("/bin/bash", "-c", cmdStr)
+ cmd.Dir = options.grafanaDir
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to run fpm: %s", output)
+ }
+
+ return nil
+}
+
+// copyPubDir copies public/ from grafanaDir to tmpDir.
+func copyPubDir(grafanaDir, tmpDir string) error {
+ srcPubDir := filepath.Join(grafanaDir, "public")
+ tgtPubDir := filepath.Join(tmpDir, "public")
+ log.Printf("Copying %q to %q...", srcPubDir, tgtPubDir)
+ if err := fs.CopyRecursive(srcPubDir, tgtPubDir); err != nil {
+ return fmt.Errorf("failed to copy %q to %q: %w", srcPubDir, tgtPubDir, err)
+ }
+
+ return nil
+}
+
+// copyBinaries copies binaries from grafanaDir into tmpDir.
+func copyBinaries(grafanaDir, tmpDir string, args grafana.BuildArgs, edition config.Edition) error {
+ tgtDir := filepath.Join(tmpDir, "bin")
+ //nolint
+ if err := os.MkdirAll(tgtDir, 0o755); err != nil {
+ return fmt.Errorf("failed to make directory %q: %w", tgtDir, err)
+ }
+
+ binDir := filepath.Join(grafanaDir, "bin", grafana.BinaryFolder(edition, args))
+
+ files, err := os.ReadDir(binDir)
+ if err != nil {
+ return fmt.Errorf("failed to list files in %q: %w", binDir, err)
+ }
+
+ if len(files) == 0 {
+ return fmt.Errorf("%w in %s", ErrorNoBinaries, binDir)
+ }
+
+ for _, file := range files {
+ srcPath := filepath.Join(binDir, file.Name())
+ tgtPath := filepath.Join(tgtDir, file.Name())
+
+ if err := fs.CopyFile(srcPath, tgtPath); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// copyConfFiles copies configuration files from grafanaDir into tmpDir.
+func copyConfFiles(grafanaDir, tmpDir string) error {
+ //nolint:gosec
+ if err := os.MkdirAll(filepath.Join(tmpDir, "conf"), 0o755); err != nil {
+ return fmt.Errorf("failed to create dir %q: %w", filepath.Join(tmpDir, "conf"), err)
+ }
+
+ confDir := filepath.Join(grafanaDir, "conf")
+ infos, err := os.ReadDir(confDir)
+ if err != nil {
+ return fmt.Errorf("failed to list files in %q: %w", confDir, err)
+ }
+ for _, info := range infos {
+ fpath := filepath.Join(confDir, info.Name())
+ if err := fs.CopyRecursive(fpath, filepath.Join(tmpDir, "conf", info.Name())); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// copyPlugins copies plugins from grafanaDir into tmpDir.
+func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir string, m pluginsManifest, shouldSign bool) error {
+ log.Printf("Copying plugins for package variant %s...", v)
+
+ variant2Sfx := map[config.Variant]string{
+ config.VariantLinuxAmd64: "linux_amd64",
+ config.VariantDarwinAmd64: "darwin_amd64",
+ config.VariantWindowsAmd64: "windows_amd64.exe",
+ }
+
+ tgtDir := filepath.Join(tmpDir, "plugins-bundled")
+ exists, err := fs.Exists(tgtDir)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ //nolint:gosec
+ if err := os.MkdirAll(tgtDir, 0o755); err != nil {
+ return err
+ }
+ }
+ pluginsDir := filepath.Join(grafanaDir, "plugins-bundled")
+
+ // External plugins.
+ for _, pm := range m.Plugins {
+ srcDir := filepath.Join(pluginsDir, fmt.Sprintf("%s-%s", pm.Name, pm.Version))
+ dstDir := filepath.Join(tgtDir, fmt.Sprintf("%s-%s", pm.Name, pm.Version))
+ log.Printf("Copying external plugin %q to %q...", srcDir, dstDir)
+
+ //nolint:gosec
+ jsonB, err := os.ReadFile(filepath.Join(srcDir, "plugin.json"))
+ if err != nil {
+ return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err)
+ }
+ var plugJSON map[string]interface{}
+ if err := json.Unmarshal(jsonB, &plugJSON); err != nil {
+ return err
+ }
+
+ plugExe, ok := plugJSON["executable"].(string)
+ var wantExe string
+ if ok && strings.TrimSpace(plugExe) != "" {
+ sfx := variant2Sfx[v]
+ if sfx == "" {
+ log.Printf("External plugin %s-%s doesn't have an executable for variant %s - ignoring",
+ pm.Name, pm.Version, v)
+ continue
+ }
+
+ wantExe = fmt.Sprintf("%s_%s", plugExe, sfx)
+ log.Printf("The external plugin should contain an executable %q", wantExe)
+ exists, err := fs.Exists(filepath.Join(srcDir, wantExe))
+ if err != nil {
+ return err
+ }
+ if !exists {
+ log.Printf("External plugin %s-%s doesn't have an executable of the right format: %q - ignoring",
+ pm.Name, pm.Version, wantExe)
+ continue
+ }
+ }
+
+ if err := filepath.Walk(srcDir, func(pth string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+
+ log.Printf("Handling %q", pth)
+
+ relPath := strings.TrimPrefix(pth, srcDir)
+ relPath = strings.TrimPrefix(relPath, "/")
+ dstPath := filepath.Join(dstDir, relPath)
+
+ if info.IsDir() {
+ log.Printf("Making directory %q", dstPath)
+ //nolint:gosec
+ return os.MkdirAll(dstPath, info.Mode())
+ }
+
+ if wantExe != "" {
+ m, err := regexp.MatchString(fmt.Sprintf(`^%s_[^/]+$`, plugExe), relPath)
+ if err != nil {
+ return err
+ }
+ if m && relPath != wantExe {
+ // Ignore other executable variants
+ log.Printf("Ignoring executable variant %q", pth)
+ return nil
+ }
+ }
+
+ log.Printf("Copying %q to %q", pth, dstPath)
+ return fs.CopyFile(pth, dstPath)
+ }); err != nil {
+ return fmt.Errorf("failed to copy external plugin %q to %q: %w", srcDir, dstDir, err)
+ }
+
+ if shouldSign {
+ if err := plugins.BuildManifest(ctx, dstDir, true); err != nil {
+ return fmt.Errorf("failed to generate signed manifest for external plugin %q: %w", dstDir, err)
+ }
+ }
+ }
+
+ return copyInternalPlugins(pluginsDir, tmpDir)
+}
+
+func copyInternalPlugins(pluginsDir, tmpDir string) error {
+ tgtDir := filepath.Join(tmpDir, "plugins-bundled", "internal")
+ srcDir := filepath.Join(pluginsDir, "dist")
+
+ exists, err := fs.Exists(tgtDir)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ //nolint:gosec
+ if err := os.MkdirAll(tgtDir, 0o755); err != nil {
+ return err
+ }
+ }
+
+ // Copy over internal plugins.
+ fis, err := os.ReadDir(srcDir)
+ if err != nil {
+ return fmt.Errorf("failed to list internal plugins in %q: %w", srcDir, err)
+ }
+ for _, fi := range fis {
+ srcPath := filepath.Join(srcDir, fi.Name())
+ if !fi.IsDir() {
+ log.Printf("Ignoring non-directory %q", srcPath)
+ continue
+ }
+
+ dstPath := filepath.Join(tgtDir, fi.Name())
+ log.Printf("Copying internal plugin %q to %q...", srcPath, dstPath)
+ if err := fs.CopyRecursive(srcPath, dstPath); err != nil {
+ return fmt.Errorf("failed to copy %q to %q: %w", srcPath, dstPath, err)
+ }
+ }
+
+ return nil
+}
+
+func realPackageVariant(ctx context.Context, v config.Variant, edition config.Edition, version, grafanaDir string,
+ m pluginsManifest, shouldSign bool) error {
+ log.Printf("Packaging Grafana %s for %s...", edition, v)
+
+ enableDeb := false
+ enableRpm := false
+ switch v {
+ case config.VariantLinuxAmd64:
+ enableDeb = true
+ enableRpm = true
+ case config.VariantArmV6:
+ enableDeb = true
+ case config.VariantArmV7:
+ enableDeb = true
+ enableRpm = true
+ case config.VariantArm64:
+ enableDeb = true
+ enableRpm = true
+ default:
+ }
+
+ tmpDir, err := os.MkdirTemp("", "")
+ if err != nil {
+ return fmt.Errorf("failed to create temporary directory: %w", err)
+ }
+ defer func() {
+ if err := os.RemoveAll(tmpDir); err != nil {
+ log.Println(err)
+ }
+ }()
+
+ args := grafana.VariantBuildArgs(v)
+
+ if err := copyPubDir(grafanaDir, tmpDir); err != nil {
+ return err
+ }
+ if err := copyBinaries(grafanaDir, tmpDir, args, edition); err != nil {
+ return err
+ }
+ if err := copyConfFiles(grafanaDir, tmpDir); err != nil {
+ return err
+ }
+ if err := copyPlugins(ctx, v, grafanaDir, tmpDir, m, shouldSign); err != nil {
+ return err
+ }
+
+ if v == config.VariantWindowsAmd64 {
+ toolsDir := filepath.Join(tmpDir, "tools")
+ //nolint:gosec
+ if err := os.MkdirAll(toolsDir, 0o755); err != nil {
+ return fmt.Errorf("failed to create tools dir %q: %w", toolsDir, err)
+ }
+
+ if err := fs.CopyFile("/usr/local/go/lib/time/zoneinfo.zip",
+ filepath.Join(tmpDir, "tools", "zoneinfo.zip")); err != nil {
+ return err
+ }
+ }
+
+ if err := os.WriteFile(filepath.Join(tmpDir, "VERSION"), []byte(version), 0664); err != nil {
+ return fmt.Errorf("failed to write %s/VERSION: %w", tmpDir, err)
+ }
+
+ if err := createArchive(tmpDir, edition, v, version, grafanaDir); err != nil {
+ return err
+ }
+
+ if enableDeb {
+ if args.DebArch == "" {
+ return fmt.Errorf("%w for %s", ErrorNoDebArch, v)
+ }
+
+ if err := createPackage(tmpDir, linuxPackageOptions{
+ edition: edition,
+ version: version,
+ grafanaDir: grafanaDir,
+ goArch: args.GoArch,
+ goArm: args.GoArm,
+ packageType: packageTypeDeb,
+ packageArch: args.DebArch,
+ homeDir: "/usr/share/grafana",
+ homeBinDir: "/usr/share/grafana/bin",
+ binPath: "/usr/sbin",
+ configDir: "/etc/grafana",
+ etcDefaultPath: "/etc/default",
+ etcDefaultFilePath: "/etc/default/grafana-server",
+ initdScriptFilePath: "/etc/init.d/grafana-server",
+ systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service",
+ postinstSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "postinst"),
+ initdScriptSrc: filepath.Join(grafanaDir, "packaging", "deb", "init.d", "grafana-server"),
+ defaultFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "default", "grafana-server"),
+ systemdFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "systemd", "grafana-server.service"),
+ wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"),
+ depends: []string{"adduser", "libfontconfig1"},
+ }); err != nil {
+ return err
+ }
+ }
+
+ if !enableRpm {
+ return nil
+ }
+
+ if args.RPMArch == "" {
+ return fmt.Errorf("%w for %s", ErrorNoRPMArch, v)
+ }
+
+ if err := createPackage(tmpDir, linuxPackageOptions{
+ edition: edition,
+ version: version,
+ grafanaDir: grafanaDir,
+ goArch: args.GoArch,
+ packageType: packageTypeRpm,
+ packageArch: args.RPMArch,
+ homeDir: "/usr/share/grafana",
+ homeBinDir: "/usr/share/grafana/bin",
+ binPath: "/usr/sbin",
+ configDir: "/etc/grafana",
+ etcDefaultPath: "/etc/sysconfig",
+ etcDefaultFilePath: "/etc/sysconfig/grafana-server",
+ initdScriptFilePath: "/etc/init.d/grafana-server",
+ systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service",
+ postinstSrc: filepath.Join(grafanaDir, "packaging", "rpm", "control", "postinst"),
+ initdScriptSrc: filepath.Join(grafanaDir, "packaging", "rpm", "init.d", "grafana-server"),
+ defaultFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "sysconfig", "grafana-server"),
+ systemdFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "systemd", "grafana-server.service"),
+ wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"),
+ // chkconfig is depended on since our systemd service wraps a SysV init script, and that requires chkconfig
+ depends: []string{"/sbin/service", "chkconfig", "fontconfig", "freetype", "urw-fonts"},
+ }); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// pluginManifest has details of an external plugin package.
+type pluginManifest struct {
+ Name string `json:"name"`
+ Version string `json:"version"`
+ Checksum string `json:"checksum"`
+}
+
+// pluginsManifest represents a manifest of Grafana's external plugins.
+type pluginsManifest struct {
+ Plugins []pluginManifest `json:"plugins"`
+}
+
+// packageVersion converts a Grafana version into the corresponding package version.
+func packageVersion(options linuxPackageOptions) string {
+ verComponents := strings.Split(options.version, "-")
+ if len(verComponents) > 2 {
+ panic(fmt.Sprintf("Version string contains more than one hyphen: %q", options.version))
+ }
+
+ switch options.packageType {
+ case packageTypeDeb, packageTypeRpm:
+ if len(verComponents) > 1 {
+ // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this
+ // is considered of lower lexical value than the empty character, and this way pre-release versions are
+ // considered to be of a lower version than the final version (which lacks this suffix).
+ return fmt.Sprintf("%s~%s", verComponents[0], verComponents[1])
+ }
+
+ return options.version
+ default:
+ panic(fmt.Sprintf("Unrecognized package type %s", options.packageType))
+ }
+}
+
+type packageType int
+
+func (pt packageType) String() string {
+ switch pt {
+ case packageTypeDeb:
+ return "Debian"
+ case packageTypeRpm:
+ return "RPM"
+ default:
+ panic(fmt.Sprintf("Unrecognized package type %d", pt))
+ }
+}
+
+const (
+ packageTypeDeb packageType = iota
+ packageTypeRpm
+)
+
+type linuxPackageOptions struct {
+ edition config.Edition
+ packageType packageType
+ version string
+ grafanaDir string
+ goArch config.Architecture
+ goArm string
+ packageArch config.Architecture
+ homeDir string
+ homeBinDir string
+ binPath string
+ configDir string
+ etcDefaultPath string
+ etcDefaultFilePath string
+ initdScriptFilePath string
+ systemdServiceFilePath string
+ postinstSrc string
+ initdScriptSrc string
+ defaultFileSrc string
+ systemdFileSrc string
+ wrapperFilePath string
+
+ depends []string
+}
+
+// createArchive makes a distribution archive.
+func createArchive(srcDir string, edition config.Edition, v config.Variant, version, grafanaDir string) error {
+ distDir := filepath.Join(grafanaDir, "dist")
+ exists, err := fs.Exists(distDir)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ log.Printf("directory %s doesn't exist - creating...", distDir)
+ //nolint:gosec
+ if err := os.MkdirAll(distDir, 0o755); err != nil {
+ return fmt.Errorf("couldn't create dist: %w", err)
+ }
+ }
+ sfx := ""
+ if edition == config.EditionEnterprise2 {
+ sfx = enterprise2Sfx
+ } else if edition == config.EditionEnterprise {
+ sfx = enterpriseSfx
+ }
+ if v != config.VariantWindowsAmd64 {
+ return createTarball(srcDir, version, string(v), sfx, grafanaDir)
+ }
+
+ return createZip(srcDir, version, string(v), sfx, grafanaDir)
+}
+
+func createZip(srcDir, version, variantStr, sfx, grafanaDir string) error {
+ fpath := filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-%s.%s.zip", sfx, version, variantStr))
+ //nolint:gosec
+ tgt, err := os.Create(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to create %q: %w", fpath, err)
+ }
+ defer func() {
+ if err := tgt.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
+ log.Println(err)
+ }
+ }()
+
+ //nolint:gosec
+ if err := os.Chmod(fpath, 0664); err != nil {
+ return fmt.Errorf("failed to set permissions on %q: %w", fpath, err)
+ }
+ zipWriter := zip.NewWriter(tgt)
+ defer func() {
+ if err := zipWriter.Close(); err != nil {
+ log.Println(err)
+ }
+ }()
+
+ for _, fname := range []string{"LICENSE", "README.md", "NOTICE.md"} {
+ fpath := filepath.Join(grafanaDir, fname)
+ fi, err := os.Lstat(fpath)
+ if err != nil {
+ return fmt.Errorf("couldn't stat %q: %w", fpath, err)
+ }
+ hdr, err := zip.FileInfoHeader(fi)
+ if err != nil {
+ return fmt.Errorf("failed to open zip header: %w", err)
+ }
+ // Enable compression, as it's disabled by default
+ hdr.Method = zip.Deflate
+ hdr.Name = fmt.Sprintf("grafana-%s/%s", version, fname)
+ w, err := zipWriter.CreateHeader(hdr)
+ if err != nil {
+ return fmt.Errorf("failed writing zip header: %w", err)
+ }
+ //nolint:gosec
+ src, err := os.Open(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to open %q: %w", fname, err)
+ }
+ if _, err := io.Copy(w, src); err != nil {
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ return fmt.Errorf("failed writing zip entry: %w", err)
+ }
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ }
+ if err := filepath.Walk(srcDir, func(fpath string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if fpath == srcDir {
+ return nil
+ }
+
+ hdr, err := zip.FileInfoHeader(fi)
+ if err != nil {
+ return fmt.Errorf("failed to open zip header: %s", err)
+ }
+ // Enable compression, as it's disabled by default
+ hdr.Method = zip.Deflate
+ hdr.Name = fmt.Sprintf("grafana-%s/%s", version, strings.TrimPrefix(fpath, fmt.Sprintf("%s/", srcDir)))
+ if fi.IsDir() {
+ // A trailing slash means it's a directory
+ if hdr.Name[len(hdr.Name)-1] != '/' {
+ hdr.Name += "/"
+ }
+ }
+ w, err := zipWriter.CreateHeader(hdr)
+ if err != nil {
+ return fmt.Errorf("failed writing zip header: %s", err)
+ }
+ if fi.IsDir() {
+ return nil
+ }
+
+ //nolint:gosec
+ src, err := os.Open(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to open %q: %w", fpath, err)
+ }
+ if _, err := io.Copy(w, src); err != nil {
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ return fmt.Errorf("failed writing zip entry: %w", err)
+ }
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ return nil
+ }); err != nil {
+ return fmt.Errorf("failed to traverse directory %q: %w", srcDir, err)
+ }
+
+ if err := zipWriter.Close(); err != nil {
+ return fmt.Errorf("failed writing %q: %w", fpath, err)
+ }
+ if err := tgt.Close(); err != nil {
+ return fmt.Errorf("failed writing %q: %w", fpath, err)
+ }
+
+ log.Printf("Successfully created %q", fpath)
+ return nil
+}
+
+// nolint
+func createTarball(srcDir, version, variantStr, sfx, grafanaDir string) error {
+ fpath := filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-%s.%s.tar.gz", sfx, version, variantStr))
+ //nolint:gosec
+ tgt, err := os.Create(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to create %q: %w", fpath, err)
+ }
+ defer func() {
+ if err := tgt.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
+ log.Println(err)
+ }
+ }()
+
+ //nolint:gosec
+ if err := os.Chmod(fpath, 0664); err != nil {
+ return fmt.Errorf("failed to set permissions on %q: %w", fpath, err)
+ }
+ gzWriter, err := gzip.NewWriterLevel(tgt, gzip.BestCompression)
+ if err != nil {
+ return fmt.Errorf("failed to create gzip writer: %w", err)
+ }
+ defer func() {
+ if err := gzWriter.Close(); err != nil {
+ log.Println(err)
+ }
+ }()
+ tarWriter := tar.NewWriter(gzWriter)
+ defer func() {
+ if err := tarWriter.Close(); err != nil {
+ log.Println(err)
+ }
+ }()
+
+ for _, fname := range []string{"LICENSE", "README.md", "NOTICE.md"} {
+ fpath := filepath.Join(grafanaDir, fname)
+ fi, err := os.Lstat(fpath)
+ if err != nil {
+ return fmt.Errorf("couldn't stat %q: %w", fpath, err)
+ }
+ hdr, err := tar.FileInfoHeader(fi, "")
+ if err != nil {
+ return fmt.Errorf("failed getting tar header: %w", err)
+ }
+ hdr.Name = fmt.Sprintf("grafana-%s/%s", version, fname)
+ if err := tarWriter.WriteHeader(hdr); err != nil {
+ return fmt.Errorf("failed writing tar header: %w", err)
+ }
+ //nolint:gosec
+ src, err := os.Open(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to open %q: %w", fname, err)
+ }
+ if _, err := io.Copy(tarWriter, src); err != nil {
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ return fmt.Errorf("failed writing tar entry: %w", err)
+ }
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ }
+ if err := filepath.Walk(srcDir, func(fpath string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if fpath == srcDir {
+ return nil
+ }
+
+ linkTgt := ""
+ if fi.Mode()&os.ModeSymlink != 0 {
+ log.Printf("reading link '%s'", fpath)
+ linkTgt, err = os.Readlink(fpath)
+ if err != nil {
+ return err
+ }
+ linkTgt = fmt.Sprintf("grafana-%s/%s", version, linkTgt)
+ }
+
+ hdr, err := tar.FileInfoHeader(fi, linkTgt)
+ if err != nil {
+ return fmt.Errorf("failed getting tar header: %w", err)
+ }
+ hdr.Name = fmt.Sprintf("grafana-%s/%s", version, strings.TrimPrefix(fpath, fmt.Sprintf("%s/", srcDir)))
+ if err := tarWriter.WriteHeader(hdr); err != nil {
+ return fmt.Errorf("failed writing tar header: %w", err)
+ }
+ if fi.IsDir() {
+ return nil
+ }
+
+ //nolint:gosec
+ src, err := os.Open(fpath)
+ if err != nil {
+ return fmt.Errorf("failed to open %q: %w", fpath, err)
+ }
+ if _, err := io.Copy(tarWriter, src); err != nil {
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+ return fmt.Errorf("failed writing tar entry: %w", err)
+ }
+ if err := src.Close(); err != nil {
+ log.Println(err)
+ }
+
+ return nil
+ }); err != nil {
+ return fmt.Errorf("failed to traverse directory %q: %w", srcDir, err)
+ }
+
+ if err := tarWriter.Close(); err != nil {
+ return fmt.Errorf("failed writing %q: %w", fpath, err)
+ }
+ if err := gzWriter.Close(); err != nil {
+ return fmt.Errorf("failed writing %q: %w", fpath, err)
+ }
+ if err := tgt.Close(); err != nil {
+ return fmt.Errorf("failed writing %q: %w", fpath, err)
+ }
+
+ st, err := os.Stat(fpath)
+ if err != nil {
+ return err
+ }
+ perms := st.Mode() & os.ModePerm
+ log.Printf("Successfully created package %q (permissions: %o)", fpath, perms)
+
+ return nil
+}
diff --git a/pkg/build/packaging/grafana_test.go b/pkg/build/packaging/grafana_test.go
new file mode 100644
index 00000000000..8f143893085
--- /dev/null
+++ b/pkg/build/packaging/grafana_test.go
@@ -0,0 +1,22 @@
+package packaging_test
+
+import (
+ "testing"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/packaging"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestPackageRegexp(t *testing.T) {
+ t.Run("It should match enterprise2 packages", func(t *testing.T) {
+ rgx := packaging.PackageRegexp(config.EditionEnterprise2)
+ matches := []string{
+ "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz",
+ "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz.sha256",
+ }
+ for _, v := range matches {
+ assert.Truef(t, rgx.MatchString(v), "'%s' should match regex '%s'", v, rgx.String())
+ }
+ })
+}
diff --git a/pkg/build/packaging/rpm.go b/pkg/build/packaging/rpm.go
new file mode 100644
index 00000000000..a4e8e557221
--- /dev/null
+++ b/pkg/build/packaging/rpm.go
@@ -0,0 +1,370 @@
+package packaging
+
+import (
+ "bytes"
+ "crypto"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ // Consider switching this over to a community fork unless there is
+ // an option to move us away from OpenPGP.
+ "golang.org/x/crypto/openpgp" //nolint:staticcheck
+ "golang.org/x/crypto/openpgp/armor" //nolint:staticcheck
+ "golang.org/x/crypto/openpgp/packet" //nolint:staticcheck
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/fsutil"
+ "github.com/grafana/grafana/pkg/infra/fs"
+)
+
+// UpdateRPMRepo updates the RPM repository with the new release.
+func UpdateRPMRepo(cfg PublishConfig, workDir string) error {
+ if cfg.ReleaseMode.Mode != config.TagMode {
+ panic(fmt.Sprintf("Unsupported version mode %s", cfg.ReleaseMode.Mode))
+ }
+
+ if cfg.ReleaseMode.IsTest && cfg.Config.RPMRepoBucket == DefaultRPMRepoBucket {
+ return fmt.Errorf("in test-release mode, the default RPM repo bucket shouldn't be used")
+ }
+
+ if err := downloadRPMs(cfg, workDir); err != nil {
+ return err
+ }
+
+ repoRoot, err := fsutil.CreateTempDir("rpm-repo")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if err := os.RemoveAll(repoRoot); err != nil {
+ log.Printf("Failed to remove %q: %s\n", repoRoot, err.Error())
+ }
+ }()
+
+ repoName := "rpm"
+ if cfg.ReleaseMode.IsBeta {
+ repoName = "rpm-beta"
+ }
+ folderURI := fmt.Sprintf("gs://%s/%s/%s", cfg.RPMRepoBucket, strings.ToLower(string(cfg.Edition)), repoName)
+
+ // Download the RPM database
+ log.Printf("Downloading RPM database from GCS (%s)...\n", folderURI)
+ //nolint:gosec
+ cmd := exec.Command("gsutil", "-m", "rsync", "-r", "-d", folderURI, repoRoot)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to download RPM database from GCS: %w\n%s", err, output)
+ }
+
+ // Add the new release to the repo
+ var sfx string
+ switch cfg.Edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = EnterpriseSfx
+ default:
+ return fmt.Errorf("unsupported edition %q", cfg.Edition)
+ }
+ allRPMs, err := filepath.Glob(filepath.Join(workDir, fmt.Sprintf("grafana%s-*.rpm", sfx)))
+ if err != nil {
+ return fmt.Errorf("failed to list RPMs in %q: %w", workDir, err)
+ }
+ rpms := []string{}
+ for _, rpm := range allRPMs {
+ if strings.Contains(rpm, "-latest") {
+ continue
+ }
+
+ rpms = append(rpms, rpm)
+ }
+ // XXX: What does the following comment mean?
+ // adds to many files for enterprise
+ for _, rpm := range rpms {
+ if err := fs.CopyFile(rpm, filepath.Join(repoRoot, filepath.Base(rpm))); err != nil {
+ return err
+ }
+ }
+
+ //nolint:gosec
+ cmd = exec.Command("createrepo", repoRoot)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to create repo at %q: %w\n%s", repoRoot, err, output)
+ }
+
+ if err := signRPMRepo(repoRoot, cfg); err != nil {
+ return err
+ }
+
+ // Update the repo in GCS
+
+ // Sync packages first to avoid cache misses
+ if cfg.DryRun {
+ log.Printf("Simulating upload of RPMs to GCS (%s)\n", folderURI)
+ } else {
+ log.Printf("Uploading RPMs to GCS (%s)...\n", folderURI)
+ args := []string{"-m", "cp"}
+ args = append(args, rpms...)
+ args = append(args, folderURI)
+ //nolint:gosec
+ cmd = exec.Command("gsutil", args...)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to upload RPMs to GCS: %w\n%s", err, output)
+ }
+ }
+
+ if cfg.DryRun {
+ log.Printf("Simulating upload of RPM repo metadata to GCS (%s)\n", folderURI)
+ } else {
+ log.Printf("Uploading RPM repo metadata to GCS (%s)...\n", folderURI)
+ //nolint:gosec
+ cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", repoRoot, folderURI)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to upload RPM repo metadata to GCS: %w\n%s", err, output)
+ }
+ allRepoResources := fmt.Sprintf("%s/**/*", folderURI)
+ log.Printf("Setting cache ttl for RPM repo resources on GCS (%s)...\n", allRepoResources)
+ //nolint:gosec
+ cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to set cache ttl for RPM repo resources on GCS: %w\n%s", err, output)
+ }
+ }
+
+ return nil
+}
+
+// downloadRPMs downloads RPM packages.
+func downloadRPMs(cfg PublishConfig, workDir string) error {
+ if !strings.HasSuffix(workDir, string(filepath.Separator)) {
+ workDir += string(filepath.Separator)
+ }
+ var version string
+ if cfg.ReleaseMode.Mode == config.TagMode {
+ if cfg.ReleaseMode.IsBeta {
+ version = strings.ReplaceAll(cfg.Version, "-", "~")
+ } else {
+ version = cfg.Version
+ }
+ }
+ if version == "" {
+ panic(fmt.Sprintf("Unrecognized version mode %s", cfg.ReleaseMode.Mode))
+ }
+
+ var sfx string
+ switch cfg.Edition {
+ case config.EditionOSS:
+ case config.EditionEnterprise:
+ sfx = EnterpriseSfx
+ default:
+ return fmt.Errorf("unrecognized edition %q", cfg.Edition)
+ }
+
+ u := fmt.Sprintf("gs://%s/%s/%s/grafana%s-%s-*.*.rpm*", cfg.Bucket,
+ strings.ToLower(string(cfg.Edition)), ReleaseFolder, sfx, version)
+ log.Printf("Downloading RPM packages %q...\n", u)
+ args := []string{
+ "-m",
+ "cp",
+ u,
+ workDir,
+ }
+ //nolint:gosec
+ cmd := exec.Command("gsutil", args...)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to download RPM packages %q: %w\n%s", u, err, output)
+ }
+
+ return nil
+}
+
+func getPublicKey(cfg PublishConfig) (*packet.PublicKey, error) {
+ f, err := os.Open(cfg.GPGPublicKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPublicKey, err)
+ }
+ defer func(f *os.File) {
+ err := f.Close()
+ if err != nil {
+ return
+ }
+ }(f)
+
+ block, err := armor.Decode(f)
+ if err != nil {
+ return nil, err
+ }
+
+ if block.Type != openpgp.PublicKeyType {
+ return nil, fmt.Errorf("invalid public key block type: %q", block.Type)
+ }
+
+ packetReader := packet.NewReader(block.Body)
+ pkt, err := packetReader.Next()
+ if err != nil {
+ return nil, err
+ }
+
+ key, ok := pkt.(*packet.PublicKey)
+ if !ok {
+ return nil, fmt.Errorf("got non-public key from packet reader: %T", pkt)
+ }
+
+ return key, nil
+}
+
+func getPrivateKey(cfg PublishConfig) (*packet.PrivateKey, error) {
+ f, err := os.Open(cfg.GPGPrivateKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open %q: %w", cfg.GPGPrivateKey, err)
+ }
+ defer func(f *os.File) {
+ err := f.Close()
+ if err != nil {
+ return
+ }
+ }(f)
+
+ passphraseB, err := os.ReadFile(cfg.GPGPassPath)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read %q: %w", cfg.GPGPrivateKey, err)
+ }
+ passphraseB = bytes.TrimSuffix(passphraseB, []byte("\n"))
+
+ block, err := armor.Decode(f)
+ if err != nil {
+ return nil, err
+ }
+
+ if block.Type != openpgp.PrivateKeyType {
+ return nil, fmt.Errorf("invalid private key block type: %q", block.Type)
+ }
+
+ packetReader := packet.NewReader(block.Body)
+ pkt, err := packetReader.Next()
+ if err != nil {
+ return nil, err
+ }
+
+ key, ok := pkt.(*packet.PrivateKey)
+ if !ok {
+ return nil, fmt.Errorf("got non-private key from packet reader: %T", pkt)
+ }
+
+ if err := key.Decrypt(passphraseB); err != nil {
+ return nil, fmt.Errorf("failed to decrypt private key: %w", err)
+ }
+ return key, nil
+}
+
+// signRPMRepo signs an RPM repository using PGP.
+// The signature gets written to the file repodata/repomd.xml.asc.
+func signRPMRepo(repoRoot string, cfg PublishConfig) error {
+ if cfg.GPGPublicKey == "" || cfg.GPGPrivateKey == "" {
+ return fmt.Errorf("private or public key is empty")
+ }
+
+ log.Printf("Signing RPM repo")
+
+ pubKey, err := getPublicKey(cfg)
+ if err != nil {
+ return err
+ }
+
+ privKey, err := getPrivateKey(cfg)
+ if err != nil {
+ return err
+ }
+
+ pcfg := packet.Config{
+ DefaultHash: crypto.SHA256,
+ DefaultCipher: packet.CipherAES256,
+ DefaultCompressionAlgo: packet.CompressionZLIB,
+ CompressionConfig: &packet.CompressionConfig{
+ Level: 9,
+ },
+ RSABits: 4096,
+ }
+ currentTime := pcfg.Now()
+ uid := packet.NewUserId("", "", "")
+
+ isPrimaryID := false
+ keyLifetimeSecs := uint32(86400 * 365)
+ signer := openpgp.Entity{
+ PrimaryKey: pubKey,
+ PrivateKey: privKey,
+ Identities: map[string]*openpgp.Identity{
+ uid.Id: {
+ Name: uid.Name,
+ UserId: uid,
+ SelfSignature: &packet.Signature{
+ CreationTime: currentTime,
+ SigType: packet.SigTypePositiveCert,
+ PubKeyAlgo: packet.PubKeyAlgoRSA,
+ Hash: pcfg.Hash(),
+ IsPrimaryId: &isPrimaryID,
+ FlagsValid: true,
+ FlagSign: true,
+ FlagCertify: true,
+ IssuerKeyId: &pubKey.KeyId,
+ },
+ },
+ },
+ Subkeys: []openpgp.Subkey{
+ {
+ PublicKey: pubKey,
+ PrivateKey: privKey,
+ Sig: &packet.Signature{
+ CreationTime: currentTime,
+ SigType: packet.SigTypeSubkeyBinding,
+ PubKeyAlgo: packet.PubKeyAlgoRSA,
+ Hash: pcfg.Hash(),
+ PreferredHash: []uint8{8}, // SHA-256
+ FlagsValid: true,
+ FlagEncryptStorage: true,
+ FlagEncryptCommunications: true,
+ IssuerKeyId: &pubKey.KeyId,
+ KeyLifetimeSecs: &keyLifetimeSecs,
+ },
+ },
+ },
+ }
+
+ // Ignore gosec G304 as this function is only used in the build process.
+ //nolint:gosec
+ freader, err := os.Open(filepath.Join(repoRoot, "repodata", "repomd.xml"))
+ if err != nil {
+ return err
+ }
+ defer func(freader *os.File) {
+ err := freader.Close()
+ if err != nil {
+ return
+ }
+ }(freader)
+
+ // Ignore gosec G304 as this function is only used in the build process.
+ //nolint:gosec
+ sigwriter, err := os.Create(filepath.Join(repoRoot, "repodata", "repomd.xml.asc"))
+ if err != nil {
+ return err
+ }
+ defer func(sigwriter *os.File) {
+ err := sigwriter.Close()
+ if err != nil {
+ return
+ }
+ }(sigwriter)
+
+ if err := openpgp.ArmoredDetachSignText(sigwriter, &signer, freader, nil); err != nil {
+ return fmt.Errorf("failed to write PGP signature: %w", err)
+ }
+
+ if err := sigwriter.Close(); err != nil {
+ return fmt.Errorf("failed to write PGP signature: %w", err)
+ }
+
+ return nil
+}
diff --git a/pkg/build/packaging/rpm_test.go b/pkg/build/packaging/rpm_test.go
new file mode 100644
index 00000000000..5a030a3093b
--- /dev/null
+++ b/pkg/build/packaging/rpm_test.go
@@ -0,0 +1,146 @@
+package packaging
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/stretchr/testify/require"
+)
+
+const pubKey = `-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: OpenPGP.js v4.10.10
+Comment: https://openpgpjs.org
+
+xsBNBGM1b9wBCADZM49X7vwOS93KbgA6yhpwrYf8ZlzksGcDaYgp1IzvqHbs
+xeU1mmBYVH/bSKRDG0tt3Qdky4Nvl4Oqd+g0e2ZGjmlEy9zUiPTTK/BtXT+5
+s8oqih2NIAkyF91BNZABAgvh/vJdYImhYeUQBDqMJgqZ/Y/Ha31N7rSW+jUt
+LHspbN0ztJYjuEd/bg2NKH7Gs/AyNvX9IQTC4k7iRRafx7q/PBCVtsk+NCwz
+BEkL93xpAdcdYiMNrRP2eIHQjBmNZ/oUCkcDsLCBvcSq6P2lGpNnpPzVoTJf
+v2qrWkVn5txJJsOkmBGpEDbECPunVilrWO6RPomP0yYkr6NE4XeCJ3QhABEB
+AAHNGWR1bW15IDxkdW1teUBob3RtYWlsLmNvbT7CwI0EEAEIACAFAmM1b9wG
+CwkHCAMCBBUICgIEFgIBAAIZAQIbAwIeAQAhCRAoJ1i5w6kkAxYhBCQv+iwt
+IFn7vj9PLygnWLnDqSQDPxkH/0Ju2Cah+bOxl09uv2Ft2BVlQh0u+wJyRVgs
+KxTxldAXFZwMrN4wK/GUoGWDiy2tzNtoVE6GpxWUj+LvSGFaVLNVjW+Le77I
+BP/sl1wKHJbQhseKc7Mz5Zj3i0F1FPM+rLik7tNk6kiEBqYVyyXahyT98Hu1
+1OKEV+8NiRG47iNgd/dpgEdVSS4DN/dL6m5q+CVy9YnlR+wXxF/2xcMmWBzR
+V2cPVw0JzunpUV8lDDQ/n1sPw61D3oL1aH0bkn8aA8pEceKOVIYOaja7LkLX
+uSlROlALA/M2fuubradW9I3FcrJNn+/xA52el2l/Hn/Syf9GQV/Ll/R+qKIo
+Z57xWd7OwE0EYzVv3AEIAJl/PNYOF2prNKY58BfZ74XurDb9mNlZ1wsIqrOu
+J/euzHEnzkCAjMUuXV7wcugjQlmpcZn6Y0QmQ2uX7SwPCMovDvngbXeAfbdd
+6FUKecQ0sG54Plm8HSMNdjetdUVl7ACxjJO8Rdc/Asx7ua7gMm42CVfqMj4L
+qN5foUBlaKJ1iGKUpQ+673UQWMYeOBuu9G8awbSzGaphN97CIX7xEMGzGeff
+yHLHK+MsfX935uDgDwJQzxJKEugIJDMKgWOLgVz1jRCsJKHlywHTWpVuMiKY
+Wnuq4tDNLBUQtaRL7uclG7Wejw/XNN0uD/zNHPgF5rmlYHVhrtDbBCP2XqTn
+WU8AEQEAAcLAdgQYAQgACQUCYzVv3AIbDAAhCRAoJ1i5w6kkAxYhBCQv+iwt
+IFn7vj9PLygnWLnDqSQDFqYH/AkdNaPUQlE7RQBigNRGOFBuqjhbLsV/rZf+
+/4K6wDHojM606lgLm57T4NUXnk53VIF3KO8+v8N11mCtPb+zBngfvVU14COC
+HEDNdOK19TlR+tH25cftfUiF+OJsgMQysErGuFEtwLE6TNzpQIcnw7SbjxMr
+EGacF9xCBKexB6MlR3GwJ2LBUJm3Lq/fvqImztoTlKDsrpk4JOH5FfYG+G2f
+1zU73fVsCCElX4qA/49rRQf0RNfhjRjmHULP8hSvCXUEhfiBggEgxof/vKlC
+qauHC55luuIeabju8HaXTjpz019cq+3IUgewX/ky0PhQXEW9SoODKabPY2yS
+yUbHFm4=
+=OCSx
+-----END PGP PUBLIC KEY BLOCK-----
+`
+
+const privKey = `-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: OpenPGP.js v4.10.10
+Comment: https://openpgpjs.org
+
+xcMGBGM1b9wBCADZM49X7vwOS93KbgA6yhpwrYf8ZlzksGcDaYgp1IzvqHbs
+xeU1mmBYVH/bSKRDG0tt3Qdky4Nvl4Oqd+g0e2ZGjmlEy9zUiPTTK/BtXT+5
+s8oqih2NIAkyF91BNZABAgvh/vJdYImhYeUQBDqMJgqZ/Y/Ha31N7rSW+jUt
+LHspbN0ztJYjuEd/bg2NKH7Gs/AyNvX9IQTC4k7iRRafx7q/PBCVtsk+NCwz
+BEkL93xpAdcdYiMNrRP2eIHQjBmNZ/oUCkcDsLCBvcSq6P2lGpNnpPzVoTJf
+v2qrWkVn5txJJsOkmBGpEDbECPunVilrWO6RPomP0yYkr6NE4XeCJ3QhABEB
+AAH+CQMIuDEg1p2Y6zbg0EQ3JvsP7VQBGsuXg9khTjktoxhwici/d+rcIW7Q
+SuKWJGqs83LTeeGmS+9etNtf3LqRdPnI7f0qbT47mAqvp2gn7Rvbrabk+5Jj
+AQS/DDLlWNiWsPrMBMZ7TZpiQ+g7gnIZaV10taFupYJr69AjtED+NPu8LOvZ
+2ItK9xBqOwl5mkNe7ps/uTT6jwYSWxeObp4ymnLDLONY3eHuaYP9QB/NSlw0
+80Wo5qBPljlU8JdbEoLFU4gY6wkEbLa/DVbEVXSHfWVtr8jZbzHW39TBxpG2
+Dxk52EVyu8Gf9XIQN2ZjDP3CzBGmlxJjLxLUD4GmRSPaDGK7LCN9ZztaXy3Y
+WtF6RJfNzEoDdCaV0kkM3AskQDsQ+CWsDVsbbQyDtfncVG6cDzqmoDrBCSq1
+Bsoz07k2hj9VP0aP2xU78qcpJWO2rmhAHy9W2NqjXSBJriy1JXrK5o2/lUUr
+94R8NLvqeVbInUw/zovVctaujHIBhNKL9wn2T0LWrA2OEJUz0HWo6ZQSaNzl
+Obtz0M8gCj/4sDYjRAiDk50FzOcZp8ijYQFVypQTVzHki5T/JfvBnMpo+4Uc
+93QB1woyiZuJCIj7DpY3MkZ5fTDtgJPa+0k8r+lPnAmE6auGUaH7JRKhbBu0
+8faDwaiSv3kD3EEDffoWX/axLLYta9jTDnitTXbf1jY03pdJeiU/ZX0BQTZi
+pehZ/6yi/qXM/F8HDVEWriSLqVsMLrXXeFIojAc3fJ/QPpAZSx6E/Fe2xh8c
+yURov5krU1zNJDwqC3SjHsHQ/UlLtamDDmmuXX+xb1CwIDd6WksGsCbe/LoN
+TxViV4hOjIeh5TwRP5jQaqsVKCT8fzoDrRXy76taT+Zaaen+J6rC51HQwyEq
+Qgf1e7WodzN3r10UV6/L/wNkfdWJgf5MzRlkdW1teSA8ZHVtbXlAaG90bWFp
+bC5jb20+wsCNBBABCAAgBQJjNW/cBgsJBwgDAgQVCAoCBBYCAQACGQECGwMC
+HgEAIQkQKCdYucOpJAMWIQQkL/osLSBZ+74/Ty8oJ1i5w6kkAz8ZB/9Cbtgm
+ofmzsZdPbr9hbdgVZUIdLvsCckVYLCsU8ZXQFxWcDKzeMCvxlKBlg4strczb
+aFROhqcVlI/i70hhWlSzVY1vi3u+yAT/7JdcChyW0IbHinOzM+WY94tBdRTz
+Pqy4pO7TZOpIhAamFcsl2ock/fB7tdTihFfvDYkRuO4jYHf3aYBHVUkuAzf3
+S+puavglcvWJ5UfsF8Rf9sXDJlgc0VdnD1cNCc7p6VFfJQw0P59bD8OtQ96C
+9Wh9G5J/GgPKRHHijlSGDmo2uy5C17kpUTpQCwPzNn7rm62nVvSNxXKyTZ/v
+8QOdnpdpfx5/0sn/RkFfy5f0fqiiKGee8Vnex8MGBGM1b9wBCACZfzzWDhdq
+azSmOfAX2e+F7qw2/ZjZWdcLCKqzrif3rsxxJ85AgIzFLl1e8HLoI0JZqXGZ
++mNEJkNrl+0sDwjKLw754G13gH23XehVCnnENLBueD5ZvB0jDXY3rXVFZewA
+sYyTvEXXPwLMe7mu4DJuNglX6jI+C6jeX6FAZWiidYhilKUPuu91EFjGHjgb
+rvRvGsG0sxmqYTfewiF+8RDBsxnn38hyxyvjLH1/d+bg4A8CUM8SShLoCCQz
+CoFji4Fc9Y0QrCSh5csB01qVbjIimFp7quLQzSwVELWkS+7nJRu1no8P1zTd
+Lg/8zRz4Bea5pWB1Ya7Q2wQj9l6k51lPABEBAAH+CQMIwr3YSD15lYrgItoy
+MDsrWqMMHJsSxusbQiK0KLgjFBuDuTolsu9zqQCHEm2dxChqT+yQ6AeeynRD
+pDMVkHEvhShvGUhB6Bu5wClHj8+xFpyprShE/KbEuppNdfIRgWVYc7UX+TYz
+6BymqhzKyIw2Q33ocrXgTRZ02HM7urKVvAhsJCEff0paByOzCspiv/TPRihi
+7GAZY0wFLDPe9qr+07ExT2ndMDX8Xb1mlg8IeaSWUaNilm7M8oW3xnUBnXeD
+XglTkObGeRVXAINim9uL4soT3lamN4QwgBus9WzFqOOCMk11fjatY8kY1zX9
+epO27igGtMwTFl11XcQLlFyvlgPBeWtFam7RiDPa3VF0XubmBYZBmqWpccWs
+xl0xHCtUK7Pd0O4kSqxsL9cB0MX9iR1yPkM8wA++Mp6pEfNcXUrGIdlie0H5
+aCq8rguYG5VuFosSUatdCbpRVGBxGnhxHes0mNTPgwAoAVNYBWXH5iq5HxKy
+i3Zy5V7ZKSyDrfg/0AajtDW5h3g+wglUI9UCdT4tNLFwYbhHqGH2xdBztYI0
+iSJ7COLmo26smkA8UXxsrlw8PWPzpbhQOG06EbMjncJimJDMI1YDC6ag7M5l
+OcG9uXZQ22ipAz5CSPtyL0/0WAp4yyn1VQRBK42n/y9ld+dMbuq6majazb15
+6sEgHUKERcwGs0Ftfj5Zamwhm7ZoIe26XEqvcshpQpv1Q9hktluVeSbiVaBe
+Nl8zUZHlo/0zUc5j7G5Up58t+ChSsyOFJGM7CGkKHHawBZYCs0EcpsdAPr3T
+1C8A0Wt9POTETYM4pZFOoLds6VTolZZcxeBN5YPoN2kbwFpOgPJN09Zz8z8S
+4psQRV4KQ92XDPZ/6q2BH5i2+F2ZwUsvCR4DwgzbVGZSRV6mM7lkjZSmnWfC
+AE7DUl7XwsB2BBgBCAAJBQJjNW/cAhsMACEJECgnWLnDqSQDFiEEJC/6LC0g
+Wfu+P08vKCdYucOpJAMWpgf8CR01o9RCUTtFAGKA1EY4UG6qOFsuxX+tl/7/
+grrAMeiMzrTqWAubntPg1ReeTndUgXco7z6/w3XWYK09v7MGeB+9VTXgI4Ic
+QM104rX1OVH60fblx+19SIX44myAxDKwSsa4US3AsTpM3OlAhyfDtJuPEysQ
+ZpwX3EIEp7EHoyVHcbAnYsFQmbcur9++oibO2hOUoOyumTgk4fkV9gb4bZ/X
+NTvd9WwIISVfioD/j2tFB/RE1+GNGOYdQs/yFK8JdQSF+IGCASDGh/+8qUKp
+q4cLnmW64h5puO7wdpdOOnPTX1yr7chSB7Bf+TLQ+FBcRb1Kg4Mpps9jbJLJ
+RscWbg==
+=KJNy
+-----END PGP PRIVATE KEY BLOCK-----
+`
+
+// Dummy GPG keys, used only for testing
+// nolint:gosec
+const passPhrase = `MkDgjkrgdGxt`
+
+func TestSignRPMRepo(t *testing.T) {
+ repoDir := t.TempDir()
+ workDir := t.TempDir()
+ pubKeyPath := filepath.Join(workDir, "pub.key")
+ err := os.WriteFile(pubKeyPath, []byte(pubKey), 0600)
+ require.NoError(t, err)
+ privKeyPath := filepath.Join(workDir, "priv.key")
+ err = os.WriteFile(privKeyPath, []byte(privKey), 0600)
+ require.NoError(t, err)
+ passPhrasePath := filepath.Join(workDir, "passphrase")
+ err = os.WriteFile(passPhrasePath, []byte(passPhrase), 0600)
+ require.NoError(t, err)
+ err = os.Mkdir(filepath.Join(repoDir, "repodata"), 0700)
+ require.NoError(t, err)
+ err = os.WriteFile(filepath.Join(repoDir, "repodata", "repomd.xml"), []byte(""), 0600)
+ require.NoError(t, err)
+
+ cfg := PublishConfig{
+ Config: config.Config{
+ GPGPrivateKey: privKeyPath,
+ GPGPublicKey: pubKeyPath,
+ GPGPassPath: passPhrasePath,
+ },
+ }
+
+ err = signRPMRepo(repoDir, cfg)
+ require.NoError(t, err)
+}
diff --git a/pkg/build/plugins/build.go b/pkg/build/plugins/build.go
new file mode 100644
index 00000000000..aa0b47c0fe6
--- /dev/null
+++ b/pkg/build/plugins/build.go
@@ -0,0 +1,66 @@
+package plugins
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+
+ "github.com/grafana/grafana/pkg/build/config"
+ "github.com/grafana/grafana/pkg/build/errutil"
+ "github.com/grafana/grafana/pkg/build/syncutil"
+ "github.com/grafana/grafana/pkg/infra/fs"
+)
+
+type PluginSigningMode = int
+
+// BuildPlugins builds internal plugins.
+// The built plugins are placed in plugins-bundled/dist/.
+func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.BuildConfig) error {
+ log.Printf("Building plugins in %q...", grafanaDir)
+
+ root := filepath.Join(grafanaDir, "plugins-bundled", "internal")
+ fis, err := os.ReadDir(root)
+ if err != nil {
+ return err
+ }
+
+ for i := range fis {
+ fi := fis[i]
+ if !fi.IsDir() {
+ continue
+ }
+
+ dpath := filepath.Join(root, fi.Name())
+
+ p.Schedule(g.Wrap(func() error {
+ log.Printf("Building plugin %q...", dpath)
+
+ cmd := exec.Command("yarn", "build")
+ cmd.Dir = dpath
+ if output, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("yarn build failed: %s", output)
+ }
+
+ dstPath := filepath.Join("plugins-bundled", "dist", fi.Name())
+ if err := fs.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil {
+ return err
+ }
+ if !verMode.PluginSignature.Sign {
+ return nil
+ }
+
+ return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign)
+ }))
+ }
+
+ if err := g.Wait(); err != nil {
+ return err
+ }
+
+ log.Printf("Built all plug-ins successfully!")
+
+ return nil
+}
diff --git a/pkg/build/plugins/download.go b/pkg/build/plugins/download.go
new file mode 100644
index 00000000000..ed8f00bf911
--- /dev/null
+++ b/pkg/build/plugins/download.go
@@ -0,0 +1,118 @@
+package plugins
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+
+ "github.com/grafana/grafana/pkg/build/errutil"
+ "github.com/grafana/grafana/pkg/build/syncutil"
+)
+
+// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
+func logCloseError(closeFunc func() error) {
+ if err := closeFunc(); err != nil {
+ log.Println(err)
+ }
+}
+
+// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
+func logError(err error) {
+ if err != nil {
+ log.Println(err)
+ }
+}
+
+// pluginManifest has details of an external plugin package.
+type pluginManifest struct {
+ Name string `json:"name"`
+ Version string `json:"version"`
+ Checksum string `json:"checksum"`
+}
+
+// pluginsManifest represents a manifest of Grafana's external plugins.
+type pluginsManifest struct {
+ Plugins []pluginManifest `json:"plugins"`
+}
+
+// downloadPlugins downloads Grafana plugins that should be bundled into packages.
+//
+// The plugin archives are downloaded into /plugins-bundled.
+func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error {
+ g, _ := errutil.GroupWithContext(ctx)
+
+ log.Println("Downloading external plugins...")
+
+ var m pluginsManifest
+ manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json")
+ //nolint:gosec
+ manifestB, err := os.ReadFile(manifestPath)
+ if err != nil {
+ return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err)
+ }
+ if err := json.Unmarshal(manifestB, &m); err != nil {
+ return err
+ }
+
+ for i := range m.Plugins {
+ pm := m.Plugins[i]
+ p.Schedule(g.Wrap(func() error {
+ tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version))
+ //nolint:gosec
+ out, err := os.Create(tgt)
+ if err != nil {
+ return err
+ }
+ defer logCloseError(out.Close)
+
+ u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name,
+ pm.Version)
+ log.Printf("Downloading plugin %q to %q...", u, tgt)
+ // nolint:gosec
+ resp, err := http.Get(u)
+ if err != nil {
+ return fmt.Errorf("downloading %q failed: %w", u, err)
+ }
+ defer logError(resp.Body.Close())
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode)
+ }
+
+ if _, err := io.Copy(out, resp.Body); err != nil {
+ return fmt.Errorf("downloading %q failed: %w", u, err)
+ }
+ if err := out.Close(); err != nil {
+ return fmt.Errorf("downloading %q failed: %w", u, err)
+ }
+
+ //nolint:gosec
+ fd, err := os.Open(tgt)
+ if err != nil {
+ return err
+ }
+ defer logCloseError(fd.Close)
+
+ h := sha256.New()
+ if _, err := io.Copy(h, fd); err != nil {
+ return err
+ }
+
+ chksum := hex.EncodeToString(h.Sum(nil))
+ if chksum != pm.Checksum {
+ return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum)
+ }
+
+ return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled"))
+ }))
+ }
+
+ return g.Wait()
+}
diff --git a/pkg/build/plugins/manifest.go b/pkg/build/plugins/manifest.go
new file mode 100644
index 00000000000..359e8ad77bc
--- /dev/null
+++ b/pkg/build/plugins/manifest.go
@@ -0,0 +1,204 @@
+package plugins
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+)
+
+type manifest struct {
+ Plugin string `json:"plugin"`
+ Version string `json:"version"`
+ Files map[string]string `json:"files"`
+}
+
+func getManifest(dpath string, chksums map[string]string) (manifest, error) {
+ m := manifest{}
+
+ type pluginInfo struct {
+ Version string `json:"version"`
+ }
+
+ type plugin struct {
+ ID string `json:"id"`
+ Info pluginInfo `json:"info"`
+ }
+
+ //nolint:gosec
+ f, err := os.Open(filepath.Join(dpath, "plugin.json"))
+ if err != nil {
+ return m, err
+ }
+ decoder := json.NewDecoder(f)
+ var p plugin
+ if err := decoder.Decode(&p); err != nil {
+ return m, err
+ }
+
+ if p.ID == "" {
+ return m, fmt.Errorf("plugin.json doesn't define id")
+ }
+ if p.Info.Version == "" {
+ return m, fmt.Errorf("plugin.json doesn't define info.version")
+ }
+
+ return manifest{
+ Plugin: p.ID,
+ Version: p.Info.Version,
+ Files: chksums,
+ }, nil
+}
+
+// BuildManifest requests a plugin's signed manifest file fromt he Grafana API.
+// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires
+// an admin API key.
+func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error {
+ log.Printf("Building manifest for plug-in at %q", dpath)
+
+ apiKey := os.Getenv("GRAFANA_API_KEY")
+ if apiKey == "" {
+ return fmt.Errorf("GRAFANA_API_KEY must be set")
+ }
+
+ manifestPath := filepath.Join(dpath, "MANIFEST.txt")
+ chksums, err := getChksums(dpath, manifestPath)
+ if err != nil {
+ return err
+ }
+ m, err := getManifest(dpath, chksums)
+ if err != nil {
+ return err
+ }
+
+ b := bytes.NewBuffer(nil)
+ encoder := json.NewEncoder(b)
+ if err := encoder.Encode(&m); err != nil {
+ return err
+ }
+ jsonB := b.Bytes()
+ u := "https://grafana.com/api/plugins/ci/sign"
+ if !signingAdmin {
+ u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin)
+ }
+ log.Printf("Requesting signed manifest from Grafana API...")
+ req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB))
+ if err != nil {
+ return err
+ }
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+ req.Header.Add("Content-Type", "application/json")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err)
+ }
+ defer func() {
+ if err := resp.Body.Close(); err != nil {
+ log.Println("failed to close response body, err: %w", err)
+ }
+ }()
+ if resp.StatusCode != 200 {
+ msg, err := io.ReadAll(resp.Body)
+ if err != nil {
+ log.Printf("Failed to read response body: %s", err)
+ msg = []byte("")
+ }
+ return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg))
+ }
+
+ log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath)
+ //nolint:gosec
+ f, err := os.Create(manifestPath)
+ if err != nil {
+ return fmt.Errorf("failed to create %s: %w", manifestPath, err)
+ }
+ defer func() {
+ if err := f.Close(); err != nil {
+ log.Println("failed to close file, err: %w", err)
+ }
+ }()
+ if _, err := io.Copy(f, resp.Body); err != nil {
+ return fmt.Errorf("failed to write %s: %w", manifestPath, err)
+ }
+ if err := f.Close(); err != nil {
+ return fmt.Errorf("failed to write %s: %w", manifestPath, err)
+ }
+
+ return nil
+}
+
+func getChksums(dpath, manifestPath string) (map[string]string, error) {
+ manifestPath = filepath.Clean(manifestPath)
+
+ chksums := map[string]string{}
+ if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+
+ if fi.IsDir() {
+ return nil
+ }
+
+ path = filepath.Clean(path)
+
+ // Handle symbolic links
+ if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+ finalPath, err := filepath.EvalSymlinks(path)
+ if err != nil {
+ return err
+ }
+
+ log.Printf("Handling symlink %q, pointing to %q", path, finalPath)
+
+ info, err := os.Stat(finalPath)
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+
+ if _, err := filepath.Rel(dpath, finalPath); err != nil {
+ return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath)
+ }
+
+ if finalPath == manifestPath {
+ return nil
+ }
+ }
+
+ if path == manifestPath {
+ return nil
+ }
+
+ h := sha256.New()
+ //nolint:gosec
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer logCloseError(f.Close)
+ if _, err := io.Copy(h, f); err != nil {
+ return err
+ }
+
+ relPath, err := filepath.Rel(dpath, path)
+ if err != nil {
+ return err
+ }
+ chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil))
+
+ return nil
+ }); err != nil {
+ return nil, err
+ }
+
+ return chksums, nil
+}
diff --git a/pkg/build/plugins/zip.go b/pkg/build/plugins/zip.go
new file mode 100644
index 00000000000..73f8e8d82f6
--- /dev/null
+++ b/pkg/build/plugins/zip.go
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+ "archive/zip"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+)
+
+// Unzip unzips a plugin.
+func Unzip(fpath, tgtDir string) error {
+ log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir)
+
+ r, err := zip.OpenReader(fpath)
+ if err != nil {
+ return err
+ }
+ defer logCloseError(r.Close)
+
+ // Closure to address file descriptors issue with all the deferred .Close() methods
+ extractAndWriteFile := func(f *zip.File) error {
+ log.Printf("Extracting zip member %q...", f.Name)
+
+ rc, err := f.Open()
+ if err != nil {
+ return err
+ }
+ defer logCloseError(rc.Close)
+
+ //nolint:gosec
+ dstPath := filepath.Join(tgtDir, f.Name)
+
+ if f.FileInfo().IsDir() {
+ return os.MkdirAll(dstPath, f.Mode())
+ }
+
+ if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil {
+ return err
+ }
+
+ //nolint:gosec
+ fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+ if err != nil {
+ return err
+ }
+ defer logCloseError(fd.Close)
+
+ // nolint:gosec
+ if _, err := io.Copy(fd, rc); err != nil {
+ return err
+ }
+
+ return fd.Close()
+ }
+
+ for _, f := range r.File {
+ if err := extractAndWriteFile(f); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/pkg/build/stringutil/contains.go b/pkg/build/stringutil/contains.go
new file mode 100644
index 00000000000..b53efe70759
--- /dev/null
+++ b/pkg/build/stringutil/contains.go
@@ -0,0 +1,10 @@
+package stringutil
+
+func Contains(arr []string, s string) bool {
+ for _, e := range arr {
+ if e == s {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/build/syncutil/pool.go b/pkg/build/syncutil/pool.go
new file mode 100644
index 00000000000..6034059d2bf
--- /dev/null
+++ b/pkg/build/syncutil/pool.go
@@ -0,0 +1,43 @@
+package syncutil
+
+import (
+ "log"
+ "runtime"
+)
+
+func worker(jobs chan func()) {
+ for j := range jobs {
+ j()
+ }
+}
+
+// WorkerPool represents a concurrent worker pool.
+type WorkerPool struct {
+ NumWorkers int
+ jobs chan func()
+}
+
+// NewWorkerPool constructs a new WorkerPool.
+func NewWorkerPool(numWorkers int) WorkerPool {
+ if numWorkers <= 0 {
+ numWorkers = runtime.NumCPU()
+ }
+ log.Printf("Creating worker pool with %d workers", numWorkers)
+ jobs := make(chan func(), 100)
+ for i := 0; i < numWorkers; i++ {
+ go worker(jobs)
+ }
+ return WorkerPool{
+ NumWorkers: numWorkers,
+ jobs: jobs,
+ }
+}
+
+// Schedule schedules a job to be executed by a worker in the pool.
+func (p WorkerPool) Schedule(job func()) {
+ p.jobs <- job
+}
+
+func (p WorkerPool) Close() {
+ close(p.jobs)
+}
diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star
index df3308d6aad..e2645cf1cc5 100644
--- a/scripts/drone/steps/lib.star
+++ b/scripts/drone/steps/lib.star
@@ -1041,6 +1041,7 @@ def publish_grafanacom_step(edition, ver_mode):
],
'environment': {
'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
+ 'GCP_KEY': from_secret('gcp_key'),
},
'commands': [
cmd,