CI: Backport CI/Release related code to v9.3.x (#62752)

* Batch-move everything

* go mod tidy

* make drone

* Remove genversions

* Bump alpine image

* Revert back pkg/build/docker/build.go

* Make sure correct enterprise branch is checked out

* Add enterprise2 version

* Remove extensions

* Bump build container

* backport node 18 test fix

(cherry picked from commit 4ff03fdbfb)

* Update scripts/drone

* Add more commands

* Fix starlark link

* Copy .drone.star

* Add drone target branch for custom events

---------
This commit is contained in:
Dimitris Sotirakis
2023-02-03 11:43:48 +02:00
committed by GitHub
parent d9b5c5f4c6
commit 03b1cf763d
85 changed files with 8622 additions and 2695 deletions
+16 -1
View File
@@ -2,7 +2,22 @@ package main
import "github.com/urfave/cli/v2"
func ArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc {
// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided.
func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() != num {
if err := cli.ShowSubcommandHelp(ctx); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
return action(ctx)
}
}
// ArgCountWrapper will cause the action to fail if there were more than `num` args provided.
func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() > max {
if err := cli.ShowSubcommandHelp(ctx); err != nil {
+1 -1
View File
@@ -13,7 +13,7 @@ import (
)
func BuildBackend(ctx *cli.Context) error {
metadata, err := GenerateMetadata(ctx)
metadata, err := config.GenerateMetadata(ctx)
if err != nil {
return err
}
+1 -1
View File
@@ -14,7 +14,7 @@ func BuildDocker(c *cli.Context) error {
return err
}
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+2 -1
View File
@@ -3,6 +3,7 @@ package main
import (
"log"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/errutil"
"github.com/grafana/grafana/pkg/build/frontend"
"github.com/grafana/grafana/pkg/build/syncutil"
@@ -10,7 +11,7 @@ import (
)
func BuildFrontend(c *cli.Context) error {
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+1 -1
View File
@@ -17,7 +17,7 @@ func BuildInternalPlugins(c *cli.Context) error {
}
const grafanaDir = "."
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"fmt"
"log"
"os"
"strconv"
"github.com/grafana/grafana/pkg/build/env"
"github.com/grafana/grafana/pkg/build/git"
"github.com/urfave/cli/v2"
)
// checkOpts are options used to create a new GitHub check for the enterprise downstream test.
type checkOpts struct {
SHA string
URL string
Branch string
PR int
}
func getCheckOpts(args []string) (*checkOpts, error) {
branch, ok := env.Lookup("DRONE_SOURCE_BRANCH", args)
if !ok {
return nil, cli.Exit("Unable to retrieve build source branch", 1)
}
var (
rgx = git.PRCheckRegexp()
matches = rgx.FindStringSubmatch(branch)
)
sha, ok := env.Lookup("SOURCE_COMMIT", args)
if !ok {
if matches == nil || len(matches) <= 1 {
return nil, cli.Exit("Unable to retrieve source commit", 1)
}
sha = matches[2]
}
url, ok := env.Lookup("DRONE_BUILD_LINK", args)
if !ok {
return nil, cli.Exit(`missing environment variable "DRONE_BUILD_LINK"`, 1)
}
prStr, ok := env.Lookup("OSS_PULL_REQUEST", args)
if !ok {
if matches == nil || len(matches) <= 1 {
return nil, cli.Exit("Unable to retrieve PR number", 1)
}
prStr = matches[1]
}
pr, err := strconv.Atoi(prStr)
if err != nil {
return nil, err
}
return &checkOpts{
Branch: branch,
PR: pr,
SHA: sha,
URL: url,
}, nil
}
// EnterpriseCheckBegin creates the GitHub check and signals the beginning of the downstream build / test process
func EnterpriseCheckBegin(c *cli.Context) error {
var (
ctx = c.Context
client = git.NewGitHubClient(ctx, c.String("github-token"))
)
opts, err := getCheckOpts(os.Environ())
if err != nil {
return err
}
if _, err = git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, "pending"); err != nil {
return err
}
return nil
}
func EnterpriseCheckSuccess(c *cli.Context) error {
return completeEnterpriseCheck(c, true)
}
func EnterpriseCheckFail(c *cli.Context) error {
return completeEnterpriseCheck(c, false)
}
func completeEnterpriseCheck(c *cli.Context, success bool) error {
var (
ctx = c.Context
client = git.NewGitHubClient(ctx, c.String("github-token"))
)
// Update the pull request labels
opts, err := getCheckOpts(os.Environ())
if err != nil {
return err
}
status := "failure"
if success {
status = "success"
}
// Update the GitHub check...
if _, err := git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, status); err != nil {
return err
}
// Delete branch if needed
log.Printf("Checking branch '%s' against '%s'", git.PRCheckRegexp().String(), opts.Branch)
if git.PRCheckRegexp().MatchString(opts.Branch) {
log.Println("Deleting branch", opts.Branch)
if err := git.DeleteEnterpriseBranch(ctx, client.Git, opts.Branch); err != nil {
return fmt.Errorf("error deleting enterprise branch: %w", err)
}
}
label := "enterprise-failed"
if success {
label = "enterprise-ok"
}
return git.AddLabelToPR(ctx, client.Issues, opts.PR, label)
}
+69
View File
@@ -0,0 +1,69 @@
package main
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetCheckOpts(t *testing.T) {
t.Run("it should return the checkOpts if the correct environment variables are set", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
"OSS_PULL_REQUEST=1",
}
opts, err := getCheckOpts(args)
require.NoError(t, err)
require.Equal(t, opts.SHA, "1234")
require.Equal(t, opts.URL, "http://example.com")
})
t.Run("it should return an error if SOURCE_COMMIT is not set", func(t *testing.T) {
args := []string{
"DRONE_BUILD_LINK=http://example.com",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
"OSS_PULL_REQUEST=1",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
t.Run("it should return an error if DRONE_BUILD_LINK is not set", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"OSS_PULL_REQUEST=1",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
t.Run("it should return an error if OSS_PULL_REQUEST is not set", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
t.Run("it should return an error if OSS_PULL_REQUEST is not an integer", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
"OSS_PULL_REQUEST=http://example.com",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
}
+2 -1
View File
@@ -4,11 +4,12 @@ import (
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/urfave/cli/v2"
)
func ExportVersion(c *cli.Context) error {
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+3 -3
View File
@@ -20,12 +20,12 @@ const (
func FetchImages(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
return cli.NewExitError("", 1)
return cli.Exit("", 1)
}
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+10
View File
@@ -46,4 +46,14 @@ var (
Usage: "Google Cloud Platform key file",
Required: true,
}
gitHubTokenFlag = cli.StringFlag{
Name: "github-token",
Value: "",
EnvVars: []string{"GITHUB_TOKEN"},
Usage: "GitHub token",
}
tagFlag = cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
}
)
-92
View File
@@ -1,92 +0,0 @@
package main
import (
"fmt"
"os"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/urfave/cli/v2"
)
func GenerateMetadata(c *cli.Context) (config.Metadata, error) {
var metadata config.Metadata
version := ""
event, err := droneutil.GetDroneEventFromEnv()
if err != nil {
return config.Metadata{}, err
}
var releaseMode config.ReleaseMode
switch event {
case string(config.PullRequestMode):
releaseMode = config.ReleaseMode{Mode: config.PullRequestMode}
case config.Push:
mode, err := config.CheckDroneTargetBranch()
if err != nil {
return config.Metadata{}, err
}
releaseMode = config.ReleaseMode{Mode: mode}
case config.Custom:
mode, err := config.CheckDroneTargetBranch()
if err != nil {
return config.Metadata{}, err
}
// if there is a custom event targeting the main branch, that's an enterprise downstream build
if mode == config.MainBranch {
releaseMode = config.ReleaseMode{Mode: config.CustomMode}
} else {
releaseMode = config.ReleaseMode{Mode: mode}
}
case config.Tag, config.Promote:
tag, ok := os.LookupEnv("DRONE_TAG")
if !ok || tag == "" {
return config.Metadata{}, err
}
version = strings.TrimPrefix(tag, "v")
mode, err := config.CheckSemverSuffix()
if err != nil {
return config.Metadata{}, err
}
releaseMode = mode
case config.Cronjob:
releaseMode = config.ReleaseMode{Mode: config.CronjobMode}
}
if version == "" {
version, err = generateVersionFromBuildID()
if err != nil {
return config.Metadata{}, err
}
}
currentCommit, err := config.GetDroneCommit()
if err != nil {
return config.Metadata{}, err
}
metadata = config.Metadata{
GrafanaVersion: version,
ReleaseMode: releaseMode,
GrabplVersion: c.App.Version,
CurrentCommit: currentCommit,
}
fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode)
return metadata, nil
}
func generateVersionFromBuildID() (string, error) {
buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER")
if !ok {
return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable")
}
var err error
version, err := config.GetGrafanaVersion(buildID, ".")
if err != nil {
return "", err
}
return version, nil
}
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"flag"
"os"
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
)
const (
DroneBuildEvent = "DRONE_BUILD_EVENT"
DroneTargetBranch = "DRONE_TARGET_BRANCH"
DroneTag = "DRONE_TAG"
DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE"
DroneBuildNumber = "DRONE_BUILD_NUMBER"
)
const (
hashedGrafanaVersion = "9.2.0-12345pre"
versionedBranch = "v9.2.x"
)
func TestGetMetadata(t *testing.T) {
tcs := []struct {
envMap map[string]string
expVersion string
mode config.ReleaseMode
}{
{map[string]string{DroneBuildEvent: config.PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.PullRequestMode}},
{map[string]string{DroneBuildEvent: config.Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: config.Push, DroneTargetBranch: config.MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.MainMode}},
{map[string]string{DroneBuildEvent: config.Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: config.Custom, DroneTargetBranch: config.MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.Custom}},
{map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", config.ReleaseMode{Mode: config.TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: true}},
{map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", config.ReleaseMode{Mode: config.TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: true}},
}
ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil)
for _, tc := range tcs {
t.Run("Should return valid metadata, ", func(t *testing.T) {
setUpEnv(t, tc.envMap)
testMetadata(t, ctx, tc.expVersion, tc.mode)
})
}
}
func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode config.ReleaseMode) {
t.Helper()
metadata, err := GenerateMetadata(ctx)
require.NoError(t, err)
t.Run("with a valid version", func(t *testing.T) {
expVersion := metadata.GrafanaVersion
require.Equal(t, expVersion, version)
})
t.Run("with a valid release mode from the built-in list", func(t *testing.T) {
expMode := metadata.ReleaseMode
require.NoError(t, err)
require.Equal(t, expMode, releaseMode)
})
}
func setUpEnv(t *testing.T, envMap map[string]string) {
t.Helper()
os.Clearenv()
err := os.Setenv("DRONE_COMMIT", "abcd12345")
require.NoError(t, err)
for k, v := range envMap {
err := os.Setenv(k, v)
require.NoError(t, err)
}
}
+8 -7
View File
@@ -15,11 +15,12 @@ import (
"strings"
"time"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/urfave/cli/v2"
)
const grafanaAPI = "https://grafana.com/api"
@@ -33,7 +34,7 @@ func GrafanaCom(c *cli.Context) error {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
@@ -66,11 +67,11 @@ func GrafanaCom(c *cli.Context) error {
grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY"))
if grafanaAPIKey == "" {
return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1)
return cli.Exit("the environment variable GRAFANA_COM_API_KEY must be set", 1)
}
whatsNewURL, releaseNotesURL, err := getReleaseURLs()
if err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
// TODO: Verify config values
@@ -89,7 +90,7 @@ func GrafanaCom(c *cli.Context) error {
}
if err := publishPackages(cfg); err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
log.Println("Successfully published packages to grafana.com!")
@@ -146,7 +147,7 @@ func publishPackages(cfg packaging.PublishConfig) error {
}
switch cfg.ReleaseMode.Mode {
case config.MainMode, config.CustomMode, config.CronjobMode:
case config.MainMode, config.DownstreamMode, config.CronjobMode:
pth = path.Join(pth, packaging.MainFolder)
default:
pth = path.Join(pth, packaging.ReleaseFolder)
@@ -177,7 +178,7 @@ func publishPackages(cfg packaging.PublishConfig) error {
Version: cfg.Version,
ReleaseDate: time.Now().UTC(),
Builds: builds,
Stable: cfg.ReleaseMode.Mode == config.TagMode,
Stable: cfg.ReleaseMode.Mode == config.TagMode && !cfg.ReleaseMode.IsBeta && !cfg.ReleaseMode.IsTest,
Beta: cfg.ReleaseMode.IsBeta,
Nightly: cfg.ReleaseMode.Mode == config.CronjobMode,
}
+206 -11
View File
@@ -5,10 +5,18 @@ import (
"os"
"strings"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/docker"
)
var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5)
//nolint:unused
func registerAppCommand(c *cli.Command) {
additionalCommands = append(additionalCommands, c)
}
func main() {
app := cli.NewApp()
app.Commands = cli.Commands{
@@ -16,7 +24,7 @@ func main() {
Name: "build-backend",
Usage: "Build one or more variants of back-end binaries",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, BuildBackend),
Action: MaxArgCountWrapper(1, BuildBackend),
Flags: []cli.Flag{
&jobsFlag,
&variantsFlag,
@@ -67,7 +75,7 @@ func main() {
Name: "build-frontend",
Usage: "Build front-end artifacts",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, BuildFrontend),
Action: MaxArgCountWrapper(1, BuildFrontend),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
@@ -77,7 +85,7 @@ func main() {
{
Name: "build-docker",
Usage: "Build Grafana Docker images",
Action: ArgCountWrapper(1, BuildDocker),
Action: MaxArgCountWrapper(1, BuildDocker),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
@@ -96,6 +104,14 @@ func main() {
},
},
},
{
Name: "upload-cdn",
Usage: "Upload public/* to a cdn bucket",
Action: UploadCDN,
Flags: []cli.Flag{
&editionFlag,
},
},
{
Name: "shellcheck",
Usage: "Run shellcheck on shell scripts",
@@ -104,7 +120,7 @@ func main() {
{
Name: "build-plugins",
Usage: "Build internal plug-ins",
Action: ArgCountWrapper(1, BuildInternalPlugins),
Action: MaxArgCountWrapper(1, BuildInternalPlugins),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
@@ -117,13 +133,19 @@ func main() {
Name: "publish-metrics",
Usage: "Publish a set of metrics from stdin",
ArgsUsage: "<api-key>",
Action: ArgCountWrapper(1, PublishMetrics),
Action: MaxArgCountWrapper(1, PublishMetrics),
},
{
Name: "verify-drone",
Usage: "Verify Drone configuration",
Action: VerifyDrone,
},
{
Name: "verify-starlark",
Usage: "Verify Starlark configuration",
ArgsUsage: "<workspace path>",
Action: VerifyStarlark,
},
{
Name: "export-version",
Usage: "Exports version in dist/grafana.version",
@@ -133,7 +155,7 @@ func main() {
Name: "package",
Usage: "Package one or more Grafana variants",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, Package),
Action: MaxArgCountWrapper(1, Package),
Flags: []cli.Flag{
&jobsFlag,
&variantsFlag,
@@ -144,7 +166,7 @@ func main() {
},
{
Name: "store-storybook",
Usage: "Integrity check for storybook build",
Usage: "Stores storybook to GCS buckets",
Action: StoreStorybook,
Flags: []cli.Flag{
&cli.StringFlag{
@@ -153,10 +175,81 @@ func main() {
},
},
},
{
Name: "verify-storybook",
Usage: "Integrity check for storybook build",
Action: VerifyStorybook,
},
{
Name: "upload-packages",
Usage: "Upload Grafana packages",
Action: UploadPackages,
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
&cli.BoolFlag{
Name: "enterprise2",
Usage: "Declare if the edition is enterprise2",
},
},
},
{
Name: "artifacts",
Usage: "Handle Grafana artifacts",
Subcommands: cli.Commands{
{
Name: "publish",
Usage: "Publish Grafana artifacts",
Action: PublishArtifactsAction,
Flags: []cli.Flag{
&editionFlag,
&cli.BoolFlag{
Name: "security",
Usage: "Security release",
},
&cli.StringFlag{
Name: "security-dest-bucket",
Usage: "Google Cloud Storage bucket for security packages (or $SECURITY_DEST_BUCKET)",
},
&cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
},
&cli.StringFlag{
Name: "src-bucket",
Value: "grafana-prerelease",
Usage: "Google Cloud Storage bucket",
},
&cli.StringFlag{
Name: "dest-bucket",
Value: "grafana-downloads",
Usage: "Google Cloud Storage bucket for published packages",
},
&cli.StringFlag{
Name: "enterprise2-dest-bucket",
Value: "grafana-downloads-enterprise2",
Usage: "Google Cloud Storage bucket for published packages",
},
&cli.StringFlag{
Name: "enterprise2-security-prefix",
Usage: "Bucket path prefix for enterprise2 security releases (or $ENTERPRISE2_SECURITY_PREFIX)",
},
&cli.StringFlag{
Name: "static-assets-bucket",
Value: "grafana-static-assets",
Usage: "Google Cloud Storage bucket for static assets",
},
&cli.StringSliceFlag{
Name: "static-asset-editions",
Usage: "All the editions of the static assets (or $STATIC_ASSET_EDITIONS)",
},
&cli.StringFlag{
Name: "storybook-bucket",
Value: "grafana-storybook",
Usage: "Google Cloud Storage bucket for storybooks",
},
},
},
{
Name: "docker",
Usage: "Handle Grafana Docker images",
@@ -165,11 +258,54 @@ func main() {
Name: "fetch",
Usage: "Fetch Grafana Docker images",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, FetchImages),
Action: MaxArgCountWrapper(1, FetchImages),
Flags: []cli.Flag{
&editionFlag,
},
},
{
Name: "publish-enterprise2",
Usage: "Handle Grafana Enterprise2 Docker images",
ArgsUsage: "[version]",
Action: Enterprise2,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "dockerhub-repo",
Usage: "DockerHub repo to push images",
},
},
},
},
},
{
Name: "npm",
Usage: "Handle Grafana npm packages",
Subcommands: cli.Commands{
{
Name: "release",
Usage: "Release npm packages",
ArgsUsage: "[version]",
Action: NpmReleaseAction,
Flags: []cli.Flag{
&tagFlag,
},
},
{
Name: "store",
Usage: "Store npm packages tarball",
Action: NpmStoreAction,
Flags: []cli.Flag{
&tagFlag,
},
},
{
Name: "retrieve",
Usage: "Retrieve npm packages tarball",
Action: NpmRetrieveAction,
Flags: []cli.Flag{
&tagFlag,
},
},
},
},
},
@@ -196,7 +332,7 @@ func main() {
{
Name: "github",
Usage: "Publish packages to GitHub releases",
Action: PublishGitHub,
Action: PublishGithub,
Flags: []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
@@ -210,7 +346,7 @@ func main() {
},
&cli.StringFlag{
Name: "tag",
Usage: "Release tag (default from metadata)ß",
Usage: "Release tag (default from metadata)",
},
&cli.BoolFlag{
Name: "create",
@@ -218,10 +354,69 @@ func main() {
},
},
},
{
Name: "aws",
Usage: "Publish image to AWS Marketplace releases",
Action: PublishAwsMarketplace,
Flags: []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
Name: "version",
Usage: "Release version (default from metadata)",
},
&cli.StringFlag{
Name: "image",
Required: true,
Usage: "Name of the image to be released",
},
&cli.StringFlag{
Name: "repo",
Required: true,
Usage: "AWS Marketplace ECR repository",
},
&cli.StringFlag{
Name: "product",
Required: true,
Usage: "AWS Marketplace product identifier",
},
},
},
},
},
{
Name: "enterprise-check",
Usage: "Commands for testing against Grafana Enterprise",
Subcommands: cli.Commands{
{
Name: "begin",
Usage: "Creates the GitHub check in a pull request and begins the tests",
Action: EnterpriseCheckBegin,
Flags: []cli.Flag{
&gitHubTokenFlag,
},
},
{
Name: "success",
Usage: "Updates the GitHub check in a pull request to show a successful build and updates the pull request labels",
Action: EnterpriseCheckSuccess,
Flags: []cli.Flag{
&gitHubTokenFlag,
},
},
{
Name: "fail",
Usage: "Updates the GitHub check in a pull request to show a failed build and updates the pull request labels",
Action: EnterpriseCheckFail,
Flags: []cli.Flag{
&gitHubTokenFlag,
},
},
},
},
}
app.Commands = append(app.Commands, additionalCommands...)
if err := app.Run(os.Args); err != nil {
log.Fatalln(err)
}
+88
View File
@@ -0,0 +1,88 @@
package main
import (
"fmt"
"os"
"os/exec"
"strings"
"github.com/grafana/grafana/pkg/build/npm"
"github.com/urfave/cli/v2"
)
func NpmRetrieveAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exitting")
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
}
err := npm.FetchNpmPackages(c.Context, tag, prereleaseBucket)
if err != nil {
return err
}
return nil
}
func NpmStoreAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exiting")
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
}
err := npm.StoreNpmPackages(c.Context, tag, prereleaseBucket)
if err != nil {
return err
}
return nil
}
func NpmReleaseAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exitting")
}
cmd := exec.Command("git", "checkout", ".")
if err := cmd.Run(); err != nil {
fmt.Println("command failed to run, err: ", err)
return err
}
err := npm.PublishNpmPackages(c.Context, tag)
if err != nil {
return err
}
return nil
}
+3 -3
View File
@@ -13,7 +13,7 @@ import (
)
func Package(c *cli.Context) error {
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
@@ -22,12 +22,12 @@ func Package(c *cli.Context) error {
releaseMode, err := metadata.GetReleaseMode()
if err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
releaseModeConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
cfg := config.Config{
+211
View File
@@ -0,0 +1,211 @@
package main
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/versions"
"github.com/urfave/cli/v2"
)
type publishConfig struct {
tag string
srcBucket string
destBucket string
enterprise2DestBucket string
enterprise2SecurityPrefix string
staticAssetsBucket string
staticAssetEditions []string
storybookBucket string
security bool
}
// requireListWithEnvFallback first checks the CLI for a flag with the required
// name. If this is empty, it falls back to taking the environment variable.
// Sadly, we cannot use cli.Flag.EnvVars for this due to it potentially leaking
// environment variables as default values in usage-errors.
func requireListWithEnvFallback(cctx *cli.Context, name string, envName string) ([]string, error) {
result := cctx.StringSlice(name)
if len(result) == 0 {
for _, v := range strings.Split(os.Getenv(envName), ",") {
value := strings.TrimSpace(v)
if value != "" {
result = append(result, value)
}
}
}
if len(result) == 0 {
return nil, cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1)
}
return result, nil
}
func requireStringWithEnvFallback(cctx *cli.Context, name string, envName string) (string, error) {
result := cctx.String(name)
if result == "" {
result = os.Getenv(envName)
}
if result == "" {
return "", cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1)
}
return result, nil
}
// Action implements the sub-command "publish-artifacts".
func PublishArtifactsAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
staticAssetEditions, err := requireListWithEnvFallback(c, "static-asset-editions", "STATIC_ASSET_EDITIONS")
if err != nil {
return err
}
securityDestBucket, err := requireStringWithEnvFallback(c, "security-dest-bucket", "SECURITY_DEST_BUCKET")
if err != nil {
return err
}
enterprise2SecurityPrefix, err := requireStringWithEnvFallback(c, "enterprise2-security-prefix", "ENTERPRISE2_SECURITY_PREFIX")
if err != nil {
return err
}
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("error connecting to gcp, %q", err)
}
cfg := publishConfig{
srcBucket: c.String("src-bucket"),
destBucket: c.String("dest-bucket"),
enterprise2DestBucket: c.String("enterprise2-dest-bucket"),
enterprise2SecurityPrefix: enterprise2SecurityPrefix,
staticAssetsBucket: c.String("static-assets-bucket"),
staticAssetEditions: staticAssetEditions,
storybookBucket: c.String("storybook-bucket"),
security: c.Bool("security"),
tag: strings.TrimPrefix(c.String("tag"), "v"),
}
if cfg.security {
cfg.destBucket = securityDestBucket
}
err = copyStaticAssets(cfg)
if err != nil {
return err
}
err = copyStorybook(cfg)
if err != nil {
return err
}
err = copyDownloads(cfg)
if err != nil {
return err
}
err = copyEnterprise2Downloads(cfg)
if err != nil {
return err
}
return nil
}
func copyStaticAssets(cfg publishConfig) error {
for _, edition := range cfg.staticAssetEditions {
log.Printf("Copying static assets for %s", edition)
srcURL := fmt.Sprintf("%s/artifacts/static-assets/%s/%s/*", cfg.srcBucket, edition, cfg.tag)
destURL := fmt.Sprintf("%s/%s/%s/", cfg.staticAssetsBucket, edition, cfg.tag)
err := gcsCopy("static assets", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying static assets, %q", err)
}
}
log.Printf("Successfully copied static assets!")
return nil
}
func copyStorybook(cfg publishConfig) error {
if cfg.security {
log.Printf("skipping storybook copy - not needed for a security release")
return nil
}
log.Printf("Copying storybooks...")
srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag)
destURL := fmt.Sprintf("%s/%s", cfg.storybookBucket, cfg.tag)
err := gcsCopy("storybook", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying storybook. %q", err)
}
stableVersion, err := versions.GetLatestVersion(versions.LatestStableVersionURL)
if err != nil {
return err
}
isLatest, err := versions.IsGreaterThanOrEqual(cfg.tag, stableVersion)
if err != nil {
return err
}
if isLatest {
log.Printf("Copying storybooks to latest...")
srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag)
destURL := fmt.Sprintf("%s/latest", cfg.storybookBucket)
err := gcsCopy("storybook (latest)", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying storybook to latest. %q", err)
}
}
log.Printf("Successfully copied storybook!")
return nil
}
func copyDownloads(cfg publishConfig) error {
for _, edition := range []string{
"oss", "enterprise",
} {
destURL := fmt.Sprintf("%s/%s/", cfg.destBucket, edition)
srcURL := fmt.Sprintf("%s/artifacts/downloads/v%s/%s/release/*", cfg.srcBucket, cfg.tag, edition)
if !cfg.security {
destURL = filepath.Join(destURL, "release")
}
log.Printf("Copying downloads for %s, from %s bucket to %s bucket", edition, srcURL, destURL)
err := gcsCopy("downloads", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying downloads, %q", err)
}
}
log.Printf("Successfully copied downloads.")
return nil
}
func copyEnterprise2Downloads(cfg publishConfig) error {
var prefix string
if cfg.security {
prefix = cfg.enterprise2SecurityPrefix
}
srcURL := fmt.Sprintf("%s/artifacts/downloads-enterprise2/v%s/enterprise2/release/*", cfg.srcBucket, cfg.tag)
destURL := fmt.Sprintf("%s/enterprise2/%srelease", cfg.enterprise2DestBucket, prefix)
log.Printf("Copying downloads for enterprise2, from %s bucket to %s bucket", srcURL, destURL)
err := gcsCopy("enterprise2 downloads", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying ")
}
return nil
}
func gcsCopy(desc, src, dest string) error {
args := strings.Split(fmt.Sprintf("-m cp -r gs://%s gs://%s", src, dest), " ")
// nolint:gosec
cmd := exec.Command("gsutil", args...)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to publish %s: %w\n%s", desc, err, out)
}
return nil
}
+308
View File
@@ -0,0 +1,308 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/marketplacecatalog"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/grafana/grafana/pkg/build/config"
"github.com/urfave/cli/v2"
)
const (
marketplaceChangeSetName = "Add new version"
marketplaceCatalogId = "AWSMarketplace"
marketplaceRegistryId = "709825985650"
marketplaceRegistryRegion = "us-east-1"
marketplaceRegistryUrl = "709825985650.dkr.ecr.us-east-1.amazonaws.com"
marketplaceRequestsUrl = "https://aws.amazon.com/marketplace/management/requests/"
releaseNotesTemplateUrl = "https://grafana.com/docs/grafana/latest/release-notes/release-notes-${TAG}/"
helmChartsUrl = "https://grafana.github.io/helm-charts/"
docsUrl = "https://grafana.com/docs/grafana/latest/enterprise/license/"
imagePlatform = "linux/amd64"
publishAwsMarketplaceTestKey publishAwsMarketplaceTestKeyType = "test-client"
)
var (
errEmptyVersion = errors.New(`failed to retrieve release version from metadata, use "--version" to set it manually`)
)
type publishAwsMarketplaceTestKeyType string
type publishAwsMarketplaceFlags struct {
dryRun bool
version string
repo string
image string
product string
}
type AwsMarketplacePublishingService struct {
auth string
docker AwsMarketplaceDocker
ecr AwsMarketplaceRegistry
mkt AwsMarketplaceCatalog
}
type AwsMarketplaceDocker interface {
ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error)
ImageTag(ctx context.Context, source string, target string) error
ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error)
}
type AwsMarketplaceRegistry interface {
GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, opts ...request.Option) (*ecr.GetAuthorizationTokenOutput, error)
}
type AwsMarketplaceCatalog interface {
DescribeEntityWithContext(ctx context.Context, input *marketplacecatalog.DescribeEntityInput, opts ...request.Option) (*marketplacecatalog.DescribeEntityOutput, error)
StartChangeSetWithContext(ctx context.Context, input *marketplacecatalog.StartChangeSetInput, opts ...request.Option) (*marketplacecatalog.StartChangeSetOutput, error)
}
func PublishAwsMarketplace(ctx *cli.Context) error {
f, err := getPublishAwsMarketplaceFlags(ctx)
if err != nil {
return err
}
if f.version == "" {
return errEmptyVersion
}
svc, err := getAwsMarketplacePublishingService()
if err != nil {
return err
}
if ctx.Context.Value(publishAwsMarketplaceTestKey) != nil {
svc = ctx.Context.Value(publishAwsMarketplaceTestKey).(*AwsMarketplacePublishingService)
}
fmt.Println("Logging in to AWS Marketplace registry")
err = svc.Login(ctx.Context)
if err != nil {
return err
}
fmt.Printf("Retrieving image '%s:%s' from Docker Hub\n", f.image, f.version)
err = svc.PullImage(ctx.Context, f.image, f.version)
if err != nil {
return err
}
fmt.Printf("Renaming image '%s:%s' to '%s/%s:%s'\n", f.image, f.version, marketplaceRegistryUrl, f.repo, f.version)
err = svc.TagImage(ctx.Context, f.image, f.repo, f.version)
if err != nil {
return err
}
if !f.dryRun {
fmt.Printf("Pushing image '%s/%s:%s' to the AWS Marketplace ECR\n", marketplaceRegistryUrl, f.repo, f.version)
err = svc.PushToMarketplace(ctx.Context, f.repo, f.version)
if err != nil {
return err
}
} else {
fmt.Printf("Dry-Run: Pushing image '%s/%s:%s' to the AWS Marketplace ECR\n", marketplaceRegistryUrl, f.repo, f.version)
}
fmt.Printf("Retrieving product identifier for product '%s'\n", f.product)
pid, err := svc.GetProductIdentifier(ctx.Context, f.product)
if err != nil {
return err
}
if !f.dryRun {
fmt.Printf("Releasing to product, you can view the progress of the release on %s\n", marketplaceRequestsUrl)
return svc.ReleaseToProduct(ctx.Context, pid, f.repo, f.version)
} else {
fmt.Printf("Dry-Run: Releasing to product, you can view the progress of the release on %s\n", marketplaceRequestsUrl)
}
return nil
}
func getAwsMarketplacePublishingService() (*AwsMarketplacePublishingService, error) {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, err
}
mySession := session.Must(session.NewSession())
ecr := ecr.New(mySession, aws.NewConfig().WithRegion(marketplaceRegistryRegion))
mkt := marketplacecatalog.New(mySession, aws.NewConfig().WithRegion(marketplaceRegistryRegion))
return &AwsMarketplacePublishingService{
docker: cli,
ecr: ecr,
mkt: mkt,
}, nil
}
func (s *AwsMarketplacePublishingService) Login(ctx context.Context) error {
out, err := s.ecr.GetAuthorizationTokenWithContext(ctx, &ecr.GetAuthorizationTokenInput{})
if err != nil {
return err
}
s.auth = *out.AuthorizationData[0].AuthorizationToken
authData, err := base64.StdEncoding.DecodeString(s.auth)
if err != nil {
return err
}
authString := strings.Split(string(authData), ":")
authData, err = json.Marshal(types.AuthConfig{
Username: authString[0],
Password: authString[1],
})
s.auth = base64.StdEncoding.EncodeToString(authData)
return err
}
func (s *AwsMarketplacePublishingService) PullImage(ctx context.Context, image string, version string) error {
reader, err := s.docker.ImagePull(ctx, fmt.Sprintf("%s:%s", image, version), types.ImagePullOptions{
Platform: imagePlatform,
})
if err != nil {
return err
}
_, err = io.Copy(os.Stdout, reader)
if err != nil {
return err
}
err = reader.Close()
if err != nil {
return err
}
return nil
}
func (s *AwsMarketplacePublishingService) TagImage(ctx context.Context, image string, repo string, version string) error {
err := s.docker.ImageTag(ctx, fmt.Sprintf("%s:%s", image, version), fmt.Sprintf("%s/%s:%s", marketplaceRegistryUrl, repo, version))
if err != nil {
return err
}
return nil
}
func (s *AwsMarketplacePublishingService) PushToMarketplace(ctx context.Context, repo string, version string) error {
reader, err := s.docker.ImagePush(ctx, fmt.Sprintf("%s/%s:%s", marketplaceRegistryUrl, repo, version), types.ImagePushOptions{
RegistryAuth: s.auth,
})
if err != nil {
return err
}
_, err = io.Copy(os.Stdout, reader)
if err != nil {
return err
}
err = reader.Close()
if err != nil {
return err
}
return nil
}
func (s *AwsMarketplacePublishingService) GetProductIdentifier(ctx context.Context, product string) (string, error) {
out, err := s.mkt.DescribeEntityWithContext(ctx, &marketplacecatalog.DescribeEntityInput{
EntityId: aws.String(product),
Catalog: aws.String(marketplaceCatalogId),
})
if err != nil {
return "", err
}
return *out.EntityIdentifier, nil
}
func (s *AwsMarketplacePublishingService) ReleaseToProduct(ctx context.Context, pid string, repo string, version string) error {
_, err := s.mkt.StartChangeSetWithContext(ctx, &marketplacecatalog.StartChangeSetInput{
Catalog: aws.String(marketplaceCatalogId),
ChangeSetName: aws.String(marketplaceChangeSetName),
ChangeSet: []*marketplacecatalog.Change{
buildAwsMarketplaceChangeSet(pid, repo, version),
},
})
return err
}
func getPublishAwsMarketplaceFlags(ctx *cli.Context) (*publishAwsMarketplaceFlags, error) {
metadata, err := config.GenerateMetadata(ctx)
if err != nil {
return nil, err
}
version := ctx.String("version")
if version == "" && metadata.GrafanaVersion != "" {
version = metadata.GrafanaVersion
}
image := ctx.String("image")
repo := ctx.String("repo")
product := ctx.String("product")
dryRun := ctx.Bool("dry-run")
return &publishAwsMarketplaceFlags{
dryRun: dryRun,
version: version,
image: image,
repo: repo,
product: product,
}, nil
}
func buildAwsMarketplaceReleaseNotesUrl(version string) string {
sanitizedVersion := strings.ReplaceAll(version, ".", "-")
return strings.ReplaceAll(releaseNotesTemplateUrl, "${TAG}", sanitizedVersion)
}
func buildAwsMarketplaceChangeSet(entityId string, repo string, version string) *marketplacecatalog.Change {
return &marketplacecatalog.Change{
ChangeType: aws.String("AddDeliveryOptions"),
Entity: &marketplacecatalog.Entity{
Type: aws.String("ContainerProduct@1.0"),
Identifier: aws.String(entityId),
},
Details: aws.String(buildAwsMarketplaceVersionDetails(repo, version)),
}
}
func buildAwsMarketplaceVersionDetails(repo string, version string) string {
releaseNotesUrl := buildAwsMarketplaceReleaseNotesUrl(version)
return fmt.Sprintf(`{
"Version": {
"ReleaseNotes": "Release notes are available on the website %s",
"VersionTitle": "v%s"
},
"DeliveryOptions": [
{
"Details": {
"EcrDeliveryOptionDetails": {
"DeploymentResources": [
{
"Name": "Helm Charts",
"Url": "%s"
}
],
"CompatibleServices": ["EKS", "ECS", "ECS-Anywhere", "EKS-Anywhere"],
"ContainerImages": ["%s/%s:%s"],
"Description": "Grafana Enterprise can be installed using the official Grafana Helm chart repository. The repository is available on Github: %s",
"UsageInstructions": "You can apply your Grafana Enterprise license to a new or existing Grafana Enterprise deployment by updating a configuration setting or environment variable. Your Grafana instance must be deployed on AWS, or have network access to AWS. For more information, see %s"
}
},
"DeliveryOptionTitle": "Helm Chart"
}
]
}`, releaseNotesUrl, version, helmChartsUrl, marketplaceRegistryUrl, repo, version, helmChartsUrl, docsUrl)
}
+207
View File
@@ -0,0 +1,207 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"errors"
"io"
"os"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/marketplacecatalog"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
"github.com/urfave/cli/v2"
)
type awsPublishTestCase struct {
name string
args []string
expectedError error
errorContains string
expectedOutput string
mockedService *AwsMarketplacePublishingService
}
func TestPublishAwsMarketplace(t *testing.T) {
t.Setenv("DRONE_BUILD_EVENT", "promote")
t.Setenv("DRONE_TAG", "v1.0.0")
t.Setenv("DRONE_COMMIT", "abcdefgh")
testApp := setupPublishAwsMarketplaceTests(t)
errShouldNotCallMock := errors.New("shouldn't call")
testCases := []awsPublishTestCase{
{
name: "try to publish without required flags",
errorContains: `Required flags "image, repo, product" not set`,
},
{
name: "try to publish without credentials",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{
GetAuthorizationTokenWithContextError: credentials.ErrNoValidProvidersFoundInChain,
},
},
expectedError: credentials.ErrNoValidProvidersFoundInChain,
},
{
name: "try to publish with valid credentials and nonexisting version",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{},
docker: &mockAwsMarketplaceDocker{},
mkt: &mockAwsMarketplaceCatalog{},
},
expectedOutput: "Releasing to product",
},
{
name: "try to publish with valid credentials and existing version",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{},
docker: &mockAwsMarketplaceDocker{},
mkt: &mockAwsMarketplaceCatalog{},
},
expectedOutput: "Releasing to product",
},
{
name: "dry run with invalid credentials",
args: []string{"--dry-run", "--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{
GetAuthorizationTokenWithContextError: credentials.ErrNoValidProvidersFoundInChain,
},
},
expectedError: credentials.ErrNoValidProvidersFoundInChain,
},
{
name: "dry run with valid credentials",
args: []string{"--dry-run", "--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{},
docker: &mockAwsMarketplaceDocker{
ImagePushError: errShouldNotCallMock,
},
mkt: &mockAwsMarketplaceCatalog{
StartChangeSetWithContextError: errShouldNotCallMock,
},
},
expectedOutput: "Dry-Run: Releasing to product",
},
}
if os.Getenv("DRONE_COMMIT") == "" {
// this test only works locally due to Drone environment
testCases = append(testCases,
awsPublishTestCase{
name: "try to publish without version",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test"},
expectedError: errEmptyVersion,
},
)
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
ctx := context.WithValue(context.Background(), publishAwsMarketplaceTestKey, test.mockedService)
args := []string{"run"}
args = append(args, test.args...)
out, err := captureStdout(t, func() error {
return testApp.RunContext(ctx, args)
})
if test.expectedOutput != "" {
assert.Contains(t, out, test.expectedOutput)
}
if test.expectedError != nil || test.errorContains != "" {
assert.Error(t, err)
if test.expectedError != nil {
assert.ErrorIs(t, err, test.expectedError)
}
if test.errorContains != "" {
assert.ErrorContains(t, err, test.errorContains)
}
} else {
assert.NoError(t, err)
}
})
}
}
func setupPublishAwsMarketplaceTests(t *testing.T) *cli.App {
t.Helper()
testApp := cli.NewApp()
testApp.Action = PublishAwsMarketplace
testApp.Flags = []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
Name: "version",
Usage: "Release version (default from metadata)",
},
&cli.StringFlag{
Name: "image",
Required: true,
Usage: "Name of the image to be released",
},
&cli.StringFlag{
Name: "repo",
Required: true,
Usage: "AWS Marketplace ECR repository",
},
&cli.StringFlag{
Name: "product",
Required: true,
Usage: "AWS Marketplace product identifier",
},
}
return testApp
}
type mockAwsMarketplaceDocker struct {
ImagePullError error
ImageTagError error
ImagePushError error
}
func (m *mockAwsMarketplaceDocker) ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader([]byte(""))), m.ImagePullError
}
func (m *mockAwsMarketplaceDocker) ImageTag(ctx context.Context, source string, target string) error {
return m.ImageTagError
}
func (m *mockAwsMarketplaceDocker) ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader([]byte(""))), m.ImagePushError
}
type mockAwsMarketplaceRegistry struct {
GetAuthorizationTokenWithContextError error
}
func (m *mockAwsMarketplaceRegistry) GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, opts ...request.Option) (*ecr.GetAuthorizationTokenOutput, error) {
return &ecr.GetAuthorizationTokenOutput{
AuthorizationData: []*ecr.AuthorizationData{
{
AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString([]byte("username:password"))),
},
},
}, m.GetAuthorizationTokenWithContextError
}
type mockAwsMarketplaceCatalog struct {
DescribeEntityWithContextError error
StartChangeSetWithContextError error
}
func (m *mockAwsMarketplaceCatalog) DescribeEntityWithContext(ctx context.Context, input *marketplacecatalog.DescribeEntityInput, opts ...request.Option) (*marketplacecatalog.DescribeEntityOutput, error) {
return &marketplacecatalog.DescribeEntityOutput{
EntityIdentifier: aws.String("productid"),
}, m.DescribeEntityWithContextError
}
func (m *mockAwsMarketplaceCatalog) StartChangeSetWithContext(ctx context.Context, input *marketplacecatalog.StartChangeSetInput, opts ...request.Option) (*marketplacecatalog.StartChangeSetOutput, error) {
return &marketplacecatalog.StartChangeSetOutput{}, m.StartChangeSetWithContextError
}
+8 -7
View File
@@ -9,6 +9,7 @@ import (
"strings"
"github.com/google/go-github/github"
"github.com/grafana/grafana/pkg/build/config"
"github.com/urfave/cli/v2"
"golang.org/x/oauth2"
)
@@ -39,9 +40,9 @@ var (
errReleaseNotFound = errors.New(`release not found, use "--create" to create the release`)
)
func PublishGitHub(ctx *cli.Context) error {
func PublishGithub(ctx *cli.Context) error {
token := os.Getenv("GH_TOKEN")
f, err := getFlags(ctx)
f, err := getPublishGithubFlags(ctx)
if err != nil {
return err
}
@@ -55,7 +56,7 @@ func PublishGitHub(ctx *cli.Context) error {
}
if f.dryRun {
return runDryRun(f, token, ctx)
return runPublishGithubDryRun(f, token, ctx)
}
client := newGithubClient(ctx.Context, token)
@@ -99,8 +100,8 @@ func githubRepositoryClient(ctx context.Context, token string) githubRepositoryS
return client.Repositories
}
func getFlags(ctx *cli.Context) (*publishGithubFlags, error) {
metadata, err := GenerateMetadata(ctx)
func getPublishGithubFlags(ctx *cli.Context) (*publishGithubFlags, error) {
metadata, err := config.GenerateMetadata(ctx)
if err != nil {
return nil, err
}
@@ -130,12 +131,12 @@ func getFlags(ctx *cli.Context) (*publishGithubFlags, error) {
}, nil
}
func runDryRun(f *publishGithubFlags, token string, ctx *cli.Context) error {
func runPublishGithubDryRun(f *publishGithubFlags, token string, ctx *cli.Context) error {
client := newGithubClient(ctx.Context, token)
fmt.Println("Dry-Run: Retrieving release on repository by tag")
release, res, err := client.GetReleaseByTag(ctx.Context, f.repo.owner, f.repo.name, f.tag)
if err != nil && res.StatusCode != 404 {
fmt.Println("Dry-Run: GitHub communication error:\n", err)
fmt.Println("Dry-Run: Github communication error:\n", err)
return nil
}
+21 -19
View File
@@ -21,17 +21,19 @@ type githubPublishTestCases struct {
expectedError error
errorContains string
expectedOutput string
mockedService *mockGitHubRepositoryServiceImpl
mockedService *mockGithubRepositoryServiceImpl
}
var mockGitHubRepositoryService = &mockGitHubRepositoryServiceImpl{}
var mockGithubRepositoryService = &mockGithubRepositoryServiceImpl{}
func mockGithubRepositoryClient(context.Context, string) githubRepositoryService {
return mockGitHubRepositoryService
return mockGithubRepositoryService
}
func TestPublishGitHub(t *testing.T) {
func TestPublishGithub(t *testing.T) {
t.Setenv("DRONE_BUILD_EVENT", "promote")
t.Setenv("DRONE_TAG", "v1.0.0")
t.Setenv("DRONE_COMMIT", "abcdefgh")
testApp, testPath := setupPublishGithubTests(t)
mockErrUnauthorized := errors.New("401")
@@ -49,21 +51,21 @@ func TestPublishGitHub(t *testing.T) {
name: "try to publish with invalid token",
token: "invalid",
args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
expectedError: mockErrUnauthorized,
},
{
name: "try to publish with valid token and nonexisting tag with create disabled",
token: "valid",
args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
expectedError: errReleaseNotFound,
},
{
name: "try to publish with valid token and nonexisting tag with create enabled",
token: "valid",
args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0", "--create"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
},
{
name: "try to publish with valid token and existing tag",
@@ -74,21 +76,21 @@ func TestPublishGitHub(t *testing.T) {
name: "dry run with invalid token",
token: "invalid",
args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
expectedOutput: "GitHub communication error",
mockedService: &mockGithubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
expectedOutput: "Github communication error",
},
{
name: "dry run with valid token and nonexisting tag with create disabled",
token: "valid",
args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
expectedOutput: "Release doesn't exist",
},
{
name: "dry run with valid token and nonexisting tag with create enabled",
token: "valid",
args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0", "--create"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
expectedOutput: "Would upload asset",
},
{
@@ -116,9 +118,9 @@ func TestPublishGitHub(t *testing.T) {
t.Setenv("GH_TOKEN", test.token)
}
if test.mockedService != nil {
mockGitHubRepositoryService = test.mockedService
mockGithubRepositoryService = test.mockedService
} else {
mockGitHubRepositoryService = &mockGitHubRepositoryServiceImpl{}
mockGithubRepositoryService = &mockGithubRepositoryServiceImpl{}
}
args := []string{"run"}
args = append(args, test.args...)
@@ -154,7 +156,7 @@ func setupPublishGithubTests(t *testing.T) (*cli.App, string) {
newGithubClient = mockGithubRepositoryClient
testApp := cli.NewApp()
testApp.Action = PublishGitHub
testApp.Action = PublishGithub
testApp.Flags = []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
@@ -165,7 +167,7 @@ func setupPublishGithubTests(t *testing.T) (*cli.App, string) {
&cli.StringFlag{
Name: "repo",
Required: true,
Usage: "GitHub repository",
Usage: "Github repository",
},
&cli.StringFlag{
Name: "tag",
@@ -194,13 +196,13 @@ func captureStdout(t *testing.T, fn func() error) (string, error) {
return string(out), err
}
type mockGitHubRepositoryServiceImpl struct {
type mockGithubRepositoryServiceImpl struct {
tagErr error
createErr error
uploadErr error
}
func (m *mockGitHubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, owner string, repo string, tag string) (*github.RepositoryRelease, *github.Response, error) {
func (m *mockGithubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, owner string, repo string, tag string) (*github.RepositoryRelease, *github.Response, error) {
var release *github.RepositoryRelease
res := &github.Response{Response: &http.Response{}}
if m.tagErr == nil {
@@ -212,12 +214,12 @@ func (m *mockGitHubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, o
return release, res, m.tagErr
}
func (m *mockGitHubRepositoryServiceImpl) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) {
func (m *mockGithubRepositoryServiceImpl) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) {
releaseID := int64(1)
return &github.RepositoryRelease{ID: &releaseID}, &github.Response{}, m.createErr
}
func (m *mockGitHubRepositoryServiceImpl) UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) {
func (m *mockGithubRepositoryServiceImpl) UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) {
assetName := "test"
assetUrl := "testurl.com.br"
return &github.ReleaseAsset{Name: &assetName, BrowserDownloadURL: &assetUrl}, &github.Response{}, m.uploadErr
+100
View File
@@ -0,0 +1,100 @@
package main
import (
"fmt"
"log"
"os"
"os/exec"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/urfave/cli/v2"
)
func Enterprise2(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return err
}
cfg := docker.Config{
Archs: buildConfig.Docker.Architectures,
Distribution: buildConfig.Docker.Distribution,
DockerHubRepo: c.String("dockerhub-repo"),
Tag: metadata.GrafanaVersion,
}
err = dockerLoginEnterprise2()
if err != nil {
return err
}
var distributionStr []string
for _, distribution := range cfg.Distribution {
switch distribution {
case alpine:
distributionStr = append(distributionStr, "")
case ubuntu:
distributionStr = append(distributionStr, "-ubuntu")
default:
return fmt.Errorf("unrecognized distribution %q", distribution)
}
}
for _, distribution := range distributionStr {
var imageFileNames []string
for _, arch := range cfg.Archs {
imageFilename := fmt.Sprintf("%s:%s%s-%s", cfg.DockerHubRepo, cfg.Tag, distribution, arch)
err := docker.PushImage(imageFilename)
if err != nil {
return err
}
imageFileNames = append(imageFileNames, imageFilename)
}
manifest := fmt.Sprintf("%s:%s%s", cfg.DockerHubRepo, cfg.Tag, distribution)
args := []string{"manifest", "create", manifest}
args = append(args, imageFileNames...)
//nolint:gosec
cmd := exec.Command("docker", args...)
cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled")
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create Docker manifest: %w\n%s", err, output)
}
err = docker.PushManifest(manifest)
if err != nil {
return err
}
}
return nil
}
func dockerLoginEnterprise2() error {
log.Println("Docker login...")
cmd := exec.Command("gcloud", "auth", "configure-docker")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("error logging in to DockerHub: %s %q", out, err)
}
log.Println("Successful login!")
return nil
}
+6 -5
View File
@@ -8,8 +8,9 @@ import (
"os"
"regexp"
"github.com/grafana/grafana/pkg/build/metrics"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/metrics"
)
func PublishMetrics(c *cli.Context) error {
@@ -17,24 +18,24 @@ func PublishMetrics(c *cli.Context) error {
input, err := io.ReadAll(os.Stdin)
if err != nil {
return cli.NewExitError(fmt.Sprintf("Reading from stdin failed: %s", err), 1)
return cli.Exit(fmt.Sprintf("Reading from stdin failed: %s", err), 1)
}
reMetrics := regexp.MustCompile(`(?ms)^Metrics: (\{.+\})`)
ms := reMetrics.FindSubmatch(input)
if len(ms) == 0 {
return cli.NewExitError(fmt.Sprintf("Input on wrong format: %q", string(input)), 1)
return cli.Exit(fmt.Sprintf("Input on wrong format: %q", string(input)), 1)
}
m := map[string]string{}
if err := json.Unmarshal(ms[1], &m); err != nil {
return cli.NewExitError(fmt.Sprintf("decoding metrics failed: %s", err), 1)
return cli.Exit(fmt.Sprintf("decoding metrics failed: %s", err), 1)
}
log.Printf("Received metrics %+v", m)
if err := metrics.Publish(m, apiKey); err != nil {
return cli.NewExitError(fmt.Sprintf("publishing metrics failed: %s", err), 1)
return cli.Exit(fmt.Sprintf("publishing metrics failed: %s", err), 1)
}
return nil
+1 -1
View File
@@ -13,7 +13,7 @@ import (
func StoreStorybook(c *cli.Context) error {
deployment := c.String("deployment")
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"fmt"
"log"
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/urfave/cli/v2"
)
// UploadCDN implements the sub-command "upload-cdn".
func UploadCDN(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
version := metadata.GrafanaVersion
if err != nil {
return cli.Exit(err.Error(), 1)
}
buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return err
}
edition := os.Getenv("EDITION")
log.Printf("Uploading Grafana CDN Assets, version %s, %s edition...", version, edition)
editionPath := ""
switch config.Edition(edition) {
case config.EditionOSS:
editionPath = "grafana-oss"
case config.EditionEnterprise:
editionPath = "grafana"
case config.EditionEnterprise2:
editionPath = os.Getenv("ENTERPRISE2_CDN_PATH")
default:
panic(fmt.Sprintf("unrecognized edition %q", edition))
}
gcs, err := storage.New()
if err != nil {
return err
}
bucket := gcs.Bucket(buildConfig.Buckets.CDNAssets)
srcPath := buildConfig.Buckets.CDNAssetsDir
srcPath = filepath.Join(srcPath, editionPath, version)
if err := gcs.DeleteDir(c.Context, bucket, srcPath); err != nil {
return err
}
log.Printf("Successfully cleaned source: %s/%s\n", buildConfig.Buckets.CDNAssets, srcPath)
if err := gcs.CopyLocalDir(c.Context, "./public", bucket, srcPath, false); err != nil {
return err
}
log.Printf("Successfully uploaded cdn static assets to: %s/%s!\n", buildConfig.Buckets.CDNAssets, srcPath)
return nil
}
+213
View File
@@ -0,0 +1,213 @@
package main
import (
"encoding/base64"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/urfave/cli/v2"
)
const releaseFolder = "release"
const mainFolder = "main"
const releaseBranchFolder = "prerelease"
type uploadConfig struct {
config.Config
edition config.Edition
versionMode config.VersionMode
gcpKey string
distDir string
versionFolder string
}
// UploadPackages implements the sub-command "upload-packages".
func UploadPackages(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
gcpKeyB64 := strings.TrimSpace(os.Getenv("GCP_KEY"))
if gcpKeyB64 == "" {
return cli.Exit("the environment variable GCP_KEY must be set", 1)
}
gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKeyB64)
if err != nil {
return cli.Exit("failed to base64 decode $GCP_KEY", 1)
}
gcpKey := string(gcpKeyB)
distDir, err := filepath.Abs("dist")
if err != nil {
return cli.Exit(err.Error(), 1)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
version := metadata.GrafanaVersion
releaseMode, err := metadata.GetReleaseMode()
if err != nil {
return cli.Exit(err.Error(), 1)
}
releaseModeConfig, err := config.GetBuildConfig(releaseMode.Mode)
if err != nil {
return cli.Exit(err.Error(), 1)
}
var edition config.Edition
if e, ok := os.LookupEnv("EDITION"); ok {
edition = config.Edition(e)
}
if c.Bool("enterprise2") {
edition = config.EditionEnterprise2
}
if edition == "" {
return fmt.Errorf("both EDITION envvar and '--enterprise2' flag are missing. At least one of those is required")
}
// TODO: Verify config values
cfg := uploadConfig{
Config: config.Config{
Version: version,
Bucket: releaseModeConfig.Buckets.Artifacts,
},
edition: edition,
versionMode: releaseMode.Mode,
gcpKey: gcpKey,
distDir: distDir,
}
event, err := droneutil.GetDroneEventFromEnv()
if err != nil {
return err
}
if cfg.edition == config.EditionEnterprise2 {
cfg.Bucket, err = bucketForEnterprise2(releaseModeConfig, event)
if err != nil {
return err
}
}
cfg.versionFolder, err = getVersionFolder(cfg, event)
if err != nil {
return err
}
if err := uploadPackages(cfg); err != nil {
return cli.Exit(err.Error(), 1)
}
log.Println("Successfully uploaded packages!")
return nil
}
// Corner case for custom enterprise2 mode
func bucketForEnterprise2(releaseModeConfig *config.BuildConfig, event string) (string, error) {
if event == config.Custom {
buildConfig, err := config.GetBuildConfig(config.ReleaseBranchMode)
if err != nil {
return "", err
}
return buildConfig.Buckets.ArtifactsEnterprise2, nil
}
if releaseModeConfig.Buckets.ArtifactsEnterprise2 != "" {
return releaseModeConfig.Buckets.ArtifactsEnterprise2, nil
}
return "", fmt.Errorf("enterprise2 bucket var doesn't exist")
}
func getVersionFolder(cfg uploadConfig, event string) (string, error) {
switch cfg.versionMode {
case config.TagMode:
return releaseFolder, nil
case config.MainMode, config.DownstreamMode:
return mainFolder, nil
case config.ReleaseBranchMode:
return releaseBranchFolder, nil
default:
// Corner case for custom enterprise2 mode
if event == config.Custom && cfg.versionMode == config.Enterprise2Mode {
return releaseFolder, nil
}
return "", fmt.Errorf("unrecognized version mode: %s", cfg.versionMode)
}
}
func uploadPackages(cfg uploadConfig) error {
log.Printf("Uploading Grafana packages, version %s, %s edition, %s mode...\n", cfg.Version, cfg.edition,
cfg.versionMode)
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
edition := strings.ToLower(string(cfg.edition))
var sfx string
switch cfg.edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = "-enterprise"
case config.EditionEnterprise2:
sfx = "-enterprise2"
default:
panic(fmt.Sprintf("unrecognized edition %q", cfg.edition))
}
matches, err := filepath.Glob(filepath.Join(cfg.distDir, fmt.Sprintf("grafana%s*", sfx)))
if err != nil {
return fmt.Errorf("failed to list packages: %w", err)
}
fpaths := []string{}
rePkg := packaging.PackageRegexp(cfg.edition)
for _, fpath := range matches {
fname := filepath.Base(fpath)
if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) {
log.Printf("Ignoring file %q\n", fpath)
continue
}
fpaths = append(fpaths, fpath)
}
var tag, gcsPath string
droneTag := strings.TrimSpace(os.Getenv("DRONE_TAG"))
if droneTag != "" {
tag = droneTag
gcsPath = fmt.Sprintf("gs://%s/%s/%s/%s", cfg.Bucket, tag, edition, cfg.versionFolder)
} else {
gcsPath = fmt.Sprintf("gs://%s/%s/%s/", cfg.Bucket, edition, cfg.versionFolder)
}
log.Printf("Uploading %d file(s) to GCS (%s)...\n", len(fpaths), gcsPath)
args := []string{"-m", "cp"}
args = append(args, fpaths...)
args = append(args, gcsPath)
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload files to GCS: %s", output)
}
return nil
}
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"errors"
"fmt"
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_getVersionFolder(t *testing.T) {
type args struct {
cfg uploadConfig
event string
versionFolder string
}
tests := []struct {
name string
args args
err error
}{
{"tag mode", args{uploadConfig{versionMode: config.TagMode}, "", releaseFolder}, nil},
{"main mode", args{uploadConfig{versionMode: config.MainMode}, "", mainFolder}, nil},
{"downstream mode", args{uploadConfig{versionMode: config.DownstreamMode}, "", mainFolder}, nil},
{"release branch mode", args{uploadConfig{versionMode: config.ReleaseBranchMode}, "", releaseBranchFolder}, nil},
{"enterprise pro mode", args{uploadConfig{versionMode: config.Enterprise2Mode}, config.Custom, releaseFolder}, nil},
{"unrecognised version mode", args{uploadConfig{versionMode: "foo"}, config.Custom, ""}, errors.New("")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
versionMode, err := getVersionFolder(tt.args.cfg, tt.args.event)
if tt.err != nil {
require.Error(t, err)
}
require.Equal(t, versionMode, tt.args.versionFolder)
})
}
}
func Test_checkForEnterprise2Edition(t *testing.T) {
type args struct {
releaseModeConfig *config.BuildConfig
event string
}
tests := []struct {
name string
args args
want string
err error
}{
{"event is not custom", args{releaseModeConfig: &config.BuildConfig{Buckets: config.Buckets{ArtifactsEnterprise2: "dummy"}}}, "dummy", nil},
{"event is not custom and string is empty", args{releaseModeConfig: &config.BuildConfig{Buckets: config.Buckets{ArtifactsEnterprise2: ""}}}, "", fmt.Errorf("enterprise2 bucket var doesn't exist")},
{"event is custom", args{releaseModeConfig: nil, event: "custom"}, "grafana-downloads-enterprise2", nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := bucketForEnterprise2(tt.args.releaseModeConfig, tt.args.event)
if tt.err != nil {
require.Error(t, err)
}
assert.Equalf(t, tt.want, got, "bucketForEnterprise2(%v, %v)", tt.args.releaseModeConfig, tt.args.event)
})
}
}
+6 -5
View File
@@ -13,10 +13,11 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/grafana/grafana/pkg/build/fsutil"
cliv1 "github.com/urfave/cli"
"github.com/urfave/cli/v2"
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/build/fsutil"
)
func VerifyDrone(c *cli.Context) error {
@@ -24,7 +25,7 @@ func VerifyDrone(c *cli.Context) error {
const backup = ".drone.yml.bak"
if err := fsutil.CopyFile(yml, backup); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to copy %s to %s: %s", yml, backup, err), 1)
return cli.Exit(fmt.Sprintf("failed to copy %s to %s: %s", yml, backup, err), 1)
}
defer func() {
if err := os.Remove(yml); err != nil {
@@ -73,7 +74,7 @@ func readConfig(fpath string) ([]map[string]interface{}, error) {
//nolint:gosec
f, err := os.Open(fpath)
if err != nil {
return nil, cli.NewExitError(fmt.Sprintf("failed to read %s: %s", fpath, err), 1)
return nil, cli.Exit(fmt.Sprintf("failed to read %s: %s", fpath, err), 1)
}
defer func() {
if err := f.Close(); err != nil {
@@ -90,7 +91,7 @@ func readConfig(fpath string) ([]map[string]interface{}, error) {
if errors.Is(err, io.EOF) {
break
}
return nil, cli.NewExitError(fmt.Sprintf("Failed to decode %s: %s", fpath, err), 1)
return nil, cli.Exit(fmt.Sprintf("Failed to decode %s: %s", fpath, err), 1)
}
if m["kind"] == "signature" {
@@ -118,7 +119,7 @@ func verifyYAML(yml, backup string) error {
}
if !cmp.Equal(c1, c2) {
return cli.NewExitError(fmt.Sprintf("%s is out of sync with .drone.star - regenerate it with drone starlark convert",
return cli.Exit(fmt.Sprintf("%s is out of sync with .drone.star - regenerate it with drone starlark convert",
yml), 1)
}
+142
View File
@@ -0,0 +1,142 @@
package main
import (
"context"
"errors"
"fmt"
"io/fs"
"os/exec"
"path/filepath"
"strings"
"github.com/urfave/cli/v2"
)
func mapSlice[I any, O any](a []I, f func(I) O) []O {
o := make([]O, len(a))
for i, e := range a {
o[i] = f(e)
}
return o
}
// VerifyStarlark is the CLI Action for verifying Starlark files in a workspace.
// It expects a single context argument which is the path to the workspace.
// The actual verification procedure can return multiple errors which are
// joined together to be one holistic error for the action.
func VerifyStarlark(c *cli.Context) error {
if c.NArg() != 1 {
var message string
if c.NArg() == 0 {
message = "ERROR: missing required argument <workspace path>"
}
if c.NArg() > 1 {
message = "ERROR: too many arguments"
}
if err := cli.ShowSubcommandHelp(c); err != nil {
return err
}
return cli.Exit(message, 1)
}
workspace := c.Args().Get(0)
verificationErrs, executionErr := verifyStarlark(c.Context, workspace, buildifierLintCommand)
if executionErr != nil {
return executionErr
}
if len(verificationErrs) == 0 {
return nil
}
noun := "file"
if len(verificationErrs) > 1 {
noun += "s"
}
return fmt.Errorf("verification failed for %d %s:\n%s",
len(verificationErrs),
noun,
strings.Join(
mapSlice(verificationErrs, func(e error) string { return e.Error() }),
"\n",
))
}
type commandFunc = func(path string) (command string, args []string)
func buildifierLintCommand(path string) (string, []string) {
return "buildifier", []string{"-lint", "warn", "-mode", "check", path}
}
// verifyStarlark walks all directories starting at provided workspace path and
// verifies any Starlark files it finds.
// Starlark files are assumed to end with the .star extension.
// The verification relies on linting frovided by the 'buildifier' binary which
// must be in the PATH.
// A slice of verification errors are returned, one for each file that failed verification.
// If any execution of the `buildifier` command fails, this is returned separately.
// commandFn is executed on every Starlark file to determine the command and arguments to be executed.
// The caller is trusted and it is the callers responsibility to ensure that the resulting command is safe to execute.
func verifyStarlark(ctx context.Context, workspace string, commandFn commandFunc) ([]error, error) {
var verificationErrs []error
// All errors from filepath.WalkDir are filtered by the fs.WalkDirFunc.
// Lstat or ReadDir errors are reported as verificationErrors.
// If any execution of the `buildifier` command fails or if the context is cancelled,
// it is reported as an error and any verification of subsequent files is skipped.
err := filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error {
// Skip verification of the file or files within the directory if there is an error
// returned by Lstat or ReadDir.
if err != nil {
verificationErrs = append(verificationErrs, err)
return nil
}
if d.IsDir() {
return nil
}
if filepath.Ext(path) == ".star" {
command, args := commandFn(path)
// The caller is trusted.
//nolint:gosec
cmd := exec.CommandContext(ctx, command, args...)
cmd.Dir = workspace
_, err = cmd.Output()
if err == nil { // No error, early return.
return nil
}
// The error returned from cmd.Output() is never wrapped.
//nolint:errorlint
if err, ok := err.(*exec.ExitError); ok {
switch err.ExitCode() {
// Case comments are informed by the output of `buildifier --help`
case 1: // syntax errors in input
verificationErrs = append(verificationErrs, errors.New(string(err.Stderr)))
return nil
case 2: // usage errors: invoked incorrectly
return fmt.Errorf("command %q: %s", cmd, err.Stderr)
case 3: // unexpected runtime errors: file I/O problems or internal bugs
return fmt.Errorf("command %q: %s", cmd, err.Stderr)
case 4: // check mode failed (reformat is needed)
verificationErrs = append(verificationErrs, errors.New(string(err.Stderr)))
return nil
default:
return fmt.Errorf("command %q: %s", cmd, err.Stderr)
}
}
// Error was not an exit error from the command.
return fmt.Errorf("command %q: %v", cmd, err)
}
return nil
})
return verificationErrs, err
}
+137
View File
@@ -0,0 +1,137 @@
//go:build requires_buildifier
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestVerifyStarlark(t *testing.T) {
t.Run("execution errors", func(t *testing.T) {
t.Run("invalid usage", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
_, executionErr := verifyStarlark(ctx, workspace, func(string) (string, []string) { return "buildifier", []string{"--invalid"} })
if executionErr == nil {
t.Fatalf("Expected execution error but got none")
}
})
t.Run("context cancellation", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
workspace := t.TempDir()
err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
err = os.WriteFile(filepath.Join(workspace, "other-ignored.star"), []byte{}, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
cancel()
_, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr == nil {
t.Fatalf("Expected execution error but got none")
}
})
})
t.Run("verification errors", func(t *testing.T) {
t.Run("a single file with lint", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
invalidContent := []byte(`load("scripts/drone/other.star", "function")
function()`)
err := os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr != nil {
t.Fatalf("Unexpected execution error: %v", executionErr)
}
if len(verificationErrs) == 0 {
t.Fatalf(`"has-lint.star" requires linting but the verifyStarlark function provided no linting error`)
}
if len(verificationErrs) > 1 {
t.Fatalf(`verifyStarlark returned multiple errors for the "has-lint.star" file but only one was expected: %v`, verificationErrs)
}
if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") {
t.Fatalf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0])
}
})
t.Run("no files with lint", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
content := []byte(`"""
This module does nothing.
"""
load("scripts/drone/other.star", "function")
function()
`)
require.NoError(t, os.WriteFile(filepath.Join(workspace, "no-lint.star"), content, os.ModePerm))
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr != nil {
t.Fatalf("Unexpected execution error: %v", executionErr)
}
if len(verificationErrs) != 0 {
t.Log(`"no-lint.star" has no lint but the verifyStarlark function provided at least one error`)
for _, err := range verificationErrs {
t.Log(err)
}
t.FailNow()
}
})
t.Run("multiple files with lint", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
invalidContent := []byte(`load("scripts/drone/other.star", "function")
function()`)
require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint2.star"), invalidContent, os.ModePerm))
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr != nil {
t.Fatalf("Unexpected execution error: %v", executionErr)
}
if len(verificationErrs) == 0 {
t.Fatalf(`Two files require linting but the verifyStarlark function provided no linting error`)
}
if len(verificationErrs) == 1 {
t.Fatalf(`Two files require linting but the verifyStarlark function provided only one linting error: %v`, verificationErrs[0])
}
if len(verificationErrs) > 2 {
t.Fatalf(`verifyStarlark returned more errors than expected: %v`, verificationErrs)
}
if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") {
t.Errorf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0])
}
if !strings.Contains(verificationErrs[1].Error(), "has-lint2.star:1: module-docstring: The file has no module docstring.") {
t.Fatalf(`"has-lint2.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0])
}
})
})
}
+33
View File
@@ -0,0 +1,33 @@
// Package verifystorybook contains the sub-command "verify-storybook".
package main
import (
"fmt"
"log"
"path/filepath"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/infra/fs"
)
// VerifyStorybook Action implements the sub-command "verify-storybook".
func VerifyStorybook(c *cli.Context) error {
const grafanaDir = "."
paths := []string{
"packages/grafana-ui/dist/storybook/index.html",
"packages/grafana-ui/dist/storybook/iframe.html"}
for _, p := range paths {
exists, err := fs.Exists(filepath.Join(grafanaDir, p))
if err != nil {
return cli.Exit(fmt.Sprintf("failed to verify Storybook build: %s", err), 1)
}
if !exists {
return fmt.Errorf("failed to verify Storybook build, missing %q", p)
}
}
log.Printf("Successfully verified Storybook integrity")
return nil
}