CI: Backport CI/Release related code to v9.3.x (#62752)

* Batch-move everything

* go mod tidy

* make drone

* Remove genversions

* Bump alpine image

* Revert back pkg/build/docker/build.go

* Make sure correct enterprise branch is checked out

* Add enterprise2 version

* Remove extensions

* Bump build container

* backport node 18 test fix

(cherry picked from commit 4ff03fdbfb)

* Update scripts/drone

* Add more commands

* Fix starlark link

* Copy .drone.star

* Add drone target branch for custom events

---------
This commit is contained in:
Dimitris Sotirakis
2023-02-03 11:43:48 +02:00
committed by GitHub
parent d9b5c5f4c6
commit 03b1cf763d
85 changed files with 8622 additions and 2695 deletions
+16 -1
View File
@@ -2,7 +2,22 @@ package main
import "github.com/urfave/cli/v2"
func ArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc {
// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided.
func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() != num {
if err := cli.ShowSubcommandHelp(ctx); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
return action(ctx)
}
}
// ArgCountWrapper will cause the action to fail if there were more than `num` args provided.
func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() > max {
if err := cli.ShowSubcommandHelp(ctx); err != nil {
+1 -1
View File
@@ -13,7 +13,7 @@ import (
)
func BuildBackend(ctx *cli.Context) error {
metadata, err := GenerateMetadata(ctx)
metadata, err := config.GenerateMetadata(ctx)
if err != nil {
return err
}
+1 -1
View File
@@ -14,7 +14,7 @@ func BuildDocker(c *cli.Context) error {
return err
}
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+2 -1
View File
@@ -3,6 +3,7 @@ package main
import (
"log"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/errutil"
"github.com/grafana/grafana/pkg/build/frontend"
"github.com/grafana/grafana/pkg/build/syncutil"
@@ -10,7 +11,7 @@ import (
)
func BuildFrontend(c *cli.Context) error {
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+1 -1
View File
@@ -17,7 +17,7 @@ func BuildInternalPlugins(c *cli.Context) error {
}
const grafanaDir = "."
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"fmt"
"log"
"os"
"strconv"
"github.com/grafana/grafana/pkg/build/env"
"github.com/grafana/grafana/pkg/build/git"
"github.com/urfave/cli/v2"
)
// checkOpts are options used to create a new GitHub check for the enterprise downstream test.
type checkOpts struct {
SHA string
URL string
Branch string
PR int
}
func getCheckOpts(args []string) (*checkOpts, error) {
branch, ok := env.Lookup("DRONE_SOURCE_BRANCH", args)
if !ok {
return nil, cli.Exit("Unable to retrieve build source branch", 1)
}
var (
rgx = git.PRCheckRegexp()
matches = rgx.FindStringSubmatch(branch)
)
sha, ok := env.Lookup("SOURCE_COMMIT", args)
if !ok {
if matches == nil || len(matches) <= 1 {
return nil, cli.Exit("Unable to retrieve source commit", 1)
}
sha = matches[2]
}
url, ok := env.Lookup("DRONE_BUILD_LINK", args)
if !ok {
return nil, cli.Exit(`missing environment variable "DRONE_BUILD_LINK"`, 1)
}
prStr, ok := env.Lookup("OSS_PULL_REQUEST", args)
if !ok {
if matches == nil || len(matches) <= 1 {
return nil, cli.Exit("Unable to retrieve PR number", 1)
}
prStr = matches[1]
}
pr, err := strconv.Atoi(prStr)
if err != nil {
return nil, err
}
return &checkOpts{
Branch: branch,
PR: pr,
SHA: sha,
URL: url,
}, nil
}
// EnterpriseCheckBegin creates the GitHub check and signals the beginning of the downstream build / test process
func EnterpriseCheckBegin(c *cli.Context) error {
var (
ctx = c.Context
client = git.NewGitHubClient(ctx, c.String("github-token"))
)
opts, err := getCheckOpts(os.Environ())
if err != nil {
return err
}
if _, err = git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, "pending"); err != nil {
return err
}
return nil
}
func EnterpriseCheckSuccess(c *cli.Context) error {
return completeEnterpriseCheck(c, true)
}
func EnterpriseCheckFail(c *cli.Context) error {
return completeEnterpriseCheck(c, false)
}
func completeEnterpriseCheck(c *cli.Context, success bool) error {
var (
ctx = c.Context
client = git.NewGitHubClient(ctx, c.String("github-token"))
)
// Update the pull request labels
opts, err := getCheckOpts(os.Environ())
if err != nil {
return err
}
status := "failure"
if success {
status = "success"
}
// Update the GitHub check...
if _, err := git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, status); err != nil {
return err
}
// Delete branch if needed
log.Printf("Checking branch '%s' against '%s'", git.PRCheckRegexp().String(), opts.Branch)
if git.PRCheckRegexp().MatchString(opts.Branch) {
log.Println("Deleting branch", opts.Branch)
if err := git.DeleteEnterpriseBranch(ctx, client.Git, opts.Branch); err != nil {
return fmt.Errorf("error deleting enterprise branch: %w", err)
}
}
label := "enterprise-failed"
if success {
label = "enterprise-ok"
}
return git.AddLabelToPR(ctx, client.Issues, opts.PR, label)
}
+69
View File
@@ -0,0 +1,69 @@
package main
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetCheckOpts(t *testing.T) {
t.Run("it should return the checkOpts if the correct environment variables are set", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
"OSS_PULL_REQUEST=1",
}
opts, err := getCheckOpts(args)
require.NoError(t, err)
require.Equal(t, opts.SHA, "1234")
require.Equal(t, opts.URL, "http://example.com")
})
t.Run("it should return an error if SOURCE_COMMIT is not set", func(t *testing.T) {
args := []string{
"DRONE_BUILD_LINK=http://example.com",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
"OSS_PULL_REQUEST=1",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
t.Run("it should return an error if DRONE_BUILD_LINK is not set", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"OSS_PULL_REQUEST=1",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
t.Run("it should return an error if OSS_PULL_REQUEST is not set", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
t.Run("it should return an error if OSS_PULL_REQUEST is not an integer", func(t *testing.T) {
args := []string{
"SOURCE_COMMIT=1234",
"DRONE_SOURCE_BRANCH=test",
"DRONE_BUILD_LINK=http://example.com",
"OSS_PULL_REQUEST=http://example.com",
}
opts, err := getCheckOpts(args)
require.Nil(t, opts)
require.Error(t, err)
})
}
+2 -1
View File
@@ -4,11 +4,12 @@ import (
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/urfave/cli/v2"
)
func ExportVersion(c *cli.Context) error {
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+3 -3
View File
@@ -20,12 +20,12 @@ const (
func FetchImages(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
return cli.NewExitError("", 1)
return cli.Exit("", 1)
}
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+10
View File
@@ -46,4 +46,14 @@ var (
Usage: "Google Cloud Platform key file",
Required: true,
}
gitHubTokenFlag = cli.StringFlag{
Name: "github-token",
Value: "",
EnvVars: []string{"GITHUB_TOKEN"},
Usage: "GitHub token",
}
tagFlag = cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
}
)
-92
View File
@@ -1,92 +0,0 @@
package main
import (
"fmt"
"os"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/urfave/cli/v2"
)
func GenerateMetadata(c *cli.Context) (config.Metadata, error) {
var metadata config.Metadata
version := ""
event, err := droneutil.GetDroneEventFromEnv()
if err != nil {
return config.Metadata{}, err
}
var releaseMode config.ReleaseMode
switch event {
case string(config.PullRequestMode):
releaseMode = config.ReleaseMode{Mode: config.PullRequestMode}
case config.Push:
mode, err := config.CheckDroneTargetBranch()
if err != nil {
return config.Metadata{}, err
}
releaseMode = config.ReleaseMode{Mode: mode}
case config.Custom:
mode, err := config.CheckDroneTargetBranch()
if err != nil {
return config.Metadata{}, err
}
// if there is a custom event targeting the main branch, that's an enterprise downstream build
if mode == config.MainBranch {
releaseMode = config.ReleaseMode{Mode: config.CustomMode}
} else {
releaseMode = config.ReleaseMode{Mode: mode}
}
case config.Tag, config.Promote:
tag, ok := os.LookupEnv("DRONE_TAG")
if !ok || tag == "" {
return config.Metadata{}, err
}
version = strings.TrimPrefix(tag, "v")
mode, err := config.CheckSemverSuffix()
if err != nil {
return config.Metadata{}, err
}
releaseMode = mode
case config.Cronjob:
releaseMode = config.ReleaseMode{Mode: config.CronjobMode}
}
if version == "" {
version, err = generateVersionFromBuildID()
if err != nil {
return config.Metadata{}, err
}
}
currentCommit, err := config.GetDroneCommit()
if err != nil {
return config.Metadata{}, err
}
metadata = config.Metadata{
GrafanaVersion: version,
ReleaseMode: releaseMode,
GrabplVersion: c.App.Version,
CurrentCommit: currentCommit,
}
fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode)
return metadata, nil
}
func generateVersionFromBuildID() (string, error) {
buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER")
if !ok {
return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable")
}
var err error
version, err := config.GetGrafanaVersion(buildID, ".")
if err != nil {
return "", err
}
return version, nil
}
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"flag"
"os"
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
)
const (
DroneBuildEvent = "DRONE_BUILD_EVENT"
DroneTargetBranch = "DRONE_TARGET_BRANCH"
DroneTag = "DRONE_TAG"
DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE"
DroneBuildNumber = "DRONE_BUILD_NUMBER"
)
const (
hashedGrafanaVersion = "9.2.0-12345pre"
versionedBranch = "v9.2.x"
)
func TestGetMetadata(t *testing.T) {
tcs := []struct {
envMap map[string]string
expVersion string
mode config.ReleaseMode
}{
{map[string]string{DroneBuildEvent: config.PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.PullRequestMode}},
{map[string]string{DroneBuildEvent: config.Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: config.Push, DroneTargetBranch: config.MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.MainMode}},
{map[string]string{DroneBuildEvent: config.Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: config.Custom, DroneTargetBranch: config.MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, config.ReleaseMode{Mode: config.Custom}},
{map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", config.ReleaseMode{Mode: config.TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: true}},
{map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", config.ReleaseMode{Mode: config.TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: config.Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", config.ReleaseMode{Mode: config.TagMode, IsBeta: false, IsTest: true}},
}
ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil)
for _, tc := range tcs {
t.Run("Should return valid metadata, ", func(t *testing.T) {
setUpEnv(t, tc.envMap)
testMetadata(t, ctx, tc.expVersion, tc.mode)
})
}
}
func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode config.ReleaseMode) {
t.Helper()
metadata, err := GenerateMetadata(ctx)
require.NoError(t, err)
t.Run("with a valid version", func(t *testing.T) {
expVersion := metadata.GrafanaVersion
require.Equal(t, expVersion, version)
})
t.Run("with a valid release mode from the built-in list", func(t *testing.T) {
expMode := metadata.ReleaseMode
require.NoError(t, err)
require.Equal(t, expMode, releaseMode)
})
}
func setUpEnv(t *testing.T, envMap map[string]string) {
t.Helper()
os.Clearenv()
err := os.Setenv("DRONE_COMMIT", "abcd12345")
require.NoError(t, err)
for k, v := range envMap {
err := os.Setenv(k, v)
require.NoError(t, err)
}
}
+8 -7
View File
@@ -15,11 +15,12 @@ import (
"strings"
"time"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/urfave/cli/v2"
)
const grafanaAPI = "https://grafana.com/api"
@@ -33,7 +34,7 @@ func GrafanaCom(c *cli.Context) error {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
@@ -66,11 +67,11 @@ func GrafanaCom(c *cli.Context) error {
grafanaAPIKey := strings.TrimSpace(os.Getenv("GRAFANA_COM_API_KEY"))
if grafanaAPIKey == "" {
return cli.NewExitError("the environment variable GRAFANA_COM_API_KEY must be set", 1)
return cli.Exit("the environment variable GRAFANA_COM_API_KEY must be set", 1)
}
whatsNewURL, releaseNotesURL, err := getReleaseURLs()
if err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
// TODO: Verify config values
@@ -89,7 +90,7 @@ func GrafanaCom(c *cli.Context) error {
}
if err := publishPackages(cfg); err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
log.Println("Successfully published packages to grafana.com!")
@@ -146,7 +147,7 @@ func publishPackages(cfg packaging.PublishConfig) error {
}
switch cfg.ReleaseMode.Mode {
case config.MainMode, config.CustomMode, config.CronjobMode:
case config.MainMode, config.DownstreamMode, config.CronjobMode:
pth = path.Join(pth, packaging.MainFolder)
default:
pth = path.Join(pth, packaging.ReleaseFolder)
@@ -177,7 +178,7 @@ func publishPackages(cfg packaging.PublishConfig) error {
Version: cfg.Version,
ReleaseDate: time.Now().UTC(),
Builds: builds,
Stable: cfg.ReleaseMode.Mode == config.TagMode,
Stable: cfg.ReleaseMode.Mode == config.TagMode && !cfg.ReleaseMode.IsBeta && !cfg.ReleaseMode.IsTest,
Beta: cfg.ReleaseMode.IsBeta,
Nightly: cfg.ReleaseMode.Mode == config.CronjobMode,
}
+206 -11
View File
@@ -5,10 +5,18 @@ import (
"os"
"strings"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/docker"
)
var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5)
//nolint:unused
func registerAppCommand(c *cli.Command) {
additionalCommands = append(additionalCommands, c)
}
func main() {
app := cli.NewApp()
app.Commands = cli.Commands{
@@ -16,7 +24,7 @@ func main() {
Name: "build-backend",
Usage: "Build one or more variants of back-end binaries",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, BuildBackend),
Action: MaxArgCountWrapper(1, BuildBackend),
Flags: []cli.Flag{
&jobsFlag,
&variantsFlag,
@@ -67,7 +75,7 @@ func main() {
Name: "build-frontend",
Usage: "Build front-end artifacts",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, BuildFrontend),
Action: MaxArgCountWrapper(1, BuildFrontend),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
@@ -77,7 +85,7 @@ func main() {
{
Name: "build-docker",
Usage: "Build Grafana Docker images",
Action: ArgCountWrapper(1, BuildDocker),
Action: MaxArgCountWrapper(1, BuildDocker),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
@@ -96,6 +104,14 @@ func main() {
},
},
},
{
Name: "upload-cdn",
Usage: "Upload public/* to a cdn bucket",
Action: UploadCDN,
Flags: []cli.Flag{
&editionFlag,
},
},
{
Name: "shellcheck",
Usage: "Run shellcheck on shell scripts",
@@ -104,7 +120,7 @@ func main() {
{
Name: "build-plugins",
Usage: "Build internal plug-ins",
Action: ArgCountWrapper(1, BuildInternalPlugins),
Action: MaxArgCountWrapper(1, BuildInternalPlugins),
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
@@ -117,13 +133,19 @@ func main() {
Name: "publish-metrics",
Usage: "Publish a set of metrics from stdin",
ArgsUsage: "<api-key>",
Action: ArgCountWrapper(1, PublishMetrics),
Action: MaxArgCountWrapper(1, PublishMetrics),
},
{
Name: "verify-drone",
Usage: "Verify Drone configuration",
Action: VerifyDrone,
},
{
Name: "verify-starlark",
Usage: "Verify Starlark configuration",
ArgsUsage: "<workspace path>",
Action: VerifyStarlark,
},
{
Name: "export-version",
Usage: "Exports version in dist/grafana.version",
@@ -133,7 +155,7 @@ func main() {
Name: "package",
Usage: "Package one or more Grafana variants",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, Package),
Action: MaxArgCountWrapper(1, Package),
Flags: []cli.Flag{
&jobsFlag,
&variantsFlag,
@@ -144,7 +166,7 @@ func main() {
},
{
Name: "store-storybook",
Usage: "Integrity check for storybook build",
Usage: "Stores storybook to GCS buckets",
Action: StoreStorybook,
Flags: []cli.Flag{
&cli.StringFlag{
@@ -153,10 +175,81 @@ func main() {
},
},
},
{
Name: "verify-storybook",
Usage: "Integrity check for storybook build",
Action: VerifyStorybook,
},
{
Name: "upload-packages",
Usage: "Upload Grafana packages",
Action: UploadPackages,
Flags: []cli.Flag{
&jobsFlag,
&editionFlag,
&cli.BoolFlag{
Name: "enterprise2",
Usage: "Declare if the edition is enterprise2",
},
},
},
{
Name: "artifacts",
Usage: "Handle Grafana artifacts",
Subcommands: cli.Commands{
{
Name: "publish",
Usage: "Publish Grafana artifacts",
Action: PublishArtifactsAction,
Flags: []cli.Flag{
&editionFlag,
&cli.BoolFlag{
Name: "security",
Usage: "Security release",
},
&cli.StringFlag{
Name: "security-dest-bucket",
Usage: "Google Cloud Storage bucket for security packages (or $SECURITY_DEST_BUCKET)",
},
&cli.StringFlag{
Name: "tag",
Usage: "Grafana version tag",
},
&cli.StringFlag{
Name: "src-bucket",
Value: "grafana-prerelease",
Usage: "Google Cloud Storage bucket",
},
&cli.StringFlag{
Name: "dest-bucket",
Value: "grafana-downloads",
Usage: "Google Cloud Storage bucket for published packages",
},
&cli.StringFlag{
Name: "enterprise2-dest-bucket",
Value: "grafana-downloads-enterprise2",
Usage: "Google Cloud Storage bucket for published packages",
},
&cli.StringFlag{
Name: "enterprise2-security-prefix",
Usage: "Bucket path prefix for enterprise2 security releases (or $ENTERPRISE2_SECURITY_PREFIX)",
},
&cli.StringFlag{
Name: "static-assets-bucket",
Value: "grafana-static-assets",
Usage: "Google Cloud Storage bucket for static assets",
},
&cli.StringSliceFlag{
Name: "static-asset-editions",
Usage: "All the editions of the static assets (or $STATIC_ASSET_EDITIONS)",
},
&cli.StringFlag{
Name: "storybook-bucket",
Value: "grafana-storybook",
Usage: "Google Cloud Storage bucket for storybooks",
},
},
},
{
Name: "docker",
Usage: "Handle Grafana Docker images",
@@ -165,11 +258,54 @@ func main() {
Name: "fetch",
Usage: "Fetch Grafana Docker images",
ArgsUsage: "[version]",
Action: ArgCountWrapper(1, FetchImages),
Action: MaxArgCountWrapper(1, FetchImages),
Flags: []cli.Flag{
&editionFlag,
},
},
{
Name: "publish-enterprise2",
Usage: "Handle Grafana Enterprise2 Docker images",
ArgsUsage: "[version]",
Action: Enterprise2,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "dockerhub-repo",
Usage: "DockerHub repo to push images",
},
},
},
},
},
{
Name: "npm",
Usage: "Handle Grafana npm packages",
Subcommands: cli.Commands{
{
Name: "release",
Usage: "Release npm packages",
ArgsUsage: "[version]",
Action: NpmReleaseAction,
Flags: []cli.Flag{
&tagFlag,
},
},
{
Name: "store",
Usage: "Store npm packages tarball",
Action: NpmStoreAction,
Flags: []cli.Flag{
&tagFlag,
},
},
{
Name: "retrieve",
Usage: "Retrieve npm packages tarball",
Action: NpmRetrieveAction,
Flags: []cli.Flag{
&tagFlag,
},
},
},
},
},
@@ -196,7 +332,7 @@ func main() {
{
Name: "github",
Usage: "Publish packages to GitHub releases",
Action: PublishGitHub,
Action: PublishGithub,
Flags: []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
@@ -210,7 +346,7 @@ func main() {
},
&cli.StringFlag{
Name: "tag",
Usage: "Release tag (default from metadata)ß",
Usage: "Release tag (default from metadata)",
},
&cli.BoolFlag{
Name: "create",
@@ -218,10 +354,69 @@ func main() {
},
},
},
{
Name: "aws",
Usage: "Publish image to AWS Marketplace releases",
Action: PublishAwsMarketplace,
Flags: []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
Name: "version",
Usage: "Release version (default from metadata)",
},
&cli.StringFlag{
Name: "image",
Required: true,
Usage: "Name of the image to be released",
},
&cli.StringFlag{
Name: "repo",
Required: true,
Usage: "AWS Marketplace ECR repository",
},
&cli.StringFlag{
Name: "product",
Required: true,
Usage: "AWS Marketplace product identifier",
},
},
},
},
},
{
Name: "enterprise-check",
Usage: "Commands for testing against Grafana Enterprise",
Subcommands: cli.Commands{
{
Name: "begin",
Usage: "Creates the GitHub check in a pull request and begins the tests",
Action: EnterpriseCheckBegin,
Flags: []cli.Flag{
&gitHubTokenFlag,
},
},
{
Name: "success",
Usage: "Updates the GitHub check in a pull request to show a successful build and updates the pull request labels",
Action: EnterpriseCheckSuccess,
Flags: []cli.Flag{
&gitHubTokenFlag,
},
},
{
Name: "fail",
Usage: "Updates the GitHub check in a pull request to show a failed build and updates the pull request labels",
Action: EnterpriseCheckFail,
Flags: []cli.Flag{
&gitHubTokenFlag,
},
},
},
},
}
app.Commands = append(app.Commands, additionalCommands...)
if err := app.Run(os.Args); err != nil {
log.Fatalln(err)
}
+88
View File
@@ -0,0 +1,88 @@
package main
import (
"fmt"
"os"
"os/exec"
"strings"
"github.com/grafana/grafana/pkg/build/npm"
"github.com/urfave/cli/v2"
)
func NpmRetrieveAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exitting")
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
}
err := npm.FetchNpmPackages(c.Context, tag, prereleaseBucket)
if err != nil {
return err
}
return nil
}
func NpmStoreAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exiting")
}
prereleaseBucket := strings.TrimSpace(os.Getenv("PRERELEASE_BUCKET"))
if prereleaseBucket == "" {
return cli.Exit("the environment variable PRERELEASE_BUCKET must be set", 1)
}
err := npm.StoreNpmPackages(c.Context, tag, prereleaseBucket)
if err != nil {
return err
}
return nil
}
func NpmReleaseAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
tag := c.String("tag")
if tag == "" {
return fmt.Errorf("no tag version specified, exitting")
}
cmd := exec.Command("git", "checkout", ".")
if err := cmd.Run(); err != nil {
fmt.Println("command failed to run, err: ", err)
return err
}
err := npm.PublishNpmPackages(c.Context, tag)
if err != nil {
return err
}
return nil
}
+3 -3
View File
@@ -13,7 +13,7 @@ import (
)
func Package(c *cli.Context) error {
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
@@ -22,12 +22,12 @@ func Package(c *cli.Context) error {
releaseMode, err := metadata.GetReleaseMode()
if err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
releaseModeConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return cli.NewExitError(err.Error(), 1)
return cli.Exit(err.Error(), 1)
}
cfg := config.Config{
+211
View File
@@ -0,0 +1,211 @@
package main
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/versions"
"github.com/urfave/cli/v2"
)
type publishConfig struct {
tag string
srcBucket string
destBucket string
enterprise2DestBucket string
enterprise2SecurityPrefix string
staticAssetsBucket string
staticAssetEditions []string
storybookBucket string
security bool
}
// requireListWithEnvFallback first checks the CLI for a flag with the required
// name. If this is empty, it falls back to taking the environment variable.
// Sadly, we cannot use cli.Flag.EnvVars for this due to it potentially leaking
// environment variables as default values in usage-errors.
func requireListWithEnvFallback(cctx *cli.Context, name string, envName string) ([]string, error) {
result := cctx.StringSlice(name)
if len(result) == 0 {
for _, v := range strings.Split(os.Getenv(envName), ",") {
value := strings.TrimSpace(v)
if value != "" {
result = append(result, value)
}
}
}
if len(result) == 0 {
return nil, cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1)
}
return result, nil
}
func requireStringWithEnvFallback(cctx *cli.Context, name string, envName string) (string, error) {
result := cctx.String(name)
if result == "" {
result = os.Getenv(envName)
}
if result == "" {
return "", cli.Exit(fmt.Sprintf("Required flag (%s) or environment variable (%s) not set", name, envName), 1)
}
return result, nil
}
// Action implements the sub-command "publish-artifacts".
func PublishArtifactsAction(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
staticAssetEditions, err := requireListWithEnvFallback(c, "static-asset-editions", "STATIC_ASSET_EDITIONS")
if err != nil {
return err
}
securityDestBucket, err := requireStringWithEnvFallback(c, "security-dest-bucket", "SECURITY_DEST_BUCKET")
if err != nil {
return err
}
enterprise2SecurityPrefix, err := requireStringWithEnvFallback(c, "enterprise2-security-prefix", "ENTERPRISE2_SECURITY_PREFIX")
if err != nil {
return err
}
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("error connecting to gcp, %q", err)
}
cfg := publishConfig{
srcBucket: c.String("src-bucket"),
destBucket: c.String("dest-bucket"),
enterprise2DestBucket: c.String("enterprise2-dest-bucket"),
enterprise2SecurityPrefix: enterprise2SecurityPrefix,
staticAssetsBucket: c.String("static-assets-bucket"),
staticAssetEditions: staticAssetEditions,
storybookBucket: c.String("storybook-bucket"),
security: c.Bool("security"),
tag: strings.TrimPrefix(c.String("tag"), "v"),
}
if cfg.security {
cfg.destBucket = securityDestBucket
}
err = copyStaticAssets(cfg)
if err != nil {
return err
}
err = copyStorybook(cfg)
if err != nil {
return err
}
err = copyDownloads(cfg)
if err != nil {
return err
}
err = copyEnterprise2Downloads(cfg)
if err != nil {
return err
}
return nil
}
func copyStaticAssets(cfg publishConfig) error {
for _, edition := range cfg.staticAssetEditions {
log.Printf("Copying static assets for %s", edition)
srcURL := fmt.Sprintf("%s/artifacts/static-assets/%s/%s/*", cfg.srcBucket, edition, cfg.tag)
destURL := fmt.Sprintf("%s/%s/%s/", cfg.staticAssetsBucket, edition, cfg.tag)
err := gcsCopy("static assets", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying static assets, %q", err)
}
}
log.Printf("Successfully copied static assets!")
return nil
}
func copyStorybook(cfg publishConfig) error {
if cfg.security {
log.Printf("skipping storybook copy - not needed for a security release")
return nil
}
log.Printf("Copying storybooks...")
srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag)
destURL := fmt.Sprintf("%s/%s", cfg.storybookBucket, cfg.tag)
err := gcsCopy("storybook", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying storybook. %q", err)
}
stableVersion, err := versions.GetLatestVersion(versions.LatestStableVersionURL)
if err != nil {
return err
}
isLatest, err := versions.IsGreaterThanOrEqual(cfg.tag, stableVersion)
if err != nil {
return err
}
if isLatest {
log.Printf("Copying storybooks to latest...")
srcURL := fmt.Sprintf("%s/artifacts/storybook/v%s/*", cfg.srcBucket, cfg.tag)
destURL := fmt.Sprintf("%s/latest", cfg.storybookBucket)
err := gcsCopy("storybook (latest)", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying storybook to latest. %q", err)
}
}
log.Printf("Successfully copied storybook!")
return nil
}
func copyDownloads(cfg publishConfig) error {
for _, edition := range []string{
"oss", "enterprise",
} {
destURL := fmt.Sprintf("%s/%s/", cfg.destBucket, edition)
srcURL := fmt.Sprintf("%s/artifacts/downloads/v%s/%s/release/*", cfg.srcBucket, cfg.tag, edition)
if !cfg.security {
destURL = filepath.Join(destURL, "release")
}
log.Printf("Copying downloads for %s, from %s bucket to %s bucket", edition, srcURL, destURL)
err := gcsCopy("downloads", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying downloads, %q", err)
}
}
log.Printf("Successfully copied downloads.")
return nil
}
func copyEnterprise2Downloads(cfg publishConfig) error {
var prefix string
if cfg.security {
prefix = cfg.enterprise2SecurityPrefix
}
srcURL := fmt.Sprintf("%s/artifacts/downloads-enterprise2/v%s/enterprise2/release/*", cfg.srcBucket, cfg.tag)
destURL := fmt.Sprintf("%s/enterprise2/%srelease", cfg.enterprise2DestBucket, prefix)
log.Printf("Copying downloads for enterprise2, from %s bucket to %s bucket", srcURL, destURL)
err := gcsCopy("enterprise2 downloads", srcURL, destURL)
if err != nil {
return fmt.Errorf("error copying ")
}
return nil
}
func gcsCopy(desc, src, dest string) error {
args := strings.Split(fmt.Sprintf("-m cp -r gs://%s gs://%s", src, dest), " ")
// nolint:gosec
cmd := exec.Command("gsutil", args...)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to publish %s: %w\n%s", desc, err, out)
}
return nil
}
+308
View File
@@ -0,0 +1,308 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/marketplacecatalog"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/grafana/grafana/pkg/build/config"
"github.com/urfave/cli/v2"
)
const (
marketplaceChangeSetName = "Add new version"
marketplaceCatalogId = "AWSMarketplace"
marketplaceRegistryId = "709825985650"
marketplaceRegistryRegion = "us-east-1"
marketplaceRegistryUrl = "709825985650.dkr.ecr.us-east-1.amazonaws.com"
marketplaceRequestsUrl = "https://aws.amazon.com/marketplace/management/requests/"
releaseNotesTemplateUrl = "https://grafana.com/docs/grafana/latest/release-notes/release-notes-${TAG}/"
helmChartsUrl = "https://grafana.github.io/helm-charts/"
docsUrl = "https://grafana.com/docs/grafana/latest/enterprise/license/"
imagePlatform = "linux/amd64"
publishAwsMarketplaceTestKey publishAwsMarketplaceTestKeyType = "test-client"
)
var (
errEmptyVersion = errors.New(`failed to retrieve release version from metadata, use "--version" to set it manually`)
)
type publishAwsMarketplaceTestKeyType string
type publishAwsMarketplaceFlags struct {
dryRun bool
version string
repo string
image string
product string
}
type AwsMarketplacePublishingService struct {
auth string
docker AwsMarketplaceDocker
ecr AwsMarketplaceRegistry
mkt AwsMarketplaceCatalog
}
type AwsMarketplaceDocker interface {
ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error)
ImageTag(ctx context.Context, source string, target string) error
ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error)
}
type AwsMarketplaceRegistry interface {
GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, opts ...request.Option) (*ecr.GetAuthorizationTokenOutput, error)
}
type AwsMarketplaceCatalog interface {
DescribeEntityWithContext(ctx context.Context, input *marketplacecatalog.DescribeEntityInput, opts ...request.Option) (*marketplacecatalog.DescribeEntityOutput, error)
StartChangeSetWithContext(ctx context.Context, input *marketplacecatalog.StartChangeSetInput, opts ...request.Option) (*marketplacecatalog.StartChangeSetOutput, error)
}
func PublishAwsMarketplace(ctx *cli.Context) error {
f, err := getPublishAwsMarketplaceFlags(ctx)
if err != nil {
return err
}
if f.version == "" {
return errEmptyVersion
}
svc, err := getAwsMarketplacePublishingService()
if err != nil {
return err
}
if ctx.Context.Value(publishAwsMarketplaceTestKey) != nil {
svc = ctx.Context.Value(publishAwsMarketplaceTestKey).(*AwsMarketplacePublishingService)
}
fmt.Println("Logging in to AWS Marketplace registry")
err = svc.Login(ctx.Context)
if err != nil {
return err
}
fmt.Printf("Retrieving image '%s:%s' from Docker Hub\n", f.image, f.version)
err = svc.PullImage(ctx.Context, f.image, f.version)
if err != nil {
return err
}
fmt.Printf("Renaming image '%s:%s' to '%s/%s:%s'\n", f.image, f.version, marketplaceRegistryUrl, f.repo, f.version)
err = svc.TagImage(ctx.Context, f.image, f.repo, f.version)
if err != nil {
return err
}
if !f.dryRun {
fmt.Printf("Pushing image '%s/%s:%s' to the AWS Marketplace ECR\n", marketplaceRegistryUrl, f.repo, f.version)
err = svc.PushToMarketplace(ctx.Context, f.repo, f.version)
if err != nil {
return err
}
} else {
fmt.Printf("Dry-Run: Pushing image '%s/%s:%s' to the AWS Marketplace ECR\n", marketplaceRegistryUrl, f.repo, f.version)
}
fmt.Printf("Retrieving product identifier for product '%s'\n", f.product)
pid, err := svc.GetProductIdentifier(ctx.Context, f.product)
if err != nil {
return err
}
if !f.dryRun {
fmt.Printf("Releasing to product, you can view the progress of the release on %s\n", marketplaceRequestsUrl)
return svc.ReleaseToProduct(ctx.Context, pid, f.repo, f.version)
} else {
fmt.Printf("Dry-Run: Releasing to product, you can view the progress of the release on %s\n", marketplaceRequestsUrl)
}
return nil
}
func getAwsMarketplacePublishingService() (*AwsMarketplacePublishingService, error) {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, err
}
mySession := session.Must(session.NewSession())
ecr := ecr.New(mySession, aws.NewConfig().WithRegion(marketplaceRegistryRegion))
mkt := marketplacecatalog.New(mySession, aws.NewConfig().WithRegion(marketplaceRegistryRegion))
return &AwsMarketplacePublishingService{
docker: cli,
ecr: ecr,
mkt: mkt,
}, nil
}
func (s *AwsMarketplacePublishingService) Login(ctx context.Context) error {
out, err := s.ecr.GetAuthorizationTokenWithContext(ctx, &ecr.GetAuthorizationTokenInput{})
if err != nil {
return err
}
s.auth = *out.AuthorizationData[0].AuthorizationToken
authData, err := base64.StdEncoding.DecodeString(s.auth)
if err != nil {
return err
}
authString := strings.Split(string(authData), ":")
authData, err = json.Marshal(types.AuthConfig{
Username: authString[0],
Password: authString[1],
})
s.auth = base64.StdEncoding.EncodeToString(authData)
return err
}
func (s *AwsMarketplacePublishingService) PullImage(ctx context.Context, image string, version string) error {
reader, err := s.docker.ImagePull(ctx, fmt.Sprintf("%s:%s", image, version), types.ImagePullOptions{
Platform: imagePlatform,
})
if err != nil {
return err
}
_, err = io.Copy(os.Stdout, reader)
if err != nil {
return err
}
err = reader.Close()
if err != nil {
return err
}
return nil
}
func (s *AwsMarketplacePublishingService) TagImage(ctx context.Context, image string, repo string, version string) error {
err := s.docker.ImageTag(ctx, fmt.Sprintf("%s:%s", image, version), fmt.Sprintf("%s/%s:%s", marketplaceRegistryUrl, repo, version))
if err != nil {
return err
}
return nil
}
func (s *AwsMarketplacePublishingService) PushToMarketplace(ctx context.Context, repo string, version string) error {
reader, err := s.docker.ImagePush(ctx, fmt.Sprintf("%s/%s:%s", marketplaceRegistryUrl, repo, version), types.ImagePushOptions{
RegistryAuth: s.auth,
})
if err != nil {
return err
}
_, err = io.Copy(os.Stdout, reader)
if err != nil {
return err
}
err = reader.Close()
if err != nil {
return err
}
return nil
}
func (s *AwsMarketplacePublishingService) GetProductIdentifier(ctx context.Context, product string) (string, error) {
out, err := s.mkt.DescribeEntityWithContext(ctx, &marketplacecatalog.DescribeEntityInput{
EntityId: aws.String(product),
Catalog: aws.String(marketplaceCatalogId),
})
if err != nil {
return "", err
}
return *out.EntityIdentifier, nil
}
func (s *AwsMarketplacePublishingService) ReleaseToProduct(ctx context.Context, pid string, repo string, version string) error {
_, err := s.mkt.StartChangeSetWithContext(ctx, &marketplacecatalog.StartChangeSetInput{
Catalog: aws.String(marketplaceCatalogId),
ChangeSetName: aws.String(marketplaceChangeSetName),
ChangeSet: []*marketplacecatalog.Change{
buildAwsMarketplaceChangeSet(pid, repo, version),
},
})
return err
}
func getPublishAwsMarketplaceFlags(ctx *cli.Context) (*publishAwsMarketplaceFlags, error) {
metadata, err := config.GenerateMetadata(ctx)
if err != nil {
return nil, err
}
version := ctx.String("version")
if version == "" && metadata.GrafanaVersion != "" {
version = metadata.GrafanaVersion
}
image := ctx.String("image")
repo := ctx.String("repo")
product := ctx.String("product")
dryRun := ctx.Bool("dry-run")
return &publishAwsMarketplaceFlags{
dryRun: dryRun,
version: version,
image: image,
repo: repo,
product: product,
}, nil
}
func buildAwsMarketplaceReleaseNotesUrl(version string) string {
sanitizedVersion := strings.ReplaceAll(version, ".", "-")
return strings.ReplaceAll(releaseNotesTemplateUrl, "${TAG}", sanitizedVersion)
}
func buildAwsMarketplaceChangeSet(entityId string, repo string, version string) *marketplacecatalog.Change {
return &marketplacecatalog.Change{
ChangeType: aws.String("AddDeliveryOptions"),
Entity: &marketplacecatalog.Entity{
Type: aws.String("ContainerProduct@1.0"),
Identifier: aws.String(entityId),
},
Details: aws.String(buildAwsMarketplaceVersionDetails(repo, version)),
}
}
func buildAwsMarketplaceVersionDetails(repo string, version string) string {
releaseNotesUrl := buildAwsMarketplaceReleaseNotesUrl(version)
return fmt.Sprintf(`{
"Version": {
"ReleaseNotes": "Release notes are available on the website %s",
"VersionTitle": "v%s"
},
"DeliveryOptions": [
{
"Details": {
"EcrDeliveryOptionDetails": {
"DeploymentResources": [
{
"Name": "Helm Charts",
"Url": "%s"
}
],
"CompatibleServices": ["EKS", "ECS", "ECS-Anywhere", "EKS-Anywhere"],
"ContainerImages": ["%s/%s:%s"],
"Description": "Grafana Enterprise can be installed using the official Grafana Helm chart repository. The repository is available on Github: %s",
"UsageInstructions": "You can apply your Grafana Enterprise license to a new or existing Grafana Enterprise deployment by updating a configuration setting or environment variable. Your Grafana instance must be deployed on AWS, or have network access to AWS. For more information, see %s"
}
},
"DeliveryOptionTitle": "Helm Chart"
}
]
}`, releaseNotesUrl, version, helmChartsUrl, marketplaceRegistryUrl, repo, version, helmChartsUrl, docsUrl)
}
+207
View File
@@ -0,0 +1,207 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"errors"
"io"
"os"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/marketplacecatalog"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
"github.com/urfave/cli/v2"
)
type awsPublishTestCase struct {
name string
args []string
expectedError error
errorContains string
expectedOutput string
mockedService *AwsMarketplacePublishingService
}
func TestPublishAwsMarketplace(t *testing.T) {
t.Setenv("DRONE_BUILD_EVENT", "promote")
t.Setenv("DRONE_TAG", "v1.0.0")
t.Setenv("DRONE_COMMIT", "abcdefgh")
testApp := setupPublishAwsMarketplaceTests(t)
errShouldNotCallMock := errors.New("shouldn't call")
testCases := []awsPublishTestCase{
{
name: "try to publish without required flags",
errorContains: `Required flags "image, repo, product" not set`,
},
{
name: "try to publish without credentials",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{
GetAuthorizationTokenWithContextError: credentials.ErrNoValidProvidersFoundInChain,
},
},
expectedError: credentials.ErrNoValidProvidersFoundInChain,
},
{
name: "try to publish with valid credentials and nonexisting version",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{},
docker: &mockAwsMarketplaceDocker{},
mkt: &mockAwsMarketplaceCatalog{},
},
expectedOutput: "Releasing to product",
},
{
name: "try to publish with valid credentials and existing version",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{},
docker: &mockAwsMarketplaceDocker{},
mkt: &mockAwsMarketplaceCatalog{},
},
expectedOutput: "Releasing to product",
},
{
name: "dry run with invalid credentials",
args: []string{"--dry-run", "--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{
GetAuthorizationTokenWithContextError: credentials.ErrNoValidProvidersFoundInChain,
},
},
expectedError: credentials.ErrNoValidProvidersFoundInChain,
},
{
name: "dry run with valid credentials",
args: []string{"--dry-run", "--image", "test/test", "--repo", "test/test", "--product", "test", "--version", "1.0.0"},
mockedService: &AwsMarketplacePublishingService{
ecr: &mockAwsMarketplaceRegistry{},
docker: &mockAwsMarketplaceDocker{
ImagePushError: errShouldNotCallMock,
},
mkt: &mockAwsMarketplaceCatalog{
StartChangeSetWithContextError: errShouldNotCallMock,
},
},
expectedOutput: "Dry-Run: Releasing to product",
},
}
if os.Getenv("DRONE_COMMIT") == "" {
// this test only works locally due to Drone environment
testCases = append(testCases,
awsPublishTestCase{
name: "try to publish without version",
args: []string{"--image", "test/test", "--repo", "test/test", "--product", "test"},
expectedError: errEmptyVersion,
},
)
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
ctx := context.WithValue(context.Background(), publishAwsMarketplaceTestKey, test.mockedService)
args := []string{"run"}
args = append(args, test.args...)
out, err := captureStdout(t, func() error {
return testApp.RunContext(ctx, args)
})
if test.expectedOutput != "" {
assert.Contains(t, out, test.expectedOutput)
}
if test.expectedError != nil || test.errorContains != "" {
assert.Error(t, err)
if test.expectedError != nil {
assert.ErrorIs(t, err, test.expectedError)
}
if test.errorContains != "" {
assert.ErrorContains(t, err, test.errorContains)
}
} else {
assert.NoError(t, err)
}
})
}
}
func setupPublishAwsMarketplaceTests(t *testing.T) *cli.App {
t.Helper()
testApp := cli.NewApp()
testApp.Action = PublishAwsMarketplace
testApp.Flags = []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
Name: "version",
Usage: "Release version (default from metadata)",
},
&cli.StringFlag{
Name: "image",
Required: true,
Usage: "Name of the image to be released",
},
&cli.StringFlag{
Name: "repo",
Required: true,
Usage: "AWS Marketplace ECR repository",
},
&cli.StringFlag{
Name: "product",
Required: true,
Usage: "AWS Marketplace product identifier",
},
}
return testApp
}
type mockAwsMarketplaceDocker struct {
ImagePullError error
ImageTagError error
ImagePushError error
}
func (m *mockAwsMarketplaceDocker) ImagePull(ctx context.Context, refStr string, options types.ImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader([]byte(""))), m.ImagePullError
}
func (m *mockAwsMarketplaceDocker) ImageTag(ctx context.Context, source string, target string) error {
return m.ImageTagError
}
func (m *mockAwsMarketplaceDocker) ImagePush(ctx context.Context, image string, options types.ImagePushOptions) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader([]byte(""))), m.ImagePushError
}
type mockAwsMarketplaceRegistry struct {
GetAuthorizationTokenWithContextError error
}
func (m *mockAwsMarketplaceRegistry) GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, opts ...request.Option) (*ecr.GetAuthorizationTokenOutput, error) {
return &ecr.GetAuthorizationTokenOutput{
AuthorizationData: []*ecr.AuthorizationData{
{
AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString([]byte("username:password"))),
},
},
}, m.GetAuthorizationTokenWithContextError
}
type mockAwsMarketplaceCatalog struct {
DescribeEntityWithContextError error
StartChangeSetWithContextError error
}
func (m *mockAwsMarketplaceCatalog) DescribeEntityWithContext(ctx context.Context, input *marketplacecatalog.DescribeEntityInput, opts ...request.Option) (*marketplacecatalog.DescribeEntityOutput, error) {
return &marketplacecatalog.DescribeEntityOutput{
EntityIdentifier: aws.String("productid"),
}, m.DescribeEntityWithContextError
}
func (m *mockAwsMarketplaceCatalog) StartChangeSetWithContext(ctx context.Context, input *marketplacecatalog.StartChangeSetInput, opts ...request.Option) (*marketplacecatalog.StartChangeSetOutput, error) {
return &marketplacecatalog.StartChangeSetOutput{}, m.StartChangeSetWithContextError
}
+8 -7
View File
@@ -9,6 +9,7 @@ import (
"strings"
"github.com/google/go-github/github"
"github.com/grafana/grafana/pkg/build/config"
"github.com/urfave/cli/v2"
"golang.org/x/oauth2"
)
@@ -39,9 +40,9 @@ var (
errReleaseNotFound = errors.New(`release not found, use "--create" to create the release`)
)
func PublishGitHub(ctx *cli.Context) error {
func PublishGithub(ctx *cli.Context) error {
token := os.Getenv("GH_TOKEN")
f, err := getFlags(ctx)
f, err := getPublishGithubFlags(ctx)
if err != nil {
return err
}
@@ -55,7 +56,7 @@ func PublishGitHub(ctx *cli.Context) error {
}
if f.dryRun {
return runDryRun(f, token, ctx)
return runPublishGithubDryRun(f, token, ctx)
}
client := newGithubClient(ctx.Context, token)
@@ -99,8 +100,8 @@ func githubRepositoryClient(ctx context.Context, token string) githubRepositoryS
return client.Repositories
}
func getFlags(ctx *cli.Context) (*publishGithubFlags, error) {
metadata, err := GenerateMetadata(ctx)
func getPublishGithubFlags(ctx *cli.Context) (*publishGithubFlags, error) {
metadata, err := config.GenerateMetadata(ctx)
if err != nil {
return nil, err
}
@@ -130,12 +131,12 @@ func getFlags(ctx *cli.Context) (*publishGithubFlags, error) {
}, nil
}
func runDryRun(f *publishGithubFlags, token string, ctx *cli.Context) error {
func runPublishGithubDryRun(f *publishGithubFlags, token string, ctx *cli.Context) error {
client := newGithubClient(ctx.Context, token)
fmt.Println("Dry-Run: Retrieving release on repository by tag")
release, res, err := client.GetReleaseByTag(ctx.Context, f.repo.owner, f.repo.name, f.tag)
if err != nil && res.StatusCode != 404 {
fmt.Println("Dry-Run: GitHub communication error:\n", err)
fmt.Println("Dry-Run: Github communication error:\n", err)
return nil
}
+21 -19
View File
@@ -21,17 +21,19 @@ type githubPublishTestCases struct {
expectedError error
errorContains string
expectedOutput string
mockedService *mockGitHubRepositoryServiceImpl
mockedService *mockGithubRepositoryServiceImpl
}
var mockGitHubRepositoryService = &mockGitHubRepositoryServiceImpl{}
var mockGithubRepositoryService = &mockGithubRepositoryServiceImpl{}
func mockGithubRepositoryClient(context.Context, string) githubRepositoryService {
return mockGitHubRepositoryService
return mockGithubRepositoryService
}
func TestPublishGitHub(t *testing.T) {
func TestPublishGithub(t *testing.T) {
t.Setenv("DRONE_BUILD_EVENT", "promote")
t.Setenv("DRONE_TAG", "v1.0.0")
t.Setenv("DRONE_COMMIT", "abcdefgh")
testApp, testPath := setupPublishGithubTests(t)
mockErrUnauthorized := errors.New("401")
@@ -49,21 +51,21 @@ func TestPublishGitHub(t *testing.T) {
name: "try to publish with invalid token",
token: "invalid",
args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
expectedError: mockErrUnauthorized,
},
{
name: "try to publish with valid token and nonexisting tag with create disabled",
token: "valid",
args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
expectedError: errReleaseNotFound,
},
{
name: "try to publish with valid token and nonexisting tag with create enabled",
token: "valid",
args: []string{"--path", testPath, "--repo", "test/test", "--tag", "v1.0.0", "--create"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
},
{
name: "try to publish with valid token and existing tag",
@@ -74,21 +76,21 @@ func TestPublishGitHub(t *testing.T) {
name: "dry run with invalid token",
token: "invalid",
args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
expectedOutput: "GitHub communication error",
mockedService: &mockGithubRepositoryServiceImpl{tagErr: mockErrUnauthorized},
expectedOutput: "Github communication error",
},
{
name: "dry run with valid token and nonexisting tag with create disabled",
token: "valid",
args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
expectedOutput: "Release doesn't exist",
},
{
name: "dry run with valid token and nonexisting tag with create enabled",
token: "valid",
args: []string{"--dry-run", "--path", testPath, "--repo", "test/test", "--tag", "v1.0.0", "--create"},
mockedService: &mockGitHubRepositoryServiceImpl{tagErr: errReleaseNotFound},
mockedService: &mockGithubRepositoryServiceImpl{tagErr: errReleaseNotFound},
expectedOutput: "Would upload asset",
},
{
@@ -116,9 +118,9 @@ func TestPublishGitHub(t *testing.T) {
t.Setenv("GH_TOKEN", test.token)
}
if test.mockedService != nil {
mockGitHubRepositoryService = test.mockedService
mockGithubRepositoryService = test.mockedService
} else {
mockGitHubRepositoryService = &mockGitHubRepositoryServiceImpl{}
mockGithubRepositoryService = &mockGithubRepositoryServiceImpl{}
}
args := []string{"run"}
args = append(args, test.args...)
@@ -154,7 +156,7 @@ func setupPublishGithubTests(t *testing.T) (*cli.App, string) {
newGithubClient = mockGithubRepositoryClient
testApp := cli.NewApp()
testApp.Action = PublishGitHub
testApp.Action = PublishGithub
testApp.Flags = []cli.Flag{
&dryRunFlag,
&cli.StringFlag{
@@ -165,7 +167,7 @@ func setupPublishGithubTests(t *testing.T) (*cli.App, string) {
&cli.StringFlag{
Name: "repo",
Required: true,
Usage: "GitHub repository",
Usage: "Github repository",
},
&cli.StringFlag{
Name: "tag",
@@ -194,13 +196,13 @@ func captureStdout(t *testing.T, fn func() error) (string, error) {
return string(out), err
}
type mockGitHubRepositoryServiceImpl struct {
type mockGithubRepositoryServiceImpl struct {
tagErr error
createErr error
uploadErr error
}
func (m *mockGitHubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, owner string, repo string, tag string) (*github.RepositoryRelease, *github.Response, error) {
func (m *mockGithubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, owner string, repo string, tag string) (*github.RepositoryRelease, *github.Response, error) {
var release *github.RepositoryRelease
res := &github.Response{Response: &http.Response{}}
if m.tagErr == nil {
@@ -212,12 +214,12 @@ func (m *mockGitHubRepositoryServiceImpl) GetReleaseByTag(ctx context.Context, o
return release, res, m.tagErr
}
func (m *mockGitHubRepositoryServiceImpl) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) {
func (m *mockGithubRepositoryServiceImpl) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) {
releaseID := int64(1)
return &github.RepositoryRelease{ID: &releaseID}, &github.Response{}, m.createErr
}
func (m *mockGitHubRepositoryServiceImpl) UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) {
func (m *mockGithubRepositoryServiceImpl) UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) {
assetName := "test"
assetUrl := "testurl.com.br"
return &github.ReleaseAsset{Name: &assetName, BrowserDownloadURL: &assetUrl}, &github.Response{}, m.uploadErr
+100
View File
@@ -0,0 +1,100 @@
package main
import (
"fmt"
"log"
"os"
"os/exec"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/docker"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/urfave/cli/v2"
)
func Enterprise2(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return err
}
cfg := docker.Config{
Archs: buildConfig.Docker.Architectures,
Distribution: buildConfig.Docker.Distribution,
DockerHubRepo: c.String("dockerhub-repo"),
Tag: metadata.GrafanaVersion,
}
err = dockerLoginEnterprise2()
if err != nil {
return err
}
var distributionStr []string
for _, distribution := range cfg.Distribution {
switch distribution {
case alpine:
distributionStr = append(distributionStr, "")
case ubuntu:
distributionStr = append(distributionStr, "-ubuntu")
default:
return fmt.Errorf("unrecognized distribution %q", distribution)
}
}
for _, distribution := range distributionStr {
var imageFileNames []string
for _, arch := range cfg.Archs {
imageFilename := fmt.Sprintf("%s:%s%s-%s", cfg.DockerHubRepo, cfg.Tag, distribution, arch)
err := docker.PushImage(imageFilename)
if err != nil {
return err
}
imageFileNames = append(imageFileNames, imageFilename)
}
manifest := fmt.Sprintf("%s:%s%s", cfg.DockerHubRepo, cfg.Tag, distribution)
args := []string{"manifest", "create", manifest}
args = append(args, imageFileNames...)
//nolint:gosec
cmd := exec.Command("docker", args...)
cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled")
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create Docker manifest: %w\n%s", err, output)
}
err = docker.PushManifest(manifest)
if err != nil {
return err
}
}
return nil
}
func dockerLoginEnterprise2() error {
log.Println("Docker login...")
cmd := exec.Command("gcloud", "auth", "configure-docker")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("error logging in to DockerHub: %s %q", out, err)
}
log.Println("Successful login!")
return nil
}
+6 -5
View File
@@ -8,8 +8,9 @@ import (
"os"
"regexp"
"github.com/grafana/grafana/pkg/build/metrics"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/metrics"
)
func PublishMetrics(c *cli.Context) error {
@@ -17,24 +18,24 @@ func PublishMetrics(c *cli.Context) error {
input, err := io.ReadAll(os.Stdin)
if err != nil {
return cli.NewExitError(fmt.Sprintf("Reading from stdin failed: %s", err), 1)
return cli.Exit(fmt.Sprintf("Reading from stdin failed: %s", err), 1)
}
reMetrics := regexp.MustCompile(`(?ms)^Metrics: (\{.+\})`)
ms := reMetrics.FindSubmatch(input)
if len(ms) == 0 {
return cli.NewExitError(fmt.Sprintf("Input on wrong format: %q", string(input)), 1)
return cli.Exit(fmt.Sprintf("Input on wrong format: %q", string(input)), 1)
}
m := map[string]string{}
if err := json.Unmarshal(ms[1], &m); err != nil {
return cli.NewExitError(fmt.Sprintf("decoding metrics failed: %s", err), 1)
return cli.Exit(fmt.Sprintf("decoding metrics failed: %s", err), 1)
}
log.Printf("Received metrics %+v", m)
if err := metrics.Publish(m, apiKey); err != nil {
return cli.NewExitError(fmt.Sprintf("publishing metrics failed: %s", err), 1)
return cli.Exit(fmt.Sprintf("publishing metrics failed: %s", err), 1)
}
return nil
+1 -1
View File
@@ -13,7 +13,7 @@ import (
func StoreStorybook(c *cli.Context) error {
deployment := c.String("deployment")
metadata, err := GenerateMetadata(c)
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"fmt"
"log"
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/urfave/cli/v2"
)
// UploadCDN implements the sub-command "upload-cdn".
func UploadCDN(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
version := metadata.GrafanaVersion
if err != nil {
return cli.Exit(err.Error(), 1)
}
buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode)
if err != nil {
return err
}
edition := os.Getenv("EDITION")
log.Printf("Uploading Grafana CDN Assets, version %s, %s edition...", version, edition)
editionPath := ""
switch config.Edition(edition) {
case config.EditionOSS:
editionPath = "grafana-oss"
case config.EditionEnterprise:
editionPath = "grafana"
case config.EditionEnterprise2:
editionPath = os.Getenv("ENTERPRISE2_CDN_PATH")
default:
panic(fmt.Sprintf("unrecognized edition %q", edition))
}
gcs, err := storage.New()
if err != nil {
return err
}
bucket := gcs.Bucket(buildConfig.Buckets.CDNAssets)
srcPath := buildConfig.Buckets.CDNAssetsDir
srcPath = filepath.Join(srcPath, editionPath, version)
if err := gcs.DeleteDir(c.Context, bucket, srcPath); err != nil {
return err
}
log.Printf("Successfully cleaned source: %s/%s\n", buildConfig.Buckets.CDNAssets, srcPath)
if err := gcs.CopyLocalDir(c.Context, "./public", bucket, srcPath, false); err != nil {
return err
}
log.Printf("Successfully uploaded cdn static assets to: %s/%s!\n", buildConfig.Buckets.CDNAssets, srcPath)
return nil
}
+213
View File
@@ -0,0 +1,213 @@
package main
import (
"encoding/base64"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/grafana/grafana/pkg/build/gcloud"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/urfave/cli/v2"
)
const releaseFolder = "release"
const mainFolder = "main"
const releaseBranchFolder = "prerelease"
type uploadConfig struct {
config.Config
edition config.Edition
versionMode config.VersionMode
gcpKey string
distDir string
versionFolder string
}
// UploadPackages implements the sub-command "upload-packages".
func UploadPackages(c *cli.Context) error {
if c.NArg() > 0 {
if err := cli.ShowSubcommandHelp(c); err != nil {
return cli.Exit(err.Error(), 1)
}
return cli.Exit("", 1)
}
gcpKeyB64 := strings.TrimSpace(os.Getenv("GCP_KEY"))
if gcpKeyB64 == "" {
return cli.Exit("the environment variable GCP_KEY must be set", 1)
}
gcpKeyB, err := base64.StdEncoding.DecodeString(gcpKeyB64)
if err != nil {
return cli.Exit("failed to base64 decode $GCP_KEY", 1)
}
gcpKey := string(gcpKeyB)
distDir, err := filepath.Abs("dist")
if err != nil {
return cli.Exit(err.Error(), 1)
}
metadata, err := config.GenerateMetadata(c)
if err != nil {
return err
}
version := metadata.GrafanaVersion
releaseMode, err := metadata.GetReleaseMode()
if err != nil {
return cli.Exit(err.Error(), 1)
}
releaseModeConfig, err := config.GetBuildConfig(releaseMode.Mode)
if err != nil {
return cli.Exit(err.Error(), 1)
}
var edition config.Edition
if e, ok := os.LookupEnv("EDITION"); ok {
edition = config.Edition(e)
}
if c.Bool("enterprise2") {
edition = config.EditionEnterprise2
}
if edition == "" {
return fmt.Errorf("both EDITION envvar and '--enterprise2' flag are missing. At least one of those is required")
}
// TODO: Verify config values
cfg := uploadConfig{
Config: config.Config{
Version: version,
Bucket: releaseModeConfig.Buckets.Artifacts,
},
edition: edition,
versionMode: releaseMode.Mode,
gcpKey: gcpKey,
distDir: distDir,
}
event, err := droneutil.GetDroneEventFromEnv()
if err != nil {
return err
}
if cfg.edition == config.EditionEnterprise2 {
cfg.Bucket, err = bucketForEnterprise2(releaseModeConfig, event)
if err != nil {
return err
}
}
cfg.versionFolder, err = getVersionFolder(cfg, event)
if err != nil {
return err
}
if err := uploadPackages(cfg); err != nil {
return cli.Exit(err.Error(), 1)
}
log.Println("Successfully uploaded packages!")
return nil
}
// Corner case for custom enterprise2 mode
func bucketForEnterprise2(releaseModeConfig *config.BuildConfig, event string) (string, error) {
if event == config.Custom {
buildConfig, err := config.GetBuildConfig(config.ReleaseBranchMode)
if err != nil {
return "", err
}
return buildConfig.Buckets.ArtifactsEnterprise2, nil
}
if releaseModeConfig.Buckets.ArtifactsEnterprise2 != "" {
return releaseModeConfig.Buckets.ArtifactsEnterprise2, nil
}
return "", fmt.Errorf("enterprise2 bucket var doesn't exist")
}
func getVersionFolder(cfg uploadConfig, event string) (string, error) {
switch cfg.versionMode {
case config.TagMode:
return releaseFolder, nil
case config.MainMode, config.DownstreamMode:
return mainFolder, nil
case config.ReleaseBranchMode:
return releaseBranchFolder, nil
default:
// Corner case for custom enterprise2 mode
if event == config.Custom && cfg.versionMode == config.Enterprise2Mode {
return releaseFolder, nil
}
return "", fmt.Errorf("unrecognized version mode: %s", cfg.versionMode)
}
}
func uploadPackages(cfg uploadConfig) error {
log.Printf("Uploading Grafana packages, version %s, %s edition, %s mode...\n", cfg.Version, cfg.edition,
cfg.versionMode)
if err := gcloud.ActivateServiceAccount(); err != nil {
return fmt.Errorf("couldn't activate service account, err: %w", err)
}
edition := strings.ToLower(string(cfg.edition))
var sfx string
switch cfg.edition {
case config.EditionOSS:
case config.EditionEnterprise:
sfx = "-enterprise"
case config.EditionEnterprise2:
sfx = "-enterprise2"
default:
panic(fmt.Sprintf("unrecognized edition %q", cfg.edition))
}
matches, err := filepath.Glob(filepath.Join(cfg.distDir, fmt.Sprintf("grafana%s*", sfx)))
if err != nil {
return fmt.Errorf("failed to list packages: %w", err)
}
fpaths := []string{}
rePkg := packaging.PackageRegexp(cfg.edition)
for _, fpath := range matches {
fname := filepath.Base(fpath)
if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) {
log.Printf("Ignoring file %q\n", fpath)
continue
}
fpaths = append(fpaths, fpath)
}
var tag, gcsPath string
droneTag := strings.TrimSpace(os.Getenv("DRONE_TAG"))
if droneTag != "" {
tag = droneTag
gcsPath = fmt.Sprintf("gs://%s/%s/%s/%s", cfg.Bucket, tag, edition, cfg.versionFolder)
} else {
gcsPath = fmt.Sprintf("gs://%s/%s/%s/", cfg.Bucket, edition, cfg.versionFolder)
}
log.Printf("Uploading %d file(s) to GCS (%s)...\n", len(fpaths), gcsPath)
args := []string{"-m", "cp"}
args = append(args, fpaths...)
args = append(args, gcsPath)
cmd := exec.Command("gsutil", args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to upload files to GCS: %s", output)
}
return nil
}
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"errors"
"fmt"
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_getVersionFolder(t *testing.T) {
type args struct {
cfg uploadConfig
event string
versionFolder string
}
tests := []struct {
name string
args args
err error
}{
{"tag mode", args{uploadConfig{versionMode: config.TagMode}, "", releaseFolder}, nil},
{"main mode", args{uploadConfig{versionMode: config.MainMode}, "", mainFolder}, nil},
{"downstream mode", args{uploadConfig{versionMode: config.DownstreamMode}, "", mainFolder}, nil},
{"release branch mode", args{uploadConfig{versionMode: config.ReleaseBranchMode}, "", releaseBranchFolder}, nil},
{"enterprise pro mode", args{uploadConfig{versionMode: config.Enterprise2Mode}, config.Custom, releaseFolder}, nil},
{"unrecognised version mode", args{uploadConfig{versionMode: "foo"}, config.Custom, ""}, errors.New("")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
versionMode, err := getVersionFolder(tt.args.cfg, tt.args.event)
if tt.err != nil {
require.Error(t, err)
}
require.Equal(t, versionMode, tt.args.versionFolder)
})
}
}
func Test_checkForEnterprise2Edition(t *testing.T) {
type args struct {
releaseModeConfig *config.BuildConfig
event string
}
tests := []struct {
name string
args args
want string
err error
}{
{"event is not custom", args{releaseModeConfig: &config.BuildConfig{Buckets: config.Buckets{ArtifactsEnterprise2: "dummy"}}}, "dummy", nil},
{"event is not custom and string is empty", args{releaseModeConfig: &config.BuildConfig{Buckets: config.Buckets{ArtifactsEnterprise2: ""}}}, "", fmt.Errorf("enterprise2 bucket var doesn't exist")},
{"event is custom", args{releaseModeConfig: nil, event: "custom"}, "grafana-downloads-enterprise2", nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := bucketForEnterprise2(tt.args.releaseModeConfig, tt.args.event)
if tt.err != nil {
require.Error(t, err)
}
assert.Equalf(t, tt.want, got, "bucketForEnterprise2(%v, %v)", tt.args.releaseModeConfig, tt.args.event)
})
}
}
+6 -5
View File
@@ -13,10 +13,11 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/grafana/grafana/pkg/build/fsutil"
cliv1 "github.com/urfave/cli"
"github.com/urfave/cli/v2"
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/build/fsutil"
)
func VerifyDrone(c *cli.Context) error {
@@ -24,7 +25,7 @@ func VerifyDrone(c *cli.Context) error {
const backup = ".drone.yml.bak"
if err := fsutil.CopyFile(yml, backup); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to copy %s to %s: %s", yml, backup, err), 1)
return cli.Exit(fmt.Sprintf("failed to copy %s to %s: %s", yml, backup, err), 1)
}
defer func() {
if err := os.Remove(yml); err != nil {
@@ -73,7 +74,7 @@ func readConfig(fpath string) ([]map[string]interface{}, error) {
//nolint:gosec
f, err := os.Open(fpath)
if err != nil {
return nil, cli.NewExitError(fmt.Sprintf("failed to read %s: %s", fpath, err), 1)
return nil, cli.Exit(fmt.Sprintf("failed to read %s: %s", fpath, err), 1)
}
defer func() {
if err := f.Close(); err != nil {
@@ -90,7 +91,7 @@ func readConfig(fpath string) ([]map[string]interface{}, error) {
if errors.Is(err, io.EOF) {
break
}
return nil, cli.NewExitError(fmt.Sprintf("Failed to decode %s: %s", fpath, err), 1)
return nil, cli.Exit(fmt.Sprintf("Failed to decode %s: %s", fpath, err), 1)
}
if m["kind"] == "signature" {
@@ -118,7 +119,7 @@ func verifyYAML(yml, backup string) error {
}
if !cmp.Equal(c1, c2) {
return cli.NewExitError(fmt.Sprintf("%s is out of sync with .drone.star - regenerate it with drone starlark convert",
return cli.Exit(fmt.Sprintf("%s is out of sync with .drone.star - regenerate it with drone starlark convert",
yml), 1)
}
+142
View File
@@ -0,0 +1,142 @@
package main
import (
"context"
"errors"
"fmt"
"io/fs"
"os/exec"
"path/filepath"
"strings"
"github.com/urfave/cli/v2"
)
func mapSlice[I any, O any](a []I, f func(I) O) []O {
o := make([]O, len(a))
for i, e := range a {
o[i] = f(e)
}
return o
}
// VerifyStarlark is the CLI Action for verifying Starlark files in a workspace.
// It expects a single context argument which is the path to the workspace.
// The actual verification procedure can return multiple errors which are
// joined together to be one holistic error for the action.
func VerifyStarlark(c *cli.Context) error {
if c.NArg() != 1 {
var message string
if c.NArg() == 0 {
message = "ERROR: missing required argument <workspace path>"
}
if c.NArg() > 1 {
message = "ERROR: too many arguments"
}
if err := cli.ShowSubcommandHelp(c); err != nil {
return err
}
return cli.Exit(message, 1)
}
workspace := c.Args().Get(0)
verificationErrs, executionErr := verifyStarlark(c.Context, workspace, buildifierLintCommand)
if executionErr != nil {
return executionErr
}
if len(verificationErrs) == 0 {
return nil
}
noun := "file"
if len(verificationErrs) > 1 {
noun += "s"
}
return fmt.Errorf("verification failed for %d %s:\n%s",
len(verificationErrs),
noun,
strings.Join(
mapSlice(verificationErrs, func(e error) string { return e.Error() }),
"\n",
))
}
type commandFunc = func(path string) (command string, args []string)
func buildifierLintCommand(path string) (string, []string) {
return "buildifier", []string{"-lint", "warn", "-mode", "check", path}
}
// verifyStarlark walks all directories starting at provided workspace path and
// verifies any Starlark files it finds.
// Starlark files are assumed to end with the .star extension.
// The verification relies on linting frovided by the 'buildifier' binary which
// must be in the PATH.
// A slice of verification errors are returned, one for each file that failed verification.
// If any execution of the `buildifier` command fails, this is returned separately.
// commandFn is executed on every Starlark file to determine the command and arguments to be executed.
// The caller is trusted and it is the callers responsibility to ensure that the resulting command is safe to execute.
func verifyStarlark(ctx context.Context, workspace string, commandFn commandFunc) ([]error, error) {
var verificationErrs []error
// All errors from filepath.WalkDir are filtered by the fs.WalkDirFunc.
// Lstat or ReadDir errors are reported as verificationErrors.
// If any execution of the `buildifier` command fails or if the context is cancelled,
// it is reported as an error and any verification of subsequent files is skipped.
err := filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error {
// Skip verification of the file or files within the directory if there is an error
// returned by Lstat or ReadDir.
if err != nil {
verificationErrs = append(verificationErrs, err)
return nil
}
if d.IsDir() {
return nil
}
if filepath.Ext(path) == ".star" {
command, args := commandFn(path)
// The caller is trusted.
//nolint:gosec
cmd := exec.CommandContext(ctx, command, args...)
cmd.Dir = workspace
_, err = cmd.Output()
if err == nil { // No error, early return.
return nil
}
// The error returned from cmd.Output() is never wrapped.
//nolint:errorlint
if err, ok := err.(*exec.ExitError); ok {
switch err.ExitCode() {
// Case comments are informed by the output of `buildifier --help`
case 1: // syntax errors in input
verificationErrs = append(verificationErrs, errors.New(string(err.Stderr)))
return nil
case 2: // usage errors: invoked incorrectly
return fmt.Errorf("command %q: %s", cmd, err.Stderr)
case 3: // unexpected runtime errors: file I/O problems or internal bugs
return fmt.Errorf("command %q: %s", cmd, err.Stderr)
case 4: // check mode failed (reformat is needed)
verificationErrs = append(verificationErrs, errors.New(string(err.Stderr)))
return nil
default:
return fmt.Errorf("command %q: %s", cmd, err.Stderr)
}
}
// Error was not an exit error from the command.
return fmt.Errorf("command %q: %v", cmd, err)
}
return nil
})
return verificationErrs, err
}
+137
View File
@@ -0,0 +1,137 @@
//go:build requires_buildifier
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestVerifyStarlark(t *testing.T) {
t.Run("execution errors", func(t *testing.T) {
t.Run("invalid usage", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
_, executionErr := verifyStarlark(ctx, workspace, func(string) (string, []string) { return "buildifier", []string{"--invalid"} })
if executionErr == nil {
t.Fatalf("Expected execution error but got none")
}
})
t.Run("context cancellation", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
workspace := t.TempDir()
err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
err = os.WriteFile(filepath.Join(workspace, "other-ignored.star"), []byte{}, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
cancel()
_, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr == nil {
t.Fatalf("Expected execution error but got none")
}
})
})
t.Run("verification errors", func(t *testing.T) {
t.Run("a single file with lint", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
invalidContent := []byte(`load("scripts/drone/other.star", "function")
function()`)
err := os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm)
if err != nil {
t.Fatalf(err.Error())
}
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr != nil {
t.Fatalf("Unexpected execution error: %v", executionErr)
}
if len(verificationErrs) == 0 {
t.Fatalf(`"has-lint.star" requires linting but the verifyStarlark function provided no linting error`)
}
if len(verificationErrs) > 1 {
t.Fatalf(`verifyStarlark returned multiple errors for the "has-lint.star" file but only one was expected: %v`, verificationErrs)
}
if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") {
t.Fatalf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0])
}
})
t.Run("no files with lint", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
content := []byte(`"""
This module does nothing.
"""
load("scripts/drone/other.star", "function")
function()
`)
require.NoError(t, os.WriteFile(filepath.Join(workspace, "no-lint.star"), content, os.ModePerm))
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr != nil {
t.Fatalf("Unexpected execution error: %v", executionErr)
}
if len(verificationErrs) != 0 {
t.Log(`"no-lint.star" has no lint but the verifyStarlark function provided at least one error`)
for _, err := range verificationErrs {
t.Log(err)
}
t.FailNow()
}
})
t.Run("multiple files with lint", func(t *testing.T) {
ctx := context.Background()
workspace := t.TempDir()
invalidContent := []byte(`load("scripts/drone/other.star", "function")
function()`)
require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm))
require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint2.star"), invalidContent, os.ModePerm))
verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand)
if executionErr != nil {
t.Fatalf("Unexpected execution error: %v", executionErr)
}
if len(verificationErrs) == 0 {
t.Fatalf(`Two files require linting but the verifyStarlark function provided no linting error`)
}
if len(verificationErrs) == 1 {
t.Fatalf(`Two files require linting but the verifyStarlark function provided only one linting error: %v`, verificationErrs[0])
}
if len(verificationErrs) > 2 {
t.Fatalf(`verifyStarlark returned more errors than expected: %v`, verificationErrs)
}
if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") {
t.Errorf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0])
}
if !strings.Contains(verificationErrs[1].Error(), "has-lint2.star:1: module-docstring: The file has no module docstring.") {
t.Fatalf(`"has-lint2.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0])
}
})
})
}
+33
View File
@@ -0,0 +1,33 @@
// Package verifystorybook contains the sub-command "verify-storybook".
package main
import (
"fmt"
"log"
"path/filepath"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/infra/fs"
)
// VerifyStorybook Action implements the sub-command "verify-storybook".
func VerifyStorybook(c *cli.Context) error {
const grafanaDir = "."
paths := []string{
"packages/grafana-ui/dist/storybook/index.html",
"packages/grafana-ui/dist/storybook/iframe.html"}
for _, p := range paths {
exists, err := fs.Exists(filepath.Join(grafanaDir, p))
if err != nil {
return cli.Exit(fmt.Sprintf("failed to verify Storybook build: %s", err), 1)
}
if !exists {
return fmt.Errorf("failed to verify Storybook build, missing %q", p)
}
}
log.Printf("Successfully verified Storybook integrity")
return nil
}
+14 -16
View File
@@ -1,20 +1,18 @@
package config
type Config struct {
Version string
Bucket string
DebRepoBucket string
DebDBBucket string
RPMRepoBucket string
GPGPassPath string
GPGPrivateKey string
GPGPublicKey string
GCPKeyFile string
NumWorkers int
GitHubUser string
GitHubToken string
PullEnterprise bool
NetworkConcurrency bool
PackageVersion string
SignPackages bool
Version string
Bucket string
DebRepoBucket string
DebDBBucket string
RPMRepoBucket string
GPGPassPath string
GPGPrivateKey string
GPGPublicKey string
NumWorkers int
GitHubUser string
GitHubToken string
PullEnterprise bool
PackageVersion string
SignPackages bool
}
+102
View File
@@ -0,0 +1,102 @@
package config
import (
"fmt"
"os"
"strings"
"github.com/grafana/grafana/pkg/build/droneutil"
"github.com/urfave/cli/v2"
)
func GenerateMetadata(c *cli.Context) (Metadata, error) {
var metadata Metadata
version := ""
event, err := droneutil.GetDroneEventFromEnv()
if err != nil {
return Metadata{}, err
}
tag, ok := os.LookupEnv("DRONE_TAG")
if !ok {
fmt.Println("DRONE_TAG envvar not present, %w", err)
}
var releaseMode ReleaseMode
switch event {
case string(PullRequestMode):
releaseMode = ReleaseMode{Mode: PullRequestMode}
case Push:
mode, err := CheckDroneTargetBranch()
if err != nil {
return Metadata{}, err
}
releaseMode = ReleaseMode{Mode: mode}
case Custom:
if edition, _ := os.LookupEnv("EDITION"); edition == string(EditionEnterprise2) {
releaseMode = ReleaseMode{Mode: Enterprise2Mode}
if tag != "" {
version = strings.TrimPrefix(tag, "v")
}
break
}
mode, err := CheckDroneTargetBranch()
if err != nil {
return Metadata{}, err
}
// if there is a custom event targeting the main branch, that's an enterprise downstream build
if mode == MainBranch {
releaseMode = ReleaseMode{Mode: DownstreamMode}
} else {
releaseMode = ReleaseMode{Mode: mode}
}
case Tag, Promote:
if tag == "" {
return Metadata{}, fmt.Errorf("DRONE_TAG envvar not present for a tag/promotion event, %w", err)
}
version = strings.TrimPrefix(tag, "v")
mode, err := CheckSemverSuffix()
if err != nil {
return Metadata{}, err
}
releaseMode = mode
case Cronjob:
releaseMode = ReleaseMode{Mode: CronjobMode}
}
if version == "" {
version, err = generateVersionFromBuildID()
if err != nil {
return Metadata{}, err
}
}
currentCommit, err := GetDroneCommit()
if err != nil {
return Metadata{}, err
}
metadata = Metadata{
GrafanaVersion: version,
ReleaseMode: releaseMode,
GrabplVersion: c.App.Version,
CurrentCommit: currentCommit,
}
fmt.Printf("building Grafana version: %s, release mode: %+v", metadata.GrafanaVersion, metadata.ReleaseMode)
return metadata, nil
}
func generateVersionFromBuildID() (string, error) {
buildID, ok := os.LookupEnv("DRONE_BUILD_NUMBER")
if !ok {
return "", fmt.Errorf("unable to get DRONE_BUILD_NUMBER environmental variable")
}
var err error
version, err := GetGrafanaVersion(buildID, ".")
if err != nil {
return "", err
}
return version, nil
}
+81
View File
@@ -0,0 +1,81 @@
package config
import (
"flag"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
)
const (
DroneBuildEvent = "DRONE_BUILD_EVENT"
DroneTargetBranch = "DRONE_TARGET_BRANCH"
DroneTag = "DRONE_TAG"
DroneSemverPrerelease = "DRONE_SEMVER_PRERELEASE"
DroneBuildNumber = "DRONE_BUILD_NUMBER"
)
const (
hashedGrafanaVersion = "9.2.0-12345pre"
versionedBranch = "v9.2.x"
)
func TestGetMetadata(t *testing.T) {
tcs := []struct {
envMap map[string]string
expVersion string
mode ReleaseMode
}{
{map[string]string{DroneBuildEvent: PullRequest, DroneTargetBranch: "", DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: PullRequestMode}},
{map[string]string{DroneBuildEvent: Push, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: Push, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: MainMode}},
{map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: versionedBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: ReleaseBranchMode}},
{map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, hashedGrafanaVersion, ReleaseMode{Mode: DownstreamMode}},
{map[string]string{DroneBuildEvent: Custom, DroneTargetBranch: MainBranch, DroneTag: "", DroneSemverPrerelease: "", DroneBuildNumber: "12345", "EDITION": string(EditionEnterprise2)}, hashedGrafanaVersion, ReleaseMode{Mode: Enterprise2Mode}},
{map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: Tag, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}},
{map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0", DroneSemverPrerelease: "", DroneBuildNumber: "12345"}, "9.2.0", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: false}},
{map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-beta", DroneSemverPrerelease: "beta", DroneBuildNumber: "12345"}, "9.2.0-beta", ReleaseMode{Mode: TagMode, IsBeta: true, IsTest: false}},
{map[string]string{DroneBuildEvent: Promote, DroneTargetBranch: "", DroneTag: "v9.2.0-test", DroneSemverPrerelease: "test", DroneBuildNumber: "12345"}, "9.2.0-test", ReleaseMode{Mode: TagMode, IsBeta: false, IsTest: true}},
}
ctx := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil)
for _, tc := range tcs {
t.Run("Should return valid metadata, ", func(t *testing.T) {
setUpEnv(t, tc.envMap)
testMetadata(t, ctx, tc.expVersion, tc.mode)
})
}
}
func testMetadata(t *testing.T, ctx *cli.Context, version string, releaseMode ReleaseMode) {
t.Helper()
metadata, err := GenerateMetadata(ctx)
require.NoError(t, err)
t.Run("with a valid version", func(t *testing.T) {
expVersion := metadata.GrafanaVersion
require.Equal(t, expVersion, version)
})
t.Run("with a valid release mode from the built-in list", func(t *testing.T) {
expMode := metadata.ReleaseMode
require.NoError(t, err)
require.Equal(t, expMode, releaseMode)
})
}
func setUpEnv(t *testing.T, envMap map[string]string) {
t.Helper()
os.Clearenv()
err := os.Setenv("DRONE_COMMIT", "abcd12345")
require.NoError(t, err)
for k, v := range envMap {
err := os.Setenv(k, v)
require.NoError(t, err)
}
}
+3
View File
@@ -0,0 +1,3 @@
{
"version": "9.2.0-pre"
}
+3 -1
View File
@@ -7,6 +7,8 @@ import (
"path/filepath"
"regexp"
"strings"
"github.com/grafana/grafana/pkg/build/git"
)
type Metadata struct {
@@ -100,8 +102,8 @@ func GetGrafanaVersion(buildID, grafanaDir string) (string, error) {
}
func CheckDroneTargetBranch() (VersionMode, error) {
rePRCheckBranch := git.PRCheckRegexp()
reRlsBranch := regexp.MustCompile(`^v\d+\.\d+\.x$`)
rePRCheckBranch := regexp.MustCompile(`^pr-check-\d+`)
target := os.Getenv("DRONE_TARGET_BRANCH")
if target == "" {
return "", fmt.Errorf("failed to get DRONE_TARGET_BRANCH environmental variable")
+2 -1
View File
@@ -8,7 +8,8 @@ const (
TagMode VersionMode = "release"
ReleaseBranchMode VersionMode = "branch"
PullRequestMode VersionMode = "pull_request"
CustomMode VersionMode = "custom"
DownstreamMode VersionMode = "downstream"
Enterprise2Mode VersionMode = "enterprise2"
CronjobMode VersionMode = "cron"
)
+39 -1
View File
@@ -59,7 +59,7 @@ var Versions = VersionMap{
Storybook: "grafana-storybook",
},
},
CustomMode: {
DownstreamMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
@@ -165,4 +165,42 @@ var Versions = VersionMap{
StorybookSrcDir: "artifacts/storybook",
},
},
Enterprise2Mode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantLinuxAmd64,
VariantLinuxAmd64Musl,
},
PluginSignature: PluginSignature{
Sign: true,
AdminSign: true,
},
Docker: Docker{
ShouldSave: true,
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7,
},
Distribution: []Distribution{
Alpine,
Ubuntu,
},
PrereleaseBucket: "grafana-prerelease/artifacts/docker",
},
Buckets: Buckets{
Artifacts: "grafana-prerelease/artifacts/downloads",
ArtifactsEnterprise2: "grafana-prerelease/artifacts/downloads-enterprise2",
CDNAssets: "grafana-prerelease",
CDNAssetsDir: "artifacts/static-assets",
Storybook: "grafana-prerelease",
StorybookSrcDir: "artifacts/storybook",
},
},
}
+9
View File
@@ -85,6 +85,11 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use
var additionalDockerRepo string
var tags []string
var imageFileBase string
var dockerEnterprise2Repo string
if repo, ok := os.LookupEnv("DOCKER_ENTERPRISE2_REPO"); ok {
dockerEnterprise2Repo = repo
}
switch edition {
case config.EditionOSS:
dockerRepo = "grafana/grafana-image-tags"
@@ -94,6 +99,10 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use
dockerRepo = "grafana/grafana-enterprise-image-tags"
imageFileBase = "grafana-enterprise"
editionStr = "-enterprise"
case config.EditionEnterprise2:
dockerRepo = dockerEnterprise2Repo
imageFileBase = "grafana-enterprise2"
editionStr = "-enterprise2"
default:
return []string{}, fmt.Errorf("unrecognized edition %s", edition)
}
+62
View File
@@ -0,0 +1,62 @@
package docker
import (
"fmt"
"log"
"os"
"os/exec"
"time"
)
const (
tries = 3
sleepTime = 30
)
func PushImage(newImage string) error {
var err error
for i := 0; i < tries; i++ {
log.Printf("push attempt #%d...", i+1)
var out []byte
cmd := exec.Command("docker", "push", newImage)
cmd.Dir = "."
out, err = cmd.CombinedOutput()
if err != nil {
log.Printf("output: %s", out)
log.Printf("sleep for %d, before retrying...", sleepTime)
time.Sleep(sleepTime * time.Second)
} else {
log.Printf("Successfully pushed %s!", newImage)
break
}
}
if err != nil {
return fmt.Errorf("error pushing images to DockerHub: %q", err)
}
return nil
}
func PushManifest(manifest string) error {
log.Printf("Pushing Docker manifest %s...", manifest)
var err error
for i := 0; i < tries; i++ {
log.Printf("push attempt #%d...", i+1)
var out []byte
cmd := exec.Command("docker", "manifest", "push", manifest)
cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled")
out, err = cmd.CombinedOutput()
if err != nil {
log.Printf("output: %s", out)
log.Printf("sleep for %d, before retrying...", sleepTime)
time.Sleep(sleepTime * time.Second)
} else {
log.Printf("Successful manifest push! %s", string(out))
break
}
}
if err != nil {
return fmt.Errorf("failed to push manifest, err: %w", err)
}
return nil
}
+18
View File
@@ -0,0 +1,18 @@
package env
import (
"strings"
)
// Lookup is the equivalent of os.LookupEnv, only you are able to provide the list of environment variables.
// To use this as os.LookupEnv would be used, simply call
// `env.Lookup("ENVIRONMENT_VARIABLE", os.Environ())`
func Lookup(name string, vars []string) (string, bool) {
for _, v := range vars {
if strings.HasPrefix(v, name) {
return strings.TrimPrefix(v, name+"="), true
}
}
return "", false
}
+43
View File
@@ -0,0 +1,43 @@
package env_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/env"
"github.com/stretchr/testify/require"
)
func TestLookup(t *testing.T) {
values := []string{"ENV_1=a", "ENV_2=b", "ENV_3=c", "ENV_4_TEST="}
{
v, ok := env.Lookup("ENV_1", values)
require.Equal(t, v, "a")
require.True(t, ok)
}
{
v, ok := env.Lookup("ENV_2", values)
require.Equal(t, v, "b")
require.True(t, ok)
}
{
v, ok := env.Lookup("ENV_3", values)
require.Equal(t, v, "c")
require.True(t, ok)
}
{
v, ok := env.Lookup("ENV_4_TEST", values)
require.Equal(t, v, "")
require.True(t, ok)
}
{
v, ok := env.Lookup("NOT_THERE", values)
require.Equal(t, v, "")
require.False(t, ok)
}
}
+142
View File
@@ -0,0 +1,142 @@
package git
import (
"context"
"errors"
"fmt"
"net/http"
"regexp"
"github.com/google/go-github/v45/github"
"github.com/grafana/grafana/pkg/build/stringutil"
"golang.org/x/oauth2"
)
const (
MainBranch = "main"
HomeDir = "."
RepoOwner = "grafana"
OSSRepo = "grafana"
EnterpriseRepo = "grafana-enterprise"
EnterpriseCheckName = "Grafana Enterprise"
EnterpriseCheckDescription = "Downstream tests to ensure that your changes are compatible with Grafana Enterprise"
)
var EnterpriseCheckLabels = []string{"enterprise-ok", "enterprise-failed", "enterprise-override"}
var (
ErrorNoDroneBuildLink = errors.New("no drone build link")
)
type GitService interface {
DeleteRef(ctx context.Context, owner string, repo string, ref string) (*github.Response, error)
}
type LabelsService interface {
ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error)
RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error)
AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error)
}
type CommentService interface {
CreateComment(ctx context.Context, owner string, repo string, number int, comment *github.IssueComment) (*github.IssueComment, *github.Response, error)
}
type StatusesService interface {
CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error)
}
// NewGitHubClient creates a new Client using the provided GitHub token if not empty.
func NewGitHubClient(ctx context.Context, token string) *github.Client {
var tc *http.Client
if token != "" {
ts := oauth2.StaticTokenSource(&oauth2.Token{
AccessToken: token,
})
tc = oauth2.NewClient(ctx, ts)
}
return github.NewClient(tc)
}
func PRCheckRegexp() *regexp.Regexp {
reBranch, err := regexp.Compile(`^prc-([0-9]+)-([A-Za-z0-9]+)\/(.+)$`)
if err != nil {
panic(fmt.Sprintf("Failed to compile regexp: %s", err))
}
return reBranch
}
func AddLabelToPR(ctx context.Context, client LabelsService, prID int, newLabel string) error {
// Check existing labels
labels, _, err := client.ListLabelsByIssue(ctx, RepoOwner, OSSRepo, prID, nil)
if err != nil {
return err
}
duplicate := false
for _, label := range labels {
if *label.Name == newLabel {
duplicate = true
continue
}
// Delete existing "enterprise-xx" labels
if stringutil.Contains(EnterpriseCheckLabels, *label.Name) {
_, err := client.RemoveLabelForIssue(ctx, RepoOwner, OSSRepo, prID, *label.Name)
if err != nil {
return err
}
}
}
if duplicate {
return nil
}
_, _, err = client.AddLabelsToIssue(ctx, RepoOwner, OSSRepo, prID, []string{newLabel})
if err != nil {
return err
}
return nil
}
func DeleteEnterpriseBranch(ctx context.Context, client GitService, branchName string) error {
ref := "heads/" + branchName
if _, err := client.DeleteRef(ctx, RepoOwner, EnterpriseRepo, ref); err != nil {
return err
}
return nil
}
// CreateEnterpriseStatus sets the status on a commit for the enterprise build check.
func CreateEnterpriseStatus(ctx context.Context, client StatusesService, sha, link, status string) (*github.RepoStatus, error) {
check, _, err := client.CreateStatus(ctx, RepoOwner, OSSRepo, sha, &github.RepoStatus{
Context: github.String(EnterpriseCheckName),
Description: github.String(EnterpriseCheckDescription),
TargetURL: github.String(link),
State: github.String(status),
})
if err != nil {
return nil, err
}
return check, nil
}
func CreateEnterpriseBuildFailedComment(ctx context.Context, client CommentService, link string, prID int) error {
body := fmt.Sprintf("Drone build failed: %s", link)
_, _, err := client.CreateComment(ctx, RepoOwner, OSSRepo, prID, &github.IssueComment{
Body: &body,
})
if err != nil {
return err
}
return nil
}
+55
View File
@@ -0,0 +1,55 @@
package git_test
import (
"context"
"errors"
"testing"
"github.com/google/go-github/v45/github"
"github.com/grafana/grafana/pkg/build/git"
"github.com/stretchr/testify/require"
)
type TestChecksService struct {
CreateCheckRunError error
}
func (s *TestChecksService) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) {
if s.CreateCheckRunError != nil {
return nil, nil, s.CreateCheckRunError
}
return &github.RepoStatus{
ID: github.Int64(1),
URL: status.URL,
}, nil, nil
}
func TestCreateEnterpriseRepoStatus(t *testing.T) {
t.Run("It should create a repo status", func(t *testing.T) {
var (
ctx = context.Background()
client = &TestChecksService{}
link = "http://example.com"
sha = "1234"
)
_, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success")
require.NoError(t, err)
})
t.Run("It should return an error if GitHub fails to create the status", func(t *testing.T) {
var (
ctx = context.Background()
createCheckError = errors.New("create check run error")
client = &TestChecksService{
CreateCheckRunError: createCheckError,
}
link = "http://example.com"
sha = "1234"
)
_, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success")
require.ErrorIs(t, err, createCheckError)
})
}
+134
View File
@@ -0,0 +1,134 @@
package git_test
import (
"context"
"errors"
"testing"
"github.com/google/go-github/v45/github"
"github.com/grafana/grafana/pkg/build/git"
"github.com/stretchr/testify/require"
)
type TestLabelsService struct {
Labels []*github.Label
ListLabelsError error
RemoveLabelError error
AddLabelsError error
}
func (s *TestLabelsService) ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) {
if s.ListLabelsError != nil {
return nil, nil, s.ListLabelsError
}
labels := s.Labels
if labels == nil {
labels = []*github.Label{}
}
return labels, nil, nil
}
func (s *TestLabelsService) RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) {
if s.RemoveLabelError != nil {
return nil, s.RemoveLabelError
}
return &github.Response{}, nil
}
func (s *TestLabelsService) AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) {
if s.AddLabelsError != nil {
return nil, nil, s.AddLabelsError
}
l := make([]*github.Label, len(labels))
for i, v := range labels {
l[i] = &github.Label{
Name: github.String(v),
}
}
return l, nil, nil
}
func TestAddLabelToPR(t *testing.T) {
t.Run("It should add a label to a pull request", func(t *testing.T) {
var (
ctx = context.Background()
client = &TestLabelsService{}
pr = 20
label = "test-label"
)
require.NoError(t, git.AddLabelToPR(ctx, client, pr, label))
})
t.Run("It should not return an error if the label already exists", func(t *testing.T) {
var (
ctx = context.Background()
client = &TestLabelsService{
Labels: []*github.Label{
{
Name: github.String("test-label"),
},
},
}
pr = 20
label = "test-label"
)
require.NoError(t, git.AddLabelToPR(ctx, client, pr, label))
})
t.Run("It should return an error if GitHub returns an error when listing labels", func(t *testing.T) {
var (
ctx = context.Background()
listLabelsError = errors.New("list labels error")
client = &TestLabelsService{
ListLabelsError: listLabelsError,
Labels: []*github.Label{},
}
pr = 20
label = "test-label"
)
require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), listLabelsError)
})
t.Run("It should not return an error if there are existing enterprise-check labels.", func(t *testing.T) {
var (
ctx = context.Background()
client = &TestLabelsService{
Labels: []*github.Label{
{
Name: github.String("enterprise-failed"),
},
},
}
pr = 20
label = "test-label"
)
require.NoError(t, git.AddLabelToPR(ctx, client, pr, label))
})
t.Run("It should return an error if GitHub returns an error when removing existing enterprise-check labels", func(t *testing.T) {
var (
ctx = context.Background()
removeLabelError = errors.New("remove label error")
client = &TestLabelsService{
RemoveLabelError: removeLabelError,
Labels: []*github.Label{
{
Name: github.String("enterprise-failed"),
},
},
}
pr = 20
label = "test-label"
)
require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), removeLabelError)
})
}
+56
View File
@@ -0,0 +1,56 @@
package git_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/git"
"github.com/stretchr/testify/assert"
)
func TestPRCheckRegexp(t *testing.T) {
type match struct {
String string
Commit string
Branch string
PR string
}
var (
shouldMatch = []match{
{
String: "prc-1-a1b2c3d4/branch-name",
Branch: "branch-name",
Commit: "a1b2c3d4",
PR: "1",
},
{
String: "prc-111-a1b2c3d4/branch/name",
Branch: "branch/name",
Commit: "a1b2c3d4",
PR: "111",
},
{
String: "prc-102930122-a1b2c3d4/branch-name",
Branch: "branch-name",
Commit: "a1b2c3d4",
PR: "102930122",
},
}
shouldNotMatch = []string{"prc-a/branch", "km/test", "test", "prc", "prc/test", "price"}
)
regex := git.PRCheckRegexp()
for _, v := range shouldMatch {
assert.Truef(t, regex.MatchString(v.String), "regex '%s' should match %s", regex.String(), v)
m := regex.FindStringSubmatch(v.String)
assert.Equal(t, m[1], v.PR)
assert.Equal(t, m[2], v.Commit)
assert.Equal(t, m[3], v.Branch)
}
for _, v := range shouldNotMatch {
assert.False(t, regex.MatchString(v), "regex '%s' should not match %s", regex.String(), v)
}
}
+28
View File
@@ -1,6 +1,7 @@
package lerna
import (
"context"
"encoding/json"
"fmt"
"os"
@@ -9,6 +10,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/fsutil"
)
// BuildFrontendPackages will bump the version for the package to the latest canary build
@@ -56,3 +58,29 @@ func GetLernaVersion(grafanaDir string) (string, error) {
}
return strings.TrimSpace(version), nil
}
func PackFrontendPackages(ctx context.Context, tag, grafanaDir, artifactsDir string) error {
exists, err := fsutil.Exists(artifactsDir)
if err != nil {
return err
}
if exists {
err = os.RemoveAll(artifactsDir)
if err != nil {
return err
}
}
// nolint:gosec
if err = os.MkdirAll(artifactsDir, 0755); err != nil {
return err
}
// nolint:gosec
cmd := exec.CommandContext(ctx, "yarn", "lerna", "exec", "--no-private", "--", "yarn", "pack", "--out", fmt.Sprintf("../../npm-artifacts/%%s-%v.tgz", tag))
cmd.Dir = grafanaDir
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), output, err)
}
return nil
}
+240
View File
@@ -0,0 +1,240 @@
package npm
import (
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/build/gcloud/storage"
"github.com/grafana/grafana/pkg/build/lerna"
"github.com/grafana/grafana/pkg/build/versions"
)
const GrafanaDir = "."
const NpmArtifactDir = "./npm-artifacts"
// TODO: could this be replaced by `yarn lerna list -p` ?
var packages = []string{
"@grafana/ui",
"@grafana/data",
"@grafana/toolkit",
"@grafana/runtime",
"@grafana/e2e",
"@grafana/e2e-selectors",
"@grafana/schema",
}
// PublishNpmPackages will publish local NPM packages to NPM registry.
func PublishNpmPackages(ctx context.Context, tag string) error {
version, err := versions.GetVersion(tag)
if err != nil {
return err
}
log.Printf("Grafana version: %s", version.Version)
if err := setNpmCredentials(); err != nil {
return err
}
npmArtifacts, err := storage.ListLocalFiles(NpmArtifactDir)
if err != nil {
return err
}
for _, packedFile := range npmArtifacts {
// nolint:gosec
cmd := exec.CommandContext(ctx, "npm", "publish", packedFile.FullPath, "--tag", version.Channel)
cmd.Dir = GrafanaDir
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err)
}
}
return updateTag(ctx, version, tag)
}
// StoreNpmPackages will store local NPM packages in GCS bucket `bucketName`.
func StoreNpmPackages(ctx context.Context, tag, bucketName string) error {
err := lerna.PackFrontendPackages(ctx, tag, GrafanaDir, NpmArtifactDir)
if err != nil {
return err
}
gcs, err := storage.New()
if err != nil {
return err
}
bucket := gcs.Bucket(bucketName)
bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag)
if err = gcs.CopyLocalDir(ctx, NpmArtifactDir, bucket, bucketPath, true); err != nil {
return err
}
log.Print("Successfully stored npm packages!")
return nil
}
// FetchNpmPackages will store NPM packages stored in GCS bucket `bucketName` on local disk in `frontend.NpmArtifactDir`.
func FetchNpmPackages(ctx context.Context, tag, bucketName string) error {
gcs, err := storage.New()
if err != nil {
return err
}
bucketPath := fmt.Sprintf("artifacts/npm/%s/", tag)
bucket := gcs.Bucket(bucketName)
err = gcs.DownloadDirectory(ctx, bucket, NpmArtifactDir, storage.FilesFilter{
Prefix: bucketPath,
FileExts: []string{".tgz"},
})
if err != nil {
return err
}
return nil
}
// updateTag will move next or latest npm dist-tags, if needed.
//
// Note: This function makes the assumption that npm dist-tags has already
// been updated and hence why move of dist-tags not always happens:
//
// If stable the dist-tag latest was used.
// If beta the dist-tag next was used.
//
// Scenarios:
//
// 1. Releasing a newer stable than the current stable
// Latest and next is 9.1.5.
// 9.1.6 is released, latest and next should point to 9.1.6.
// The next dist-tag is moved to point to 9.1.6.
//
// 2. Releasing during an active beta period:
// Latest and next is 9.1.6.
// 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1
// No move of dist-tags
// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1
// No move of dist-tags
// Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2
// No move of dist-tags
// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0.
// The next dist-tag is moved to point to 9.2.0.
//
// 3. Releasing an older stable than the current stable
// Latest and next is 9.2.0.
// Next 9.1.8 is released, latest should point to 9.2.0, next should point to 9.2.0
// The latest dist-tag is moved to point to 9.2.0.
func updateTag(ctx context.Context, version *versions.Version, releaseVersion string) error {
if version.Channel != versions.Latest {
return nil
}
latestStableVersion, err := getLatestStableVersion()
if err != nil {
return err
}
betaVersion, err := getLatestBetaVersion()
if err != nil {
return err
}
isLatest, err := versions.IsGreaterThanOrEqual(releaseVersion, latestStableVersion)
if err != nil {
return err
}
isNewerThanLatestBeta, err := versions.IsGreaterThanOrEqual(releaseVersion, betaVersion)
if err != nil {
return err
}
for _, pkg := range packages {
if !isLatest {
err = runMoveLatestNPMTagCommand(ctx, pkg, latestStableVersion)
if err != nil {
return err
}
}
if isLatest && isNewerThanLatestBeta {
err = runMoveNextNPMTagCommand(ctx, pkg, version.Version)
if err != nil {
return err
}
}
}
return nil
}
func getLatestStableVersion() (string, error) {
return versions.GetLatestVersion(versions.LatestStableVersionURL)
}
func getLatestBetaVersion() (string, error) {
return versions.GetLatestVersion(versions.LatestBetaVersionURL)
}
func runMoveNextNPMTagCommand(ctx context.Context, pkg string, packageVersion string) error {
// nolint:gosec
cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, packageVersion), "next")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err)
}
return nil
}
func runMoveLatestNPMTagCommand(ctx context.Context, pkg string, latestStableVersion string) error {
// nolint:gosec
cmd := exec.CommandContext(ctx, "npm", "dist-tag", "add", fmt.Sprintf("%s@%s", pkg, latestStableVersion), "latest")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("command '%s' failed to run, output: %s, err: %q", cmd.String(), out, err)
}
return nil
}
// setNpmCredentials Creates a .npmrc file in the users home folder and writes the
// necessary credentials to it for publishing packages to the NPM registry.
func setNpmCredentials() error {
npmToken := strings.TrimSpace(os.Getenv("NPM_TOKEN"))
if npmToken == "" {
return fmt.Errorf("npm token is not set")
}
homeDir, err := os.UserHomeDir()
if err != nil {
return fmt.Errorf("failed to obtain home directory, err: %q", err)
}
npmPath := filepath.Join(homeDir, ".npmrc")
registry := []byte(fmt.Sprintf("//registry.npmjs.org/:_authToken=%s", npmToken))
if _, err = os.Stat(npmPath); os.IsNotExist(err) {
// nolint:gosec
f, err := os.Create(npmPath)
if err != nil {
return fmt.Errorf("couldn't create npmrc file, err: %q", err)
}
_, err = f.Write(registry)
if err != nil {
return fmt.Errorf("failed to write to file, err: %q", err)
}
defer func() {
if err := f.Close(); err != nil {
log.Printf("Failed to close file: %s", err.Error())
}
}()
} else {
err = os.WriteFile(npmPath, registry, 0644)
if err != nil {
return fmt.Errorf("error writing to file, err: %q", err)
}
}
return nil
}
+7 -6
View File
@@ -8,10 +8,11 @@ import (
"path/filepath"
"strings"
"github.com/urfave/cli/v2"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/grafana/grafana/pkg/infra/fs"
"github.com/urfave/cli/v2"
)
func writeAptlyConf(dbDir, repoDir string) error {
@@ -166,7 +167,7 @@ func UpdateDebRepo(cfg PublishConfig, workDir string) error {
cmd := exec.Command("aptly", "publish", "update", "-batch", passArg, "-force-overwrite", tp,
"filesystem:repo:grafana")
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1)
return cli.Exit(fmt.Sprintf("failed to update Debian %q repository: %s", tp, output), 1)
}
}
@@ -179,7 +180,7 @@ func UpdateDebRepo(cfg PublishConfig, workDir string) error {
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", dbDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1)
return cli.Exit(fmt.Sprintf("failed to upload Debian repo database to GCS: %s", output), 1)
}
}
@@ -193,14 +194,14 @@ func UpdateDebRepo(cfg PublishConfig, workDir string) error {
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "rsync", "-r", "-d", grafDir, u)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1)
return cli.Exit(fmt.Sprintf("failed to upload Debian repo resources to GCS: %s", output), 1)
}
allRepoResources := fmt.Sprintf("%s/**/*", u)
log.Printf("Setting cache ttl for Debian repo resources on GCS (%s)...\n", allRepoResources)
//nolint:gosec
cmd = exec.Command("gsutil", "-m", "setmeta", "-h", CacheSettings+cfg.TTL, allRepoResources)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1)
return cli.Exit(fmt.Sprintf("failed to set cache ttl for Debian repo resources on GCS: %s", output), 1)
}
}
@@ -238,7 +239,7 @@ func addPkgsToRepo(cfg PublishConfig, workDir, tmpDir, repoName string) error {
//nolint:gosec
cmd := exec.Command("aptly", "repo", "add", "-force-replace", repoName, tmpDir)
if output, err := cmd.CombinedOutput(); err != nil {
return cli.NewExitError(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1)
return cli.Exit(fmt.Sprintf("failed to add packages to local Debian repository: %s", output), 1)
}
return nil
+22
View File
@@ -0,0 +1,22 @@
package packaging_test
import (
"testing"
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/packaging"
"github.com/stretchr/testify/assert"
)
func TestPackageRegexp(t *testing.T) {
t.Run("It should match enterprise2 packages", func(t *testing.T) {
rgx := packaging.PackageRegexp(config.EditionEnterprise2)
matches := []string{
"grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz",
"grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz.sha256",
}
for _, v := range matches {
assert.Truef(t, rgx.MatchString(v), "'%s' should match regex '%s'", v, rgx.String())
}
})
}
+6 -3
View File
@@ -10,12 +10,15 @@ import (
"path/filepath"
"strings"
// Consider switching this over to a community fork unless there is
// an option to move us away from OpenPGP.
"golang.org/x/crypto/openpgp" //nolint:staticcheck
"golang.org/x/crypto/openpgp/armor" //nolint:staticcheck
"golang.org/x/crypto/openpgp/packet" //nolint:staticcheck
"github.com/grafana/grafana/pkg/build/config"
"github.com/grafana/grafana/pkg/build/fsutil"
"github.com/grafana/grafana/pkg/infra/fs"
"golang.org/x/crypto/openpgp"
"golang.org/x/crypto/openpgp/armor"
"golang.org/x/crypto/openpgp/packet"
)
// UpdateRPMRepo updates the RPM repository with the new release.
+10
View File
@@ -0,0 +1,10 @@
package stringutil
func Contains(arr []string, s string) bool {
for _, e := range arr {
if e == s {
return true
}
}
return false
}
+160
View File
@@ -0,0 +1,160 @@
package versions
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"regexp"
"strconv"
"github.com/Masterminds/semver/v3"
)
var (
reGrafanaTag = regexp.MustCompile(`^v(\d+\.\d+\.\d+$)`)
reGrafanaTagBeta = regexp.MustCompile(`^v(\d+\.\d+\.\d+-beta)`)
reGrafanaTagCustom = regexp.MustCompile(`^v(\d+\.\d+\.\d+-\w+)`)
)
const (
Latest = "latest"
Next = "next"
Test = "test"
)
type Version struct {
Version string
Channel string
}
type VersionFromAPI struct {
Version string `json:"version"`
}
type LatestGcomAPI = string
const (
LatestStableVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/stable"
LatestBetaVersionURL LatestGcomAPI = "https://grafana.com/api/grafana/versions/beta"
)
func GetLatestVersion(url LatestGcomAPI) (string, error) {
// nolint:gosec
resp, err := http.Get(url)
if err != nil {
return "", err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Printf("Failed to close body: %s", err.Error())
}
}()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("server returned non 200 status code: %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
var apiResponse VersionFromAPI
err = json.Unmarshal(body, &apiResponse)
if err != nil {
return "", err
}
return apiResponse.Version, nil
}
// IsGreaterThanOrEqual semantically checks whether newVersion is greater than or equal to stableVersion.
func IsGreaterThanOrEqual(newVersion, stableVersion string) (bool, error) {
v1SemVer, err := semver.NewVersion(newVersion)
if err != nil {
return isGreaterThanOrEqualFourDigit(newVersion, stableVersion)
}
v2SemVer, err := semver.NewVersion(stableVersion)
if err != nil {
return isGreaterThanOrEqualFourDigit(newVersion, stableVersion)
}
comp := v1SemVer.Compare(v2SemVer)
switch comp {
case -1:
return false, nil
case 1, 0:
return true, nil
default:
return true, fmt.Errorf("unknown comparison value between scemantic versions, err: %q", err)
}
}
var fourDigitRe = regexp.MustCompile(`(\d+\.\d+\.\d+)\.(\d+)`)
func parseFourDigit(version string) (*semver.Version, int, error) {
matches := fourDigitRe.FindStringSubmatch(version)
if len(matches) < 2 {
semVer, err := semver.NewVersion(version)
if err != nil {
return nil, 0, err
}
return semVer, 0, nil
}
semVer, err := semver.NewVersion(matches[1])
if err != nil {
return nil, 0, err
}
i, err := strconv.Atoi(matches[2])
if err != nil {
return nil, 0, err
}
return semVer, i, nil
}
func isGreaterThanOrEqualFourDigit(newVersion, stableVersion string) (bool, error) {
newVersionSemVer, newVersionSemVerNo, err := parseFourDigit(newVersion)
if err != nil {
return false, err
}
stableVersionSemVer, stableVersionSemVerNo, err := parseFourDigit(stableVersion)
if err != nil {
return false, err
}
if stableVersionSemVer.Original() != newVersionSemVer.Original() {
return IsGreaterThanOrEqual(newVersionSemVer.Original(), stableVersionSemVer.Original())
}
return newVersionSemVerNo >= stableVersionSemVerNo, nil
}
func GetVersion(tag string) (*Version, error) {
var version Version
switch {
case reGrafanaTag.MatchString(tag):
version = Version{
Version: reGrafanaTag.FindStringSubmatch(tag)[1],
Channel: Latest,
}
case reGrafanaTagBeta.MatchString(tag):
version = Version{
Version: reGrafanaTagBeta.FindStringSubmatch(tag)[1],
Channel: Next,
}
case reGrafanaTagCustom.MatchString(tag):
version = Version{
Version: reGrafanaTagCustom.FindStringSubmatch(tag)[1],
Channel: Test,
}
default:
return nil, fmt.Errorf("%s not a supported Grafana version, exitting", tag)
}
return &version, nil
}
+69
View File
@@ -0,0 +1,69 @@
package versions
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
func TestIsGreaterThanOrEqual(t *testing.T) {
testCases := []struct {
newVersion string
stableVersion string
expected bool
}{
{newVersion: "9.0.0", stableVersion: "8.0.0", expected: true},
{newVersion: "6.0.0", stableVersion: "6.0.0", expected: true},
{newVersion: "7.0.0", stableVersion: "8.0.0", expected: false},
{newVersion: "8.5.0-beta1", stableVersion: "8.0.0", expected: true},
{newVersion: "8.5.0", stableVersion: "8.5.0-beta1", expected: true},
{newVersion: "9.0.0.1", stableVersion: "9.0.0", expected: true},
{newVersion: "9.0.0.2", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.1.0", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.1-0-beta1", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.0.0.1", stableVersion: "9.0.1.1", expected: false},
{newVersion: "9.0.1.1", stableVersion: "9.0.0.1", expected: true},
{newVersion: "9.0.0.1", stableVersion: "9.0.0.1", expected: true},
{newVersion: "7.0.0.1", stableVersion: "8.0.0", expected: false},
{newVersion: "9.1-0-beta1", stableVersion: "9.1-0-beta2", expected: false},
{newVersion: "9.1-0-beta3", stableVersion: "9.1-0-beta2", expected: true},
}
for _, tc := range testCases {
name := fmt.Sprintf("newVersion %s greater than or equal stableVersion %s = %v", tc.newVersion, tc.stableVersion, tc.expected)
t.Run(name, func(t *testing.T) {
result, err := IsGreaterThanOrEqual(tc.newVersion, tc.stableVersion)
require.NoError(t, err)
require.Equal(t, tc.expected, result)
})
}
}
func TestGetLatestVersion(t *testing.T) {
t.Run("it returns a version", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
response := VersionFromAPI{
Version: "8.4.0",
}
jsonRes, err := json.Marshal(&response)
require.NoError(t, err)
_, err = w.Write(jsonRes)
require.NoError(t, err)
}))
version, err := GetLatestVersion(server.URL)
require.NoError(t, err)
require.Equal(t, "8.4.0", version)
})
t.Run("it handles non 200 responses", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
_, err := GetLatestVersion(server.URL)
require.Error(t, err)
})
}