public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v9.5.x] CI: Update CI/CD tooling and pipelines from main (#76875)

Browse Source 
* CI: Update CI/CD tooling and pipelines from main (#76814)

* CI: Update CI/CD tooling and pipelines from main

* Update Makefile

* Comment out validate_openapi_spec_step

* Update broken frontend tests

* Fix validate-npm-packages regex to work without suffix

* Fix cypress image version

(cherry picked from commit 03ecb1db39)

* Comment out kindsysreport/codegen/report.go on Makefile

* Disable steps that don't work for older versions

* Fix permissions of validate-npm-packages.sh
This commit is contained in:
Guilherme Caulada
2023-10-20 10:26:39 -03:00
committed by GitHub
parent 0b55a53a47
commit 9598e8783b
49 changed files with 3000 additions and 2883 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/build/cmd.go
+11 -1
View File
@@ -222,15 +222,25 @@ func ldflags(opts BuildOpts) (string, error) {
commitSha = v
}
var enterpriseCommitSha string
if opts.enterprise {
enterpriseCommitSha = getGitEnterpriseSha()
if v := os.Getenv("ENTERPRISE_COMMIT_SHA"); v != "" {
enterpriseCommitSha = v
}
}
buildBranch := getGitBranch()
if v := os.Getenv("BUILD_BRANCH"); v != "" {
buildBranch = v
}
var b bytes.Buffer
b.WriteString("-w")
b.WriteString(fmt.Sprintf(" -X main.version=%s", opts.version))
b.WriteString(fmt.Sprintf(" -X main.commit=%s", commitSha))
if enterpriseCommitSha != "" {
b.WriteString(fmt.Sprintf(" -X main.enterpriseCommit=%s", enterpriseCommitSha))
}
b.WriteString(fmt.Sprintf(" -X main.buildstamp=%d", buildStamp))
b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", buildBranch))
if v := os.Getenv("LDFLAGS"); v != "" {
pkg/build/cmd/grafanacom.go
+1 -1
View File
@@ -232,7 +232,7 @@ func getSHA256(u string) ([]byte, error) {
return sha256, nil
}
func postRequest(cfg packaging.PublishConfig, pth string, obj interface{}, descr string) error {
func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) error {
var sfx string
switch cfg.Edition {
case config.EditionOSS:
pkg/build/cmd/verifydrone.go
+3 -3
View File
@@ -68,7 +68,7 @@ func VerifyDrone(c *cli.Context) error {
return nil
}
func readConfig(fpath string) ([]map[string]interface{}, error) {
func readConfig(fpath string) ([]map[string]any, error) {
//nolint:gosec
f, err := os.Open(fpath)
if err != nil {
@@ -82,9 +82,9 @@ func readConfig(fpath string) ([]map[string]interface{}, error) {
// The YAML stream may contain multiple pipeline configurations, read them all
dec := yaml.NewDecoder(f)
var c []map[string]interface{}
var c []map[string]any
for {
var m map[string]interface{}
var m map[string]any
if err := dec.Decode(&m); err != nil {
if errors.Is(err, io.EOF) {
break
pkg/build/config/genmetadata_test.go
+2 -4
View File
@@ -72,10 +72,8 @@ func setUpEnv(t *testing.T, envMap map[string]string) {
t.Helper()
os.Clearenv()
err := os.Setenv("DRONE_COMMIT", "abcd12345")
require.NoError(t, err)
t.Setenv("DRONE_COMMIT", "abcd12345")
for k, v := range envMap {
err := os.Setenv(k, v)
require.NoError(t, err)
t.Setenv(k, v)
}
}
pkg/build/config/revision.go
+20 -6
View File
@@ -3,6 +3,7 @@ package config
import (
"context"
"fmt"
"log"
"strconv"
"time"
@@ -10,9 +11,10 @@ import (
)
type Revision struct {
Timestamp int64
SHA256 string
Branch string
Timestamp int64
SHA256 string
EnterpriseCommit string
Branch string
}
func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) {
@@ -42,14 +44,26 @@ func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) {
return Revision{}, err
}
enterpriseCommit, err := executil.OutputAt(ctx, grafanaDir, "git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD")
if err != nil {
enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "..", "rev-parse", "--short", "HEAD")
if err != nil {
enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "/tmp/grafana-enterprise", "rev-parse", "--short", "HEAD")
if err != nil {
log.Println("Could not get enterprise commit. Error:", err)
}
}
}
branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD")
if err != nil {
return Revision{}, err
}
return Revision{
SHA256: sha,
Branch: branch,
Timestamp: stamp,
SHA256: sha,
EnterpriseCommit: enterpriseCommit,
Branch: branch,
Timestamp: stamp,
}, nil
}
pkg/build/config/version.go
+1 -1
View File
@@ -94,7 +94,7 @@ func GetPackageJSONVersion(grafanaDir string) (string, error) {
if err != nil {
return "", fmt.Errorf("failed to read %q: %w", pkgJSONPath, err)
}
pkgObj := map[string]interface{}{}
pkgObj := map[string]any{}
if err := json.Unmarshal(pkgJSONB, &pkgObj); err != nil {
return "", fmt.Errorf("failed decoding %q: %w", pkgJSONPath, err)
}
pkg/build/config/versions.go
+21 -16
View File
@@ -1,7 +1,5 @@
package config
const PublicBucket = "grafana-downloads"
var Versions = VersionMap{
PullRequestMode: {
Variants: []Variant{
@@ -9,8 +7,9 @@ var Versions = VersionMap{
VariantLinuxAmd64Musl,
VariantDarwinAmd64,
VariantWindowsAmd64,
VariantArm64,
VariantArm64Musl,
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// VariantArm64,
// VariantArm64Musl,
},
PluginSignature: PluginSignature{
Sign: false,
@@ -29,9 +28,10 @@ var Versions = VersionMap{
},
MainMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// VariantArmV6,
// VariantArmV7,
// VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
@@ -48,7 +48,8 @@ var Versions = VersionMap{
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
},
Distribution: []Distribution{
Alpine,
@@ -64,9 +65,10 @@ var Versions = VersionMap{
},
DownstreamMode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// VariantArmV6,
//VariantArmV7,
// VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
@@ -83,7 +85,8 @@ var Versions = VersionMap{
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// ArchARMv7, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable.
},
Distribution: []Distribution{
Alpine,
@@ -171,9 +174,10 @@ var Versions = VersionMap{
},
Enterprise2Mode: {
Variants: []Variant{
VariantArmV6,
VariantArmV7,
VariantArmV7Musl,
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// VariantArmV6,
// VariantArmV7,
// VariantArmV7Musl,
VariantArm64,
VariantArm64Musl,
VariantDarwinAmd64,
@@ -190,7 +194,8 @@ var Versions = VersionMap{
Architectures: []Architecture{
ArchAMD64,
ArchARM64,
ArchARMv7,
// https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved
// ArchARMv7,
},
Distribution: []Distribution{
Alpine,
pkg/build/docker/build.go
+1 -1
View File
@@ -74,7 +74,7 @@ func BuildImage(version string, arch config.Architecture, grafanaDir string, use
tagSuffix := ""
if useUbuntu {
libc = ""
baseImage = fmt.Sprintf("%subuntu:20.04", baseArch)
baseImage = fmt.Sprintf("%subuntu:22.04", baseArch)
tagSuffix = "-ubuntu"
}
pkg/build/env/fallback_test.go
Vendored
+14 -13
View File
@@ -15,6 +15,11 @@ const (
flag2 = "flag2"
)
type flagObj struct {
name string
value string
}
func TestRequireListWithEnvFallback(t *testing.T) {
var app = cli.NewApp()
tests := []struct {
@@ -73,7 +78,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) {
}{
{
testName: "string present in the context",
ctx: cli.NewContext(app, setFlags(t, flag1, flag2, flag.NewFlagSet("test", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError), flagObj{name: flag1, value: "a"}), nil),
name: flag1,
envName: "",
expected: "a",
@@ -81,7 +86,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) {
},
{
testName: "string present in env",
ctx: cli.NewContext(app, setFlags(t, "", "", flag.NewFlagSet("test", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError)), nil),
name: flag1,
envName: setEnv(t, flag1, "a"),
expected: "a",
@@ -89,7 +94,7 @@ func TestRequireStringWithEnvFallback(t *testing.T) {
},
{
testName: "string absent from both context and env",
ctx: cli.NewContext(app, setFlags(t, "", flag2, flag.NewFlagSet("test", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("test", flag.ContinueOnError), flagObj{name: flag2, value: "b"}), nil),
name: flag1,
envName: "",
expected: "",
@@ -120,13 +125,12 @@ func applyFlagSet(t *testing.T, aFlag, aValue string) *flag.FlagSet {
return set
}
func setFlags(t *testing.T, flag1, flag2 string, flagSet *flag.FlagSet) *flag.FlagSet {
func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet {
t.Helper()
if flag1 != "" {
flagSet.StringVar(&flag1, "flag1", "a", "")
}
if flag2 != "" {
flagSet.StringVar(&flag2, "flag2", "b", "")
for _, f := range flags {
if f.name != "" {
flagSet.StringVar(&f.name, f.name, f.value, "")
}
}
return flagSet
}
@@ -135,9 +139,6 @@ func setEnv(t *testing.T, key, value string) string {
t.Helper()
os.Clearenv()
err := os.Setenv(key, value)
if err != nil {
require.NoError(t, err)
}
t.Setenv(key, value)
return key
}
pkg/build/frontend/config_test.go
+15 -14
View File
@@ -21,6 +21,11 @@ type packageJson struct {
Version string `json:"version"`
}
type flagObj struct {
name string
value string
}
var app = cli.NewApp()
func TestGetConfig(t *testing.T) {
@@ -32,35 +37,35 @@ func TestGetConfig(t *testing.T) {
wantErr bool
}{
{
ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil),
name: "package.json matches tag",
packageJsonVersion: "10.0.0",
metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}},
wantErr: false,
},
{
ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil),
name: "custom tag, package.json doesn't match",
packageJsonVersion: "10.0.0",
metadata: config.Metadata{GrafanaVersion: "10.0.0-abcd123pre", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}},
wantErr: false,
},
{
ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil),
name: "package.json doesn't match tag",
packageJsonVersion: "10.1.0",
metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}},
wantErr: true,
},
{
ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, "", flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil),
name: "test tag event, check should be skipped",
packageJsonVersion: "10.1.0",
metadata: config.Metadata{GrafanaVersion: "10.1.0-test", ReleaseMode: config.ReleaseMode{Mode: config.TagMode, IsTest: true}},
wantErr: false,
},
{
ctx: cli.NewContext(app, setFlags(t, jobs, githubToken, buildID, flag.NewFlagSet("flagSet", flag.ContinueOnError)), nil),
ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}, flagObj{name: buildID, value: "12345"}), nil),
name: "non-tag event",
packageJsonVersion: "10.1.0-pre",
metadata: config.Metadata{GrafanaVersion: "10.1.0-12345pre", ReleaseMode: config.ReleaseMode{Mode: config.PullRequestMode}},
@@ -85,16 +90,12 @@ func TestGetConfig(t *testing.T) {
}
}
func setFlags(t *testing.T, flag1, flag2, flag3 string, flagSet *flag.FlagSet) *flag.FlagSet {
func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet {
t.Helper()
if flag1 != "" {
flagSet.StringVar(&flag1, jobs, "2", "")
}
if flag2 != "" {
flagSet.StringVar(&flag2, githubToken, "token", "")
}
if flag3 != "" {
flagSet.StringVar(&flag3, buildID, "12345", "")
for _, f := range flags {
if f.name != "" {
flagSet.StringVar(&f.name, f.name, f.value, "")
}
}
return flagSet
}
pkg/build/gcloud/storage/gsutil.go
+16 -6
View File
@@ -389,11 +389,17 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path
return "", ErrorNilBucket
}
it := bucket.Objects(ctx, &storage.Query{
query := &storage.Query{
Prefix: path,
})
}
err := query.SetAttrSelection([]string{"Name", "Generation"})
if err != nil {
return "", fmt.Errorf("failed to set attribute selector, err: %q", err)
}
it := bucket.Objects(ctx, query)
var files []string
var oldGeneration int64
for {
attrs, err := it.Next()
if errors.Is(err, iterator.Done) {
@@ -402,13 +408,17 @@ func GetLatestMainBuild(ctx context.Context, bucket *storage.BucketHandle, path
if err != nil {
return "", fmt.Errorf("failed to iterate through bucket, err: %w", err)
}
files = append(files, attrs.Name)
if attrs.Generation >= oldGeneration {
files = append([]string{attrs.Name}, files...)
oldGeneration = attrs.Generation
} else {
files = append(files, attrs.Name)
}
}
var latestVersion string
for i := len(files) - 1; i >= 0; i-- {
captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+pre)`)
for i := 0; i < len(files); i++ {
captureVersion := regexp.MustCompile(`(\d+\.\d+\.\d+-\d+)`)
if captureVersion.MatchString(files[i]) {
latestVersion = captureVersion.FindString(files[i])
break
pkg/build/git.go
+13
View File
@@ -15,3 +15,16 @@ func getGitSha() string {
}
return string(v)
}
func getGitEnterpriseSha() string {
// supporting the old way of dev setup
v, err := runError("git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD")
if err != nil {
// supporting the new way of dev setup
v, err = runError("git", "-C", "..", "rev-parse", "--short", "HEAD")
if err != nil {
return ""
}
}
return string(v)
}
pkg/build/grafana/build.go
+7 -1
View File
@@ -23,13 +23,19 @@ const (
)
func GrafanaLDFlags(version string, r config.Revision) []string {
return []string{
cmd := []string{
"-w",
fmt.Sprintf("-X main.version=%s", version),
fmt.Sprintf("-X main.commit=%s", r.SHA256),
fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp),
fmt.Sprintf("-X main.buildBranch=%s", r.Branch),
}
if r.EnterpriseCommit != "" {
cmd = append(cmd, fmt.Sprintf("-X main.enterpriseCommit=%s", r.EnterpriseCommit))
}
return cmd
}
// BinaryFolder returns the path to where the Grafana binary is build given the provided arguments.
pkg/build/lerna/lerna.go
+1 -1
View File
@@ -47,7 +47,7 @@ func GetLernaVersion(grafanaDir string) (string, error) {
if err != nil {
return "", fmt.Errorf("failed to read %q: %w", lernaJSONPath, err)
}
pkgObj := map[string]interface{}{}
pkgObj := map[string]any{}
if err := json.Unmarshal(lernaJSONB, &pkgObj); err != nil {
return "", fmt.Errorf("failed decoding %q: %w", lernaJSONPath, err)
}
pkg/build/npm/npm.go
+3 -3
View File
@@ -21,11 +21,11 @@ const NpmArtifactDir = "./npm-artifacts"
var packages = []string{
"@grafana/ui",
"@grafana/data",
"@grafana/toolkit",
"@grafana/runtime",
"@grafana/e2e",
"@grafana/e2e-selectors",
"@grafana/schema",
"@grafana/flamegraph",
}
// PublishNpmPackages will publish local NPM packages to NPM registry.
@@ -117,11 +117,11 @@ func FetchNpmPackages(ctx context.Context, tag, bucketName string) error {
// Latest and next is 9.1.6.
// 9.2.0-beta1 is released, the latest should stay on 9.1.6, next should point to 9.2.0-beta1
// No move of dist-tags
// 9.1.7 is relased, the latest should point to 9.1.7, next should stay to 9.2.0-beta1
// 9.1.7 is released, the latest should point to 9.1.7, next should stay to 9.2.0-beta1
// No move of dist-tags
// Next week 9.2.0-beta2 is released, the latest should point to 9.1.7, next should point to 9.2.0-beta2
// No move of dist-tags
// In two weeks 9.2.0 stable is relased, the latest and next should point to 9.2.0.
// In two weeks 9.2.0 stable is released, the latest and next should point to 9.2.0.
// The next dist-tag is moved to point to 9.2.0.
//
// 3. Releasing an older stable than the current stable
pkg/build/packaging/grafana.go
+2 -3
View File
@@ -533,7 +533,7 @@ func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir strin
if err != nil {
return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err)
}
var plugJSON map[string]interface{}
var plugJSON map[string]any
if err := json.Unmarshal(jsonB, &plugJSON); err != nil {
return err
}
@@ -771,8 +771,7 @@ func realPackageVariant(ctx context.Context, v config.Variant, edition config.Ed
defaultFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "sysconfig", "grafana-server"),
systemdFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "systemd", "grafana-server.service"),
wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"),
// chkconfig is depended on since our systemd service wraps a SysV init script, and that requires chkconfig
depends: []string{"/sbin/service", "chkconfig", "fontconfig", "freetype"},
depends: []string{"/sbin/service", "fontconfig", "freetype"},
}); err != nil {
return err
}
pkg/build/version.go
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
@@ -14,7 +15,7 @@ type PackageJSON struct {
// Opens the package.json file in the provided directory and returns a struct that represents its contents
func OpenPackageJSON(dir string) (PackageJSON, error) {
reader, err := os.Open("package.json")
reader, err := os.Open(filepath.Clean(dir + "/package.json"))
if err != nil {
return PackageJSON{}, err
}