diff --git a/.drone.yml b/.drone.yml index 84d7898fab5..74ae5e4371b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -74,6 +74,12 @@ steps: - grabpl image: grafana/build-container:1.5.9 name: yarn-install +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - yarn run prettier:check - yarn run lint @@ -138,6 +144,12 @@ steps: - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} depends_on: @@ -238,6 +250,12 @@ steps: - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} depends_on: @@ -279,10 +297,11 @@ steps: token: from_secret: drone_token - commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend - commands: @@ -495,6 +514,12 @@ steps: - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.15 @@ -641,6 +666,12 @@ steps: - build-frontend-docs image: grafana/docs-base:latest name: build-docs-website +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd trigger: event: - pull_request @@ -731,6 +762,12 @@ steps: - build-frontend-docs image: grafana/docs-base:latest name: build-docs-website +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd trigger: branch: main event: @@ -773,6 +810,12 @@ steps: - grabpl image: grafana/build-container:1.5.9 name: yarn-install +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - yarn run prettier:check - yarn run lint @@ -829,6 +872,12 @@ steps: - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} depends_on: @@ -924,6 +973,12 @@ steps: - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER} depends_on: @@ -990,10 +1045,11 @@ steps: token: from_secret: drone_token - commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend - commands: @@ -1342,6 +1398,12 @@ steps: - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.15 @@ -1526,6 +1588,12 @@ steps: - grabpl image: grafana/build-container:1.5.9 name: gen-version +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME image: alpine:3.15 @@ -1639,10 +1707,17 @@ steps: image: grafana/build-container:1.5.9 name: yarn-install - commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss ${DRONE_TAG} + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd +- commands: + - ./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend - commands: @@ -1935,6 +2010,12 @@ steps: - grabpl image: grafana/build-container:1.5.9 name: yarn-install +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - ./bin/grabpl shellcheck depends_on: @@ -2225,6 +2306,12 @@ steps: from_secret: github_token image: grafana/build-container:1.5.9 name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - make gen-go depends_on: @@ -2254,10 +2341,11 @@ steps: image: grafana/build-container:1.5.9 name: verify-gen-cue - commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise ${DRONE_TAG} + - ./bin/build build-backend --jobs 8 --edition enterprise ${DRONE_TAG} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend - commands: @@ -2289,10 +2377,11 @@ steps: image: grafana/build-container:1.5.9 name: build-plugins - commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} + - ./bin/build build-backend --jobs 8 --edition enterprise2 ${DRONE_TAG} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend-enterprise2 - commands: @@ -2571,6 +2660,12 @@ steps: from_secret: github_token image: grafana/build-container:1.5.9 name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - make gen-go depends_on: @@ -3523,10 +3618,17 @@ steps: image: grafana/build-container:1.5.9 name: yarn-install - commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd +- commands: + - ./bin/build build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend - commands: @@ -3790,6 +3892,12 @@ steps: - grabpl image: grafana/build-container:1.5.9 name: yarn-install +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - ./bin/grabpl shellcheck depends_on: @@ -4054,6 +4162,12 @@ steps: environment: {} image: grafana/build-container:1.5.9 name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - make gen-go depends_on: @@ -4083,10 +4197,11 @@ steps: image: grafana/build-container:1.5.9 name: verify-gen-cue - commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-backend --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend - commands: @@ -4119,11 +4234,12 @@ steps: image: grafana/build-container:1.5.9 name: build-plugins - commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} + - ./bin/build build-backend --jobs 8 --edition enterprise2 --build-id ${DRONE_BUILD_NUMBER} --variants linux-amd64 depends_on: - gen-version - wire-install + - compile-build-cmd image: grafana/build-container:1.5.9 name: build-backend-enterprise2 - commands: @@ -4390,6 +4506,12 @@ steps: environment: {} image: grafana/build-container:1.5.9 name: init-enterprise +- commands: + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + environment: + CGO_ENABLED: 0 + image: golang:1.17 + name: compile-build-cmd - commands: - make gen-go depends_on: @@ -4864,6 +4986,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 419f84644385caa3d64cdfdf1f5dcd2b4ddfd02dfb5a24589de45e3ed03cb0bd +hmac: 63ee096e5fd4d4147f43465937c4976e0d64a1dffa74206282cd9c8edf6494c0 ... diff --git a/pkg/build/action.go b/pkg/build/action.go new file mode 100644 index 00000000000..5c93a935341 --- /dev/null +++ b/pkg/build/action.go @@ -0,0 +1 @@ +package build diff --git a/pkg/build/cmd/argcount_wrapper.go b/pkg/build/cmd/argcount_wrapper.go new file mode 100644 index 00000000000..d3a3cfd67f6 --- /dev/null +++ b/pkg/build/cmd/argcount_wrapper.go @@ -0,0 +1,16 @@ +package main + +import "github.com/urfave/cli/v2" + +func ArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { + return func(ctx *cli.Context) error { + if ctx.NArg() > max { + if err := cli.ShowSubcommandHelp(ctx); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + return action(ctx) + } +} diff --git a/pkg/build/cmd/buildbackend.go b/pkg/build/cmd/buildbackend.go new file mode 100644 index 00000000000..ca3e7466962 --- /dev/null +++ b/pkg/build/cmd/buildbackend.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "log" + "path/filepath" + "strings" + + "github.com/urfave/cli/v2" + + "github.com/grafana/grafana/pkg/build/compilers" + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/errutil" + "github.com/grafana/grafana/pkg/build/grafana" + "github.com/grafana/grafana/pkg/build/syncutil" +) + +func BuildBackend(ctx *cli.Context) error { + metadata, err := config.GetMetadata(filepath.Join("dist", "version.json")) + if err != nil { + return err + } + + var version string + + // ./ci build-backend v1.0.0 + if ctx.NArg() == 1 { + version = strings.TrimPrefix(ctx.Args().Get(0), "v") + } else { + version = metadata.GrafanaVersion + } + + var ( + edition = config.Edition(ctx.String("edition")) + cfg = config.Config{ + NumWorkers: ctx.Int("jobs"), + } + ) + + mode, err := config.GetVersion(metadata.ReleaseMode) + if err != nil { + return fmt.Errorf("could not get version / package info for mode '%s': %w", metadata.ReleaseMode, err) + } + + const grafanaDir = "." + + log.Printf("Building Grafana back-end, version %q, %s edition, variants [%v]", + version, edition, mode.Variants) + + p := syncutil.NewWorkerPool(cfg.NumWorkers) + defer p.Close() + + if err := compilers.Install(); err != nil { + return cli.Exit(err.Error(), 1) + } + + g, _ := errutil.GroupWithContext(ctx.Context) + for _, variant := range mode.Variants { + variant := variant + + opts := grafana.BuildVariantOpts{ + Variant: variant, + Edition: edition, + Version: version, + GrafanaDir: grafanaDir, + } + + p.Schedule(g.Wrap(func() error { + return grafana.BuildVariant(ctx.Context, opts) + })) + } + if err := g.Wait(); err != nil { + return cli.Exit(err.Error(), 1) + } + + log.Println("Successfully built back-end binaries!") + return nil +} diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go new file mode 100644 index 00000000000..89af88a0b90 --- /dev/null +++ b/pkg/build/cmd/flags.go @@ -0,0 +1,23 @@ +package main + +import "github.com/urfave/cli/v2" + +var ( + jobsFlag = cli.IntFlag{ + Name: "jobs", + Usage: "Number of parallel jobs", + } + buildIDFlag = cli.StringFlag{ + Name: "build-id", + Usage: "Optionally supply a build ID to be part of the version", + } + editionFlag = cli.StringFlag{ + Name: "edition", + Usage: "The edition of Grafana to build (oss or enterprise)", + Value: "oss", + } + variantsFlag = cli.StringFlag{ + Name: "variants", + Usage: "Comma-separated list of variants to build", + } +) diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go new file mode 100644 index 00000000000..6ee2db3c950 --- /dev/null +++ b/pkg/build/cmd/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "log" + "os" + + "github.com/urfave/cli/v2" +) + +func main() { + app := cli.NewApp() + + app.Commands = cli.Commands{ + { + Name: "build-backend", + Usage: "Build one or more variants of back-end binaries", + ArgsUsage: "[version]", + Action: ArgCountWrapper(1, BuildBackend), + Flags: []cli.Flag{ + &jobsFlag, + &variantsFlag, + &editionFlag, + &buildIDFlag, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatalln(err) + } +} diff --git a/pkg/build/compilers/install.go b/pkg/build/compilers/install.go new file mode 100644 index 00000000000..9a45ba48c0b --- /dev/null +++ b/pkg/build/compilers/install.go @@ -0,0 +1,50 @@ +package compilers + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +const ( + ArmV6 = "/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc" + Armv7 = "arm-linux-gnueabihf-gcc" + Armv7Musl = "/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc" + Arm64 = "aarch64-linux-gnu-gcc" + Arm64Musl = "/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc" + Osx64 = "/tmp/osxcross/target/bin/o64-clang" + Win64 = "x86_64-w64-mingw32-gcc" + LinuxX64 = "/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc" + LinuxX64Musl = "/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc" +) + +func Install() error { + // From the os.TempDir documentation: + // On Unix systems, it returns $TMPDIR if non-empty, + // else /tmp. On Windows, it uses GetTempPath, + // returning the first non-empty value from %TMP%, %TEMP%, %USERPROFILE%, + // or the Windows directory. On Plan 9, it returns /tmp. + tmp := os.TempDir() + + var ( + centosArchive = "x86_64-centos6-linux-gnu.tar.xz" + osxArchive = "osxcross.tar.xz" + ) + + for _, fname := range []string{centosArchive, osxArchive} { + path := filepath.Join(tmp, fname) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("stat error: %w", err) + } + // Ignore gosec G204 as this function is only used in the build process. + //nolint:gosec + cmd := exec.Command("tar", "xfJ", fname) + cmd.Dir = tmp + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to unpack %q: %q, %w", fname, output, err) + } + } + + return nil +} diff --git a/pkg/build/config/config.go b/pkg/build/config/config.go new file mode 100644 index 00000000000..2f7dbe7358f --- /dev/null +++ b/pkg/build/config/config.go @@ -0,0 +1,19 @@ +package config + +type Config struct { + Version string + Bucket string + DebRepoBucket string + DebDBBucket string + RPMRepoBucket string + GPGPassPath string + GPGPrivateKey string + GPGPublicKey string + GCPKeyFile string + NumWorkers int + GitHubUser string + GitHubToken string + PullEnterprise bool + NetworkConcurrency bool + PackageVersion string +} diff --git a/pkg/build/config/edition.go b/pkg/build/config/edition.go new file mode 100644 index 00000000000..48d7375f1cd --- /dev/null +++ b/pkg/build/config/edition.go @@ -0,0 +1,9 @@ +package config + +type Edition string + +const ( + EditionOSS Edition = "oss" + EditionEnterprise Edition = "enterprise" + EditionEnterprise2 Edition = "enterprise2" +) diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go new file mode 100644 index 00000000000..aa104916652 --- /dev/null +++ b/pkg/build/config/revision.go @@ -0,0 +1,55 @@ +package config + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/grafana/grafana/pkg/build/executil" +) + +type Revision struct { + Timestamp int64 + SHA256 string + Branch string +} + +func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) { + out, err := executil.OutputAt(ctx, dir, "git", "show", "-s", "--format=%ct") + if err != nil { + return time.Now().Unix(), nil + } + + stamp, err := strconv.ParseInt(out, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse output from git show: %q", out) + } + + return stamp, nil +} + +// GrafanaRevision uses git commands to get information about the checked out Grafana code located at 'grafanaDir'. +// This could maybe be a more generic "Describe" function in the "git" package. +func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) { + stamp, err := GrafanaTimestamp(ctx, grafanaDir) + if err != nil { + return Revision{}, err + } + + sha, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--short", "HEAD") + if err != nil { + return Revision{}, err + } + + branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD") + if err != nil { + return Revision{}, err + } + + return Revision{ + SHA256: sha, + Branch: branch, + Timestamp: stamp, + }, nil +} diff --git a/pkg/build/config/variant.go b/pkg/build/config/variant.go new file mode 100644 index 00000000000..8cbe3dc172f --- /dev/null +++ b/pkg/build/config/variant.go @@ -0,0 +1,41 @@ +package config + +// Variant is the OS / Architecture combination that Grafana can be compiled for. +type Variant string + +const ( + VariantLinuxAmd64 Variant = "linux-amd64" + VariantLinuxAmd64Musl Variant = "linux-amd64-musl" + VariantArmV6 Variant = "linux-armv6" + VariantArmV7 Variant = "linux-armv7" + VariantArmV7Musl Variant = "linux-armv7-musl" + VariantArm64 Variant = "linux-arm64" + VariantArm64Musl Variant = "linux-arm64-musl" + VariantDarwinAmd64 Variant = "darwin-amd64" + VariantWindowsAmd64 Variant = "windows-amd64" +) + +// Architecture is an allowed value in the GOARCH environment variable. +type Architecture string + +const ( + ArchAMD64 Architecture = "amd64" + ArchARMv6 Architecture = "armv6" + ArchARM64 Architecture = "arm64" + ArchARMHF Architecture = "armhf" + ArchARM Architecture = "arm" +) + +type OS string + +const ( + OSWindows OS = "windows" + OSDarwin OS = "darwin" + OSLinux OS = "linux" +) + +type LibC string + +const ( + LibCMusl = "musl" +) diff --git a/pkg/build/config/version.go b/pkg/build/config/version.go new file mode 100644 index 00000000000..171a2d18cab --- /dev/null +++ b/pkg/build/config/version.go @@ -0,0 +1,87 @@ +package config + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" +) + +type Metadata struct { + GrafanaVersion string `json:"version,omitempty"` + ReleaseMode VersionMode `json:"releaseMode,omitempty"` + GrabplVersion string `json:"grabplVersion,omitempty"` +} + +type PluginSignature struct { + Sign bool `json:"sign,omitempty"` + AdminSign bool `json:"adminSign,omitempty"` +} + +type Docker struct { + ShouldSave bool `json:"shouldSave,omitempty"` + Architectures []Architecture `json:"archs,omitempty"` +} + +// Version represents the "version.json" that defines all of the different variables used to build Grafana +type Version struct { + Variants []Variant `json:"variants,omitempty"` + PluginSignature PluginSignature `json:"pluginSignature,omitempty"` + Docker Docker `json:"docker,omitempty"` + PackagesBucket string `json:"packagesBucket,omitempty"` + PackagesBucketEnterprise2 string `json:"packagesBucketEnterprise2,omitempty"` + CDNAssetsBucket string `json:"CDNAssetsBucket,omitempty"` + CDNAssetsDir string `json:"CDNAssetsDir,omitempty"` + StorybookBucket string `json:"storybookBucket,omitempty"` + StorybookSrcDir string `json:"storybookSrcDir,omitempty"` +} + +// Versions is a map of versions. Each key of the Versions map is an event that uses the the config as the value for that key. +// For example, the 'pull_request' key will have data in it that might cause Grafana to be built differently in a pull request, +// than the way it will be built in 'main' +type VersionMap map[VersionMode]Version + +// GetMetadata attempts to read the JSON file located at 'path' and decode it as a Metadata{} type. +// If the provided path deos not exist, then an error is not returned. Instead, an empty metadata is returned with no error. +func GetMetadata(path string) (*Metadata, error) { + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return &Metadata{}, nil + } + return nil, err + } + // Ignore gosec G304 as this function is only used in the build process. + //nolint:gosec + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { + if err := file.Close(); err != nil { + log.Printf("error closing file at '%s': %s", path, err.Error()) + } + }() + + return DecodeMetadata(file) +} + +// DecodeMetadata decodes the data in the io.Reader 'r' as Metadata. +func DecodeMetadata(r io.Reader) (*Metadata, error) { + m := &Metadata{} + if err := json.NewDecoder(r).Decode(m); err != nil { + return nil, err + } + + return m, nil +} + +// GetVersions reads the embedded config.json and decodes it. +func GetVersion(mode VersionMode) (*Version, error) { + if v, ok := Versions[mode]; ok { + return &v, nil + } + + return nil, fmt.Errorf("mode not found in version list") +} diff --git a/pkg/build/config/version_json_test.go b/pkg/build/config/version_json_test.go new file mode 100644 index 00000000000..4f64ea3eaff --- /dev/null +++ b/pkg/build/config/version_json_test.go @@ -0,0 +1,167 @@ +package config_test + +var configJSON = []byte(`{ + "pull_request": { + "variants": [ + "linux-amd64", + "linux-amd64-musl", + "darwin-amd64", + "windows-amd64" + ], + "pluginSignature": { + "sign": false, + "adminSign": false + }, + "docker": { + "shouldSave": false, + "archs": [ + "amd64" + ] + } + }, + "main": { + "variants": [ + "linux-armv6", + "linux-armv7", + "linux-armv7-musl", + "linux-arm64", + "linux-arm64-musl", + "darwin-amd64", + "windows-amd64", + "linux-amd64", + "linux-amd64-musl" + ], + "pluginSignature": { + "sign": true, + "adminSign": true + }, + "docker": { + "shouldSave": false, + "archs": [ + "amd64", + "arm64", + "arm" + ] + }, + "packagesBucket": "grafana-downloads", + "CDNAssetsBucket": "grafana-static-assets" + }, + "branch": { + "variants": [ + "linux-armv6", + "linux-armv7", + "linux-armv7-musl", + "linux-arm64", + "linux-arm64-musl", + "darwin-amd64", + "windows-amd64", + "linux-amd64", + "linux-amd64-musl" + ], + "pluginSignature": { + "sign": true, + "adminSign": true + }, + "docker": { + "shouldSave": true, + "archs": [ + "amd64", + "arm64", + "arm" + ] + }, + "packagesBucket": "grafana-downloads", + "packagesBucketEnterprise2": "grafana-downloads-enterprise2", + "CDNAssetsBucket": "grafana-static-assets" + }, + "release": { + "variants": [ + "linux-armv6", + "linux-armv7", + "linux-armv7-musl", + "linux-arm64", + "linux-arm64-musl", + "darwin-amd64", + "windows-amd64", + "linux-amd64", + "linux-amd64-musl" + ], + "pluginSignature": { + "sign": true, + "adminSign": true + }, + "docker": { + "shouldSave": true, + "archs": [ + "amd64", + "arm64", + "arm" + ] + }, + "packagesBucket": "grafana-prerelease/artifacts/downloads", + "CDNAssetsBucket": "grafana-prerelease", + "CDNAssetsDir": "artifacts/static-assets", + "storybookBucket": "grafana-prerelease", + "storybookSrcDir": "artifacts/storybook" + }, + "beta": { + "variants": [ + "linux-armv6", + "linux-armv7", + "linux-armv7-musl", + "linux-arm64", + "linux-arm64-musl", + "darwin-amd64", + "windows-amd64", + "linux-amd64", + "linux-amd64-musl" + ], + "pluginSignature": { + "sign": true, + "adminSign": true + }, + "docker": { + "shouldSave": true, + "archs": [ + "amd64", + "arm64", + "arm" + ] + }, + "packagesBucket": "grafana-prerelease/artifacts/downloads", + "CDNAssetsBucket": "grafana-prerelease", + "CDNAssetsDir": "artifacts/static-assets", + "storybookBucket": "grafana-prerelease", + "storybookSrcDir": "artifacts/storybook" + }, + "test": { + "variants": [ + "linux-armv6", + "linux-armv7", + "linux-armv7-musl", + "linux-arm64", + "linux-arm64-musl", + "darwin-amd64", + "windows-amd64", + "linux-amd64", + "linux-amd64-musl" + ], + "pluginSignature": { + "sign": true, + "adminSign": true + }, + "docker": { + "shouldSave": true, + "archs": [ + "amd64", + "arm64", + "arm" + ] + }, + "packagesBucket": "grafana-prerelease/artifacts/downloads", + "CDNAssetsBucket": "grafana-prerelease", + "CDNAssetsDir": "artifacts/static-assets", + "storybookBucket": "grafana-prerelease", + "storybookSrcDir": "artifacts/storybook" + } +}`) diff --git a/pkg/build/config/version_mode.go b/pkg/build/config/version_mode.go new file mode 100644 index 00000000000..9bda62a4669 --- /dev/null +++ b/pkg/build/config/version_mode.go @@ -0,0 +1,13 @@ +package config + +// VersionMode defines the source event that created a release or published version +type VersionMode string + +const ( + MainMode VersionMode = "main" + ReleaseMode VersionMode = "release" + BetaReleaseMode VersionMode = "beta" + TestReleaseMode VersionMode = "test" + ReleaseBranchMode VersionMode = "branch" + PullRequestMode VersionMode = "pull_request" +) diff --git a/pkg/build/config/version_test.go b/pkg/build/config/version_test.go new file mode 100644 index 00000000000..955a140aa04 --- /dev/null +++ b/pkg/build/config/version_test.go @@ -0,0 +1,96 @@ +package config_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/stretchr/testify/require" +) + +func TestGetMetadata(t *testing.T) { + tcs := []struct { + version string + mode config.VersionMode + }{ + {"v1.2.3", config.ReleaseMode}, + {"v1.2.3-12345pre", config.PullRequestMode}, + {"v1.2.3-beta1", config.BetaReleaseMode}, + {"v1.2.3-test1", config.TestReleaseMode}, + {"v1.2.3-foobar", config.ReleaseBranchMode}, + {"v1.2.3-foobar", config.MainMode}, + } + + t.Run("Should return empty metadata, dist/ is not present", func(t *testing.T) { + dir := t.TempDir() + metadata, err := config.GetMetadata(filepath.Join(dir, "dist")) + require.NoError(t, err) + require.Equal(t, metadata, &config.Metadata{}) + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + }) + + for _, tc := range tcs { + dir := t.TempDir() + t.Run("Should return valid metadata, tag mode, ", func(t *testing.T) { + testMetadata(t, dir, tc.version, tc.mode) + }) + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + } +} + +func testMetadata(t *testing.T, dir string, version string, mode config.VersionMode) { + t.Helper() + file := filepath.Join(dir, "version.json") + createVersionJSON(t, version, file, mode) + + metadata, err := config.GetMetadata(file) + require.NoError(t, err) + t.Run("with a valid version", func(t *testing.T) { + expVersion := metadata.GrafanaVersion + require.Equal(t, expVersion, version) + }) + + t.Run("with a valid release mode from the built-in list", func(t *testing.T) { + expMode := metadata.ReleaseMode + require.NoError(t, err) + require.Equal(t, expMode, mode) + }) + + t.Run("with a valid configuration from a JSON file", func(t *testing.T) { + version, err := config.GetVersion(metadata.ReleaseMode) + require.NoError(t, err) + parsed := verModeFromConfig(t, metadata) + require.EqualValues(t, parsed, *version) + }) +} + +func verModeFromConfig(t *testing.T, metadata *config.Metadata) config.Version { + t.Helper() + + metadataComp := config.VersionMap{} + + require.NoError(t, json.Unmarshal(configJSON, &metadataComp)) + + return metadataComp[metadata.ReleaseMode] +} + +func createVersionJSON(t *testing.T, version string, file string, mode config.VersionMode) { + t.Helper() + + metadata := &config.Metadata{ + GrafanaVersion: version, + ReleaseMode: mode, + } + + //nolint:gosec + f, err := os.Create(file) + require.NoError(t, err) + + require.NoError(t, json.NewEncoder(f).Encode(metadata)) +} diff --git a/pkg/build/config/versions.go b/pkg/build/config/versions.go new file mode 100644 index 00000000000..61287c6c13e --- /dev/null +++ b/pkg/build/config/versions.go @@ -0,0 +1,167 @@ +package config + +var Versions = VersionMap{ + PullRequestMode: { + Variants: []Variant{ + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + }, + PluginSignature: PluginSignature{ + Sign: false, + AdminSign: false, + }, + Docker: Docker{ + ShouldSave: false, + Architectures: []Architecture{ + ArchAMD64, + }, + }, + }, + MainMode: { + Variants: []Variant{ + VariantArmV6, + VariantArmV7, + VariantArmV7Musl, + VariantArm64, + VariantArm64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + }, + PluginSignature: PluginSignature{ + Sign: true, + AdminSign: true, + }, + Docker: Docker{ + ShouldSave: false, + Architectures: []Architecture{ + ArchAMD64, + ArchARM64, + ArchARM, // GOARCH=ARM is used for both armv6 and armv7. They are differentiated by the GOARM variable. + }, + }, + PackagesBucket: "grafana-downloads", + CDNAssetsBucket: "grafana-static-assets", + }, + ReleaseBranchMode: { + Variants: []Variant{ + VariantArmV6, + VariantArmV7, + VariantArmV7Musl, + VariantArm64, + VariantArm64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + }, + PluginSignature: PluginSignature{ + Sign: true, + AdminSign: true, + }, + Docker: Docker{ + ShouldSave: true, + Architectures: []Architecture{ + ArchAMD64, + ArchARM64, + ArchARM, + }, + }, + PackagesBucket: "grafana-downloads", + PackagesBucketEnterprise2: "grafana-downloads-enterprise2", + CDNAssetsBucket: "grafana-static-assets", + }, + ReleaseMode: { + Variants: []Variant{ + VariantArmV6, + VariantArmV7, + VariantArmV7Musl, + VariantArm64, + VariantArm64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + }, + PluginSignature: PluginSignature{ + Sign: true, + AdminSign: true, + }, + Docker: Docker{ + ShouldSave: true, + Architectures: []Architecture{ + ArchAMD64, + ArchARM64, + ArchARM, + }, + }, + PackagesBucket: "grafana-prerelease/artifacts/downloads", + CDNAssetsBucket: "grafana-prerelease", + CDNAssetsDir: "artifacts/static-assets", + StorybookBucket: "grafana-prerelease", + StorybookSrcDir: "artifacts/storybook", + }, + BetaReleaseMode: { + Variants: []Variant{ + VariantArmV6, + VariantArmV7, + VariantArmV7Musl, + VariantArm64, + VariantArm64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + }, + PluginSignature: PluginSignature{ + Sign: true, + AdminSign: true, + }, + Docker: Docker{ + ShouldSave: true, + Architectures: []Architecture{ + ArchAMD64, + ArchARM64, + ArchARM, + }, + }, + PackagesBucket: "grafana-prerelease/artifacts/downloads", + CDNAssetsBucket: "grafana-prerelease", + CDNAssetsDir: "artifacts/static-assets", + StorybookBucket: "grafana-prerelease", + StorybookSrcDir: "artifacts/storybook", + }, + TestReleaseMode: { + Variants: []Variant{ + VariantArmV6, + VariantArmV7, + VariantArmV7Musl, + VariantArm64, + VariantArm64Musl, + VariantDarwinAmd64, + VariantWindowsAmd64, + VariantLinuxAmd64, + VariantLinuxAmd64Musl, + }, + PluginSignature: PluginSignature{ + Sign: true, + AdminSign: true, + }, + Docker: Docker{ + ShouldSave: true, + Architectures: []Architecture{ + ArchAMD64, + ArchARM64, + ArchARM, + }, + }, + PackagesBucket: "grafana-prerelease/artifacts/downloads", + CDNAssetsBucket: "grafana-prerelease", + CDNAssetsDir: "artifacts/static-assets", + StorybookBucket: "grafana-prerelease", + StorybookSrcDir: "artifacts/storybook", + }, +} diff --git a/pkg/build/cryptoutil/md5.go b/pkg/build/cryptoutil/md5.go new file mode 100644 index 00000000000..68dba9233b0 --- /dev/null +++ b/pkg/build/cryptoutil/md5.go @@ -0,0 +1,36 @@ +package cryptoutil + +import ( + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "log" + "os" +) + +func MD5File(fpath string) error { + // Ignore gosec G304 as this function is only used in the build process. + //nolint:gosec + fd, err := os.Open(fpath) + if err != nil { + return err + } + defer func() { + if err := fd.Close(); err != nil { + log.Printf("error closing file at '%s': %s", fpath, err.Error()) + } + }() + + h := md5.New() // nolint:gosec + if _, err = io.Copy(h, fd); err != nil { + return err + } + + // nolint:gosec + if err := ioutil.WriteFile(fpath+".md5", []byte(fmt.Sprintf("%x\n", h.Sum(nil))), 0664); err != nil { + return err + } + + return nil +} diff --git a/pkg/build/droneutil/docs.go b/pkg/build/droneutil/docs.go new file mode 100644 index 00000000000..8bf3753936e --- /dev/null +++ b/pkg/build/droneutil/docs.go @@ -0,0 +1,2 @@ +// Package droneutil provides utility functions for working with Drone. +package droneutil diff --git a/pkg/build/droneutil/event.go b/pkg/build/droneutil/event.go new file mode 100644 index 00000000000..707b87383c2 --- /dev/null +++ b/pkg/build/droneutil/event.go @@ -0,0 +1,34 @@ +package droneutil + +import ( + "fmt" + "os" + "strings" +) + +// Lookup is the equivalent of os.LookupEnv, but also accepts a list of strings rather than only checking os.Environ() +func Lookup(values []string, val string) (string, bool) { + for _, v := range values { + prefix := val + "=" + if strings.HasPrefix(v, prefix) { + return strings.TrimPrefix(v, prefix), true + } + } + + return "", false +} + +// GetDroneEvent looks for the "DRONE_BUILD_EVENT" in the provided env list and returns the value. +// if it was not found, then an error is returned. +func GetDroneEvent(env []string) (string, error) { + event, ok := Lookup(env, "DRONE_BUILD_EVENT") + if !ok { + return "", fmt.Errorf("failed to get DRONE_BUILD_EVENT environmental variable") + } + return event, nil +} + +// GetDroneEventFromEnv returns the value of DRONE_BUILD_EVENT from os.Environ() +func GetDroneEventFromEnv() (string, error) { + return GetDroneEvent(os.Environ()) +} diff --git a/pkg/build/droneutil/event_test.go b/pkg/build/droneutil/event_test.go new file mode 100644 index 00000000000..f0efcfa284e --- /dev/null +++ b/pkg/build/droneutil/event_test.go @@ -0,0 +1,36 @@ +package droneutil_test + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/droneutil" + "github.com/stretchr/testify/require" +) + +func TestGetDroneEvent(t *testing.T) { + t.Run("Should return the Drone Event", func(t *testing.T) { + env := []string{"DRONE_BUILD_EVENT=pull_request"} + droneEvent, err := droneutil.GetDroneEvent(env) + require.NoError(t, err) + require.Equal(t, droneEvent, "pull_request") + }) + t.Run("Should return error, Drone Event env var is missing", func(t *testing.T) { + droneEvent, err := droneutil.GetDroneEvent([]string{}) + require.Error(t, err) + require.Empty(t, droneEvent) + }) +} + +func TestLookup(t *testing.T) { + env := []string{"", "EXAMPLE_KEY=value", "EXAMPLE_KEY"} + t.Run("A valid lookup should return a string and no error", func(t *testing.T) { + val, ok := droneutil.Lookup(env, "EXAMPLE_KEY") + require.True(t, ok) + require.Equal(t, val, "value") + }) + + t.Run("An invalid lookup should return an error", func(t *testing.T) { + _, ok := droneutil.Lookup(env, "EXAMPLE_KEY_DOES_NOT_EXIST") + require.False(t, ok) + }) +} diff --git a/pkg/build/errutil/group.go b/pkg/build/errutil/group.go new file mode 100644 index 00000000000..0844616e579 --- /dev/null +++ b/pkg/build/errutil/group.go @@ -0,0 +1,61 @@ +package errutil + +import ( + "context" + "log" + "sync" +) + +type Group struct { + cancel func() + wg sync.WaitGroup + errOnce sync.Once + err error +} + +func GroupWithContext(ctx context.Context) (*Group, context.Context) { + ctx, cancel := context.WithCancel(ctx) + return &Group{cancel: cancel}, ctx +} + +// Wait waits for any wrapped goroutines to finish and returns any error having occurred in one of them. +func (g *Group) Wait() error { + log.Println("Waiting on Group") + g.wg.Wait() + if g.cancel != nil { + log.Println("Group canceling its context after waiting") + g.cancel() + } + return g.err +} + +// Cancel cancels the associated context. +func (g *Group) Cancel() { + log.Println("Group's Cancel method being called") + g.cancel() +} + +// Wrap wraps a function to be executed in a goroutine. +func (g *Group) Wrap(f func() error) func() { + g.wg.Add(1) + return func() { + defer g.wg.Done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + log.Printf("An error occurred in Group: %s", err) + g.err = err + if g.cancel != nil { + log.Println("Group canceling its context due to error") + g.cancel() + } + }) + } + } +} + +// Go wraps the provided function and executes it in a goroutine. +func (g *Group) Go(f func() error) { + wrapped := g.Wrap(f) + go wrapped() +} diff --git a/pkg/build/executil/exec.go b/pkg/build/executil/exec.go new file mode 100644 index 00000000000..e46f568a997 --- /dev/null +++ b/pkg/build/executil/exec.go @@ -0,0 +1,46 @@ +package executil + +import ( + "context" + "fmt" + "os/exec" + "strings" +) + +func RunAt(ctx context.Context, dir, cmd string, args ...string) error { + // Ignore gosec G204 as this function is only used in the build process. + //nolint:gosec + c := exec.CommandContext(ctx, cmd, args...) + c.Dir = dir + + b, err := c.CombinedOutput() + + if err != nil { + return fmt.Errorf("%w. '%s %v': %s", err, cmd, args, string(b)) + } + + return nil +} + +func Run(ctx context.Context, cmd string, args ...string) error { + return RunAt(ctx, ".", cmd, args...) +} + +func OutputAt(ctx context.Context, dir, cmd string, args ...string) (string, error) { + // Ignore gosec G204 as this function is only used in the build process. + //nolint:gosec + c := exec.CommandContext(ctx, cmd, args...) + c.Dir = dir + + b, err := c.CombinedOutput() + + if err != nil { + return "", err + } + + return strings.TrimSpace(string(b)), nil +} + +func Output(ctx context.Context, cmd string, args ...string) (string, error) { + return OutputAt(ctx, ".", cmd, args...) +} diff --git a/pkg/build/golangutils/build.go b/pkg/build/golangutils/build.go new file mode 100644 index 00000000000..14da56a7765 --- /dev/null +++ b/pkg/build/golangutils/build.go @@ -0,0 +1,124 @@ +package golangutils + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + + "github.com/grafana/grafana/pkg/build/config" +) + +type BuildOpts struct { + // Package refers to the path to the `main` package containing `func main` + Package string + + // Output is used as the -o argument in the go build command + Output string + + // Workdir should define some place in the module where the package path resolves. + // Go commands need to be ran inside a the Go module directory. + Workdir string + + GoOS config.OS + GoArch config.Architecture + GoArm string + Go386 string + CC string + LibC string + + CGoEnabled bool + CGoCFlags string + + // LdFlags are joined by a space character and provided to the -ldflags argument. + // A valid element here would be `-X 'main.version=1.0.0'`. + LdFlags []string + + Stdout io.ReadWriter + Stderr io.ReadWriter + Stdin io.ReadWriter + + // ExtraEnv allows consumers to provide extra env args that are not defined above. + // A single element should be formatted using like so: {NAME}={VALUE}. Example: GOOS=linux. + ExtraEnv []string + + // ExtraArgs allows consumers to provide extra arguments that are not defined above. + // Flag names and values should be two separate elements. + // These flags will be appended to the command arguments before the package path in "go build". + ExtraArgs []string +} + +// Env constructs a list of key/value pairs for setting a build command's environment. +// Should we consider using something to unmarshal the struct to env? +func (opts BuildOpts) Env() []string { + env := []string{} + if opts.CGoEnabled { + env = append(env, "CGO_ENABLED=1") + } + + if opts.GoOS != "" { + env = append(env, fmt.Sprintf("GOOS=%s", opts.GoOS)) + } + + if opts.GoArch != "" { + env = append(env, fmt.Sprintf("GOARCH=%s", opts.GoArch)) + } + + if opts.CC != "" { + env = append(env, fmt.Sprintf("CC=%s", opts.CC)) + } + + if opts.CGoCFlags != "" { + env = append(env, fmt.Sprintf("CGO_CFLAGS=%s", opts.CGoCFlags)) + } + + if opts.GoArm != "" { + env = append(env, fmt.Sprintf("GOARM=%s", opts.GoArm)) + } + + if opts.ExtraEnv != nil { + return append(opts.ExtraEnv, env...) + } + + return env +} + +// Args constructs a list of flags and values for use with the exec.Command type when running "go build". +func (opts BuildOpts) Args() []string { + args := []string{} + + if opts.LdFlags != nil { + args = append(args, "-ldflags", strings.Join(opts.LdFlags, " ")) + } + + if opts.Output != "" { + args = append(args, "-o", opts.Output) + } + + if opts.ExtraArgs != nil { + args = append(args, opts.ExtraArgs...) + } + + args = append(args, opts.Package) + + return args +} + +// Build runs the go build process in the current shell given the opts. +// This function will panic if no Stdout/Stderr/Stdin is provided in the opts. +func RunBuild(ctx context.Context, opts BuildOpts) error { + env := opts.Env() + args := append([]string{"build"}, opts.Args()...) + // Ignore gosec G304 as this function is only used in the build process. + //nolint:gosec + cmd := exec.CommandContext(ctx, "go", args...) + cmd.Env = env + + cmd.Stdout = opts.Stdout + cmd.Stderr = opts.Stderr + cmd.Stdin = opts.Stdin + cmd.Dir = opts.Workdir + + return cmd.Run() +} diff --git a/pkg/build/golangutils/doc.go b/pkg/build/golangutils/doc.go new file mode 100644 index 00000000000..2cb33af05e1 --- /dev/null +++ b/pkg/build/golangutils/doc.go @@ -0,0 +1,2 @@ +// Package golangutils holds utility functions, wrappers, and types for building Go binaries for Grafana. +package golangutils diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go new file mode 100644 index 00000000000..b8fa022c396 --- /dev/null +++ b/pkg/build/grafana/build.go @@ -0,0 +1,123 @@ +package grafana + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/cryptoutil" + "github.com/grafana/grafana/pkg/build/golangutils" +) + +var binaries = []string{"grafana-server", "grafana-cli"} + +const ( + SuffixEnterprise2 = "-enterprise2" +) + +const ( + ExtensionExe = ".exe" +) + +func GrafanaLDFlags(version string, r config.Revision) []string { + return []string{ + "-w", + fmt.Sprintf("-X main.version=%s", version), + fmt.Sprintf("-X main.commit=%s", r.SHA256), + fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp), + fmt.Sprintf("-X main.buildBranch=%s", r.Branch), + } +} + +// BinaryFolder returns the path to where the Grafana binary is build given the provided arguments. +func BinaryFolder(edition config.Edition, args BuildArgs) string { + sfx := "" + if edition == config.EditionEnterprise2 { + sfx = SuffixEnterprise2 + } + + arch := string(args.GoArch) + if args.GoArch == config.ArchARM { + arch = string(args.GoArch) + "v" + args.GoArm + } + + format := fmt.Sprintf("%s-%s", args.GoOS, arch) + if args.LibC != "" { + format += fmt.Sprintf("-%s", args.LibC) + } + format += sfx + + if args.GoOS == config.OSWindows { + format += ExtensionExe + } + + return format +} + +func GrafanaDescriptor(opts golangutils.BuildOpts) string { + libcPart := "" + if opts.LibC != "" { + libcPart = fmt.Sprintf("/%s", opts.LibC) + } + arch := string(opts.GoArch) + if opts.GoArch == config.ArchARM { + arch = string(opts.GoArch) + "v" + opts.GoArm + } + + return fmt.Sprintf("%s/%s%s", opts.GoOS, arch, libcPart) +} + +// BuildGrafanaBinary builds a certain binary according to certain parameters. +func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArgs, edition config.Edition) error { + opts := args.BuildOpts + opts.ExtraEnv = os.Environ() + + revision, err := config.GrafanaRevision(ctx, opts.Workdir) + if err != nil { + return err + } + + folder := BinaryFolder(edition, args) + + if opts.GoOS == config.OSWindows { + name += ExtensionExe + } + + binary := filepath.Join(opts.Workdir, "bin", folder, name) + opts.Output = binary + + if err := os.RemoveAll(binary); err != nil { + return fmt.Errorf("failed to remove %q: %w", binary, err) + } + + if err := os.RemoveAll(binary + ".md5"); err != nil { + return fmt.Errorf("failed to remove %q: %w", binary+".md5", err) + } + + descriptor := GrafanaDescriptor(opts) + + log.Printf("Building %q for %s\nwith env: %v", binary, descriptor, opts.Env()) + + opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...) + + if edition == config.EditionEnterprise2 { + opts.ExtraArgs = []string{"-tags=pro"} + } + + log.Printf("Running command 'go %s'", opts.Args()) + + if err := golangutils.RunBuild(ctx, opts); err != nil { + return err + } + + // Create an MD5 checksum of the binary, to be included in the archive for + // automatic upgrades. + if err := cryptoutil.MD5File(binary); err != nil { + return err + } + + return nil +} diff --git a/pkg/build/grafana/variant.go b/pkg/build/grafana/variant.go new file mode 100644 index 00000000000..40c9a50c05b --- /dev/null +++ b/pkg/build/grafana/variant.go @@ -0,0 +1,160 @@ +package grafana + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + + "github.com/grafana/grafana/pkg/build/compilers" + "github.com/grafana/grafana/pkg/build/config" + "github.com/grafana/grafana/pkg/build/golangutils" +) + +// BuildArgs represent the build parameters that define the "go build" behavior of a single variant. +// These arguments are applied as environment variables and arguments to the "go build" command. +type BuildArgs struct { + golangutils.BuildOpts + DebArch config.Architecture + RPMArch config.Architecture +} + +type BuildVariantOpts struct { + Variant config.Variant + Edition config.Edition + + Version string + GrafanaDir string +} + +// BuildVariant builds a certain variant of the grafana-server and grafana-cli binaries sequentially. +func BuildVariant(ctx context.Context, opts BuildVariantOpts) error { + grafanaDir, err := filepath.Abs(opts.GrafanaDir) + if err != nil { + return err + } + + var ( + args = VariantBuildArgs(opts.Variant) + ) + + for _, binary := range binaries { + // Note that for Golang cmd paths we must use the relative path and the Linux file separators (/) even for Windows users. + var ( + pkg = fmt.Sprintf("./pkg/cmd/%s", binary) + stdout = bytes.NewBuffer(nil) + stderr = bytes.NewBuffer(nil) + ) + + args.BuildOpts.Workdir = grafanaDir + args.BuildOpts.Stdout = stdout + args.BuildOpts.Stderr = stderr + args.Package = pkg + + if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil { + return fmt.Errorf("failed to build %s for %s: %w\nstdout: %s\nstderr: %s", pkg, opts.Variant, err, stdout.String(), stderr.String()) + } + } + + return nil +} + +var ldFlagsStatic = []string{"-linkmode=external", "-extldflags=-static"} + +var variantArgs = map[config.Variant]BuildArgs{ + config.VariantArmV6: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + CGoEnabled: true, + GoArch: config.ArchARM, + GoArm: "6", + CC: compilers.ArmV6, + }, + DebArch: config.ArchARMHF, + }, + config.VariantArmV7: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + CGoEnabled: true, + GoArch: config.ArchARM, + GoArm: "7", + CC: compilers.Armv7, + }, + DebArch: config.ArchARMHF, + RPMArch: config.ArchARMHF, + }, + config.VariantArmV7Musl: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + CGoEnabled: true, + GoArch: config.ArchARM, + GoArm: "7", + LibC: config.LibCMusl, + CC: compilers.Armv7Musl, + LdFlags: ldFlagsStatic, + }, + }, + config.VariantArm64: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + CGoEnabled: true, + GoArch: config.ArchARM64, + CC: compilers.Arm64, + }, + DebArch: config.ArchARM64, + RPMArch: "aarch64", + }, + config.VariantArm64Musl: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + GoArch: config.ArchARM64, + CGoEnabled: true, + CC: compilers.Arm64Musl, + LibC: config.LibCMusl, + LdFlags: ldFlagsStatic, + }, + }, + config.VariantDarwinAmd64: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSDarwin, + CGoEnabled: true, + GoArch: config.ArchAMD64, + CC: compilers.Osx64, + }, + }, + config.VariantWindowsAmd64: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSWindows, + GoArch: config.ArchAMD64, + CC: compilers.Win64, + CGoEnabled: true, + CGoCFlags: "-D_WIN32_WINNT=0x0601", + }, + }, + config.VariantLinuxAmd64: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + GoArch: config.ArchAMD64, + CC: compilers.LinuxX64, + }, + DebArch: config.ArchAMD64, + RPMArch: config.ArchAMD64, + }, + config.VariantLinuxAmd64Musl: { + BuildOpts: golangutils.BuildOpts{ + GoOS: config.OSLinux, + GoArch: config.ArchAMD64, + CC: compilers.LinuxX64Musl, + LibC: config.LibCMusl, + LdFlags: ldFlagsStatic, + }, + }, +} + +func VariantBuildArgs(v config.Variant) BuildArgs { + if val, ok := variantArgs[v]; ok { + return val + } + + return BuildArgs{} +} diff --git a/pkg/build/syncutil/pool.go b/pkg/build/syncutil/pool.go new file mode 100644 index 00000000000..6034059d2bf --- /dev/null +++ b/pkg/build/syncutil/pool.go @@ -0,0 +1,43 @@ +package syncutil + +import ( + "log" + "runtime" +) + +func worker(jobs chan func()) { + for j := range jobs { + j() + } +} + +// WorkerPool represents a concurrent worker pool. +type WorkerPool struct { + NumWorkers int + jobs chan func() +} + +// NewWorkerPool constructs a new WorkerPool. +func NewWorkerPool(numWorkers int) WorkerPool { + if numWorkers <= 0 { + numWorkers = runtime.NumCPU() + } + log.Printf("Creating worker pool with %d workers", numWorkers) + jobs := make(chan func(), 100) + for i := 0; i < numWorkers; i++ { + go worker(jobs) + } + return WorkerPool{ + NumWorkers: numWorkers, + jobs: jobs, + } +} + +// Schedule schedules a job to be executed by a worker in the pool. +func (p WorkerPool) Schedule(job func()) { + p.jobs <- job +} + +func (p WorkerPool) Close() { + close(p.jobs) +} diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 5d4aadb25da..31e63ae3762 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -42,7 +42,8 @@ load( 'verify_gen_cue_step', 'publish_images_step', 'trigger_oss', - 'artifacts_page_step' + 'artifacts_page_step', + 'compile_build_cmd', ) load( @@ -59,6 +60,7 @@ load( 'failure_template', 'drone_change_template', ) + load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket') def store_npm_packages_step(): @@ -162,6 +164,7 @@ def get_steps(edition, ver_mode): verify_gen_cue_step(edition), wire_install_step(), yarn_install_step(), + compile_build_cmd(), ] test_steps = [] @@ -302,7 +305,8 @@ def get_enterprise_pipelines(trigger, ver_mode): download_grabpl_step(), identify_runner_step(), clone_enterprise_step(ver_mode), - init_enterprise_step(ver_mode) + init_enterprise_step(ver_mode), + compile_build_cmd(), ] for step in [wire_install_step(), yarn_install_step(), gen_version_step(ver_mode), verify_gen_cue_step(edition)]: step.update(deps_on_clone_enterprise_step) diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index 6e5b246e822..9839162c34c 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -1,41 +1,42 @@ load( 'scripts/drone/steps/lib.star', 'download_grabpl_step', - 'build_image', - 'identify_runner_step', - 'gen_version_step', - 'wire_install_step', - 'yarn_install_step', - 'build_backend_step', - 'build_frontend_step', - 'build_frontend_package_step', - 'build_plugins_step', - 'package_step', - 'grafana_server_step', - 'e2e_tests_step', - 'e2e_tests_artifacts', - 'build_storybook_step', - 'copy_packages_for_docker_step', - 'build_docker_images_step', - 'publish_images_step', - 'postgres_integration_tests_step', - 'mysql_integration_tests_step', - 'redis_integration_tests_step', - 'memcached_integration_tests_step', - 'get_windows_steps', - 'benchmark_ldap_step', - 'enterprise_downstream_step', - 'frontend_metrics_step', - 'store_storybook_step', - 'release_canary_npm_packages_step', - 'upload_packages_step', - 'store_packages_step', - 'upload_cdn_step', - 'verify_gen_cue_step', - 'test_a11y_frontend_step', - 'trigger_oss', - 'betterer_frontend_step', - 'trigger_test_release' + 'build_image', + 'identify_runner_step', + 'gen_version_step', + 'wire_install_step', + 'yarn_install_step', + 'build_backend_step', + 'build_frontend_step', + 'build_frontend_package_step', + 'build_plugins_step', + 'package_step', + 'grafana_server_step', + 'e2e_tests_step', + 'e2e_tests_artifacts', + 'build_storybook_step', + 'copy_packages_for_docker_step', + 'build_docker_images_step', + 'publish_images_step', + 'postgres_integration_tests_step', + 'mysql_integration_tests_step', + 'redis_integration_tests_step', + 'memcached_integration_tests_step', + 'get_windows_steps', + 'benchmark_ldap_step', + 'enterprise_downstream_step', + 'frontend_metrics_step', + 'store_storybook_step', + 'release_canary_npm_packages_step', + 'upload_packages_step', + 'store_packages_step', + 'upload_cdn_step', + 'verify_gen_cue_step', + 'test_a11y_frontend_step', + 'trigger_oss', + 'betterer_frontend_step', + 'trigger_test_release', + 'compile_build_cmd', ) load( @@ -48,6 +49,7 @@ def build_e2e(trigger, ver_mode, edition): init_steps = [ identify_runner_step(), download_grabpl_step(), + compile_build_cmd(), gen_version_step(ver_mode), verify_gen_cue_step(edition="oss"), wire_install_step(), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index d24573dcfe2..36117d0c7e9 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -12,6 +12,7 @@ load( 'build_frontend_docs_step', 'build_frontend_package_step', 'build_docs_website_step', + 'compile_build_cmd', ) load( @@ -37,6 +38,7 @@ def docs_pipelines(edition, ver_mode, trigger): build_frontend_package_step(edition=edition, ver_mode=ver_mode), build_frontend_docs_step(edition=edition), build_docs_website_step(), + compile_build_cmd(), ] return pipeline( diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index 72d1d8a0489..ee4d77ef9f1 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -6,6 +6,7 @@ load( 'wire_install_step', 'postgres_integration_tests_step', 'mysql_integration_tests_step', + 'compile_build_cmd', ) load( @@ -25,6 +26,7 @@ def integration_tests(trigger, ver_mode, edition): volumes = integration_test_services_volumes() init_steps = [ download_grabpl_step(), + compile_build_cmd(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), diff --git a/scripts/drone/pipelines/publish.star b/scripts/drone/pipelines/publish.star index 1f75ba89271..827d16dff80 100644 --- a/scripts/drone/pipelines/publish.star +++ b/scripts/drone/pipelines/publish.star @@ -3,7 +3,8 @@ load( 'identify_runner_step', 'gen_version_step', 'download_grabpl_step', - 'store_packages_step' + 'store_packages_step', + 'compile_build_cmd', ) load( @@ -15,6 +16,7 @@ def publish(trigger, ver_mode, edition): steps = [ download_grabpl_step(), gen_version_step(ver_mode), + compile_build_cmd(), identify_runner_step(), store_packages_step(edition=edition, ver_mode=ver_mode), ] diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star index 25b7193a272..2f82fce1fdf 100644 --- a/scripts/drone/pipelines/test_backend.star +++ b/scripts/drone/pipelines/test_backend.star @@ -11,6 +11,7 @@ load( 'test_backend_step', 'test_backend_integration_step', 'verify_gen_cue_step', + 'compile_build_cmd', ) load( @@ -22,6 +23,7 @@ def test_backend(trigger, ver_mode): init_steps = [ identify_runner_step(), download_grabpl_step(), + compile_build_cmd(), gen_version_step(ver_mode), verify_gen_cue_step(edition="oss"), wire_install_step(), diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star index 0231f19fb6d..81416cae4d3 100644 --- a/scripts/drone/pipelines/test_frontend.star +++ b/scripts/drone/pipelines/test_frontend.star @@ -7,6 +7,7 @@ load( 'lint_frontend_step', 'betterer_frontend_step', 'test_frontend_step', + 'compile_build_cmd', ) load( @@ -20,6 +21,7 @@ def test_frontend(trigger, ver_mode): download_grabpl_step(), gen_version_step(ver_mode), yarn_install_step(), + compile_build_cmd(), ] test_steps = [ lint_frontend_step(), diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 8f8c2794367..6cf7c94716b 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -376,14 +376,14 @@ def build_backend_step(edition, ver_mode, variants=None): # TODO: Convert number of jobs to percentage if ver_mode == 'release': cmds = [ - './bin/grabpl build-backend --jobs 8 --edition {} ${{DRONE_TAG}}'.format( + './bin/build build-backend --jobs 8 --edition {} ${{DRONE_TAG}}'.format( edition, ), ] else: build_no = '${DRONE_BUILD_NUMBER}' cmds = [ - './bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{}'.format( + './bin/build build-backend --jobs 8 --edition {} --build-id {}{}'.format( edition, build_no, variants_str, ), ] @@ -394,6 +394,7 @@ def build_backend_step(edition, ver_mode, variants=None): 'depends_on': [ 'gen-version', 'wire-install', + 'compile-build-cmd', ], 'commands': cmds, } @@ -1211,3 +1212,15 @@ def end_to_end_tests_deps(edition): 'end-to-end-tests-smoke-tests-suite' + enterprise2_suffix(edition), 'end-to-end-tests-various-suite' + enterprise2_suffix(edition), ] + +def compile_build_cmd(): + return { + 'name': 'compile-build-cmd', + 'image': 'golang:1.17', + 'commands': [ + "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", + ], + 'environment': { + 'CGO_ENABLED': 0, + }, + }