From bf06bb3b926efd24ec015b51fcc216fbe2894715 Mon Sep 17 00:00:00 2001 From: Kevin Minehart <5140827+kminehart@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:12:15 -0500 Subject: [PATCH] [v11.1.x] CI: use linux to build msi installers (#95297) CI: use linux to build msi installers (#95215) * Build the MSI installers using Linux and wine (cherry picked from commit 66c728d26bbe34b28bf62703c448ef539e4f9299) --- .drone.star | 18 +- .drone.yml | 430 +++++++++++------------- scripts/drone/events/main.star | 8 +- scripts/drone/pipelines/ci_images.star | 66 ---- scripts/drone/pipelines/windows.star | 85 ----- scripts/drone/rgm.star | 34 +- scripts/drone/utils/images.star | 1 + scripts/drone/utils/windows_images.star | 17 - scripts/drone/variables.star | 2 +- scripts/drone/windows.star | 149 ++++++++ 10 files changed, 360 insertions(+), 450 deletions(-) delete mode 100644 scripts/drone/pipelines/ci_images.star delete mode 100644 scripts/drone/pipelines/windows.star delete mode 100644 scripts/drone/utils/windows_images.star create mode 100644 scripts/drone/windows.star diff --git a/.drone.star b/.drone.star index 725aeec7850..033b481e4eb 100644 --- a/.drone.star +++ b/.drone.star @@ -17,23 +17,19 @@ load( "publish_npm_pipelines", "publish_packages_pipeline", ) -load( - "scripts/drone/pipelines/ci_images.star", - "publish_ci_windows_test_image_pipeline", -) load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", ) -load( - "scripts/drone/pipelines/windows.star", - "windows_test_backend", -) load( "scripts/drone/rgm.star", "rgm", ) load("scripts/drone/vault.star", "secrets") +load( + "scripts/drone/windows.star", + "windows_manual_pipeline", +) def main(_ctx): return ( @@ -44,12 +40,8 @@ def main(_ctx): publish_npm_pipelines() + publish_packages_pipeline() + rgm() + - [windows_test_backend({ - "event": ["promote"], - "target": ["test-windows"], - }, "oss", "testing")] + integration_test_pipelines() + - publish_ci_windows_test_image_pipeline() + cronjobs() + + [windows_manual_pipeline()] + secrets() ) diff --git a/.drone.yml b/.drone.yml index fb1d270e985..e0dbfa24d7f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -482,7 +482,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -889,7 +889,7 @@ steps: name: clone-enterprise - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1791,7 +1791,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2295,7 +2295,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2476,53 +2476,6 @@ volumes: clone: retries: 3 depends_on: -- main-test-frontend -- main-test-backend -- main-build-e2e-publish -- main-integration-tests -environment: - EDITION: oss -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: main-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -trigger: - branch: main - event: - - push - paths: - exclude: - - '*.md' - - docs/** - - latest.json - repo: - - grafana/grafana -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- -clone: - retries: 3 -depends_on: - main-build-e2e-publish - main-integration-tests environment: @@ -2574,7 +2527,6 @@ depends_on: - main-test-backend - main-build-e2e-publish - main-integration-tests -- main-windows kind: pipeline name: main-notify platform: @@ -2629,7 +2581,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2761,7 +2713,7 @@ steps: name: identify-runner - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -3464,51 +3416,6 @@ volumes: clone: retries: 3 depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: release-whatsnew-checker -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.22.7-alpine - name: compile-build-cmd -- commands: - - ./bin/build whatsnew-checker - depends_on: - - compile-build-cmd - image: golang:1.22.7-alpine - name: whats-new-checker -trigger: - event: - exclude: - - promote - ref: - exclude: - - refs/tags/*-cloud* - include: - - refs/tags/v* -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] image_pull_secrets: - gcr - gar @@ -3579,53 +3486,34 @@ volumes: --- clone: retries: 3 -depends_on: -- rgm-tag-prerelease +depends_on: [] +environment: + EDITION: oss image_pull_secrets: - gcr - gar kind: pipeline -name: rgm-tag-prerelease-windows +name: release-whatsnew-checker +node: + type: no-parallel platform: arch: amd64 - os: windows - version: "1809" + os: linux services: [] steps: - commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: windows-init -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --target gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/grafana-${DRONE_TAG:1}.windows-amd64.zip - --edition oss ${DRONE_TAG} - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - - gsutil cp "$$fname.sha256" gs://grafana-prerelease/artifacts/downloads/${DRONE_TAG}/oss/release/ - depends_on: - - windows-init + - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd + depends_on: [] environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer + CGO_ENABLED: 0 + image: golang:1.22.7-alpine + name: compile-build-cmd +- commands: + - ./bin/build whatsnew-checker + depends_on: + - compile-build-cmd + image: golang:1.22.7-alpine + name: whats-new-checker trigger: event: exclude: @@ -3638,14 +3526,96 @@ trigger: type: docker volumes: - host: - path: //./pipe/docker_engine/ + path: /var/run/docker.sock name: docker --- clone: retries: 3 depends_on: - rgm-tag-prerelease -- rgm-tag-prerelease-windows +image_pull_secrets: +- gcr +- gar +kind: pipeline +name: prerelease-windows-msi +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - curl -L0 https://nssm.cc/release/nssm-2.24.zip -o nssm-2.24.zip + image: byrnedo/alpine-curl:0.1.8 + name: downlad-nssm +- commands: + - mkdir wix3 && cd wix3 + - curl -L0 https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip + -o wix3.zip + - unzip wix3.zip + image: byrnedo/alpine-curl:0.1.8 + name: download-wix3 +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release/grafana-$${DRONE_TAG:1}.windows-amd64.zip + grafana.zip' + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: download-zip +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - export WINEPATH=$(winepath ./wix3) + - ./bin/grabpl windows-installer --target grafana.zip --edition oss + depends_on: + - downlad-nssm + - download-wix3 + - grabpl + - download-zip + entrypoint: + - /bin/bash + image: scottyhardy/docker-wine:stable-9.0 + name: build-msi +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp *.msi gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + - bash -c 'gcloud storage cp *.msi.sha256 gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + depends_on: + - build-msi + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: upload-msi-installer +trigger: + event: + exclude: + - promote + ref: + exclude: + - refs/tags/*-cloud* + include: + - refs/tags/v* +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: +- prerelease-windows-msi +- rgm-tag-prerelease image_pull_secrets: - gcr - gar @@ -4282,59 +4252,6 @@ volumes: path: /var/run/docker.sock name: docker --- -clone: - disable: true -depends_on: [] -environment: - EDITION: oss -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: testing-test-backend-windows -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - git clone "https://$$env:GITHUB_TOKEN@github.com/$$env:DRONE_REPO.git" . - - git checkout -f $$env:DRONE_COMMIT - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: [] - depends_on: - - clone - image: golang:1.22.7-windowsservercore-1809 - name: windows-init -- commands: - - go install github.com/google/wire/cmd/wire@v0.5.0 - - wire gen -tags oss ./pkg/server - depends_on: - - windows-init - image: golang:1.22.7-windowsservercore-1809 - name: wire-install -- commands: - - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... - depends_on: - - wire-install - image: golang:1.22.7-windowsservercore-1809 - name: test-backend -trigger: - event: - - promote - target: - - test-windows -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- clone: retries: 3 depends_on: [] @@ -4399,7 +4316,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.56/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4563,55 +4480,6 @@ volumes: temp: medium: memory --- -clone: - disable: true -depends_on: [] -image_pull_secrets: -- gcr -- gar -kind: pipeline -name: publish-ci-windows-test-image -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-ci-sandbox.git" - . - - git checkout -f $$env:DRONE_COMMIT - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: - - cd scripts\build\ci-windows-test - - docker login -u $$env:DOCKER_USERNAME -p $$env:DOCKER_PASSWORD - - docker build -t grafana/grafana-ci-windows-test:$$env:TAG . - - docker push grafana/grafana-ci-windows-test:$$env:TAG - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USERNAME: - from_secret: docker_username - image: docker:windowsservercore-1809 - name: build-and-publish - volumes: - - name: docker - path: //./pipe/docker_engine/ -trigger: - event: - - promote - target: - - ci-windows-test-image -type: docker -volumes: -- host: - path: //./pipe/docker_engine/ - name: docker ---- clone: retries: 3 kind: pipeline @@ -4922,6 +4790,7 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM rockylinux:9 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM scottyhardy/docker-wine:stable-9.0 depends_on: - authenticate-gcr image: aquasec/trivy:0.21.0 @@ -4959,6 +4828,7 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1 - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable - trivy --exit-code 1 --severity HIGH,CRITICAL rockylinux:9 + - trivy --exit-code 1 --severity HIGH,CRITICAL scottyhardy/docker-wine:stable-9.0 depends_on: - authenticate-gcr environment: @@ -4991,6 +4861,82 @@ volumes: - name: config temp: {} --- +clone: + retries: 3 +depends_on: [] +image_pull_secrets: +- gcr +- gar +kind: pipeline +name: windows-pipeline-manual +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - curl -L0 https://nssm.cc/release/nssm-2.24.zip -o nssm-2.24.zip + image: byrnedo/alpine-curl:0.1.8 + name: downlad-nssm +- commands: + - mkdir wix3 && cd wix3 + - curl -L0 https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip + -o wix3.zip + - unzip wix3.zip + image: byrnedo/alpine-curl:0.1.8 + name: download-wix3 +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release/grafana-$${DRONE_TAG:1}.windows-amd64.zip + grafana.zip' + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: download-zip +- commands: + - mkdir -p bin + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.1.1/grabpl + - chmod +x bin/grabpl + image: byrnedo/alpine-curl:0.1.8 + name: grabpl +- commands: + - export WINEPATH=$(winepath ./wix3) + - ./bin/grabpl windows-installer --target grafana.zip --edition oss + depends_on: + - downlad-nssm + - download-wix3 + - grabpl + - download-zip + entrypoint: + - /bin/bash + image: scottyhardy/docker-wine:stable-9.0 + name: build-msi +- commands: + - printenv GCP_KEY | base64 -d > /tmp/key.json + - gcloud auth activate-service-account --key-file=/tmp/key.json + - bash -c 'gcloud storage cp *.msi gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + - bash -c 'gcloud storage cp *.msi.sha256 gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release' + depends_on: + - build-msi + environment: + GCP_KEY: + from_secret: gcp_key_base64 + image: google/cloud-sdk:431.0.0 + name: upload-msi-installer +trigger: + event: + - promote + target: build-msi +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- get: name: credentials.json path: infra/data/ci/grafana-release-eng/grafanauploads @@ -5190,6 +5136,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: ec762bbe26644048a3c3b06db1da7a391b8ab9cd3d88607a86d0795af6d6158b +hmac: 30f8d5ff1b13b984dfbbee83dcdea98c3bc6721b10b6172ce50f73d81a31c0c8 ... diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star index ee47392ee5c..54be77084c4 100644 --- a/scripts/drone/events/main.star +++ b/scripts/drone/events/main.star @@ -35,10 +35,6 @@ load( "scripts/drone/pipelines/trigger_downstream.star", "enterprise_downstream_pipeline", ) -load( - "scripts/drone/pipelines/windows.star", - "windows", -) load( "scripts/drone/utils/utils.star", "failure_template", @@ -64,6 +60,8 @@ trigger = { } def main_pipelines(): + # This is how we should define any new pipelines. At some point we should update existing ones. + # Let's make an effort to reduce the amount of string constants in "depends_on" lists. pipelines = [ docs_pipelines(ver_mode, trigger_docs_main()), test_frontend(trigger, ver_mode), @@ -72,7 +70,6 @@ def main_pipelines(): lint_backend_pipeline(trigger, ver_mode), build_e2e(trigger, ver_mode), integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode), - windows(trigger, ver_mode = ver_mode), enterprise_downstream_pipeline(), notify_pipeline( name = "main-notify", @@ -83,7 +80,6 @@ def main_pipelines(): "main-test-backend", "main-build-e2e-publish", "main-integration-tests", - "main-windows", ], template = failure_template, secret = "slack_webhook", diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star deleted file mode 100644 index 1304cdedebd..00000000000 --- a/scripts/drone/pipelines/ci_images.star +++ /dev/null @@ -1,66 +0,0 @@ -""" -This module contains steps and pipelines relating to creating CI Docker images. -""" - -load( - "scripts/drone/utils/utils.star", - "pipeline", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) -load( - "scripts/drone/vault.star", - "from_secret", -) - -def publish_ci_windows_test_image_pipeline(): - trigger = { - "event": ["promote"], - "target": ["ci-windows-test-image"], - } - pl = pipeline( - name = "publish-ci-windows-test-image", - trigger = trigger, - platform = "windows", - steps = [ - { - "name": "clone", - "image": windows_images["wix"], - "environment": { - "GITHUB_TOKEN": from_secret("github_token"), - }, - "commands": [ - 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-ci-sandbox.git" .', - "git checkout -f $$env:DRONE_COMMIT", - ], - }, - { - "name": "build-and-publish", - "image": windows_images["windows_server_core"], - "environment": { - "DOCKER_USERNAME": from_secret("docker_username"), - "DOCKER_PASSWORD": from_secret("docker_password"), - }, - "commands": [ - "cd scripts\\build\\ci-windows-test", - "docker login -u $$env:DOCKER_USERNAME -p $$env:DOCKER_PASSWORD", - "docker build -t grafana/grafana-ci-windows-test:$$env:TAG .", - "docker push grafana/grafana-ci-windows-test:$$env:TAG", - ], - "volumes": [ - { - "name": "docker", - "path": "//./pipe/docker_engine/", - }, - ], - }, - ], - ) - - pl["clone"] = { - "disable": True, - } - - return [pl] diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star deleted file mode 100644 index 59c12a19536..00000000000 --- a/scripts/drone/pipelines/windows.star +++ /dev/null @@ -1,85 +0,0 @@ -""" -This module returns the pipeline used for building Grafana on Windows. -""" - -load( - "scripts/drone/steps/lib_windows.star", - "clone_step_windows", - "get_windows_steps", - "test_backend_step_windows", - "wire_install_step_windows", -) -load( - "scripts/drone/utils/utils.star", - "pipeline", -) -load( - "scripts/drone/utils/windows_images.star", - "windows_images", -) - -def windows_test_backend(trigger, edition, ver_mode): - """ Generates a pipeline that runs backend tests on Windows - - Args: - trigger: a Drone trigger for the pipeline - edition: controls whether enterprise code is included or not - ver_mode: controls whether a pre-release or actual release pipeline is generated. - Returns: - A single pipeline running backend tests for Windows - """ - environment = {"EDITION": edition} - steps = [ - clone_step_windows(), - ] - - steps.extend([{ - "name": "windows-init", - "image": windows_images["go"], - "depends_on": ["clone"], - "commands": [], - }]) - - steps.extend([ - wire_install_step_windows(edition), - test_backend_step_windows(), - ]) - pl = pipeline( - name = "{}-test-backend-windows".format(ver_mode), - trigger = trigger, - steps = steps, - depends_on = [], - platform = "windows", - environment = environment, - ) - pl["clone"] = { - "disable": True, - } - return pl - -def windows(trigger, ver_mode): - """Generates the pipeline used for building Grafana on Windows. - - Args: - trigger: a Drone trigger for the pipeline. - ver_mode: controls whether a pre-release or actual release pipeline is generated. - Also indirectly controls which version of enterprise code is used. - - Returns: - Drone pipeline. - """ - environment = {"EDITION": "oss"} - - return pipeline( - name = "main-windows", - trigger = dict(trigger, repo = ["grafana/grafana"]), - steps = get_windows_steps(ver_mode), - depends_on = [ - "main-test-frontend", - "main-test-backend", - "main-build-e2e-publish", - "main-integration-tests", - ], - platform = "windows", - environment = environment, - ) diff --git a/scripts/drone/rgm.star b/scripts/drone/rgm.star index 7efbb35d88a..85e956d57a8 100644 --- a/scripts/drone/rgm.star +++ b/scripts/drone/rgm.star @@ -20,10 +20,6 @@ load( "scripts/drone/pipelines/whats_new_checker.star", "whats_new_checker_pipeline", ) -load( - "scripts/drone/steps/lib_windows.star", - "get_windows_steps", -) load( "scripts/drone/utils/images.star", "images", @@ -49,6 +45,10 @@ load( "rgm_github_token", "rgm_storybook_destination", ) +load( + "scripts/drone/windows.star", + "windows_pipeline_release", +) docs_paths = { "exclude": [ @@ -228,18 +228,6 @@ def rgm_tag(): steps = rgm_run("rgm-build", "drone_build_tag_grafana.sh"), ) -def rgm_tag_windows(): - return pipeline( - name = "rgm-tag-prerelease-windows", - trigger = tag_trigger, - steps = get_windows_steps( - ver_mode = "release", - bucket = "grafana-prerelease", - ), - depends_on = ["rgm-tag-prerelease"], - platform = "windows", - ) - def rgm_version_branch(): # Runs a package / build proces (with all distros) when a commit lands on a version branch return pipeline( @@ -299,17 +287,23 @@ def rgm_nightly_pipeline(): ] def rgm_tag_pipeline(): + build = rgm_tag() + + # the Windows step requires an uploaded .zip file to base its compilation on + windows = windows_pipeline_release(trigger = tag_trigger, depends_on = [ + build["name"], + ]) return [ + build, whats_new_checker_pipeline(tag_trigger), - rgm_tag(), - rgm_tag_windows(), + windows, verify_release_pipeline( trigger = tag_trigger, name = "rgm-tag-verify-prerelease-assets", bucket = "grafana-prerelease", depends_on = [ - "rgm-tag-prerelease", - "rgm-tag-prerelease-windows", + windows["name"], + build["name"], ], ), ] diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index 8ee8f14f667..85308604d20 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -36,4 +36,5 @@ images = { "dockerize": "jwilder/dockerize:0.6.1", "shellcheck": "koalaman/shellcheck:stable", "rocky": "rockylinux:9", + "wine": "scottyhardy/docker-wine:stable-9.0", } diff --git a/scripts/drone/utils/windows_images.star b/scripts/drone/utils/windows_images.star deleted file mode 100644 index a25c637a11e..00000000000 --- a/scripts/drone/utils/windows_images.star +++ /dev/null @@ -1,17 +0,0 @@ -""" -This module contains all the windows docker images that are used to build test and publish Grafana. -All the windows images needed to be in a different file than the other images, since they cannot be scanned -by trivy. Related issue: https://github.com/aquasecurity/trivy/issues/1392 -""" - -load( - "scripts/drone/variables.star", - "golang_version", -) - -windows_images = { - "1809": "mcr.microsoft.com/windows:1809", - "wix": "grafana/ci-wix:0.1.1", - "windows_server_core": "docker:windowsservercore-1809", - "go": "golang:{}-windowsservercore-1809".format(golang_version), -} diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star index 0171396effb..4e3d8a8aad7 100644 --- a/scripts/drone/variables.star +++ b/scripts/drone/variables.star @@ -2,7 +2,7 @@ global variables """ -grabpl_version = "v3.0.56" +grabpl_version = "v3.1.1" golang_version = "1.22.7" # nodejs_version should match what's in ".nvmrc", but without the v prefix. diff --git a/scripts/drone/windows.star b/scripts/drone/windows.star new file mode 100644 index 00000000000..2c0943b9dc2 --- /dev/null +++ b/scripts/drone/windows.star @@ -0,0 +1,149 @@ +""" +This module is a library of Drone steps that exclusively run on windows machines. +""" + +load( + "scripts/drone/steps/lib.star", + "download_grabpl_step", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) +load( + "scripts/drone/vault.star", + "from_secret", + "rgm_gcp_key_base64", +) + +def download_nssm_step(): + return { + "name": "downlad-nssm", + "image": images["curl"], + "commands": [ + # We don't need to extract nssm-2.24 because the wix / build process extracts it. It just needs to be in + # PWD and be named `nssm-2.24`. + "curl -L0 https://nssm.cc/release/nssm-2.24.zip -o nssm-2.24.zip", + ], + } + +def download_wix_step(): + return { + "name": "download-wix3", + "image": images["curl"], + "commands": [ + "mkdir wix3 && cd wix3", + "curl -L0 https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip -o wix3.zip", + "unzip wix3.zip", + ], + } + +def download_zip_step(target = ""): + path = "{}/grafana-$${{DRONE_TAG:1}}.windows-amd64.zip".format(target) + return { + "name": "download-zip", + "image": images["cloudsdk"], + "commands": [ + "printenv GCP_KEY | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "bash -c 'gcloud storage cp {} grafana.zip'".format(path), + ], + "environment": { + "GCP_KEY": from_secret(rgm_gcp_key_base64), + }, + } + +def windows_msi_pipeline(target = "", name = "", trigger = {}, depends_on = [], environment = {}): + """windows_msi_pipeline is a pipeline which creates an MSI from a .zip file. + + Args: + target: GCS path (with gs:// scheme) to the oflder containing the zip file + name: Name of the pipeline, should be unique. + trigger: The conditions which trigger the pipeline + depends_on: dependencies (strings) + environment: map of environment variables + Returns: + Drone step. + """ + nssm = download_nssm_step() + wix = download_wix_step() + grabpl = download_grabpl_step() + zip = download_zip_step(target = target) + build = build_msi_step( + depends_on = [ + nssm["name"], + wix["name"], + grabpl["name"], + zip["name"], + ], + ) + upload = upload_msi_step( + depends_on = [ + build["name"], + ], + target = target, + ) + + return pipeline( + name = name, + steps = [ + nssm, + wix, + zip, + grabpl, + build, + upload, + ], + trigger = trigger, + depends_on = depends_on, + environment = environment, + ) + +def windows_pipeline_release(name = "prerelease-windows-msi", depends_on = [], trigger = {}, environment = {}): + target = "gs://grafana-prerelease/artifacts/downloads/$${DRONE_TAG}/oss/release" + return windows_msi_pipeline(name = name, target = target, depends_on = depends_on, trigger = trigger, environment = environment) + +def windows_pipeline_main(depends_on = [], trigger = {}, environment = {}): + target = "gs://grafana-downloads/oss/main" + return windows_msi_pipeline(name = "main-windows-msi", target = target, depends_on = depends_on, trigger = trigger, environment = environment) + +def upload_msi_step(depends_on = [], target = ""): + return { + "name": "upload-msi-installer", + "image": images["cloudsdk"], + "commands": [ + "printenv GCP_KEY | base64 -d > /tmp/key.json", + "gcloud auth activate-service-account --key-file=/tmp/key.json", + "bash -c 'gcloud storage cp *.msi {}'".format(target), + "bash -c 'gcloud storage cp *.msi.sha256 {}'".format(target), + ], + "depends_on": depends_on, + "environment": { + "GCP_KEY": from_secret(rgm_gcp_key_base64), + }, + } + +def build_msi_step(depends_on = []): + return { + "name": "build-msi", + "image": images["wine"], + "entrypoint": ["/bin/bash"], + "commands": [ + "export WINEPATH=$(winepath ./wix3)", + "./bin/grabpl windows-installer --target grafana.zip --edition oss", + ], + "depends_on": depends_on, + } + +def windows_manual_pipeline(): + return windows_pipeline_release( + name = "windows-pipeline-manual", + trigger = { + "event": ["promote"], + "target": "build-msi", + }, + )