From 0850edc1378b5a907b5d3d1d4fe9dbd28ab23efd Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Mon, 12 Jun 2023 17:32:09 +0300 Subject: [PATCH] [v9.4.x] Security Scans: Add trivy scans to every docker image used for building/testing/publishing (#69911) (#69920) * Security Scans: Add trivy scans to every docker image used for building/testing/publishing (#69911) * Created images.star * Fix typo * Add cronjobs for build-images (cherry picked from commit 2cda971796fb23f64575b139b74dd3cd46dc5635) # Conflicts: # .drone.yml # scripts/drone/services/services.star # scripts/drone/steps/lib.star * Fix rebasing issue --- .drone.yml | 84 +++++++++++- scripts/drone/events/cron.star | 73 ++++++++-- scripts/drone/events/release.star | 21 +-- scripts/drone/pipelines/aws_marketplace.star | 7 +- scripts/drone/pipelines/ci_images.star | 12 +- scripts/drone/pipelines/docs.star | 7 +- scripts/drone/pipelines/github.star | 7 +- scripts/drone/pipelines/shellcheck.star | 8 +- scripts/drone/pipelines/windows.star | 7 +- scripts/drone/services/services.star | 11 +- scripts/drone/steps/lib.star | 136 +++++++++---------- scripts/drone/utils/images.star | 29 ++++ 12 files changed, 284 insertions(+), 118 deletions(-) create mode 100644 scripts/drone/utils/images.star diff --git a/.drone.yml b/.drone.yml index f5a4a19972d..c682bea4639 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6289,7 +6289,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest image: aquasec/trivy:0.21.0 @@ -6319,7 +6319,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main image: aquasec/trivy:0.21.0 @@ -6349,7 +6349,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu image: aquasec/trivy:0.21.0 @@ -6380,7 +6380,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu image: aquasec/trivy:0.21.0 @@ -6403,6 +6403,80 @@ type: docker clone: retries: 3 kind: pipeline +name: scan-build-test-and-publish-docker-images +platform: + arch: amd64 + os: linux +steps: +- commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.4 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.17.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mcr.microsoft.com/windows:1809 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/ci-wix:0.1.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.4 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-windows-test:0.1.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM memcached:1.6.9-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM docker:windowsservercore-1809 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us.gcr.io/kubernetes-dev/package-publish:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM osixia/openldap:1.4.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/drone-downstream + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: aquasec/trivy:0.21.0 + name: scan-unknown-low-medium-vulnerabilities +- commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.4 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.17.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 + - trivy --exit-code 1 --severity HIGH,CRITICAL mcr.microsoft.com/windows:1809 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/ci-wix:0.1.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.4 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-windows-test:0.1.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack + - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 + - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 + - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL memcached:1.6.9-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL docker:windowsservercore-1809 + - trivy --exit-code 1 --severity HIGH,CRITICAL us.gcr.io/kubernetes-dev/package-publish:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL osixia/openldap:1.4.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/drone-downstream + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 + - trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: aquasec/trivy:0.21.0 + name: scan-high-critical-vulnerabilities +- image: plugins/slack + name: slack-notify-failure + settings: + channel: grafana-backend-ops + template: 'Nightly docker image scan job for build-images failed: {{build.link}}' + webhook: + from_secret: slack_webhook_backend + when: + status: failure +trigger: + cron: nightly + event: cron +type: docker +--- +clone: + retries: 3 +kind: pipeline name: grafana-com-nightly platform: arch: amd64 @@ -6582,6 +6656,6 @@ kind: secret name: delivery-bot-app-private-key --- kind: signature -hmac: d436ab210d1a6476bb6ce0daa7206f1644156f2d36cf6c30f14305c01e70f838 +hmac: 763badebc2faf7414837407bb4ef9809bcc857538c29b2caae8da16db8f0855c ... diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index 9b718f79572..d73e1074562 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -6,7 +6,10 @@ load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/steps/lib.star", "compile_build_cmd", - "publish_image", +) +load( + "scripts/drone/utils/images.star", + "images", ) aquasec_trivy_image = "aquasec/trivy:0.21.0" @@ -17,6 +20,7 @@ def cronjobs(): scan_docker_image_pipeline("main"), scan_docker_image_pipeline("latest-ubuntu"), scan_docker_image_pipeline("main-ubuntu"), + scan_build_test_publish_docker_image_pipeline(), grafana_com_nightly_pipeline(), ] @@ -54,34 +58,77 @@ def scan_docker_image_pipeline(tag): cronName = "nightly", name = "scan-" + docker_image + "-image", steps = [ - scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image), + scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image), scan_docker_image_high_critical_vulnerabilities_step(docker_image), slack_job_failed_step("grafana-backend-ops", docker_image), ], ) -def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image): - return { - "name": "scan-unkown-low-medium-vulnerabilities", - "image": aquasec_trivy_image, - "commands": [ - "trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image, +def scan_build_test_publish_docker_image_pipeline(): + """Generates a cronjob pipeline for nightly scans of grafana Docker images. + + Returns: + Drone cronjob pipeline. + """ + + return cron_job_pipeline( + cronName = "nightly", + name = "scan-build-test-and-publish-docker-images", + steps = [ + scan_docker_image_unknown_low_medium_vulnerabilities_step("all"), + scan_docker_image_high_critical_vulnerabilities_step("all"), + slack_job_failed_step("grafana-backend-ops", "build-images"), ], + ) + +def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image): + """Generates a step for scans of Grafana Docker images. + + Args: + docker_image: determines which image is scanned. + + Returns: + Drone cronjob step . + """ + + cmds = [] + if docker_image == "all": + for key in images: + cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]] + else: + cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] + return { + "name": "scan-unknown-low-medium-vulnerabilities", + "image": aquasec_trivy_image, + "commands": cmds, } def scan_docker_image_high_critical_vulnerabilities_step(docker_image): + """Generates a step for scans of Grafana Docker images. + + Args: + docker_image: determines which image is scanned. + + Returns: + Drone cronjob step . + """ + + cmds = [] + if docker_image == "all": + for key in images: + cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]] + else: + cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image] return { "name": "scan-high-critical-vulnerabilities", "image": aquasec_trivy_image, - "commands": [ - "trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image, - ], + "commands": cmds, } def slack_job_failed_step(channel, image): return { "name": "slack-notify-failure", - "image": "plugins/slack", + "image": images["plugins_slack_image"], "settings": { "webhook": from_secret("slack_webhook_backend"), "channel": channel, @@ -95,7 +142,7 @@ def slack_job_failed_step(channel, image): def post_to_grafana_com_step(): return { "name": "post-to-grafana-com", - "image": publish_image, + "image": images["publish_image"], "environment": { "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), "GCP_KEY": from_secret("gcp_key"), diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index aa4a0756047..45be46e954f 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -9,11 +9,9 @@ load( "build_docker_images_step", "build_frontend_package_step", "build_frontend_step", - "build_image", "build_plugins_step", "build_storybook_step", "clone_enterprise_step", - "cloudsdk_image", "compile_build_cmd", "copy_packages_for_docker_step", "download_grabpl_step", @@ -29,7 +27,6 @@ load( "package_step", "postgres_integration_tests_step", "publish_grafanacom_step", - "publish_image", "publish_images_step", "publish_linux_packages_step", "redis_integration_tests_step", @@ -63,6 +60,10 @@ load( "test_backend_enterprise", ) load("scripts/drone/vault.star", "from_secret", "prerelease_bucket") +load( + "scripts/drone/utils/images.star", + "images", +) ver_mode = "release" release_trigger = { @@ -84,7 +85,7 @@ release_trigger = { def store_npm_packages_step(): return { "name": "store-npm-packages", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", "build-frontend-packages", @@ -99,7 +100,7 @@ def store_npm_packages_step(): def retrieve_npm_packages_step(): return { "name": "retrieve-npm-packages", - "image": publish_image, + "image": images["publish_image"], "depends_on": [ "compile-build-cmd", "yarn-install", @@ -115,7 +116,7 @@ def retrieve_npm_packages_step(): def release_npm_packages_step(): return { "name": "release-npm-packages", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", "retrieve-npm-packages", @@ -548,7 +549,7 @@ def publish_artifacts_step(mode): security = "--security " return { "name": "publish-artifacts", - "image": publish_image, + "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -566,7 +567,7 @@ def publish_artifacts_step(mode): def publish_static_assets_step(): return { "name": "publish-static-assets", - "image": publish_image, + "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -581,7 +582,7 @@ def publish_static_assets_step(): def publish_storybook_step(): return { "name": "publish-storybook", - "image": publish_image, + "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -812,7 +813,7 @@ def verify_release_pipeline( step = { "name": "gsutil-stat", "depends_on": ["clone"], - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "environment": { "BUCKET": bucket, "GCP_KEY": gcp_key, diff --git a/scripts/drone/pipelines/aws_marketplace.star b/scripts/drone/pipelines/aws_marketplace.star index a9ca6c57ab4..82d40a4fa57 100644 --- a/scripts/drone/pipelines/aws_marketplace.star +++ b/scripts/drone/pipelines/aws_marketplace.star @@ -5,18 +5,21 @@ This module contains steps and pipelines publishing to AWS Marketplace. load( "scripts/drone/steps/lib.star", "compile_build_cmd", - "publish_image", ) load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/utils/images.star", + "images", +) def publish_aws_marketplace_step(): return { "name": "publish-aws-marketplace", - "image": publish_image, + "image": images["publish_image"], "commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"], "depends_on": ["compile-build-cmd"], "environment": { diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index fa6f458cd2b..734b731c2a8 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -2,10 +2,6 @@ This module contains steps and pipelines relating to creating CI Docker images. """ -load( - "scripts/drone/steps/lib.star", - "wix_image", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -14,6 +10,10 @@ load( "scripts/drone/vault.star", "from_secret", ) +load( + "scripts/drone/utils/images.star", + "images", +) def publish_ci_windows_test_image_pipeline(): trigger = { @@ -28,7 +28,7 @@ def publish_ci_windows_test_image_pipeline(): steps = [ { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -39,7 +39,7 @@ def publish_ci_windows_test_image_pipeline(): }, { "name": "build-and-publish", - "image": "docker:windowsservercore-1809", + "image": images["windows_server_core_image"], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index 5113fc09de2..d60fcdb6cad 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -5,11 +5,14 @@ This module returns all the pipelines used in the event of documentation changes load( "scripts/drone/steps/lib.star", "build_docs_website_step", - "build_image", "codespell_step", "identify_runner_step", "yarn_install_step", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -46,7 +49,7 @@ def docs_pipelines(ver_mode, trigger): def lint_docs(): return { "name": "lint-docs", - "image": build_image, + "image": images["build_image"], "depends_on": [ "yarn-install", ], diff --git a/scripts/drone/pipelines/github.star b/scripts/drone/pipelines/github.star index d3ccb6b30c9..fc43638c504 100644 --- a/scripts/drone/pipelines/github.star +++ b/scripts/drone/pipelines/github.star @@ -6,18 +6,21 @@ load( "scripts/drone/steps/lib.star", "compile_build_cmd", "fetch_images_step", - "publish_image", ) load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/utils/images.star", + "images", +) def publish_github_step(): return { "name": "publish-github", - "image": publish_image, + "image": images["publish_image"], "commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"], "depends_on": ["fetch-images-enterprise2"], "environment": { diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 2a2aa55149e..b8ef57265d7 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -2,11 +2,15 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ -load("scripts/drone/steps/lib.star", "build_image", "compile_build_cmd") +load("scripts/drone/steps/lib.star", "compile_build_cmd") load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/utils/images.star", + "images", +) trigger = { "event": [ @@ -25,7 +29,7 @@ trigger = { def shellcheck_step(): return { "name": "shellcheck", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", ], diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index f00b0a16fd1..e2ecd7a4f22 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -10,11 +10,14 @@ load( "scripts/drone/steps/lib.star", "get_windows_steps", "windows_clone_step", - "windows_go_image", "windows_init_enterprise_steps", "windows_test_backend_step", "windows_wire_install_step", ) +load( + "scripts/drone/utils/images.star", + "images", +) def windows_test_backend(trigger, edition, ver_mode): """ Generates a pipeline that runs backend tests on Windows @@ -36,7 +39,7 @@ def windows_test_backend(trigger, edition, ver_mode): else: steps.extend([{ "name": "windows-init", - "image": windows_go_image, + "image": images["windows_go_image"], "depends_on": ["clone"], "commands": [], }]) diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index 9d7d04a2ded..827eb161200 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -2,6 +2,11 @@ This module has functions for Drone services to be used in pipelines. """ +load( + "scripts/drone/utils/images.star", + "images", +) + def integration_test_services_volumes(): return [ {"name": "postgres", "temp": {"medium": "memory"}}, @@ -12,7 +17,7 @@ def integration_test_services(edition): services = [ { "name": "postgres", - "image": "postgres:12.3-alpine", + "image": images["postgres_alpine_image"], "environment": { "POSTGRES_USER": "grafanatest", "POSTGRES_PASSWORD": "grafanatest", @@ -25,7 +30,7 @@ def integration_test_services(edition): }, { "name": "mysql", - "image": "mysql:5.7.39", + "image": images["mysql5_image"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -57,7 +62,7 @@ def integration_test_services(edition): def ldap_service(): return { "name": "ldap", - "image": "osixia/openldap:1.4.0", + "image": images["openldap_image"], "environment": { "LDAP_ADMIN_PASSWORD": "grafana", "LDAP_DOMAIN": "grafana.org", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index c7827275b97..51b4b403fec 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -7,18 +7,12 @@ load( "from_secret", "prerelease_bucket", ) +load( + "scripts/drone/utils/images.star", + "images", +) grabpl_version = "v3.0.38" -cloudsdk_image = "google/cloud-sdk:431.0.0" -build_image = "grafana/build-container:1.7.4" -publish_image = "grafana/grafana-ci-deploy:1.3.3" -deploy_docker_image = "us.gcr.io/kubernetes-dev/drone/plugins/deploy-image" -alpine_image = "alpine:3.17.1" -curl_image = "byrnedo/alpine-curl:0.1.8" -windows_image = "mcr.microsoft.com/windows:1809" -wix_image = "grafana/ci-wix:0.1.1" -go_image = "golang:1.20.4" -windows_go_image = "grafana/grafana-ci-windows-test:0.1.0" trigger_oss = { "repo": [ @@ -29,7 +23,7 @@ trigger_oss = { def slack_step(channel, template, secret): return { "name": "slack", - "image": "plugins/slack", + "image": images["plugins_slack_image"], "settings": { "webhook": from_secret(secret), "channel": channel, @@ -40,7 +34,7 @@ def slack_step(channel, template, secret): def yarn_install_step(): return { "name": "yarn-install", - "image": build_image, + "image": images["build_image"], "commands": [ "yarn install --immutable", ], @@ -50,7 +44,7 @@ def yarn_install_step(): def wire_install_step(): return { "name": "wire-install", - "image": build_image, + "image": images["build_image"], "commands": [ "make gen-go", ], @@ -62,7 +56,7 @@ def wire_install_step(): def windows_wire_install_step(edition): return { "name": "wire-install", - "image": windows_go_image, + "image": images["windows_go_image"], "commands": [ "go install github.com/google/wire/cmd/wire@v0.5.0", "wire gen -tags {} ./pkg/server".format(edition), @@ -76,7 +70,7 @@ def identify_runner_step(platform = "linux"): if platform == "linux": return { "name": "identify-runner", - "image": alpine_image, + "image": images["alpine_image"], "commands": [ "echo $DRONE_RUNNER_NAME", ], @@ -84,7 +78,7 @@ def identify_runner_step(platform = "linux"): else: return { "name": "identify-runner", - "image": windows_image, + "image": images["windows_image"], "commands": [ "echo $env:DRONE_RUNNER_NAME", ], @@ -101,7 +95,7 @@ def clone_enterprise_step(committish = "${DRONE_COMMIT}"): """ return { "name": "clone-enterprise", - "image": build_image, + "image": images["build_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -140,7 +134,7 @@ def init_enterprise_step(ver_mode): token = "" return { "name": "init-enterprise", - "image": build_image, + "image": images["build_image"], "depends_on": [ "clone-enterprise", "grabpl", @@ -198,7 +192,7 @@ def windows_init_enterprise_steps(ver_mode): download_grabpl_step(platform = "windows"), { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -206,7 +200,7 @@ def windows_init_enterprise_steps(ver_mode): }, { "name": "windows-init", - "image": wix_image, + "image": images["wix_image"], "commands": init_cmds, "depends_on": ["clone"], "environment": {"GITHUB_TOKEN": from_secret("github_token")}, @@ -219,7 +213,7 @@ def download_grabpl_step(platform = "linux"): if platform == "windows": return { "name": "grabpl", - "image": wix_image, + "image": images["wix_image"], "commands": [ '$$ProgressPreference = "SilentlyContinue"', "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( @@ -230,7 +224,7 @@ def download_grabpl_step(platform = "linux"): return { "name": "grabpl", - "image": curl_image, + "image": images["curl_image"], "commands": [ "mkdir -p bin", "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( @@ -243,7 +237,7 @@ def download_grabpl_step(platform = "linux"): def lint_drone_step(): return { "name": "lint-drone", - "image": curl_image, + "image": images["curl_image"], "commands": [ "./bin/build verify-drone", ], @@ -255,7 +249,7 @@ def lint_drone_step(): def lint_starlark_step(): return { "name": "lint-starlark", - "image": build_image, + "image": images["build_image"], "commands": [ "./bin/build verify-starlark .", ], @@ -282,7 +276,7 @@ def enterprise_downstream_step(ver_mode): step = { "name": "trigger-enterprise-downstream", - "image": "grafana/drone-downstream", + "image": images["drone_downstream_image"], "settings": { "server": "https://drone.grafana.net", "token": from_secret("drone_token"), @@ -306,7 +300,7 @@ def lint_backend_step(): return { "name": "lint-backend", # TODO: build_image or go_image? - "image": go_image, + "image": images["go_image"], "environment": { # We need CGO because of go-sqlite3 "CGO_ENABLED": "1", @@ -324,7 +318,7 @@ def lint_backend_step(): def benchmark_ldap_step(): return { "name": "benchmark-ldap", - "image": build_image, + "image": images["build_image"], "environment": { "LDAP_HOSTNAME": "ldap", }, @@ -337,7 +331,7 @@ def benchmark_ldap_step(): def build_storybook_step(ver_mode): return { "name": "build-storybook", - "image": build_image, + "image": images["build_image"], "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged "build-frontend", @@ -381,7 +375,7 @@ def store_storybook_step(ver_mode, trigger = None): step = { "name": "store-storybook", - "image": publish_image, + "image": images["publish_image"], "depends_on": [ "build-storybook", ] + @@ -411,7 +405,7 @@ def store_storybook_step(ver_mode, trigger = None): def e2e_tests_artifacts(): return { "name": "e2e-tests-artifacts-upload", - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "depends_on": [ "end-to-end-tests-dashboards-suite", "end-to-end-tests-panels-suite", @@ -473,7 +467,7 @@ def upload_cdn_step(edition, ver_mode, trigger = None): step = { "name": "upload-cdn-assets" + enterprise2_suffix(edition), - "image": publish_image, + "image": images["publish_image"], "depends_on": deps, "environment": { "GCP_KEY": from_secret("gcp_key"), @@ -525,7 +519,7 @@ def build_backend_step(edition, ver_mode, variants = None): return { "name": "build-backend" + enterprise2_suffix(edition), - "image": build_image, + "image": images["build_image"], "depends_on": [ "wire-install", "compile-build-cmd", @@ -560,7 +554,7 @@ def build_frontend_step(edition, ver_mode): return { "name": "build-frontend", - "image": build_image, + "image": images["build_image"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -598,7 +592,7 @@ def build_frontend_package_step(edition, ver_mode): return { "name": "build-frontend-packages", - "image": build_image, + "image": images["build_image"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -618,7 +612,7 @@ def build_plugins_step(edition, ver_mode): env = None return { "name": "build-plugins", - "image": build_image, + "image": images["build_image"], "environment": env, "depends_on": [ "compile-build-cmd", @@ -630,7 +624,7 @@ def build_plugins_step(edition, ver_mode): ], } -def test_backend_step(image = build_image): +def test_backend_step(image = images["build_image"]): return { "name": "test-backend", "image": image, @@ -643,13 +637,13 @@ def test_backend_step(image = build_image): } def windows_test_backend_step(): - step = test_backend_step(image = windows_go_image) + step = test_backend_step(image = images["windows_go_image"]) return step def test_backend_integration_step(): return { "name": "test-backend-integration", - "image": build_image, + "image": images["build_image"], "depends_on": [ "wire-install", ], @@ -674,7 +668,7 @@ def betterer_frontend_step(edition = "oss"): deps.extend(["yarn-install"]) return { "name": "betterer-frontend", - "image": build_image, + "image": images["build_image"], "depends_on": deps, "commands": [ "yarn betterer ci", @@ -697,7 +691,7 @@ def test_frontend_step(edition = "oss"): deps.extend(["yarn-install"]) return { "name": "test-frontend", - "image": build_image, + "image": images["build_image"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -710,7 +704,7 @@ def test_frontend_step(edition = "oss"): def lint_frontend_step(): return { "name": "lint-frontend", - "image": build_image, + "image": images["build_image"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -758,7 +752,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001): return { "name": "test-a11y-frontend", # TODO which image should be used? - "image": "grafana/docker-puppeteer:1.1.0", + "image": images["docker_puppeteer_image"], "depends_on": [ "grafana-server", ], @@ -783,7 +777,7 @@ def frontend_metrics_step(trigger = None): """ step = { "name": "publish-frontend-metrics", - "image": build_image, + "image": images["build_image"], "depends_on": [ "test-a11y-frontend", ], @@ -802,7 +796,7 @@ def frontend_metrics_step(trigger = None): def codespell_step(): return { "name": "codespell", - "image": build_image, + "image": images["build_image"], "commands": [ # Important: all words have to be in lowercase, and separated by "\n". 'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt', @@ -861,7 +855,7 @@ def package_step(edition, ver_mode): return { "name": "package" + enterprise2_suffix(edition), - "image": build_image, + "image": images["build_image"], "depends_on": deps, "environment": env, "commands": cmds, @@ -884,7 +878,7 @@ def grafana_server_step(edition, port = 3001): return { "name": "grafana-server", - "image": build_image, + "image": images["build_image"], "detach": True, "depends_on": [ "build-plugins", @@ -904,7 +898,7 @@ def e2e_tests_step(suite, port = 3001, tries = None): cmd += " --tries {}".format(tries) return { "name": "end-to-end-tests-{}".format(suite), - "image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "image": images["cypress_image"], "depends_on": [ "grafana-server", ], @@ -956,7 +950,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-") step = { "name": "end-to-end-tests-{}-{}".format(suite, cloud), - "image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", + "image": images["cloud_datasources_e2e_image"], "depends_on": [ "grafana-server", ], @@ -970,7 +964,7 @@ def build_docs_website_step(): return { "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - "image": "grafana/docs-base:latest", + "image": images["docs_image"], "commands": [ "mkdir -p /hugo/content/docs/grafana", "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", @@ -981,7 +975,7 @@ def build_docs_website_step(): def copy_packages_for_docker_step(edition = None): return { "name": "copy-packages-for-docker", - "image": build_image, + "image": images["build_image"], "depends_on": [ "package" + enterprise2_suffix(edition), ], @@ -1029,7 +1023,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa return { "name": "build-docker-images" + ubuntu_sfx, - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "depends_on": [ "copy-packages-for-docker", "compile-build-cmd", @@ -1042,7 +1036,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa def fetch_images_step(edition): return { "name": "fetch-images-{}".format(edition), - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "DOCKER_USER": from_secret("docker_username"), @@ -1121,7 +1115,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): step = { "name": "publish-images-{}".format(name), - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "environment": environment, "commands": [cmd], "depends_on": deps, @@ -1147,7 +1141,7 @@ def postgres_integration_tests_step(): ] return { "name": "postgres-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "PGPASSWORD": "grafanatest", @@ -1169,7 +1163,7 @@ def mysql_integration_tests_step(): ] return { "name": "mysql-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "GRAFANA_TEST_DB": "mysql", @@ -1181,7 +1175,7 @@ def mysql_integration_tests_step(): def redis_integration_tests_step(): return { "name": "redis-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "REDIS_URL": "redis://redis:6379/0", @@ -1196,7 +1190,7 @@ def redis_integration_tests_step(): def memcached_integration_tests_step(): return { "name": "memcached-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "MEMCACHED_HOSTS": "memcached:11211", @@ -1220,7 +1214,7 @@ def release_canary_npm_packages_step(trigger = None): """ step = { "name": "release-canary-npm-packages", - "image": build_image, + "image": images["build_image"], "depends_on": end_to_end_tests_deps(), "environment": { "NPM_TOKEN": from_secret("npm_token"), @@ -1253,7 +1247,7 @@ def upload_packages_step(edition, ver_mode, trigger = None): """ step = { "name": "upload-packages" + enterprise2_suffix(edition), - "image": publish_image, + "image": images["publish_image"], "depends_on": end_to_end_tests_deps(), "environment": { "GCP_KEY": from_secret("gcp_key"), @@ -1295,7 +1289,7 @@ def publish_grafanacom_step(edition, ver_mode): return { "name": "publish-grafanacom-{}".format(edition), - "image": publish_image, + "image": images["publish_image"], "depends_on": [ "publish-linux-packages-deb", "publish-linux-packages-rpm", @@ -1313,7 +1307,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - "image": "us.gcr.io/kubernetes-dev/package-publish:latest", + "image": images["package_publish_image"], "depends_on": ["compile-build-cmd"], "privileged": True, "settings": { @@ -1335,7 +1329,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"): def windows_clone_step(): return { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -1395,7 +1389,7 @@ def get_windows_steps(edition, ver_mode): [ { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -1403,7 +1397,7 @@ def get_windows_steps(edition, ver_mode): }, { "name": "windows-init", - "image": wix_image, + "image": images["wix_image"], "commands": init_cmds, "depends_on": ["clone"], "environment": {"GITHUB_TOKEN": from_secret("github_token")}, @@ -1422,7 +1416,7 @@ def get_windows_steps(edition, ver_mode): [ { "name": "windows-init", - "image": wix_image, + "image": images["wix_image"], "commands": init_cmds, }, ], @@ -1497,7 +1491,7 @@ def get_windows_steps(edition, ver_mode): steps.append( { "name": "build-windows-installer", - "image": wix_image, + "image": images["wix_image"], "depends_on": [ "windows-init", ], @@ -1515,7 +1509,7 @@ def get_windows_steps(edition, ver_mode): def verify_gen_cue_step(): return { "name": "verify-gen-cue", - "image": build_image, + "image": images["build_image"], "depends_on": [], "commands": [ "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", @@ -1527,7 +1521,7 @@ def verify_gen_cue_step(): def verify_gen_jsonnet_step(): return { "name": "verify-gen-jsonnet", - "image": build_image, + "image": images["build_image"], "depends_on": [], "commands": [ "# It is required that generated jsonnet is committed and in sync with its inputs.", @@ -1539,7 +1533,7 @@ def verify_gen_jsonnet_step(): def trigger_test_release(): return { "name": "trigger-test-release", - "image": build_image, + "image": images["build_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token_pr"), "TEST_TAG": "v0.0.0-test", @@ -1573,7 +1567,7 @@ def trigger_test_release(): def artifacts_page_step(): return { "name": "artifacts-page", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", ], @@ -1601,7 +1595,7 @@ def compile_build_cmd(edition = "oss"): ] return { "name": "compile-build-cmd", - "image": go_image, + "image": images["go_image"], "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star new file mode 100644 index 00000000000..0450301c1c2 --- /dev/null +++ b/scripts/drone/utils/images.star @@ -0,0 +1,29 @@ +""" +This module contains all the docker images that are used to build test and publish Grafana. +""" + +images = { + "cloudsdk_image": "google/cloud-sdk:431.0.0", + "build_image": "grafana/build-container:1.7.4", + "publish_image": "grafana/grafana-ci-deploy:1.3.3", + "alpine_image": "alpine:3.17.1", + "curl_image": "byrnedo/alpine-curl:0.1.8", + "windows_image": "mcr.microsoft.com/windows:1809", + "wix_image": "grafana/ci-wix:0.1.1", + "go_image": "golang:1.20.4", + "windows_go_image": "grafana/grafana-ci-windows-test:0.1.0", + "plugins_slack_image": "plugins/slack", + "postgres_alpine_image": "postgres:12.3-alpine", + "mysql5_image": "mysql:5.7.39", + "mysql8_image": "mysql:8.0.32", + "redis_alpine_image": "redis:6.2.11-alpine", + "memcached_alpine_image": "memcached:1.6.9-alpine", + "windows_server_core_image": "docker:windowsservercore-1809", + "package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest", + "openldap_image": "osixia/openldap:1.4.0", + "drone_downstream_image": "grafana/drone-downstream", + "docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0", + "docs_image": "grafana/docs-base:latest", + "cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", +}