public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Update CI/CD tooling and pipelines from main (#76814)

Browse Source 
* CI: Update CI/CD tooling and pipelines from main

* Update Makefile

* Comment out validate_openapi_spec_step

* Update broken frontend tests

* Fix validate-npm-packages regex to work without suffix

* Fix cypress image version
This commit is contained in:
Guilherme Caulada
2023-10-19 17:02:22 -03:00
committed by GitHub
parent 8757cb124b
commit 03ecb1db39
46 changed files with 2237 additions and 2858 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/events/cron.star
+14 -22
View File
@@ -2,15 +2,11 @@
This module provides functions for cronjob pipelines and steps used within.
"""
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load("scripts/drone/vault.star", "from_secret")
aquasec_trivy_image = "aquasec/trivy:0.21.0"
@@ -21,7 +17,6 @@ def cronjobs():
scan_docker_image_pipeline("latest-ubuntu"),
scan_docker_image_pipeline("main-ubuntu"),
scan_build_test_publish_docker_image_pipeline(),
grafana_com_nightly_pipeline(),
]
def authenticate_gcr_step():
@@ -59,6 +54,10 @@ def cron_job_pipeline(cronName, name, steps):
"path": "/var/run/docker.sock",
},
},
{
"name": "config",
"temp": {},
},
],
}
@@ -117,13 +116,13 @@ def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image):
for key in images:
cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]]
else:
cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image]
cmds = ["trivy image --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image]
return {
"name": "scan-unknown-low-medium-vulnerabilities",
"image": aquasec_trivy_image,
"commands": cmds,
"depends_on": ["authenticate-gcr"],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}],
}
def scan_docker_image_high_critical_vulnerabilities_step(docker_image):
@@ -141,19 +140,22 @@ def scan_docker_image_high_critical_vulnerabilities_step(docker_image):
for key in images:
cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]]
else:
cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image]
cmds = ["trivy image --exit-code 1 --severity HIGH,CRITICAL " + docker_image]
return {
"name": "scan-high-critical-vulnerabilities",
"image": aquasec_trivy_image,
"commands": cmds,
"depends_on": ["authenticate-gcr"],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
"environment": {
"GOOGLE_APPLICATION_CREDENTIALS": from_secret("gcr_credentials_json"),
},
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}, {"name": "config", "path": "/root/.docker/"}],
}
def slack_job_failed_step(channel, image):
return {
"name": "slack-notify-failure",
"image": images["plugins_slack_image"],
"image": images["plugins_slack"],
"settings": {
"webhook": from_secret("slack_webhook_backend"),
"channel": channel,
@@ -167,7 +169,7 @@ def slack_job_failed_step(channel, image):
def post_to_grafana_com_step():
return {
"name": "post-to-grafana-com",
"image": images["publish_image"],
"image": images["publish"],
"environment": {
"GRAFANA_COM_API_KEY": from_secret("grafana_api_key"),
"GCP_KEY": from_secret("gcp_key"),
@@ -175,13 +177,3 @@ def post_to_grafana_com_step():
"depends_on": ["compile-build-cmd"],
"commands": ["./bin/build publish grafana-com --edition oss"],
}
def grafana_com_nightly_pipeline():
return cron_job_pipeline(
cronName = "grafana-com-nightly",
name = "grafana-com-nightly",
steps = [
compile_build_cmd(),
post_to_grafana_com_step(),
],
)
scripts/drone/events/main.star
+23 -49
View File
@@ -3,40 +3,18 @@ This module returns all the pipelines used in the event of pushes to the main br
"""
load(
"scripts/drone/utils/utils.star",
"drone_change_template",
"failure_template",
"notify_pipeline",
"scripts/drone/pipelines/build.star",
"build_e2e",
)
load(
"scripts/drone/pipelines/docs.star",
"docs_pipelines",
"trigger_docs_main",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
)
load(
"scripts/drone/pipelines/integration_tests.star",
"integration_tests",
)
load(
"scripts/drone/pipelines/build.star",
"build_e2e",
)
load(
"scripts/drone/pipelines/windows.star",
"windows",
)
load(
"scripts/drone/pipelines/trigger_downstream.star",
"enterprise_downstream_pipeline",
)
load(
"scripts/drone/pipelines/lint_backend.star",
"lint_backend_pipeline",
@@ -45,6 +23,27 @@ load(
"scripts/drone/pipelines/lint_frontend.star",
"lint_frontend_pipeline",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
)
load(
"scripts/drone/pipelines/trigger_downstream.star",
"enterprise_downstream_pipeline",
)
load(
"scripts/drone/pipelines/windows.star",
"windows",
)
load(
"scripts/drone/utils/utils.star",
"failure_template",
"notify_pipeline",
)
ver_mode = "main"
trigger = {
@@ -65,24 +64,6 @@ trigger = {
}
def main_pipelines():
drone_change_trigger = {
"event": [
"push",
],
"branch": "main",
"repo": [
"grafana/grafana",
],
"paths": {
"include": [
".drone.yml",
],
"exclude": [
"exclude",
],
},
}
pipelines = [
docs_pipelines(ver_mode, trigger_docs_main()),
test_frontend(trigger, ver_mode),
@@ -92,13 +73,6 @@ def main_pipelines():
build_e2e(trigger, ver_mode),
integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode),
windows(trigger, ver_mode = ver_mode),
notify_pipeline(
name = "notify-drone-changes",
slack_channel = "slack-webhooks-test",
trigger = drone_change_trigger,
template = drone_change_template,
secret = "drone-changes-webhook",
),
enterprise_downstream_pipeline(),
notify_pipeline(
name = "main-notify",
scripts/drone/events/pr.star
+22 -22
View File
@@ -4,37 +4,21 @@ It also includes a function generating a PR trigger from a list of included and
"""
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
)
load(
"scripts/drone/pipelines/integration_tests.star",
"integration_tests",
"scripts/drone/pipelines/benchmarks.star",
"integration_benchmarks",
)
load(
"scripts/drone/pipelines/build.star",
"build_e2e",
)
load(
"scripts/drone/pipelines/verify_drone.star",
"verify_drone",
)
load(
"scripts/drone/pipelines/verify_starlark.star",
"verify_starlark",
)
load(
"scripts/drone/pipelines/docs.star",
"docs_pipelines",
"trigger_docs_pr",
)
load(
"scripts/drone/pipelines/shellcheck.star",
"shellcheck_pipeline",
"scripts/drone/pipelines/integration_tests.star",
"integration_tests",
)
load(
"scripts/drone/pipelines/lint_backend.star",
@@ -45,8 +29,24 @@ load(
"lint_frontend_pipeline",
)
load(
"scripts/drone/pipelines/benchmarks.star",
"integration_benchmarks",
"scripts/drone/pipelines/shellcheck.star",
"shellcheck_pipeline",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
)
load(
"scripts/drone/pipelines/verify_drone.star",
"verify_drone",
)
load(
"scripts/drone/pipelines/verify_starlark.star",
"verify_starlark",
)
ver_mode = "pr"
scripts/drone/events/release.star
+34 -228
View File
@@ -2,55 +2,36 @@
This module returns all the pipelines used in the event of a release along with supporting functions.
"""
load(
"scripts/drone/steps/lib.star",
"build_backend_step",
"build_docker_images_step",
"build_frontend_package_step",
"build_frontend_step",
"build_plugins_step",
"build_storybook_step",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
"e2e_tests_artifacts",
"e2e_tests_step",
"get_windows_steps",
"grafana_server_step",
"identify_runner_step",
"memcached_integration_tests_step",
"mysql_integration_tests_step",
"package_step",
"postgres_integration_tests_step",
"publish_grafanacom_step",
"publish_linux_packages_step",
"redis_integration_tests_step",
"store_storybook_step",
"trigger_oss",
"upload_cdn_step",
"upload_packages_step",
"verify_gen_cue_step",
"verify_gen_jsonnet_step",
"wire_install_step",
"yarn_install_step",
)
load(
"scripts/drone/services/services.star",
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"identify_runner_step",
"memcached_integration_tests_steps",
"mysql_integration_tests_steps",
"postgres_integration_tests_steps",
"publish_grafanacom_step",
"publish_linux_packages_step",
"redis_integration_tests_steps",
"remote_alertmanager_integration_tests_steps",
"verify_gen_cue_step",
"verify_gen_jsonnet_step",
"wire_install_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
)
load(
"scripts/drone/vault.star",
"from_secret",
@@ -59,51 +40,13 @@ load(
"prerelease_bucket",
"rgm_gcp_key_base64",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/pipelines/whats_new_checker.star",
"whats_new_checker_pipeline",
)
ver_mode = "release"
release_trigger = {
"event": {
"exclude": [
"promote",
],
},
"ref": {
"include": [
"refs/tags/v*",
],
"exclude": [
"refs/tags/*-cloud*",
],
},
}
def store_npm_packages_step():
return {
"name": "store-npm-packages",
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
"build-frontend-packages",
],
"environment": {
"GCP_KEY": from_secret(gcp_grafanauploads_base64),
"PRERELEASE_BUCKET": from_secret(prerelease_bucket),
},
"commands": ["./bin/build artifacts npm store --tag ${DRONE_TAG}"],
}
def retrieve_npm_packages_step():
return {
"name": "retrieve-npm-packages",
"image": images["publish_image"],
"image": images["publish"],
"depends_on": [
"compile-build-cmd",
"yarn-install",
@@ -119,7 +62,7 @@ def retrieve_npm_packages_step():
def release_npm_packages_step():
return {
"name": "release-npm-packages",
"image": images["build_image"],
"image": images["go"],
"depends_on": [
"compile-build-cmd",
"retrieve-npm-packages",
@@ -131,146 +74,10 @@ def release_npm_packages_step():
"commands": ["./bin/build artifacts npm release --tag ${DRONE_TAG}"],
}
def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
"""Generates all pipelines used for Grafana OSS.
Args:
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
environment = {"EDITION": "oss"}
services = integration_test_services()
volumes = integration_test_services_volumes()
init_steps = [
identify_runner_step(),
download_grabpl_step(),
verify_gen_cue_step(),
wire_install_step(),
yarn_install_step(),
compile_build_cmd(),
]
build_steps = [
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(publish = True),
build_docker_images_step(
publish = True,
ubuntu = True,
),
grafana_server_step(),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
e2e_tests_step("various-suite", tries = 3),
e2e_tests_artifacts(),
build_storybook_step(ver_mode = ver_mode),
]
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
publish_steps.extend(
[
upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss),
upload_packages_step(
ver_mode = ver_mode,
trigger = trigger_oss,
),
],
)
if ver_mode in ("release",):
publish_steps.extend(
[
store_storybook_step(ver_mode = ver_mode),
store_npm_packages_step(),
],
)
integration_test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step("mysql57", "5.7"),
mysql_integration_tests_step("mysql80", "8.0"),
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
pipelines = []
# We don't need to run integration tests at release time since they have
# been run multiple times before:
if ver_mode in ("release"):
pipelines.append(whats_new_checker_pipeline(release_trigger))
integration_test_steps = []
volumes = []
windows_pipeline_dependencies = [
"{}-build-e2e-publish".format(ver_mode),
"{}-test-frontend".format(ver_mode),
]
pipelines.extend([
pipeline(
name = "{}-build-e2e-publish".format(ver_mode),
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
environment = environment,
volumes = volumes,
),
test_frontend(trigger, ver_mode),
test_backend(trigger, ver_mode),
])
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-integration-tests".format(ver_mode),
trigger = trigger,
services = services,
steps = [
download_grabpl_step(),
identify_runner_step(),
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
wire_install_step(),
] +
integration_test_steps,
environment = environment,
volumes = volumes,
))
windows_pipeline = pipeline(
name = "{}-windows".format(ver_mode),
trigger = trigger,
steps = get_windows_steps(ver_mode = ver_mode),
platform = "windows",
depends_on = windows_pipeline_dependencies,
environment = environment,
)
pipelines.append(windows_pipeline)
return pipelines
def publish_artifacts_step():
return {
"name": "publish-artifacts",
"image": images["publish_image"],
"image": images["publish"],
"environment": {
"GCP_KEY": from_secret(gcp_grafanauploads_base64),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@@ -284,7 +91,7 @@ def publish_artifacts_step():
def publish_static_assets_step():
return {
"name": "publish-static-assets",
"image": images["publish_image"],
"image": images["publish"],
"environment": {
"GCP_KEY": from_secret(gcp_grafanauploads_base64),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@@ -299,7 +106,7 @@ def publish_static_assets_step():
def publish_storybook_step():
return {
"name": "publish-storybook",
"image": images["publish_image"],
"image": images["publish"],
"environment": {
"GCP_KEY": from_secret(gcp_grafanauploads_base64),
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
@@ -411,13 +218,12 @@ def integration_test_pipelines():
}
pipelines = []
volumes = integration_test_services_volumes()
integration_test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step("mysql57", "5.7"),
mysql_integration_tests_step("mysql80", "8.0"),
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
integration_test_steps = postgres_integration_tests_steps() + \
mysql_integration_tests_steps("mysql57", "5.7") + \
mysql_integration_tests_steps("mysql80", "8.0") + \
redis_integration_tests_steps() + \
memcached_integration_tests_steps() + \
remote_alertmanager_integration_tests_steps()
pipelines.append(pipeline(
name = "integration-tests",
@@ -442,7 +248,7 @@ def verify_release_pipeline(
bucket = from_secret(prerelease_bucket),
gcp_key = from_secret(rgm_gcp_key_base64),
version = "${DRONE_TAG}",
trigger = release_trigger,
trigger = {},
depends_on = [
"release-build-e2e-publish",
"release-windows",
@@ -456,7 +262,7 @@ def verify_release_pipeline(
step = {
"name": "gsutil-stat",
"depends_on": ["clone"],
"image": images["cloudsdk_image"],
"image": images["cloudsdk"],
"environment": {
"BUCKET": bucket,
"GCP_KEY": gcp_key,