public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Update CI/CD tooling and pipelines from main (#76814)

Browse Source 
* CI: Update CI/CD tooling and pipelines from main

* Update Makefile

* Comment out validate_openapi_spec_step

* Update broken frontend tests

* Fix validate-npm-packages regex to work without suffix

* Fix cypress image version
This commit is contained in:
Guilherme Caulada
2023-10-19 17:02:22 -03:00
committed by GitHub
parent 8757cb124b
commit 03ecb1db39
46 changed files with 2237 additions and 2858 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/pipelines/benchmarks.star
+19 -21
View File
@@ -2,6 +2,11 @@
This module returns the pipeline used for integration benchmarks.
"""
load(
"scripts/drone/services/services.star",
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
@@ -11,11 +16,6 @@ load(
"verify_gen_jsonnet_step",
"wire_install_step",
)
load(
"scripts/drone/services/services.star",
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@@ -52,22 +52,20 @@ def integration_benchmarks(prefix):
wire_install_step(),
]
benchmark_steps = [
integration_benchmarks_step("sqlite"),
integration_benchmarks_step("postgres", {
"PGPASSWORD": "grafanatest",
"GRAFANA_TEST_DB": "postgres",
"POSTGRES_HOST": "postgres",
}),
integration_benchmarks_step("mysql-5.7", {
"GRAFANA_TEST_DB": "mysql",
"MYSQL_HOST": "mysql57",
}),
integration_benchmarks_step("mysql-8.0", {
"GRAFANA_TEST_DB": "mysql",
"MYSQL_HOST": "mysql80",
}),
]
benchmark_steps = integration_benchmarks_step("sqlite") + \
integration_benchmarks_step("postgres", {
"PGPASSWORD": "grafanatest",
"GRAFANA_TEST_DB": "postgres",
"POSTGRES_HOST": "postgres",
}) + \
integration_benchmarks_step("mysql-5.7", {
"GRAFANA_TEST_DB": "mysql",
"MYSQL_HOST": "mysql57",
}) + \
integration_benchmarks_step("mysql-8.0", {
"GRAFANA_TEST_DB": "mysql",
"MYSQL_HOST": "mysql80",
})
return pipeline(
name = "{}-integration-benchmarks".format(prefix),
scripts/drone/pipelines/build.star
+30 -29
View File
@@ -2,15 +2,10 @@
load(
"scripts/drone/steps/lib.star",
"build_backend_step",
"build_docker_images_step",
"build_frontend_package_step",
"build_frontend_step",
"build_plugins_step",
"build_storybook_step",
"cloud_plugins_e2e_tests_step",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
"e2e_tests_artifacts",
"e2e_tests_step",
@@ -18,12 +13,12 @@ load(
"frontend_metrics_step",
"grafana_server_step",
"identify_runner_step",
"package_step",
"publish_images_step",
"release_canary_npm_packages_step",
"store_storybook_step",
"test_a11y_frontend_step",
"trigger_oss",
"update_package_json_version",
"upload_cdn_step",
"upload_packages_step",
"verify_gen_cue_step",
@@ -31,6 +26,15 @@ load(
"wire_install_step",
"yarn_install_step",
)
load(
"scripts/drone/steps/rgm.star",
"rgm_build_docker_step",
"rgm_package_step",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@@ -42,7 +46,7 @@ def build_e2e(trigger, ver_mode):
Args:
trigger: controls which events can trigger the pipeline execution.
ver_mode: used in the naming of the pipeline.
ver_mode: used in the naming of the pipeline. Either 'pr' or 'main'.
Returns:
Drone pipeline.
@@ -64,17 +68,19 @@ def build_e2e(trigger, ver_mode):
if ver_mode == "pr":
build_steps.extend(
[
build_frontend_package_step(),
enterprise_downstream_step(ver_mode = ver_mode),
],
)
else:
build_steps.extend([
update_package_json_version(),
build_frontend_package_step(depends_on = ["update-package-json-version"]),
])
build_steps.extend(
[
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
rgm_package_step(distros = "linux/amd64,linux/arm64", file = "packages.txt"),
grafana_server_step(),
e2e_tests_step("dashboards-suite"),
e2e_tests_step("smoke-tests-suite"),
@@ -87,7 +93,6 @@ def build_e2e(trigger, ver_mode):
),
e2e_tests_artifacts(),
build_storybook_step(ver_mode = ver_mode),
copy_packages_for_docker_step(),
test_a11y_frontend_step(ver_mode = ver_mode),
],
)
@@ -97,12 +102,12 @@ def build_e2e(trigger, ver_mode):
[
store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
frontend_metrics_step(trigger = trigger_oss),
build_docker_images_step(
publish = False,
),
build_docker_images_step(
publish = False,
ubuntu = True,
rgm_build_docker_step(
"packages.txt",
images["ubuntu"],
images["alpine"],
tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
),
publish_images_step(
docker_repo = "grafana",
@@ -128,16 +133,12 @@ def build_e2e(trigger, ver_mode):
elif ver_mode == "pr":
build_steps.extend(
[
build_docker_images_step(
archs = [
"amd64",
],
),
build_docker_images_step(
archs = [
"amd64",
],
ubuntu = True,
rgm_build_docker_step(
"packages.txt",
images["ubuntu"],
images["alpine"],
tag_format = "{{ .version_base }}-{{ .buildID }}-{{ .arch }}",
ubuntu_tag_format = "{{ .version_base }}-{{ .buildID }}-ubuntu-{{ .arch }}",
),
publish_images_step(
docker_repo = "grafana",
scripts/drone/pipelines/ci_images.star
+12 -12
View File
@@ -3,21 +3,21 @@ This module contains steps and pipelines relating to creating CI Docker images.
"""
load(
"scripts/drone/utils/utils.star",
"pipeline",
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/vault.star",
"from_secret",
"gcp_download_build_container_assets_key",
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/windows_images.star",
"windows_images",
)
load(
"scripts/drone/utils/images.star",
"images",
"scripts/drone/vault.star",
"from_secret",
"gcp_download_build_container_assets_key",
)
def publish_ci_windows_test_image_pipeline():
@@ -32,7 +32,7 @@ def publish_ci_windows_test_image_pipeline():
steps = [
{
"name": "clone",
"image": windows_images["wix_image"],
"image": windows_images["wix"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
@@ -43,7 +43,7 @@ def publish_ci_windows_test_image_pipeline():
},
{
"name": "build-and-publish",
"image": windows_images["windows_server_core_image"],
"image": windows_images["windows_server_core"],
"environment": {
"DOCKER_USERNAME": from_secret("docker_username"),
"DOCKER_PASSWORD": from_secret("docker_password"),
@@ -81,14 +81,14 @@ def publish_ci_build_container_image_pipeline():
steps = [
{
"name": "validate-version",
"image": images["alpine_image"],
"image": images["alpine"],
"commands": [
"if [ -z \"${BUILD_CONTAINER_VERSION}\" ]; then echo Missing BUILD_CONTAINER_VERSION; false; fi",
],
},
{
"name": "download-macos-sdk",
"image": images["cloudsdk_image"],
"image": images["cloudsdk"],
"environment": {
"GCP_KEY": from_secret(gcp_download_build_container_assets_key),
},
@@ -100,7 +100,7 @@ def publish_ci_build_container_image_pipeline():
},
{
"name": "build-and-publish", # Consider splitting the build and the upload task.
"image": images["cloudsdk_image"],
"image": images["cloudsdk"],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
"environment": {
"DOCKER_USERNAME": from_secret("docker_username"),
scripts/drone/pipelines/docs.star
+1 -1
View File
@@ -50,7 +50,7 @@ def docs_pipelines(ver_mode, trigger):
def lint_docs():
return {
"name": "lint-docs",
"image": images["build_image"],
"image": images["node"],
"depends_on": [
"yarn-install",
],
scripts/drone/pipelines/integration_tests.star
+17 -16
View File
@@ -2,25 +2,26 @@
This module returns the pipeline used for integration tests.
"""
load(
"scripts/drone/services/services.star",
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"memcached_integration_tests_step",
"mysql_integration_tests_step",
"postgres_integration_tests_step",
"redis_integration_tests_step",
"memcached_integration_tests_steps",
"mysql_integration_tests_steps",
"postgres_integration_tests_steps",
"redis_integration_tests_steps",
"remote_alertmanager_integration_tests_steps",
"verify_gen_cue_step",
"verify_gen_jsonnet_step",
"wire_install_step",
)
load(
"scripts/drone/services/services.star",
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@@ -60,13 +61,13 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
wire_install_step(),
]
test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step("mysql57", "5.7"),
mysql_integration_tests_step("mysql80", "8.0"),
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
# test_steps = [a, b] + [c, d] + [e, f]...
test_steps = postgres_integration_tests_steps() + \
mysql_integration_tests_steps("mysql57", "5.7") + \
mysql_integration_tests_steps("mysql80", "8.0") + \
redis_integration_tests_steps() + \
memcached_integration_tests_steps() + \
remote_alertmanager_integration_tests_steps()
return pipeline(
name = "{}-integration-tests".format(prefix),
scripts/drone/pipelines/lint_backend.star
+2
View File
@@ -46,6 +46,8 @@ def lint_backend_pipeline(trigger, ver_mode):
test_steps = [
lint_backend_step(),
validate_modfile_step(),
# OpenAPI spec validation is disabled for versions below 10.2.x.
# validate_openapi_spec_step(),
]
if ver_mode == "main":
scripts/drone/pipelines/shellcheck.star
+7 -9
View File
@@ -3,14 +3,14 @@ This module returns a Drone step and pipeline for linting with shellcheck.
"""
load("scripts/drone/steps/lib.star", "compile_build_cmd")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
trigger = {
"event": [
@@ -29,12 +29,10 @@ trigger = {
def shellcheck_step():
return {
"name": "shellcheck",
"image": images["build_image"],
"depends_on": [
"compile-build-cmd",
],
"image": images["ubuntu"],
"commands": [
"./bin/build shellcheck",
"apt-get update -yq && apt-get install shellcheck",
"shellcheck -e SC1071 -e SC2162 scripts/**/*.sh",
],
}
scripts/drone/pipelines/test_backend.star
-2
View File
@@ -4,7 +4,6 @@ This module returns the pipeline used for testing backend code.
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"enterprise_setup_step",
"identify_runner_step",
"test_backend_integration_step",
@@ -41,7 +40,6 @@ def test_backend(trigger, ver_mode):
steps += [
identify_runner_step(),
compile_build_cmd(),
verify_step,
verify_jsonnet_step,
wire_install_step(),
scripts/drone/pipelines/test_frontend.star
-2
View File
@@ -5,7 +5,6 @@ This module returns the pipeline used for testing backend code.
load(
"scripts/drone/steps/lib.star",
"betterer_frontend_step",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"test_frontend_step",
@@ -30,7 +29,6 @@ def test_frontend(trigger, ver_mode):
steps = [
identify_runner_step(),
download_grabpl_step(),
yarn_install_step(),
betterer_frontend_step(),
]
scripts/drone/pipelines/whats_new_checker.star
+5 -5
View File
@@ -2,14 +2,14 @@
This module contains logic for checking if the package.json whats new url matches with the in-flight tag.
"""
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
)
load(
"scripts/drone/utils/images.star",
"images",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
@@ -18,7 +18,7 @@ load(
def whats_new_checker_step():
return {
"name": "whats-new-checker",
"image": images["go_image"],
"image": images["go"],
"depends_on": [
"compile-build-cmd",
],
scripts/drone/pipelines/windows.star
+11 -11
View File
@@ -3,15 +3,15 @@ This module returns the pipeline used for building Grafana on Windows.
"""
load(
"scripts/drone/utils/utils.star",
"pipeline",
"scripts/drone/steps/lib_windows.star",
"clone_step_windows",
"get_windows_steps",
"test_backend_step_windows",
"wire_install_step_windows",
)
load(
"scripts/drone/steps/lib.star",
"get_windows_steps",
"windows_clone_step",
"windows_test_backend_step",
"windows_wire_install_step",
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/windows_images.star",
@@ -30,19 +30,19 @@ def windows_test_backend(trigger, edition, ver_mode):
"""
environment = {"EDITION": edition}
steps = [
windows_clone_step(),
clone_step_windows(),
]
steps.extend([{
"name": "windows-init",
"image": windows_images["windows_go_image"],
"image": windows_images["go"],
"depends_on": ["clone"],
"commands": [],
}])
steps.extend([
windows_wire_install_step(edition),
windows_test_backend_step(),
wire_install_step_windows(edition),
test_backend_step_windows(),
])
pl = pipeline(
name = "{}-test-backend-windows".format(ver_mode),