public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v10.0.x] CI: Removes enterprise specific pipelines and steps (#71782)

Browse Source 
[WIP] CI: Removes enterprise specific pipelines and steps (#70815)

* Removes enterprise specific pipelines and steps (#123)

* Comment out enterprise related pipelines and steps

* Suppress unused variable warning

* Removes all edition arguments

* Remove leftover comments

* Remove redundant oss on pipelines and steps names

* Remove leftover unused variable

* Remove leftovers

* Remove pipeline dependencies

* Rename pipelines

* Fix starlark

---------

Co-authored-by: dsotirakis <dimitrios.sotirakis@grafana.com>
(cherry picked from commit 642a81ba75e79138246797302aba5c35575f030d)

* Add editions for static assets

(cherry picked from commit b13939b9af)

Co-authored-by: Dimitris Sotirakis <dimitrios.sotirakis@grafana.com>
This commit is contained in:
Guilherme Caulada
2023-07-18 11:21:11 -03:00
committed by GitHub
co-authored by Dimitris Sotirakis
parent f877bb4f05
commit cf45b836ab
26 changed files with 178 additions and 3570 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/events/main.star
+1 -1
View File
@@ -88,7 +88,7 @@ def main_pipelines():
lint_backend_pipeline(trigger, ver_mode),
build_e2e(trigger, ver_mode),
integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode),
windows(trigger, edition = "oss", ver_mode = ver_mode),
windows(trigger, ver_mode = ver_mode),
notify_pipeline(
name = "notify-drone-changes",
slack_channel = "slack-webhooks-test",
scripts/drone/events/release.star
+27 -372
View File
@@ -10,23 +10,19 @@ load(
"build_frontend_step",
"build_plugins_step",
"build_storybook_step",
"clone_enterprise_step",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
"e2e_tests_artifacts",
"e2e_tests_step",
"fetch_images_step",
"get_windows_steps",
"grafana_server_step",
"identify_runner_step",
"init_enterprise_step",
"memcached_integration_tests_step",
"mysql_integration_tests_step",
"package_step",
"postgres_integration_tests_step",
"publish_grafanacom_step",
"publish_images_step",
"publish_linux_packages_step",
"redis_integration_tests_step",
"store_storybook_step",
@@ -46,17 +42,14 @@ load(
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
"test_frontend_enterprise",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
"test_backend_enterprise",
)
load("scripts/drone/vault.star", "from_secret", "prerelease_bucket")
load(
@@ -159,19 +152,18 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
]
build_steps = [
build_backend_step(edition = "oss", ver_mode = ver_mode),
build_frontend_step(edition = "oss", ver_mode = ver_mode),
build_frontend_package_step(edition = "oss", ver_mode = ver_mode),
build_plugins_step(edition = "oss", ver_mode = ver_mode),
package_step(edition = "oss", ver_mode = ver_mode),
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(edition = "oss", publish = True),
build_docker_images_step(publish = True),
build_docker_images_step(
edition = "oss",
publish = True,
ubuntu = True,
),
grafana_server_step(edition = "oss"),
grafana_server_step(),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
@@ -188,9 +180,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
):
publish_steps.extend(
[
upload_cdn_step(edition = "oss", ver_mode = ver_mode, trigger = trigger_oss),
upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss),
upload_packages_step(
edition = "oss",
ver_mode = ver_mode,
trigger = trigger_oss,
),
@@ -222,13 +213,12 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
volumes = []
windows_pipeline_dependencies = [
"{}-oss-build-e2e-publish".format(ver_mode),
"{}-oss-test-frontend".format(ver_mode),
"{}-build-e2e-publish".format(ver_mode),
"{}-test-frontend".format(ver_mode),
]
pipelines.extend([
pipeline(
name = "{}-oss-build-e2e-publish".format(ver_mode),
edition = "oss",
name = "{}-build-e2e-publish".format(ver_mode),
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
@@ -241,8 +231,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-oss-integration-tests".format(ver_mode),
edition = "oss",
name = "{}-integration-tests".format(ver_mode),
trigger = trigger,
services = services,
steps = [
@@ -258,10 +247,9 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
))
windows_pipeline = pipeline(
name = "{}-oss-windows".format(ver_mode),
edition = "oss",
name = "{}-windows".format(ver_mode),
trigger = trigger,
steps = get_windows_steps(edition = "oss", ver_mode = ver_mode),
steps = get_windows_steps(ver_mode = ver_mode),
platform = "windows",
depends_on = windows_pipeline_dependencies,
environment = environment,
@@ -271,287 +259,6 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
return pipelines
def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):
"""Generates all pipelines used for Grafana Enterprise.
Args:
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "${DRONE_BRANCH}"
else:
source = "${DRONE_COMMIT}"
environment = {"EDITION": "enterprise"}
services = integration_test_services()
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
compile_build_cmd("enterprise"),
] + with_deps(
[
wire_install_step(),
yarn_install_step(),
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
)
build_steps = [
build_backend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode),
build_plugins_step(edition = "enterprise", ver_mode = ver_mode),
package_step(
edition = "enterprise",
ver_mode = ver_mode,
),
copy_packages_for_docker_step(),
build_docker_images_step(edition = "enterprise", publish = True),
build_docker_images_step(
edition = "enterprise",
publish = True,
ubuntu = True,
),
grafana_server_step(edition = "enterprise"),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
e2e_tests_step("various-suite", tries = 3),
e2e_tests_artifacts(),
]
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
upload_packages_enterprise = upload_packages_step(
edition = "enterprise",
ver_mode = ver_mode,
trigger = trigger_oss,
)
upload_packages_enterprise["depends_on"] = ["package"]
publish_steps.extend(
[
upload_cdn_step(
edition = "enterprise",
ver_mode = ver_mode,
trigger = trigger_oss,
),
upload_packages_enterprise,
],
)
integration_test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step(),
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
# We don't need to run integration tests at release time since they have
# been run multiple times before:
if ver_mode in ("release"):
integration_test_steps = []
volumes = []
windows_pipeline = pipeline(
name = "{}-enterprise-windows".format(ver_mode),
edition = "enterprise",
trigger = trigger,
steps = get_windows_steps(edition = "enterprise", ver_mode = ver_mode),
platform = "windows",
depends_on = [
"{}-enterprise-build-e2e-publish".format(ver_mode),
"{}-enterprise-test-frontend".format(ver_mode),
"{}-enterprise-test-backend".format(ver_mode),
],
environment = environment,
)
pipelines = [
pipeline(
name = "{}-enterprise-build-e2e-publish".format(ver_mode),
edition = "enterprise",
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
environment = environment,
volumes = volumes,
),
test_frontend_enterprise(trigger, ver_mode, source = source),
test_backend_enterprise(trigger, ver_mode, source = source),
]
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-enterprise-integration-tests".format(ver_mode),
edition = "enterprise",
trigger = trigger,
services = services,
steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
] +
integration_test_steps,
environment = environment,
volumes = volumes,
))
pipelines.append(windows_pipeline)
return pipelines
def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):
"""Generate the next generation of pipelines for Grafana Enterprise.
Args:
prefix: a prefix for the pipeline name used to differentiate multiple instances of
the same pipeline.
Defaults to ''.
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "${DRONE_BRANCH}"
else:
source = "${DRONE_COMMIT}"
environment = {
"EDITION": "enterprise2",
}
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
compile_build_cmd("enterprise"),
] + with_deps(
[
wire_install_step(),
yarn_install_step(),
verify_gen_cue_step(),
],
[
"init-enterprise",
],
)
build_steps = [
build_frontend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode),
build_plugins_step(edition = "enterprise", ver_mode = ver_mode),
build_backend_step(
edition = "enterprise2",
ver_mode = ver_mode,
variants = ["linux-amd64"],
),
]
fetch_images = fetch_images_step("enterprise2")
fetch_images.update(
{"depends_on": ["build-docker-images", "build-docker-images-ubuntu"]},
)
upload_cdn = upload_cdn_step(edition = "enterprise2", ver_mode = ver_mode)
upload_cdn["environment"].update(
{"ENTERPRISE2_CDN_PATH": from_secret("enterprise2-cdn-path")},
)
build_steps.extend(
[
package_step(
edition = "enterprise2",
ver_mode = ver_mode,
),
upload_cdn,
copy_packages_for_docker_step(edition = "enterprise2"),
build_docker_images_step(
edition = "enterprise2",
publish = True,
),
build_docker_images_step(
edition = "enterprise2",
publish = True,
ubuntu = True,
),
fetch_images,
publish_images_step(
"enterprise2",
"release",
docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}",
),
],
)
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
step = upload_packages_step(edition = "enterprise2", ver_mode = ver_mode)
step["depends_on"] = ["package-enterprise2"]
publish_steps.append(step)
pipelines = [
pipeline(
name = "{}{}-enterprise2-build-e2e-publish".format(prefix, ver_mode),
edition = "enterprise",
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
volumes = volumes,
environment = environment,
),
]
return pipelines
def publish_artifacts_step():
return {
"name": "publish-artifacts",
@@ -576,7 +283,7 @@ def publish_static_assets_step():
"STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"),
},
"commands": [
"./bin/build artifacts static-assets --tag ${DRONE_TAG}",
"./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss",
],
"depends_on": ["compile-build-cmd"],
}
@@ -621,13 +328,12 @@ def publish_artifacts_pipelines(mode):
name = "publish-artifacts-{}".format(mode),
trigger = trigger,
steps = steps,
edition = "all",
environment = {"EDITION": "all"},
environment = {"EDITION": "oss"},
),
]
def publish_packages_pipeline():
"""Generates pipelines used for publishing packages for both OSS and enterprise.
"""Generates pipelines used for publishing packages for OSS.
Returns:
List of Drone pipelines. One for each of OSS and enterprise packages.
@@ -639,40 +345,24 @@ def publish_packages_pipeline():
}
oss_steps = [
compile_build_cmd(),
publish_linux_packages_step(edition = "oss", package_manager = "deb"),
publish_linux_packages_step(edition = "oss", package_manager = "rpm"),
publish_grafanacom_step(edition = "oss", ver_mode = "release"),
publish_linux_packages_step(package_manager = "deb"),
publish_linux_packages_step(package_manager = "rpm"),
publish_grafanacom_step(ver_mode = "release"),
]
enterprise_steps = [
compile_build_cmd(),
publish_linux_packages_step(edition = "enterprise", package_manager = "deb"),
publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"),
publish_grafanacom_step(edition = "enterprise", ver_mode = "release"),
]
deps = [
"publish-artifacts-public",
"publish-docker-oss-public",
"publish-docker-enterprise-public",
"publish-docker-public",
]
return [
pipeline(
name = "publish-packages-oss",
name = "publish-packages",
trigger = trigger,
steps = oss_steps,
edition = "all",
depends_on = deps,
environment = {"EDITION": "oss"},
),
pipeline(
name = "publish-packages-enterprise",
trigger = trigger,
steps = enterprise_steps,
edition = "all",
depends_on = deps,
environment = {"EDITION": "enterprise"},
),
]
def publish_npm_pipelines():
@@ -692,8 +382,7 @@ def publish_npm_pipelines():
name = "publish-npm-packages-public",
trigger = trigger,
steps = steps,
edition = "all",
environment = {"EDITION": "all"},
environment = {"EDITION": "oss"},
),
]
@@ -706,7 +395,7 @@ def integration_test_pipelines():
particular build.
Returns:
List of Drone pipelines (one for enterprise and one for oss integration tests)
List of Drone pipelines
"""
trigger = {
"event": ["promote"],
@@ -720,11 +409,9 @@ def integration_test_pipelines():
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
source = "${DRONE_TAG}"
pipelines.append(pipeline(
name = "integration-tests-oss",
edition = "oss",
name = "integration-tests",
trigger = trigger,
services = integration_test_services(),
steps = [
@@ -739,34 +426,6 @@ def integration_test_pipelines():
volumes = volumes,
))
pipelines.append(pipeline(
name = "integration-tests-enterprise",
edition = "enterprise",
trigger = trigger,
services = integration_test_services(),
steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(source = source),
init_enterprise_step(ver_mode),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
] +
integration_test_steps,
environment = {"EDITION": "enterprise"},
volumes = volumes,
))
return pipelines
def verify_release_pipeline(
@@ -776,11 +435,8 @@ def verify_release_pipeline(
version = "${DRONE_TAG}",
trigger = release_trigger,
depends_on = [
"release-oss-build-e2e-publish",
"release-enterprise-build-e2e-publish",
"release-enterprise2-build-e2e-publish",
"release-oss-windows",
"release-enterprise-windows",
"release-build-e2e-publish",
"release-windows",
]):
"""
Runs a script that 'gsutil stat's every artifact that should have been produced by the pre-release process.
@@ -806,7 +462,6 @@ def verify_release_pipeline(
return pipeline(
depends_on = depends_on,
name = name,
edition = "all",
trigger = trigger,
steps = [step],
)