public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v9.4.x] CI: Removes enterprise specific pipelines and steps (#71775)

Browse Source 
[WIP] CI: Removes enterprise specific pipelines and steps (#70815)

* Removes enterprise specific pipelines and steps (#123)

* Comment out enterprise related pipelines and steps

* Suppress unused variable warning

* Removes all edition arguments

* Remove leftover comments

* Remove redundant oss on pipelines and steps names

* Remove leftover unused variable

* Remove leftovers

* Remove pipeline dependencies

* Rename pipelines

* Fix starlark

---------

Co-authored-by: dsotirakis <dimitrios.sotirakis@grafana.com>
(cherry picked from commit 642a81ba75e79138246797302aba5c35575f030d)

* Add editions for static assets

(cherry picked from commit b13939b9af)

Co-authored-by: Dimitris Sotirakis <dimitrios.sotirakis@grafana.com>
This commit is contained in:
Guilherme Caulada
2023-07-18 11:17:11 -03:00
committed by GitHub
co-authored by Dimitris Sotirakis
parent c465fa856e
commit 20f001649e
27 changed files with 362 additions and 3527 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/events/main.star
+2 -2
View File
@@ -87,8 +87,8 @@ def main_pipelines():
test_backend(trigger, ver_mode),
lint_backend_pipeline(trigger, ver_mode),
build_e2e(trigger, ver_mode),
integration_tests(trigger, prefix = ver_mode),
windows(trigger, edition = "oss", ver_mode = ver_mode),
integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode),
windows(trigger, ver_mode = ver_mode),
notify_pipeline(
name = "notify-drone-changes",
slack_channel = "slack-webhooks-test",
scripts/drone/events/pr.star
+1
View File
@@ -130,6 +130,7 @@ def pr_pipelines():
],
),
prefix = ver_mode,
ver_mode = ver_mode,
),
docs_pipelines(ver_mode, trigger_docs_pr()),
shellcheck_pipeline(),
scripts/drone/events/release.star
+30 -380
View File
@@ -10,25 +10,19 @@ load(
"build_frontend_step",
"build_plugins_step",
"build_storybook_step",
"clone_enterprise_step",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
"e2e_tests_artifacts",
"e2e_tests_step",
"fetch_images_step",
"get_windows_steps",
"grafana_server_step",
"identify_runner_step",
"init_enterprise_step",
"memcached_integration_tests_step",
"mysql_integration_tests_step",
"package_step",
"postgres_integration_tests_step",
"publish_grafanacom_step",
"publish_images_step",
"publish_linux_packages_step",
"redis_integration_tests_step",
"store_storybook_step",
"trigger_oss",
"upload_cdn_step",
@@ -46,17 +40,14 @@ load(
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
load(
"scripts/drone/pipelines/test_frontend.star",
"test_frontend",
"test_frontend_enterprise",
)
load(
"scripts/drone/pipelines/test_backend.star",
"test_backend",
"test_backend_enterprise",
)
load("scripts/drone/vault.star", "from_secret", "prerelease_bucket")
load(
@@ -158,19 +149,18 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
]
build_steps = [
build_backend_step(edition = "oss", ver_mode = ver_mode),
build_frontend_step(edition = "oss", ver_mode = ver_mode),
build_frontend_package_step(edition = "oss", ver_mode = ver_mode),
build_plugins_step(edition = "oss", ver_mode = ver_mode),
package_step(edition = "oss", ver_mode = ver_mode),
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(edition = "oss", publish = True),
build_docker_images_step(publish = True),
build_docker_images_step(
edition = "oss",
publish = True,
ubuntu = True,
),
grafana_server_step(edition = "oss"),
grafana_server_step(),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
@@ -187,9 +177,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
):
publish_steps.extend(
[
upload_cdn_step(edition = "oss", ver_mode = ver_mode, trigger = trigger_oss),
upload_cdn_step(ver_mode = ver_mode, trigger = trigger_oss),
upload_packages_step(
edition = "oss",
ver_mode = ver_mode,
trigger = trigger_oss,
),
@@ -218,23 +207,13 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
integration_test_steps = []
volumes = []
windows_pipeline = pipeline(
name = "{}-oss-windows".format(ver_mode),
edition = "oss",
trigger = trigger,
steps = get_windows_steps(edition = "oss", ver_mode = ver_mode),
platform = "windows",
depends_on = [
"{}-oss-build-e2e-publish".format(ver_mode),
"{}-oss-test-frontend".format(ver_mode),
],
environment = environment,
)
windows_pipeline_dependencies = [
"{}-build-e2e-publish".format(ver_mode),
"{}-test-frontend".format(ver_mode),
]
pipelines.extend([
pipeline(
name = "{}-oss-build-e2e-publish".format(ver_mode),
edition = "oss",
name = "{}-build-e2e-publish".format(ver_mode),
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
@@ -247,8 +226,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-oss-integration-tests".format(ver_mode),
edition = "oss",
name = "{}-integration-tests".format(ver_mode),
trigger = trigger,
services = services,
steps = [
@@ -263,291 +241,19 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
volumes = volumes,
))
pipelines.append(windows_pipeline)
return pipelines
def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):
"""Generates all pipelines used for Grafana Enterprise.
Args:
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
if ver_mode == "release":
committish = "${DRONE_TAG}"
elif ver_mode == "release-branch":
committish = "${DRONE_BRANCH}"
else:
committish = "${DRONE_COMMIT}"
environment = {"EDITION": "enterprise"}
services = integration_test_services(edition = "enterprise")
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(committish = committish),
init_enterprise_step(ver_mode),
compile_build_cmd("enterprise"),
] + with_deps(
[
wire_install_step(),
yarn_install_step(),
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
)
build_steps = [
build_backend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode),
build_plugins_step(edition = "enterprise", ver_mode = ver_mode),
package_step(
edition = "enterprise",
ver_mode = ver_mode,
),
copy_packages_for_docker_step(),
build_docker_images_step(edition = "enterprise", publish = True),
build_docker_images_step(
edition = "enterprise",
publish = True,
ubuntu = True,
),
grafana_server_step(edition = "enterprise"),
e2e_tests_step("dashboards-suite", tries = 3),
e2e_tests_step("smoke-tests-suite", tries = 3),
e2e_tests_step("panels-suite", tries = 3),
e2e_tests_step("various-suite", tries = 3),
e2e_tests_artifacts(),
]
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
upload_packages_enterprise = upload_packages_step(
edition = "enterprise",
ver_mode = ver_mode,
trigger = trigger_oss,
)
upload_packages_enterprise["depends_on"] = ["package"]
publish_steps.extend(
[
upload_cdn_step(
edition = "enterprise",
ver_mode = ver_mode,
trigger = trigger_oss,
),
upload_packages_enterprise,
],
)
integration_test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step(),
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
# We don't need to run integration tests at release time since they have
# been run multiple times before:
if ver_mode in ("release"):
integration_test_steps = []
volumes = []
windows_pipeline = pipeline(
name = "{}-enterprise-windows".format(ver_mode),
edition = "enterprise",
name = "{}-windows".format(ver_mode),
trigger = trigger,
steps = get_windows_steps(edition = "enterprise", ver_mode = ver_mode),
steps = get_windows_steps(ver_mode = ver_mode),
platform = "windows",
depends_on = [
"{}-enterprise-build-e2e-publish".format(ver_mode),
"{}-enterprise-test-frontend".format(ver_mode),
"{}-enterprise-test-backend".format(ver_mode),
],
depends_on = windows_pipeline_dependencies,
environment = environment,
)
pipelines = [
pipeline(
name = "{}-enterprise-build-e2e-publish".format(ver_mode),
edition = "enterprise",
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
environment = environment,
volumes = volumes,
),
test_frontend_enterprise(trigger, ver_mode, committish = committish),
test_backend_enterprise(trigger, ver_mode, committish = committish),
]
if ver_mode not in ("release"):
pipelines.append(pipeline(
name = "{}-enterprise-integration-tests".format(ver_mode),
edition = "enterprise",
trigger = trigger,
services = services,
steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(committish = committish),
init_enterprise_step(ver_mode),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
] +
integration_test_steps,
environment = environment,
volumes = volumes,
))
pipelines.append(windows_pipeline)
return pipelines
def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_trigger):
"""Generate the next generation of pipelines for Grafana Enterprise.
Args:
prefix: a prefix for the pipeline name used to differentiate multiple instances of
the same pipeline.
Defaults to ''.
ver_mode: controls which steps are included in the pipeline.
Defaults to 'release'.
trigger: controls which events can trigger the pipeline execution.
Defaults to tag events for tags with a 'v' prefix.
Returns:
List of Drone pipelines.
"""
if ver_mode == "release":
committish = "${DRONE_TAG}"
elif ver_mode == "release-branch":
committish = "${DRONE_BRANCH}"
else:
committish = "${DRONE_COMMIT}"
environment = {
"EDITION": "enterprise2",
}
volumes = integration_test_services_volumes()
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(committish = committish),
init_enterprise_step(ver_mode),
compile_build_cmd("enterprise"),
] + with_deps(
[
wire_install_step(),
yarn_install_step(),
verify_gen_cue_step(),
],
[
"init-enterprise",
],
)
build_steps = [
build_frontend_step(edition = "enterprise", ver_mode = ver_mode),
build_frontend_package_step(edition = "enterprise", ver_mode = ver_mode),
build_plugins_step(edition = "enterprise", ver_mode = ver_mode),
build_backend_step(
edition = "enterprise2",
ver_mode = ver_mode,
variants = ["linux-amd64"],
),
]
fetch_images = fetch_images_step("enterprise2")
fetch_images.update(
{"depends_on": ["build-docker-images", "build-docker-images-ubuntu"]},
)
upload_cdn = upload_cdn_step(edition = "enterprise2", ver_mode = ver_mode)
upload_cdn["environment"].update(
{"ENTERPRISE2_CDN_PATH": from_secret("enterprise2-cdn-path")},
)
build_steps.extend(
[
package_step(
edition = "enterprise2",
ver_mode = ver_mode,
),
upload_cdn,
copy_packages_for_docker_step(edition = "enterprise2"),
build_docker_images_step(
edition = "enterprise2",
publish = True,
),
build_docker_images_step(
edition = "enterprise2",
publish = True,
ubuntu = True,
),
fetch_images,
publish_images_step(
"enterprise2",
"release",
docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}",
),
],
)
publish_steps = []
if ver_mode in (
"release",
"release-branch",
):
step = upload_packages_step(edition = "enterprise2", ver_mode = ver_mode)
step["depends_on"] = ["package-enterprise2"]
publish_steps.append(step)
pipelines = [
pipeline(
name = "{}{}-enterprise2-build-e2e-publish".format(prefix, ver_mode),
edition = "enterprise",
trigger = trigger,
services = [],
steps = init_steps + build_steps + publish_steps,
volumes = volumes,
environment = environment,
),
]
return pipelines
def publish_artifacts_step():
return {
"name": "publish-artifacts",
@@ -572,7 +278,7 @@ def publish_static_assets_step():
"STATIC_ASSET_EDITIONS": from_secret("static_asset_editions"),
},
"commands": [
"./bin/build artifacts static-assets --tag ${DRONE_TAG}",
"./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss",
],
"depends_on": ["compile-build-cmd"],
}
@@ -617,13 +323,12 @@ def publish_artifacts_pipelines(mode):
name = "publish-artifacts-{}".format(mode),
trigger = trigger,
steps = steps,
edition = "all",
environment = {"EDITION": "all"},
environment = {"EDITION": "oss"},
),
]
def publish_packages_pipeline():
"""Generates pipelines used for publishing packages for both OSS and enterprise.
"""Generates pipelines used for publishing packages for OSS.
Returns:
List of Drone pipelines. One for each of OSS and enterprise packages.
@@ -635,40 +340,24 @@ def publish_packages_pipeline():
}
oss_steps = [
compile_build_cmd(),
publish_linux_packages_step(edition = "oss", package_manager = "deb"),
publish_linux_packages_step(edition = "oss", package_manager = "rpm"),
publish_grafanacom_step(edition = "oss", ver_mode = "release"),
publish_linux_packages_step(package_manager = "deb"),
publish_linux_packages_step(package_manager = "rpm"),
publish_grafanacom_step(ver_mode = "release"),
]
enterprise_steps = [
compile_build_cmd(),
publish_linux_packages_step(edition = "enterprise", package_manager = "deb"),
publish_linux_packages_step(edition = "enterprise", package_manager = "rpm"),
publish_grafanacom_step(edition = "enterprise", ver_mode = "release"),
]
deps = [
"publish-artifacts-public",
"publish-docker-oss-public",
"publish-docker-enterprise-public",
"publish-docker-public",
]
return [
pipeline(
name = "publish-packages-oss",
name = "publish-packages",
trigger = trigger,
steps = oss_steps,
edition = "all",
depends_on = deps,
environment = {"EDITION": "oss"},
),
pipeline(
name = "publish-packages-enterprise",
trigger = trigger,
steps = enterprise_steps,
edition = "all",
depends_on = deps,
environment = {"EDITION": "enterprise"},
),
]
def publish_npm_pipelines():
@@ -688,8 +377,7 @@ def publish_npm_pipelines():
name = "publish-npm-packages-public",
trigger = trigger,
steps = steps,
edition = "all",
environment = {"EDITION": "all"},
environment = {"EDITION": "oss"},
),
]
@@ -702,7 +390,7 @@ def integration_test_pipelines():
particular build.
Returns:
List of Drone pipelines (one for enterprise and one for oss integration tests)
List of Drone pipelines
"""
trigger = {
"event": ["promote"],
@@ -714,15 +402,9 @@ def integration_test_pipelines():
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
enterprise_integration_test_steps = oss_integration_test_steps + [
redis_integration_tests_step(),
memcached_integration_tests_step(),
]
source = "${DRONE_TAG}"
pipelines.append(pipeline(
name = "integration-tests-oss",
edition = "oss",
name = "integration-tests",
trigger = trigger,
services = integration_test_services(edition = "oss"),
steps = [
@@ -737,34 +419,6 @@ def integration_test_pipelines():
volumes = volumes,
))
pipelines.append(pipeline(
name = "integration-tests-enterprise",
edition = "enterprise",
trigger = trigger,
services = integration_test_services(edition = "enterprise"),
steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(committish = source),
init_enterprise_step(ver_mode),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
] +
enterprise_integration_test_steps,
environment = {"EDITION": "enterprise"},
volumes = volumes,
))
return pipelines
def verify_release_pipeline(
@@ -774,11 +428,8 @@ def verify_release_pipeline(
version = "${DRONE_TAG}",
trigger = release_trigger,
depends_on = [
"release-oss-build-e2e-publish",
"release-enterprise-build-e2e-publish",
"release-enterprise2-build-e2e-publish",
"release-oss-windows",
"release-enterprise-windows",
"release-build-e2e-publish",
"release-windows",
]):
"""
Runs a script that 'gsutil stat's every artifact that should have been produced by the pre-release process.
@@ -804,7 +455,6 @@ def verify_release_pipeline(
return pipeline(
depends_on = depends_on,
name = name,
edition = "all",
trigger = trigger,
steps = [step],
)
scripts/drone/pipelines/aws_marketplace.star
-45
View File
@@ -1,45 +0,0 @@
"""
This module contains steps and pipelines publishing to AWS Marketplace.
"""
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_aws_marketplace_step():
return {
"name": "publish-aws-marketplace",
"image": images["publish_image"],
"commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"],
"depends_on": ["compile-build-cmd"],
"environment": {
"AWS_REGION": from_secret("aws_region"),
"AWS_ACCESS_KEY_ID": from_secret("aws_access_key_id"),
"AWS_SECRET_ACCESS_KEY": from_secret("aws_secret_access_key"),
},
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
def publish_aws_marketplace_pipeline(mode):
trigger = {
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name = "publish-aws-marketplace-{}".format(mode),
trigger = trigger,
steps = [compile_build_cmd(), publish_aws_marketplace_step()],
edition = "",
depends_on = ["publish-docker-enterprise-public"],
environment = {"EDITION": "enterprise2"},
)]
scripts/drone/pipelines/build.star
+8 -18
View File
@@ -48,8 +48,8 @@ def build_e2e(trigger, ver_mode):
Returns:
Drone pipeline.
"""
edition = "oss"
environment = {"EDITION": edition}
environment = {"EDITION": "oss"}
init_steps = [
identify_runner_step(),
download_grabpl_step(),
@@ -72,12 +72,12 @@ def build_e2e(trigger, ver_mode):
build_steps.extend(
[
build_backend_step(edition = edition, ver_mode = ver_mode),
build_frontend_step(edition = edition, ver_mode = ver_mode),
build_frontend_package_step(edition = edition, ver_mode = ver_mode),
build_plugins_step(edition = edition, ver_mode = ver_mode),
package_step(edition = edition, ver_mode = ver_mode),
grafana_server_step(edition = edition),
build_backend_step(ver_mode = ver_mode),
build_frontend_step(ver_mode = ver_mode),
build_frontend_package_step(ver_mode = ver_mode),
build_plugins_step(ver_mode = ver_mode),
package_step(ver_mode = ver_mode),
grafana_server_step(),
e2e_tests_step("dashboards-suite"),
e2e_tests_step("smoke-tests-suite"),
e2e_tests_step("panels-suite"),
@@ -100,34 +100,28 @@ def build_e2e(trigger, ver_mode):
store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
frontend_metrics_step(trigger = trigger_oss),
build_docker_images_step(
edition = edition,
publish = False,
),
build_docker_images_step(
edition = edition,
publish = False,
ubuntu = True,
),
publish_images_step(
docker_repo = "grafana",
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
publish_images_step(
docker_repo = "grafana-oss",
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
release_canary_npm_packages_step(trigger = trigger_oss),
upload_packages_step(
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
upload_cdn_step(
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
@@ -140,18 +134,15 @@ def build_e2e(trigger, ver_mode):
archs = [
"amd64",
],
edition = edition,
),
build_docker_images_step(
archs = [
"amd64",
],
edition = edition,
ubuntu = True,
),
publish_images_step(
docker_repo = "grafana",
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
@@ -164,7 +155,6 @@ def build_e2e(trigger, ver_mode):
return pipeline(
name = "{}-build-e2e{}".format(ver_mode, publish_suffix),
edition = "oss",
environment = environment,
services = [],
steps = init_steps + build_steps,
scripts/drone/pipelines/ci_images.star
-2
View File
@@ -28,7 +28,6 @@ def publish_ci_windows_test_image_pipeline():
pl = pipeline(
name = "publish-ci-windows-test-image",
trigger = trigger,
edition = "",
platform = "windows",
steps = [
{
@@ -79,7 +78,6 @@ def publish_ci_build_container_image_pipeline():
pl = pipeline(
name = "publish-ci-build-container-image",
trigger = trigger,
edition = "",
steps = [
{
"name": "validate-version",
scripts/drone/pipelines/docs.star
-1
View File
@@ -39,7 +39,6 @@ def docs_pipelines(ver_mode, trigger):
return pipeline(
name = "{}-docs".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
scripts/drone/pipelines/github.star
-43
View File
@@ -1,43 +0,0 @@
"""
This module contains steps and pipelines relating to GitHub.
"""
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"fetch_images_step",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/utils/images.star",
"images",
)
def publish_github_step():
return {
"name": "publish-github",
"image": images["publish_image"],
"commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"],
"depends_on": ["fetch-images-enterprise2"],
"environment": {
"GH_TOKEN": from_secret("github_token"),
"GH_REGISTRY": from_secret("gh_registry"),
},
}
def publish_github_pipeline(mode):
trigger = {
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name = "publish-github-{}".format(mode),
trigger = trigger,
steps = [compile_build_cmd(), fetch_images_step("enterprise2"), publish_github_step()],
edition = "",
environment = {"EDITION": "enterprise2"},
)]
scripts/drone/pipelines/integration_tests.star
+10 -3
View File
@@ -6,6 +6,7 @@ load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"mysql_integration_tests_step",
"postgres_integration_tests_step",
@@ -23,12 +24,13 @@ load(
"pipeline",
)
def integration_tests(trigger, prefix):
def integration_tests(trigger, prefix, ver_mode):
"""Generate a pipeline for integration tests.
Args:
trigger: controls which events can trigger the pipeline execution.
prefix: used in the naming of the pipeline.
ver_mode: defines the event / origin of this build. In this function, if it is set to pr, then it will attempt to clone grafana-enterprise. Otherwise it has no effect.
Returns:
Drone pipeline.
@@ -38,7 +40,13 @@ def integration_tests(trigger, prefix):
services = integration_test_services(edition = "oss")
volumes = integration_test_services_volumes()
init_steps = [
init_steps = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(enterprise_setup_step())
init_steps += [
download_grabpl_step(),
compile_build_cmd(),
identify_runner_step(),
@@ -54,7 +62,6 @@ def integration_tests(trigger, prefix):
return pipeline(
name = "{}-integration-tests".format(prefix),
edition = "oss",
trigger = trigger,
environment = environment,
services = services,
scripts/drone/pipelines/lint_backend.star
+7 -2
View File
@@ -5,6 +5,7 @@ This module returns the pipeline used for linting backend code.
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"enterprise_setup_step",
"identify_runner_step",
"lint_backend_step",
"lint_drone_step",
@@ -33,9 +34,14 @@ def lint_backend_pipeline(trigger, ver_mode):
init_steps = [
identify_runner_step(),
compile_build_cmd(),
wire_step,
]
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(enterprise_setup_step())
init_steps.append(wire_step)
test_steps = [
lint_backend_step(),
]
@@ -45,7 +51,6 @@ def lint_backend_pipeline(trigger, ver_mode):
return pipeline(
name = "{}-lint-backend".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = init_steps + test_steps,
scripts/drone/pipelines/lint_frontend.star
+8 -2
View File
@@ -4,6 +4,7 @@ This module returns the pipeline used for linting frontend code.
load(
"scripts/drone/steps/lib.star",
"enterprise_setup_step",
"identify_runner_step",
"lint_frontend_step",
"yarn_install_step",
@@ -25,7 +26,13 @@ def lint_frontend_pipeline(trigger, ver_mode):
"""
environment = {"EDITION": "oss"}
init_steps = [
init_steps = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps = [enterprise_setup_step()]
init_steps += [
identify_runner_step(),
yarn_install_step(),
]
@@ -36,7 +43,6 @@ def lint_frontend_pipeline(trigger, ver_mode):
return pipeline(
name = "{}-lint-frontend".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = init_steps + test_steps,
scripts/drone/pipelines/publish_images.star
+6 -24
View File
@@ -15,13 +15,10 @@ load(
"pipeline",
)
def publish_image_steps(edition, docker_repo):
def publish_image_steps(docker_repo):
"""Generates the steps used for publising Docker images using grabpl.
Args:
edition: controls which version of an image is fetched in the case of a release.
It also controls which publishing implementation is used.
If edition == 'oss', it additionally publishes the grafana/grafana-oss repository.
docker_repo: the Docker image name.
It is combined with the 'grafana/' library prefix.
@@ -32,15 +29,11 @@ def publish_image_steps(edition, docker_repo):
identify_runner_step(),
download_grabpl_step(),
compile_build_cmd(),
fetch_images_step(edition),
publish_images_step(edition, "release", docker_repo),
fetch_images_step(),
publish_images_step("release", docker_repo),
publish_images_step("release", "grafana-oss"),
]
if edition == "oss":
steps.append(
publish_images_step(edition, "release", "grafana-oss"),
)
return steps
def publish_image_pipelines_public():
@@ -56,20 +49,9 @@ def publish_image_pipelines_public():
}
return [
pipeline(
name = "publish-docker-oss-{}".format(mode),
name = "publish-docker-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(edition = "oss", docker_repo = "grafana"),
edition = "",
steps = publish_image_steps(docker_repo = "grafana"),
environment = {"EDITION": "oss"},
),
pipeline(
name = "publish-docker-enterprise-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(
edition = "enterprise",
docker_repo = "grafana-enterprise",
),
edition = "",
environment = {"EDITION": "enterprise"},
),
]
scripts/drone/pipelines/shellcheck.star
-1
View File
@@ -46,7 +46,6 @@ def shellcheck_pipeline():
]
return pipeline(
name = "pr-shellcheck",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
scripts/drone/pipelines/test_backend.star
+10 -62
View File
@@ -5,15 +5,12 @@ This module returns the pipeline used for testing backend code.
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
load(
"scripts/drone/steps/lib.star",
"clone_enterprise_step",
"compile_build_cmd",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"init_enterprise_step",
"test_backend_integration_step",
"test_backend_step",
"verify_gen_cue_step",
@@ -33,9 +30,15 @@ def test_backend(trigger, ver_mode):
"""
environment = {"EDITION": "oss"}
steps = [
steps = []
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(enterprise_setup_step())
steps += [
identify_runner_step(),
compile_build_cmd(edition = "oss"),
compile_build_cmd(),
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
wire_install_step(),
@@ -43,63 +46,8 @@ def test_backend(trigger, ver_mode):
test_backend_integration_step(),
]
pipeline_name = "{}-test-backend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-backend".format(ver_mode, "oss")
return pipeline(
name = pipeline_name,
edition = "oss",
trigger = trigger,
steps = steps,
environment = environment,
)
def test_backend_enterprise(trigger, ver_mode, committish, edition = "enterprise"):
"""Generates the pipeline used for testing backend enterprise code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
committish: controls what revision of enterprise code to test with.
edition: affects the clone step in the pipeline and also affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
steps = (
[
clone_enterprise_step(committish),
download_grabpl_step(),
init_enterprise_step(ver_mode),
identify_runner_step(),
compile_build_cmd(edition),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
test_backend_step(),
test_backend_integration_step(),
]
)
pipeline_name = "{}-test-backend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-backend".format(ver_mode, edition)
return pipeline(
name = pipeline_name,
edition = edition,
name = "{}-test-backend".format(ver_mode),
trigger = trigger,
steps = steps,
environment = environment,
scripts/drone/pipelines/test_frontend.star
+10 -50
View File
@@ -5,15 +5,13 @@ This module returns the pipeline used for testing backend code.
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
load(
"scripts/drone/steps/lib.star",
"betterer_frontend_step",
"clone_enterprise_step",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"init_enterprise_step",
"test_frontend_step",
"yarn_install_step",
)
@@ -34,57 +32,19 @@ def test_frontend(trigger, ver_mode):
identify_runner_step(),
download_grabpl_step(),
yarn_install_step(),
betterer_frontend_step(edition = "oss"),
test_frontend_step(edition = "oss"),
betterer_frontend_step(),
]
pipeline_name = "{}-test-frontend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-frontend".format(ver_mode, "oss")
test_step = test_frontend_step()
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(enterprise_setup_step())
steps.append(test_step)
return pipeline(
name = pipeline_name,
edition = "oss",
trigger = trigger,
steps = steps,
environment = environment,
)
def test_frontend_enterprise(trigger, ver_mode, committish, edition = "enterprise"):
"""Generates the pipeline used for testing frontend enterprise code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
committish: controls what revision of enterprise code to test with.
edition: affects the clone step in the pipeline and also affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
steps = (
[
download_grabpl_step(),
clone_enterprise_step(committish),
init_enterprise_step(ver_mode),
identify_runner_step(),
] +
with_deps([yarn_install_step()], ["init-enterprise"]) +
[
betterer_frontend_step(edition),
test_frontend_step(edition),
]
)
pipeline_name = "{}-test-frontend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-frontend".format(ver_mode, edition)
return pipeline(
name = pipeline_name,
edition = edition,
name = "{}-test-frontend".format(ver_mode),
trigger = trigger,
steps = steps,
environment = environment,
scripts/drone/pipelines/trigger_downstream.star
-1
View File
@@ -36,7 +36,6 @@ def enterprise_downstream_pipeline():
]
return pipeline(
name = "main-trigger-downstream",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
scripts/drone/pipelines/verify_drone.star
-1
View File
@@ -22,7 +22,6 @@ def verify_drone(trigger, ver_mode):
]
return pipeline(
name = "{}-verify-drone".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
scripts/drone/pipelines/verify_starlark.star
-1
View File
@@ -22,7 +22,6 @@ def verify_starlark(trigger, ver_mode):
]
return pipeline(
name = "{}-verify-starlark".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
scripts/drone/pipelines/whats_new_checker.star
-1
View File
@@ -35,7 +35,6 @@ def whats_new_checker_pipeline(trigger):
]
return pipeline(
name = "release-whatsnew-checker",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
scripts/drone/pipelines/windows.star
+10 -17
View File
@@ -10,7 +10,6 @@ load(
"scripts/drone/steps/lib.star",
"get_windows_steps",
"windows_clone_step",
"windows_init_enterprise_steps",
"windows_test_backend_step",
"windows_wire_install_step",
)
@@ -34,23 +33,19 @@ def windows_test_backend(trigger, edition, ver_mode):
windows_clone_step(),
]
if edition == "enterprise":
steps.extend(windows_init_enterprise_steps(ver_mode))
else:
steps.extend([{
"name": "windows-init",
"image": windows_images["windows_go_image"],
"depends_on": ["clone"],
"commands": [],
}])
steps.extend([{
"name": "windows-init",
"image": windows_images["windows_go_image"],
"depends_on": ["clone"],
"commands": [],
}])
steps.extend([
windows_wire_install_step(edition),
windows_test_backend_step(),
])
pl = pipeline(
name = "{}-{}-test-backend-windows".format(ver_mode, edition),
edition = edition,
name = "{}-test-backend-windows".format(ver_mode),
trigger = trigger,
steps = steps,
depends_on = [],
@@ -62,25 +57,23 @@ def windows_test_backend(trigger, edition, ver_mode):
}
return pl
def windows(trigger, edition, ver_mode):
def windows(trigger, ver_mode):
"""Generates the pipeline used for building Grafana on Windows.
Args:
trigger: a Drone trigger for the pipeline.
edition: controls whether enterprise code is included in the pipeline steps.
ver_mode: controls whether a pre-release or actual release pipeline is generated.
Also indirectly controls which version of enterprise code is used.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
environment = {"EDITION": "oss"}
return pipeline(
name = "main-windows",
edition = edition,
trigger = dict(trigger, repo = ["grafana/grafana"]),
steps = get_windows_steps(edition, ver_mode),
steps = get_windows_steps(ver_mode),
depends_on = [
"main-test-frontend",
"main-test-backend",
scripts/drone/rgm.star
-2
View File
@@ -70,7 +70,6 @@ def rgm_main():
return pipeline(
name = "rgm-main-prerelease",
edition = "all",
trigger = trigger,
steps = rgm_build(),
depends_on = ["main-test-backend", "main-test-frontend"],
@@ -95,7 +94,6 @@ tag_trigger = {
def rgm_tag():
return pipeline(
name = "rgm-tag-prerelease",
edition = "all",
trigger = tag_trigger,
steps = rgm_build(script = "drone_publish_tag.sh"),
depends_on = [],
scripts/drone/steps/lib.star
+122 -317
View File
@@ -88,130 +88,65 @@ def identify_runner_step(platform = "linux"):
],
}
def clone_enterprise_step(committish = "${DRONE_COMMIT}"):
def enterprise_setup_step(source = "${DRONE_SOURCE_BRANCH}", canFail = True, isPromote = False):
"""Setup the enterprise source into the ./grafana-enterprise directory.
Args:
source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request.
canFail: controls whether the step can fail. This is useful for pull requests where the enterprise source may not exist.
isPromote: controls whether or not this step is being used in a promote pipeline. If it is, then the clone enterprise step will not check if the pull request is a fork.
Returns:
Drone step.
"""
step = clone_enterprise_step_pr(source = source, target = "${DRONE_TARGET_BRANCH}", canFail = canFail, location = "../grafana-enterprise", isPromote = isPromote)
step["commands"] += [
"cd ../",
"ln -s src grafana",
"cd ./grafana-enterprise",
"./build.sh",
]
return step
def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFail = False, location = "grafana-enterprise", isPromote = False):
"""Clone the enterprise source into the ./grafana-enterprise directory.
Args:
committish: controls which revision of grafana-enterprise is cloned.
source: controls which revision of grafana-enterprise is checked out, if it exists. The name 'source' derives from the 'source branch' of a pull request.
target: controls which revision of grafana-enterprise is checked out, if it 'source' does not exist. The name 'target' derives from the 'target branch' of a pull request. If this does not exist, then 'main' will be checked out.
canFail: controls whether or not this step is allowed to fail. If it fails and this is true, then the pipeline will continue. canFail is used in pull request pipelines where enterprise may be cloned but may not clone in forks.
location: the path where grafana-enterprise is cloned.
isPromote: controls whether or not this step is being used in a promote pipeline. If it is, then the step will not check if the pull request is a fork.
Returns:
Drone step.
"""
return {
if isPromote:
check = []
else:
check = [
'is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" | jq .head.repo.fork)',
'if [ "$is_fork" != false ]; then return 1; fi', # Only clone if we're confident that 'fork' is 'false'. Fail if it's also empty.
]
step = {
"name": "clone-enterprise",
"image": images["build_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": [
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"',
"cd grafana-enterprise",
"git checkout {}".format(committish),
] + check + [
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" ' + location,
"cd {}".format(location),
'if git checkout {0}; then echo "checked out {0}"; elif git checkout {1}; then echo "git checkout {1}"; else git checkout main; fi'.format(source, target),
],
}
def init_enterprise_step(ver_mode):
"""Adds the enterprise deployment configuration into the source directory.
if canFail:
step["failure"] = "ignore"
Args:
ver_mode: controls what revision of the OSS source to use.
If ver_mode is 'release', the step uses the tagged revision.
Otherwise, the DRONE_SOURCE_BRANCH is used.
Returns:
Drone step.
"""
source_commit = ""
if ver_mode == "release":
source_commit = " ${DRONE_TAG}"
environment = {
"GITHUB_TOKEN": from_secret("github_token"),
}
token = "--github-token $${GITHUB_TOKEN}"
elif ver_mode == "release-branch":
environment = {
"GITHUB_TOKEN": from_secret("github_token"),
}
token = "--github-token $${GITHUB_TOKEN}"
else:
environment = {}
token = ""
return {
"name": "init-enterprise",
"image": images["build_image"],
"depends_on": [
"clone-enterprise",
"grabpl",
],
"environment": environment,
"commands": [
"mv bin/grabpl /tmp/",
"rmdir bin",
"mv grafana-enterprise /tmp/",
"/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}".format(
token,
source_commit,
).rstrip(),
"mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json",
"mkdir bin",
"mv /tmp/grabpl bin/",
],
}
def windows_init_enterprise_steps(ver_mode):
"""Performs init-enterprise steps in a Windows environment
Args:
ver_mode: in what mode should this be run
Returns:
A list of steps setting up an enterprise folder
"""
if ver_mode == "release":
source = "${DRONE_TAG}"
elif ver_mode == "release-branch":
source = "$$env:DRONE_BRANCH"
else:
source = "main"
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
"cd grafana-enterprise",
"git checkout {}".format(source),
]
init_cmds = [
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
"cp -r grafana-enterprise C:\\App\\grafana-enterprise",
"rm -r -force grafana-enterprise",
"cp grabpl.exe C:\\App\\grabpl.exe",
"rm -force grabpl.exe",
"C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise {}".format(source),
"cp C:\\App\\grabpl.exe grabpl.exe",
]
steps = []
steps.extend(
[
download_grabpl_step(platform = "windows"),
{
"name": "clone",
"image": windows_images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": clone_cmds,
},
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
},
],
)
return steps
return step
def download_grabpl_step(platform = "linux"):
if platform == "windows":
@@ -443,11 +378,10 @@ def e2e_tests_artifacts():
],
}
def upload_cdn_step(edition, ver_mode, trigger = None):
def upload_cdn_step(ver_mode, trigger = None):
"""Uploads CDN assets using the Grafana build tool.
Args:
edition: controls the output directory for the CDN assets.
ver_mode: only uses the step trigger when ver_mode == 'release-branch' or 'main'
trigger: a Drone trigger for the step.
Defaults to None.
@@ -455,41 +389,29 @@ def upload_cdn_step(edition, ver_mode, trigger = None):
Returns:
Drone step.
"""
deps = []
if edition in "enterprise2":
deps.extend(
[
"package" + enterprise2_suffix(edition),
],
)
else:
deps.extend(
[
"grafana-server",
],
)
step = {
"name": "upload-cdn-assets" + enterprise2_suffix(edition),
"name": "upload-cdn-assets",
"image": images["publish_image"],
"depends_on": deps,
"depends_on": [
"grafana-server",
],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"PRERELEASE_BUCKET": from_secret(prerelease_bucket),
},
"commands": [
"./bin/build upload-cdn --edition {}".format(edition),
"./bin/build upload-cdn --edition oss",
],
}
if trigger and ver_mode in ("release-branch", "main"):
step = dict(step, when = trigger)
return step
def build_backend_step(edition, ver_mode, variants = None):
def build_backend_step(ver_mode, variants = None):
"""Build the backend code using the Grafana build tool.
Args:
edition: controls which edition of the backend is built.
ver_mode: if ver_mode != 'release', pass the DRONE_BUILD_NUMBER environment
variable as the value for the --build-id option.
TODO: is this option actually used by the build-backend subcommand?
@@ -507,22 +429,19 @@ def build_backend_step(edition, ver_mode, variants = None):
# TODO: Convert number of jobs to percentage
if ver_mode == "release":
cmds = [
"./bin/build build-backend --jobs 8 --edition {} ${{DRONE_TAG}}".format(
edition,
),
"./bin/build build-backend --jobs 8 --edition oss ${DRONE_TAG}",
]
else:
build_no = "${DRONE_BUILD_NUMBER}"
cmds = [
"./bin/build build-backend --jobs 8 --edition {} --build-id {}{}".format(
edition,
"./bin/build build-backend --jobs 8 --edition oss --build-id {}{}".format(
build_no,
variants_str,
),
]
return {
"name": "build-backend" + enterprise2_suffix(edition),
"name": "build-backend",
"image": images["build_image"],
"depends_on": [
"wire-install",
@@ -531,11 +450,10 @@ def build_backend_step(edition, ver_mode, variants = None):
"commands": cmds,
}
def build_frontend_step(edition, ver_mode):
def build_frontend_step(ver_mode):
"""Build the frontend code using the Grafana build tool.
Args:
edition: controls which edition of the frontend is built.
ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment
variable as a build identifier.
@@ -548,11 +466,11 @@ def build_frontend_step(edition, ver_mode):
if ver_mode == "release":
cmds = [
"./bin/build build-frontend --jobs 8 " +
"--edition {} ${{DRONE_TAG}}".format(edition),
"--edition oss ${DRONE_TAG}",
]
else:
cmds = [
"./bin/build build-frontend --jobs 8 --edition {} ".format(edition) +
"./bin/build build-frontend --jobs 8 --edition oss" +
"--build-id {}".format(build_no),
]
@@ -569,11 +487,10 @@ def build_frontend_step(edition, ver_mode):
"commands": cmds,
}
def build_frontend_package_step(edition, ver_mode):
def build_frontend_package_step(ver_mode):
"""Build the frontend packages using the Grafana build tool.
Args:
edition: controls which edition of the frontend is built.
ver_mode: if ver_mode != 'release', use the DRONE_BUILD_NUMBER environment
variable as a build identifier.
@@ -586,11 +503,11 @@ def build_frontend_package_step(edition, ver_mode):
if ver_mode == "release":
cmds = [
"./bin/build build-frontend-packages --jobs 8 " +
"--edition {} ${{DRONE_TAG}}".format(edition),
"--edition oss ${DRONE_TAG}",
]
else:
cmds = [
"./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) +
"./bin/build build-frontend-packages --jobs 8 --edition oss" +
"--build-id {}".format(build_no),
]
@@ -607,7 +524,7 @@ def build_frontend_package_step(edition, ver_mode):
"commands": cmds,
}
def build_plugins_step(edition, ver_mode):
def build_plugins_step(ver_mode):
if ver_mode != "pr":
env = {
"GRAFANA_API_KEY": from_secret("grafana_api_key"),
@@ -624,7 +541,7 @@ def build_plugins_step(edition, ver_mode):
],
"commands": [
# TODO: Use percentage for num jobs
"./bin/build build-plugins --jobs 8 --edition {}".format(edition),
"./bin/build build-plugins --jobs 8 --edition oss",
],
}
@@ -656,50 +573,40 @@ def test_backend_integration_step():
],
}
def betterer_frontend_step(edition = "oss"):
def betterer_frontend_step():
"""Run betterer on frontend code.
Args:
edition: controls whether enterprise code is also included in the source.
Defaults to 'oss'.
Returns:
Drone step.
"""
deps = []
if edition == "enterprise":
deps.extend(["init-enterprise"])
deps.extend(["yarn-install"])
return {
"name": "betterer-frontend",
"image": images["build_image"],
"depends_on": deps,
"depends_on": [
"yarn-install",
],
"commands": [
"yarn betterer ci",
],
}
def test_frontend_step(edition = "oss"):
def test_frontend_step():
"""Runs tests on frontend code.
Args:
edition: controls whether enterprise code is also included in the source.
Defaults to 'oss'.
Returns:
Drone step.
"""
deps = []
if edition == "enterprise":
deps.extend(["init-enterprise"])
deps.extend(["yarn-install"])
return {
"name": "test-frontend",
"image": images["build_image"],
"environment": {
"TEST_MAX_WORKERS": "50%",
},
"depends_on": deps,
"depends_on": [
"yarn-install",
],
"commands": [
"yarn run ci:test-frontend",
],
@@ -809,11 +716,10 @@ def codespell_step():
],
}
def package_step(edition, ver_mode):
def package_step(ver_mode):
"""Packages Grafana with the Grafana build tool.
Args:
edition: controls which edition of Grafana is packaged.
ver_mode: controls whether the packages are signed for a release.
If ver_mode != 'release', use the DRONE_BUILD_NUMBER environment
variable as a build identifier.
@@ -823,7 +729,7 @@ def package_step(edition, ver_mode):
"""
deps = [
"build-plugins",
"build-backend" + enterprise2_suffix(edition),
"build-backend",
"build-frontend",
"build-frontend-packages",
]
@@ -847,29 +753,28 @@ def package_step(edition, ver_mode):
# TODO: Use percentage for jobs
if ver_mode == "release":
cmds = [
"{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) +
"{} ${{DRONE_TAG}}".format(sign_args),
"{}./bin/build package --jobs 8 --edition oss ".format(test_args) +
"{} $${{DRONE_TAG}}".format(sign_args),
]
else:
build_no = "${DRONE_BUILD_NUMBER}"
cmds = [
"{}./bin/build package --jobs 8 --edition {} ".format(test_args, edition) +
"{}./bin/build package --jobs 8 --edition oss ".format(test_args) +
"--build-id {}{}".format(build_no, sign_args),
]
return {
"name": "package" + enterprise2_suffix(edition),
"name": "package",
"image": images["build_image"],
"depends_on": deps,
"environment": env,
"commands": cmds,
}
def grafana_server_step(edition, port = 3001):
def grafana_server_step(port = 3001):
"""Runs the grafana-server binary as a service.
Args:
edition: controls which edition of grafana-server to run.
port: port to listen on.
Defaults to 3001.
@@ -877,8 +782,6 @@ def grafana_server_step(edition, port = 3001):
Drone step.
"""
environment = {"PORT": port, "ARCH": "linux-amd64"}
if edition == "enterprise":
environment["RUNDIR"] = "scripts/grafana-server/tmp-grafana-enterprise"
return {
"name": "grafana-server",
@@ -976,12 +879,12 @@ def build_docs_website_step():
],
}
def copy_packages_for_docker_step(edition = None):
def copy_packages_for_docker_step():
return {
"name": "copy-packages-for-docker",
"image": images["build_image"],
"depends_on": [
"package" + enterprise2_suffix(edition),
"package",
],
"commands": [
"ls dist/*.tar.gz*",
@@ -989,11 +892,10 @@ def copy_packages_for_docker_step(edition = None):
],
}
def build_docker_images_step(edition, archs = None, ubuntu = False, publish = False):
def build_docker_images_step(archs = None, ubuntu = False, publish = False):
"""Build Docker images using the Grafana build tool.
Args:
edition: controls which repository the image is published to.
archs: a list of architectures to build the image for.
Defaults to None.
ubuntu: controls whether the final image is built from an Ubuntu base image.
@@ -1004,7 +906,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
Returns:
Drone step.
"""
cmd = "./bin/build build-docker --edition {}".format(edition)
cmd = "./bin/build build-docker --edition oss"
if publish:
cmd += " --shouldSave"
@@ -1020,11 +922,6 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
"GCP_KEY": from_secret("gcp_key"),
}
if edition == "enterprise2":
environment.update(
{"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo")},
)
return {
"name": "build-docker-images" + ubuntu_sfx,
"image": images["cloudsdk_image"],
@@ -1037,27 +934,24 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa
"environment": environment,
}
def fetch_images_step(edition):
def fetch_images_step():
return {
"name": "fetch-images-{}".format(edition),
"name": "fetch-images",
"image": images["cloudsdk_image"],
"environment": {
"GCP_KEY": from_secret("gcp_key"),
"DOCKER_USER": from_secret("docker_username"),
"DOCKER_PASSWORD": from_secret("docker_password"),
"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"),
},
"commands": ["./bin/build artifacts docker fetch --edition {}".format(edition)],
"commands": ["./bin/build artifacts docker fetch --edition oss"],
"depends_on": ["compile-build-cmd"],
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
def publish_images_step(edition, ver_mode, docker_repo, trigger = None):
def publish_images_step(ver_mode, docker_repo, trigger = None):
"""Generates a step for publishing public Docker images with grabpl.
Args:
edition: controls which version of an image is fetched in the case of a release.
It also controls which publishing implementation is used.
ver_mode: controls whether the image needs to be built or retrieved from a previous build.
If ver_mode == 'release', the previously built image is fetched instead of being built again.
docker_repo: the Docker image name.
@@ -1086,22 +980,9 @@ def publish_images_step(edition, ver_mode, docker_repo, trigger = None):
deps = ["build-docker-images", "build-docker-images-ubuntu"]
if ver_mode == "release":
deps = ["fetch-images-{}".format(edition)]
deps = ["fetch-images"]
cmd += " --version-tag ${DRONE_TAG}"
if edition == "enterprise2":
name = edition
docker_repo = "$${DOCKER_ENTERPRISE2_REPO}"
environment.update(
{
"GCP_KEY": from_secret("gcp_key_hg"),
"DOCKER_ENTERPRISE2_REPO": from_secret("docker_enterprise2_repo"),
},
)
cmd = "./bin/build artifacts docker publish-enterprise2 --dockerhub-repo {}".format(
docker_repo,
)
if ver_mode == "pr":
environment = {
"DOCKER_USER": from_secret("docker_username_pr"),
@@ -1225,16 +1106,10 @@ def release_canary_npm_packages_step(trigger = None):
step = dict(step, when = trigger)
return step
def enterprise2_suffix(edition):
if edition == "enterprise2":
return "-{}".format(edition)
return ""
def upload_packages_step(edition, ver_mode, trigger = None):
def upload_packages_step(ver_mode, trigger = None):
"""Upload packages to object storage.
Args:
edition: controls which edition of Grafana packages to upload.
ver_mode: when ver_mode == 'main', inhibit upload of enterprise
edition packages when executed.
trigger: a Drone trigger for the step.
@@ -1244,7 +1119,7 @@ def upload_packages_step(edition, ver_mode, trigger = None):
Drone step.
"""
step = {
"name": "upload-packages" + enterprise2_suffix(edition),
"name": "upload-packages",
"image": images["publish_image"],
"depends_on": end_to_end_tests_deps(),
"environment": {
@@ -1252,18 +1127,17 @@ def upload_packages_step(edition, ver_mode, trigger = None):
"PRERELEASE_BUCKET": from_secret("prerelease_bucket"),
},
"commands": [
"./bin/build upload-packages --edition {}".format(edition),
"./bin/build upload-packages --edition oss",
],
}
if trigger and ver_mode in ("release-branch", "main"):
step = dict(step, when = trigger)
return step
def publish_grafanacom_step(edition, ver_mode):
def publish_grafanacom_step(ver_mode):
"""Publishes Grafana packages to grafana.com.
Args:
edition: controls which edition of Grafana to publish to.
ver_mode: if ver_mode == 'main', pass the DRONE_BUILD_NUMBER environment
variable as the value for the --build-id option.
TODO: is this actually used by the grafanacom subcommand? I think it might
@@ -1273,20 +1147,17 @@ def publish_grafanacom_step(edition, ver_mode):
Drone step.
"""
if ver_mode == "release":
cmd = "./bin/build publish grafana-com --edition {} ${{DRONE_TAG}}".format(
edition,
)
cmd = "./bin/build publish grafana-com --edition oss ${DRONE_TAG}"
elif ver_mode == "main":
build_no = "${DRONE_BUILD_NUMBER}"
cmd = "./bin/build publish grafana-com --edition {} --build-id {}".format(
edition,
cmd = "./bin/build publish grafana-com --edition oss --build-id {}".format(
build_no,
)
else:
fail("Unexpected version mode {}".format(ver_mode))
return {
"name": "publish-grafanacom-{}".format(edition),
"name": "publish-grafanacom",
"image": images["publish_image"],
"depends_on": [
"publish-linux-packages-deb",
@@ -1301,7 +1172,7 @@ def publish_grafanacom_step(edition, ver_mode):
],
}
def publish_linux_packages_step(edition, package_manager = "deb"):
def publish_linux_packages_step(package_manager = "deb"):
return {
"name": "publish-linux-packages-{}".format(package_manager),
# See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image
@@ -1317,8 +1188,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"):
"gpg_passphrase": from_secret("packages_gpg_passphrase"),
"gpg_public_key": from_secret("packages_gpg_public_key"),
"gpg_private_key": from_secret("packages_gpg_private_key"),
"package_path": "gs://grafana-prerelease/artifacts/downloads/*${{DRONE_TAG}}/{}/**.{}".format(
edition,
"package_path": "gs://grafana-prerelease/artifacts/downloads/*$${{DRONE_TAG}}/oss/**.{}".format(
package_manager,
),
},
@@ -1337,11 +1207,10 @@ def windows_clone_step():
],
}
def get_windows_steps(edition, ver_mode):
def get_windows_steps(ver_mode):
"""Generate the list of Windows steps.
Args:
edition: used to differentiate steps for different Grafana editions.
ver_mode: used to differentiate steps for different version modes.
Returns:
@@ -1351,80 +1220,24 @@ def get_windows_steps(edition, ver_mode):
identify_runner_step("windows"),
]
if edition in ("enterprise", "enterprise2"):
if ver_mode == "release":
committish = "${DRONE_TAG}"
elif ver_mode == "release-branch":
committish = "$$env:DRONE_BRANCH"
else:
committish = "$$env:DRONE_COMMIT"
init_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
grabpl_version,
),
]
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
download_grabpl_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
grabpl_version,
),
]
steps.extend(
[
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
},
],
)
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
"cd grafana-enterprise",
"git checkout {}".format(committish),
]
init_cmds = [
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
"cp -r grafana-enterprise C:\\App\\grafana-enterprise",
"rm -r -force grafana-enterprise",
"cp grabpl.exe C:\\App\\grabpl.exe",
"rm -force grabpl.exe",
"C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise {}".format(committish),
"cp C:\\App\\grabpl.exe grabpl.exe",
]
steps.extend(
[
{
"name": "clone",
"image": windows_images["wix_image"],
"environment": {
"GITHUB_TOKEN": from_secret("github_token"),
},
"commands": download_grabpl_cmds + clone_cmds,
},
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
"depends_on": ["clone"],
"environment": {"GITHUB_TOKEN": from_secret("github_token")},
},
],
)
else:
init_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
"Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format(
grabpl_version,
),
]
steps.extend(
[
{
"name": "windows-init",
"image": windows_images["wix_image"],
"commands": init_cmds,
},
],
)
# TODO: Run windows backend tests
if (
ver_mode == "main" and (edition not in ("enterprise", "enterprise2"))
) or ver_mode in (
if ver_mode in (
"release",
"release-branch",
):
@@ -1446,13 +1259,11 @@ def get_windows_steps(edition, ver_mode):
"rm gcpkey.json",
"cp C:\\App\\nssm-2.24.zip .",
]
if (
ver_mode == "main" and (edition not in ("enterprise", "enterprise2"))
) or ver_mode in ("release",):
if ver_mode in ("release",):
installer_commands.extend(
[
".\\grabpl.exe windows-installer --edition {} {}".format(
edition,
".\\grabpl.exe windows-installer --edition oss {}".format(
ver_part,
),
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
@@ -1461,10 +1272,9 @@ def get_windows_steps(edition, ver_mode):
if ver_mode == "main":
installer_commands.extend(
[
"gsutil cp $$fname gs://{}/{}/{}/".format(bucket, edition, dir),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(
"gsutil cp $$fname gs://{}/oss/{}/".format(bucket, dir),
'gsutil cp "$$fname.sha256" gs://{}/oss/{}/'.format(
bucket,
edition,
dir,
),
],
@@ -1472,16 +1282,14 @@ def get_windows_steps(edition, ver_mode):
else:
installer_commands.extend(
[
"gsutil cp $$fname gs://{}/{}/{}/{}/".format(
"gsutil cp $$fname gs://{}/{}/oss/{}/".format(
bucket,
ver_part,
edition,
dir,
),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(
'gsutil cp "$$fname.sha256" gs://{}/{}/oss/{}/'.format(
bucket,
ver_part,
edition,
dir,
),
],
@@ -1570,12 +1378,9 @@ def end_to_end_tests_deps():
"end-to-end-tests-various-suite",
]
def compile_build_cmd(edition = "oss"):
def compile_build_cmd():
dependencies = []
if edition in ("enterprise", "enterprise2"):
dependencies = [
"init-enterprise",
]
return {
"name": "compile-build-cmd",
"image": images["go_image"],
scripts/drone/utils/utils.star
-8
View File
@@ -13,7 +13,6 @@ drone_change_template = "`.drone.yml` and `starlark` files have been changed on
def pipeline(
name,
edition,
trigger,
steps,
services = [],
@@ -30,7 +29,6 @@ def pipeline(
Args:
name: controls the pipeline name.
edition: used to differentiate the pipeline for enterprise builds.
trigger: a Drone trigger for the pipeline.
steps: the Drone steps for the pipeline.
services: auxilliary services used during the pipeline.
@@ -98,12 +96,6 @@ def pipeline(
pipeline["volumes"].extend(volumes)
pipeline.update(platform_conf)
if edition in ("enterprise", "enterprise2"):
# We have a custom clone step for enterprise
pipeline["clone"] = {
"disable": True,
}
return pipeline
def notify_pipeline(
scripts/drone/vault.star
-15
View File
@@ -90,21 +90,6 @@ def secrets():
"infra/data/ci/packages-publish/bucket-credentials",
"Secret",
),
vault_secret(
"aws_region",
"secret/data/common/aws-marketplace",
"aws_region",
),
vault_secret(
"aws_access_key_id",
"secret/data/common/aws-marketplace",
"aws_access_key_id",
),
vault_secret(
"aws_secret_access_key",
"secret/data/common/aws-marketplace",
"aws_secret_access_key",
),
vault_secret(
"static_asset_editions",
"infra/data/ci/grafana-release-eng/artifact-publishing",
scripts/drone/version.star
+1 -5
View File
@@ -4,8 +4,6 @@ This module returns the pipeline used for version branches.
load(
"scripts/drone/events/release.star",
"enterprise2_pipelines",
"enterprise_pipelines",
"oss_pipelines",
)
@@ -14,7 +12,5 @@ trigger = {"ref": ["refs/heads/v[0-9]*"]}
def version_branch_pipelines():
return (
oss_pipelines(ver_mode = ver_mode, trigger = trigger) +
enterprise_pipelines(ver_mode = ver_mode, trigger = trigger) +
enterprise2_pipelines(ver_mode = ver_mode, trigger = trigger)
oss_pipelines(ver_mode = ver_mode, trigger = trigger)
)