public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Backport CI/Release related code to v9.3.x (#62752)

Browse Source 
* Batch-move everything

* go mod tidy

* make drone

* Remove genversions

* Bump alpine image

* Revert back pkg/build/docker/build.go

* Make sure correct enterprise branch is checked out

* Add enterprise2 version

* Remove extensions

* Bump build container

* backport node 18 test fix

(cherry picked from commit 4ff03fdbfb)

* Update scripts/drone

* Add more commands

* Fix starlark link

* Copy .drone.star

* Add drone target branch for custom events

---------
This commit is contained in:
Dimitris Sotirakis
2023-02-03 11:43:48 +02:00
committed by GitHub
parent d9b5c5f4c6
commit 03b1cf763d
85 changed files with 8622 additions and 2695 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/pipelines/aws_marketplace.star
+43
View File
@@ -0,0 +1,43 @@
"""
This module contains steps and pipelines publishing to AWS Marketplace.
"""
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"fetch_images_step",
"publish_image",
)
load("scripts/drone/vault.star", "from_secret")
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def publish_aws_marketplace_step():
return {
"name": "publish-aws-marketplace",
"image": publish_image,
"commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"],
"depends_on": ["fetch-images-enterprise"],
"environment": {
"AWS_REGION": from_secret("aws_region"),
"AWS_ACCESS_KEY_ID": from_secret("aws_access_key_id"),
"AWS_SECRET_ACCESS_KEY": from_secret("aws_secret_access_key"),
},
"volumes": [{"name": "docker", "path": "/var/run/docker.sock"}],
}
def publish_aws_marketplace_pipeline(mode):
trigger = {
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name = "publish-aws-marketplace-{}".format(mode),
trigger = trigger,
steps = [compile_build_cmd(), fetch_images_step("enterprise"), publish_aws_marketplace_step()],
edition = "",
depends_on = ["publish-docker-enterprise-public"],
environment = {"EDITION": "enterprise2"},
)]
scripts/drone/pipelines/build.star
+152 -92
View File
@@ -1,110 +1,170 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'build_image',
'identify_runner_step',
'wire_install_step',
'yarn_install_step',
'build_backend_step',
'build_frontend_step',
'build_frontend_package_step',
'build_plugins_step',
'package_step',
'grafana_server_step',
'e2e_tests_step',
'e2e_tests_artifacts',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'publish_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'enterprise_downstream_step',
'frontend_metrics_step',
'store_storybook_step',
'release_canary_npm_packages_step',
'upload_packages_step',
'upload_cdn_step',
'verify_gen_cue_step',
'verify_gen_jsonnet_step',
'test_a11y_frontend_step',
'trigger_oss',
'betterer_frontend_step',
'trigger_test_release',
'compile_build_cmd',
'cloud_plugins_e2e_tests_step',
)
"""This module contains the comprehensive build pipeline."""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"build_backend_step",
"build_docker_images_step",
"build_frontend_package_step",
"build_frontend_step",
"build_plugins_step",
"build_storybook_step",
"cloud_plugins_e2e_tests_step",
"compile_build_cmd",
"copy_packages_for_docker_step",
"download_grabpl_step",
"e2e_tests_artifacts",
"e2e_tests_step",
"enterprise_downstream_step",
"frontend_metrics_step",
"grafana_server_step",
"identify_runner_step",
"package_step",
"publish_images_step",
"release_canary_npm_packages_step",
"store_storybook_step",
"test_a11y_frontend_step",
"trigger_oss",
"trigger_test_release",
"upload_cdn_step",
"upload_packages_step",
"verify_gen_cue_step",
"verify_gen_jsonnet_step",
"wire_install_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def build_e2e(trigger, ver_mode, edition):
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
# @unused
def build_e2e(trigger, ver_mode):
"""Perform e2e building, testing, and publishing.
Args:
trigger: controls which events can trigger the pipeline execution.
ver_mode: used in the naming of the pipeline.
Returns:
Drone pipeline.
"""
edition = "oss"
environment = {"EDITION": edition}
init_steps = [
identify_runner_step(),
download_grabpl_step(),
compile_build_cmd(),
verify_gen_cue_step(edition="oss"),
verify_gen_jsonnet_step(edition="oss"),
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
wire_install_step(),
yarn_install_step(),
]
build_steps = []
if ver_mode == 'pr':
build_steps.extend([trigger_test_release()])
build_steps.extend([enterprise_downstream_step(edition=edition, ver_mode=ver_mode)])
build_steps.extend([
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, ver_mode=ver_mode),
])
if ver_mode == 'main':
build_steps.extend([package_step(edition=edition, ver_mode=ver_mode)])
elif ver_mode == 'pr':
build_steps.extend([package_step(edition=edition, ver_mode=ver_mode, variants=variants)])
variants = None
build_steps.extend([
grafana_server_step(edition=edition),
e2e_tests_step('dashboards-suite', edition=edition),
e2e_tests_step('smoke-tests-suite', edition=edition),
e2e_tests_step('panels-suite', edition=edition),
e2e_tests_step('various-suite', edition=edition),
cloud_plugins_e2e_tests_step('cloud-plugins-suite', edition=edition, cloud='azure', trigger=trigger_oss),
e2e_tests_artifacts(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
copy_packages_for_docker_step(),
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
])
if ver_mode == 'main':
build_steps.extend([
store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
frontend_metrics_step(edition=edition, trigger=trigger_oss)
])
if ver_mode == "pr":
build_steps.extend(
[
trigger_test_release(),
enterprise_downstream_step(ver_mode = ver_mode),
],
)
if ver_mode == 'main':
build_steps.extend([
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana', trigger=trigger_oss),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana-oss', trigger=trigger_oss),
release_canary_npm_packages_step(edition, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
])
elif ver_mode == 'pr':
build_steps.extend([build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ])])
variants = [
"linux-amd64",
"linux-amd64-musl",
"darwin-amd64",
"windows-amd64",
]
publish_suffix = ''
if ver_mode == 'main':
publish_suffix = '-publish'
build_steps.extend(
[
build_backend_step(edition = edition, ver_mode = ver_mode),
build_frontend_step(edition = edition, ver_mode = ver_mode),
build_frontend_package_step(edition = edition, ver_mode = ver_mode),
build_plugins_step(edition = edition, ver_mode = ver_mode),
package_step(edition = edition, variants = variants, ver_mode = ver_mode),
grafana_server_step(edition = edition),
e2e_tests_step("dashboards-suite"),
e2e_tests_step("smoke-tests-suite"),
e2e_tests_step("panels-suite"),
e2e_tests_step("various-suite"),
cloud_plugins_e2e_tests_step(
"cloud-plugins-suite",
cloud = "azure",
trigger = trigger_oss,
),
e2e_tests_artifacts(),
build_storybook_step(ver_mode = ver_mode),
copy_packages_for_docker_step(),
test_a11y_frontend_step(ver_mode = ver_mode),
],
)
if ver_mode == "main":
build_steps.extend(
[
store_storybook_step(trigger = trigger_oss, ver_mode = ver_mode),
frontend_metrics_step(trigger = trigger_oss),
build_docker_images_step(
edition = edition,
publish = False,
),
build_docker_images_step(
edition = edition,
publish = False,
ubuntu = True,
),
publish_images_step(
docker_repo = "grafana",
edition = edition,
mode = "",
trigger = trigger_oss,
ver_mode = ver_mode,
),
publish_images_step(
docker_repo = "grafana-oss",
edition = edition,
mode = "",
trigger = trigger_oss,
ver_mode = ver_mode,
),
release_canary_npm_packages_step(trigger = trigger_oss),
upload_packages_step(
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
upload_cdn_step(
edition = edition,
trigger = trigger_oss,
ver_mode = ver_mode,
),
],
)
elif ver_mode == "pr":
build_steps.extend(
[
build_docker_images_step(
archs = [
"amd64",
],
edition = edition,
),
],
)
publish_suffix = ""
if ver_mode == "main":
publish_suffix = "-publish"
return pipeline(
name='{}-build-e2e{}'.format(ver_mode, publish_suffix), edition="oss", trigger=trigger, services=[], steps=init_steps + build_steps,
name = "{}-build-e2e{}".format(ver_mode, publish_suffix),
edition = "oss",
environment = environment,
services = [],
steps = init_steps + build_steps,
trigger = trigger,
)
scripts/drone/pipelines/docs.star
+40 -41
View File
@@ -1,37 +1,32 @@
load(
'scripts/drone/steps/lib.star',
'build_image',
'yarn_install_step',
'identify_runner_step',
'download_grabpl_step',
'lint_frontend_step',
'codespell_step',
'test_frontend_step',
'build_storybook_step',
'build_docs_website_step',
)
"""
This module returns all the pipelines used in the event of documentation changes along with supporting functions.
"""
load(
'scripts/drone/services/services.star',
'integration_test_services',
'ldap_service',
"scripts/drone/steps/lib.star",
"build_docs_website_step",
"build_image",
"codespell_step",
"download_grabpl_step",
"identify_runner_step",
"yarn_install_step",
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/utils/utils.star",
"pipeline",
)
docs_paths = {
'include': [
'*.md',
'docs/**',
'packages/**/*.md',
'latest.json',
"include": [
"*.md",
"docs/**",
"packages/**/*.md",
"latest.json",
],
}
def docs_pipelines(edition, ver_mode, trigger):
def docs_pipelines(ver_mode, trigger):
environment = {"EDITION": "oss"}
steps = [
download_grabpl_step(),
identify_runner_step(),
@@ -42,38 +37,42 @@ def docs_pipelines(edition, ver_mode, trigger):
]
return pipeline(
name='{}-docs'.format(ver_mode), edition=edition, trigger=trigger, services=[], steps=steps,
name = "{}-docs".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
environment = environment,
)
def lint_docs():
return {
'name': 'lint-docs',
'image': build_image,
'depends_on': [
'yarn-install',
"name": "lint-docs",
"image": build_image,
"depends_on": [
"yarn-install",
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',
"environment": {
"NODE_OPTIONS": "--max_old_space_size=8192",
},
'commands': [
'yarn run prettier:checkDocs',
"commands": [
"yarn run prettier:checkDocs",
],
}
def trigger_docs_main():
return {
'branch': 'main',
'event': [
'push',
"branch": "main",
"event": [
"push",
],
'paths': docs_paths,
"paths": docs_paths,
}
def trigger_docs_pr():
return {
'event': [
'pull_request',
"event": [
"pull_request",
],
'paths': docs_paths,
"paths": docs_paths,
}
scripts/drone/pipelines/github.star
+26 -22
View File
@@ -1,36 +1,40 @@
"""
This module contains steps and pipelines relating to GitHub.
"""
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'publish_images_step',
'compile_build_cmd',
'fetch_images_step',
'publish_image',
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"fetch_images_step",
"publish_image",
)
load('scripts/drone/vault.star', 'from_secret')
load("scripts/drone/vault.star", "from_secret")
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/utils/utils.star",
"pipeline",
)
def publish_github_step():
return {
'name': 'publish-github',
'image': publish_image,
'commands': ['./bin/build publish github --repo $${GH_REGISTRY} --create'],
'depends_on': ['fetch-images-enterprise2'],
'environment': {
'GH_TOKEN': from_secret('github_token'),
'GH_REGISTRY': from_secret('gh_registry'),
"name": "publish-github",
"image": publish_image,
"commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"],
"depends_on": ["fetch-images-enterprise2"],
"environment": {
"GH_TOKEN": from_secret("github_token"),
"GH_REGISTRY": from_secret("gh_registry"),
},
}
def publish_github_pipeline(mode):
trigger = {
'event': ['promote'],
'target': [mode],
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name='publish-github-{}'.format(mode), trigger=trigger, steps=[compile_build_cmd(), fetch_images_step('enterprise2'), publish_github_step()], edition="", environment = {'EDITION': 'enterprise2'}
),]
name = "publish-github-{}".format(mode),
trigger = trigger,
steps = [compile_build_cmd(), fetch_images_step("enterprise2"), publish_github_step()],
edition = "",
environment = {"EDITION": "enterprise2"},
)]
scripts/drone/pipelines/integration_tests.star
+62 -27
View File
@@ -1,44 +1,79 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'verify_gen_cue_step',
'verify_gen_jsonnet_step',
'wire_install_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'compile_build_cmd',
)
"""
This module returns the pipeline used for integration tests.
"""
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"mysql_integration_tests_step",
"postgres_integration_tests_step",
"verify_gen_cue_step",
"verify_gen_jsonnet_step",
"wire_install_step",
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/services/services.star",
"integration_test_services",
"integration_test_services_volumes",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def integration_tests(trigger, ver_mode, edition):
services = integration_test_services(edition)
def integration_tests(trigger, prefix, ver_mode = "pr"):
"""Generate a pipeline for integration tests.
Args:
trigger: controls which events can trigger the pipeline execution.
prefix: used in the naming of the pipeline.
ver_mode: defines the event / origin of this build. In this function, if it is set to pr, then it will attempt to clone grafana-enterprise. Otherwise it has no effect.
Returns:
Drone pipeline.
"""
environment = {"EDITION": "oss"}
services = integration_test_services(edition = "oss")
volumes = integration_test_services_volumes()
init_steps = [
init_steps = []
verify_step = verify_gen_cue_step()
verify_jsonnet_step = verify_gen_jsonnet_step()
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(enterprise_setup_step())
# Ensure that verif_gen_cue happens after we clone enterprise
# At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on.
verify_step["depends_on"].append("clone-enterprise")
verify_jsonnet_step["depends_on"].append("clone-enterprise")
init_steps += [
download_grabpl_step(),
compile_build_cmd(),
identify_runner_step(),
verify_gen_cue_step(edition="oss"),
verify_gen_jsonnet_step(edition="oss"),
verify_step,
verify_jsonnet_step,
wire_install_step(),
]
test_steps = [
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
return pipeline(
name='{}-integration-tests'.format(ver_mode), edition="oss", trigger=trigger, services=services, steps=init_steps + test_steps,
volumes=volumes
name = "{}-integration-tests".format(prefix),
edition = "oss",
trigger = trigger,
environment = environment,
services = services,
volumes = volumes,
steps = init_steps + test_steps,
)
scripts/drone/pipelines/lint_backend.star
+45 -16
View File
@@ -1,31 +1,60 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'wire_install_step',
'lint_backend_step',
'lint_drone_step',
'compile_build_cmd',
)
"""
This module returns the pipeline used for linting backend code.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"enterprise_setup_step",
"identify_runner_step",
"lint_backend_step",
"lint_drone_step",
"wire_install_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def lint_backend_pipeline(trigger, ver_mode):
"""Generates the pipelines used linting backend code.
Args:
trigger: controls which events can trigger the pipeline execution.
ver_mode: used in the naming of the pipeline.
Returns:
Drone pipeline.
"""
environment = {"EDITION": "oss"}
wire_step = wire_install_step()
wire_step.update({ 'depends_on': [] })
wire_step.update({"depends_on": []})
init_steps = [
identify_runner_step(),
compile_build_cmd(),
wire_step,
]
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps.append(enterprise_setup_step())
wire_step["depends_on"].append("clone-enterprise")
init_steps.append(wire_step)
test_steps = [
lint_backend_step(edition="oss"),
lint_backend_step(),
]
if ver_mode == 'main':
test_steps.extend([lint_drone_step()])
if ver_mode == "main":
test_steps.append(lint_drone_step())
return pipeline(
name='{}-lint-backend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
name = "{}-lint-backend".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = init_steps + test_steps,
environment = environment,
)
scripts/drone/pipelines/lint_frontend.star
+43 -14
View File
@@ -1,25 +1,54 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'yarn_install_step',
'lint_frontend_step',
)
"""
This module returns the pipeline used for linting frontend code.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"enterprise_setup_step",
"identify_runner_step",
"lint_frontend_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def lint_frontend_pipeline(trigger, ver_mode):
yarn_step = yarn_install_step()
yarn_step.update({ 'depends_on': [] })
init_steps = [
"""Generates the pipelines used linting frontend code.
Args:
trigger: controls which events can trigger the pipeline execution.
ver_mode: used in the naming of the pipeline.
Returns:
Drone pipeline.
"""
environment = {"EDITION": "oss"}
init_steps = []
lint_step = lint_frontend_step()
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
init_steps = [enterprise_setup_step()]
# Ensure the lint step happens after the clone-enterprise step
lint_step["depends_on"].append("clone-enterprise")
init_steps += [
identify_runner_step(),
yarn_step,
yarn_install_step(),
]
test_steps = [
lint_frontend_step(),
lint_step,
]
return pipeline(
name='{}-lint-frontend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
name = "{}-lint-frontend".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = init_steps + test_steps,
environment = environment,
)
scripts/drone/pipelines/publish_images.star
+77 -30
View File
@@ -1,50 +1,97 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'publish_images_step',
'compile_build_cmd',
'fetch_images_step',
)
"""
This module returns the pipeline used for publishing Docker images and its steps.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"fetch_images_step",
"identify_runner_step",
"publish_images_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def publish_image_steps(edition, mode, docker_repo):
additional_docker_repo = ""
if edition == 'oss':
additional_docker_repo='grafana/grafana-oss'
"""Generates the steps used for publising Docker images using grabpl.
Args:
edition: controls which version of an image is fetched in the case of a release.
It also controls which publishing implementation is used.
If edition == 'oss', it additionally publishes the grafana/grafana-oss repository.
mode: uses to control the publishing of security images when mode == 'security'.
docker_repo: the Docker image name.
It is combined with the 'grafana/' library prefix.
Returns:
List of Drone steps.
"""
steps = [
identify_runner_step(),
download_grabpl_step(),
compile_build_cmd(),
fetch_images_step(edition),
publish_images_step(edition, 'release', mode, docker_repo),
publish_images_step(edition, "release", mode, docker_repo),
]
if additional_docker_repo != "":
steps.extend([publish_images_step(edition, 'release', mode, additional_docker_repo)])
if edition == "oss":
steps.append(
publish_images_step(edition, "release", mode, "grafana-oss"),
)
return steps
def publish_image_pipelines_public():
mode='public'
"""Generates the pipeline used for publising public Docker images.
Returns:
Drone pipeline
"""
mode = "public"
trigger = {
'event': ['promote'],
'target': [mode],
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana/grafana'), edition=""
), pipeline(
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition=""
),]
return [
pipeline(
name = "publish-docker-oss-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(edition = "oss", mode = mode, docker_repo = "grafana"),
edition = "",
environment = {"EDITION": "oss"},
),
pipeline(
name = "publish-docker-enterprise-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(
edition = "enterprise",
mode = mode,
docker_repo = "grafana-enterprise",
),
edition = "",
environment = {"EDITION": "enterprise"},
),
]
def publish_image_pipelines_security():
mode='security'
mode = "security"
trigger = {
'event': ['promote'],
'target': [mode],
"event": ["promote"],
"target": [mode],
}
return [pipeline(
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition=""
),]
return [
pipeline(
name = "publish-docker-enterprise-{}".format(mode),
trigger = trigger,
steps = publish_image_steps(
edition = "enterprise",
mode = mode,
docker_repo = "grafana-enterprise",
),
edition = "",
environment = {"EDITION": "enterprise"},
),
]
scripts/drone/pipelines/shellcheck.star
+27 -25
View File
@@ -1,48 +1,50 @@
load(
'scripts/drone/steps/lib.star',
'build_image',
'compile_build_cmd'
)
"""
This module returns a Drone step and pipeline for linting with shellcheck.
"""
load("scripts/drone/steps/lib.star", "build_image", "compile_build_cmd")
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/utils/utils.star",
"pipeline",
)
trigger = {
'event': [
'pull_request',
"event": [
"pull_request",
],
'paths': {
'exclude': [
'*.md',
'docs/**',
'latest.json',
],
'include': [
'scripts/**/*.sh'
"paths": {
"exclude": [
"*.md",
"docs/**",
"latest.json",
],
"include": ["scripts/**/*.sh"],
},
}
def shellcheck_step():
return {
'name': 'shellcheck',
'image': build_image,
'depends_on': [
'compile-build-cmd',
"name": "shellcheck",
"image": build_image,
"depends_on": [
"compile-build-cmd",
],
'commands': [
'./bin/build shellcheck',
"commands": [
"./bin/build shellcheck",
],
}
def shellcheck_pipeline():
environment = {"EDITION": "oss"}
steps = [
compile_build_cmd(),
shellcheck_step(),
]
return pipeline(
name='pr-shellcheck', edition="oss", trigger=trigger, services=[], steps=steps,
name = "pr-shellcheck",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
environment = environment,
)
scripts/drone/pipelines/test_backend.star
+110 -31
View File
@@ -1,42 +1,121 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'wire_install_step',
'test_backend_step',
'test_backend_integration_step',
'verify_gen_cue_step',
'verify_gen_jsonnet_step',
'compile_build_cmd',
'clone_enterprise_step',
'init_enterprise_step',
)
"""
This module returns the pipeline used for testing backend code.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"clone_enterprise_step",
"compile_build_cmd",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"init_enterprise_step",
"test_backend_integration_step",
"test_backend_step",
"verify_gen_cue_step",
"verify_gen_jsonnet_step",
"wire_install_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
def test_backend(trigger, ver_mode, edition="oss"):
environment = {'EDITION': edition}
init_steps = []
if edition != 'oss':
init_steps.extend([clone_enterprise_step(ver_mode), download_grabpl_step(), init_enterprise_step(ver_mode),])
init_steps.extend([
def test_backend(trigger, ver_mode):
"""Generates the pipeline used for testing OSS backend code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": "oss"}
steps = []
verify_step = verify_gen_cue_step()
verify_jsonnet_step = verify_gen_jsonnet_step()
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(enterprise_setup_step())
# Ensure that verif_gen_cue happens after we clone enterprise
# At the time of writing this, very_gen_cue is depended on by the wire step which is what everything else depends on.
verify_step["depends_on"].append("clone-enterprise")
verify_jsonnet_step["depends_on"].append("clone-enterprise")
steps += [
identify_runner_step(),
compile_build_cmd(edition),
verify_gen_cue_step(edition),
verify_gen_jsonnet_step(edition),
compile_build_cmd(edition = "oss"),
verify_step,
verify_jsonnet_step,
wire_install_step(),
])
test_steps = [
test_backend_step(edition),
test_backend_integration_step(edition),
test_backend_step(),
test_backend_integration_step(),
]
pipeline_name = '{}-test-backend'.format(ver_mode)
pipeline_name = "{}-test-backend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = '{}-{}-test-backend'.format(ver_mode, edition)
pipeline_name = "{}-{}-test-backend".format(ver_mode, "oss")
return pipeline(
name=pipeline_name, edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps, environment=environment
name = pipeline_name,
edition = "oss",
trigger = trigger,
steps = steps,
environment = environment,
)
def test_backend_enterprise(trigger, ver_mode, source, edition = "enterprise"):
"""Generates the pipeline used for testing backend enterprise code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
source: controls what revision of enterprise code to test with. The source of the PR, usually.
edition: affects the clone step in the pipeline and also affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
steps = (
[
clone_enterprise_step(source),
download_grabpl_step(),
init_enterprise_step(ver_mode),
identify_runner_step(),
compile_build_cmd(edition),
] +
with_deps(
[
verify_gen_cue_step(),
verify_gen_jsonnet_step(),
],
[
"init-enterprise",
],
) +
[
wire_install_step(),
test_backend_step(),
test_backend_integration_step(),
]
)
pipeline_name = "{}-test-backend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-backend".format(ver_mode, edition)
return pipeline(
name = pipeline_name,
edition = edition,
trigger = trigger,
steps = steps,
environment = environment,
)
scripts/drone/pipelines/test_frontend.star
+93 -26
View File
@@ -1,36 +1,103 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'clone_enterprise_step',
'init_enterprise_step',
'download_grabpl_step',
'yarn_install_step',
'betterer_frontend_step',
'test_frontend_step',
)
"""
This module returns the pipeline used for testing backend code.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"betterer_frontend_step",
"clone_enterprise_step",
"download_grabpl_step",
"enterprise_setup_step",
"identify_runner_step",
"init_enterprise_step",
"test_frontend_step",
"yarn_install_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
"with_deps",
)
def test_frontend(trigger, ver_mode, edition="oss"):
environment = {'EDITION': edition}
init_steps = []
if edition != 'oss':
init_steps.extend([clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode),])
init_steps.extend([
def test_frontend(trigger, ver_mode):
"""Generates the pipeline used for testing frontend code.
Args:
trigger: a Drone trigger for the pipeline
ver_mode: indirectly controls which revision of enterprise code to use.
Returns:
Drone pipeline.
"""
environment = {"EDITION": "oss"}
steps = [
identify_runner_step(),
download_grabpl_step(),
yarn_install_step(edition),
])
test_steps = [
betterer_frontend_step(edition),
test_frontend_step(edition),
yarn_install_step(),
betterer_frontend_step(edition = "oss"),
]
pipeline_name = '{}-test-frontend'.format(ver_mode)
pipeline_name = "{}-test-frontend".format(ver_mode)
test_step = test_frontend_step(edition = "oss")
if ver_mode == "pr":
# In pull requests, attempt to clone grafana enterprise.
steps.append(enterprise_setup_step())
# Also, make the test step depend on 'clone-enterprise
test_step["depends_on"].append("clone-enterprise")
steps.append(test_step)
pipeline_name = "{}-test-frontend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = '{}-{}-test-frontend'.format(ver_mode, edition)
pipeline_name = "{}-oss-test-frontend".format(ver_mode)
return pipeline(
name=pipeline_name, edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps,
name = pipeline_name,
edition = "oss",
trigger = trigger,
steps = steps,
environment = environment,
)
def test_frontend_enterprise(trigger, ver_mode, source, edition = "enterprise"):
"""Generates the pipeline used for testing frontend enterprise code.
Args:
trigger: a Drone trigger for the pipeline.
ver_mode: affects the pipeline name.
source: controls what revision of Grafana code to test with.
edition: affects the clone step in the pipeline and also affects the pipeline name.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
steps = (
[
clone_enterprise_step(source),
init_enterprise_step(ver_mode),
identify_runner_step(),
download_grabpl_step(),
] +
with_deps([yarn_install_step()], ["init-enterprise"]) +
[
betterer_frontend_step(edition),
test_frontend_step(edition),
]
)
pipeline_name = "{}-test-frontend".format(ver_mode)
if ver_mode in ("release-branch", "release"):
pipeline_name = "{}-{}-test-frontend".format(ver_mode, edition)
return pipeline(
name = pipeline_name,
edition = edition,
trigger = trigger,
steps = steps,
environment = environment,
)
scripts/drone/pipelines/trigger_downstream.star
+35 -18
View File
@@ -1,28 +1,45 @@
load(
'scripts/drone/steps/lib.star',
'enterprise_downstream_step',
)
"""
This module returns the pipeline used for triggering a downstream pipeline for Grafana Enterprise.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"enterprise_downstream_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
trigger = {
'event': ['push',],
'branch': 'main',
'paths': {
'exclude': [
'*.md',
'docs/**',
'latest.json',
"event": [
"push",
],
"branch": "main",
"paths": {
"exclude": [
"*.md",
"docs/**",
"latest.json",
],
},
}
def enterprise_downstream_pipeline(edition, ver_mode):
steps = [enterprise_downstream_step(edition, ver_mode)]
deps = ['main-build-e2e-publish', 'main-integration-tests']
def enterprise_downstream_pipeline():
environment = {"EDITION": "oss"}
steps = [
enterprise_downstream_step(ver_mode = "main"),
]
deps = [
"main-build-e2e-publish",
"main-integration-tests",
]
return pipeline(
name='main-trigger-downstream', edition=edition, trigger=trigger, services=[], steps=steps, depends_on=deps
)
name = "main-trigger-downstream",
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
depends_on = deps,
environment = environment,
)
scripts/drone/pipelines/verify_drone.star
+19 -10
View File
@@ -1,17 +1,21 @@
load(
'scripts/drone/steps/lib.star',
'identify_runner_step',
'download_grabpl_step',
'lint_drone_step',
'compile_build_cmd',
)
"""
This module returns the pipeline used for verifying Drone configuration.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"identify_runner_step",
"lint_drone_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def verify_drone(trigger, ver_mode):
environment = {"EDITION": "oss"}
steps = [
identify_runner_step(),
download_grabpl_step(),
@@ -19,5 +23,10 @@ def verify_drone(trigger, ver_mode):
lint_drone_step(),
]
return pipeline(
name='{}-verify-drone'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=steps,
name = "{}-verify-drone".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
environment = environment,
)
scripts/drone/pipelines/verify_starlark.star
+32
View File
@@ -0,0 +1,32 @@
"""
This module returns a Drone pipeline that verifies all Starlark files are linted.
"""
load(
"scripts/drone/steps/lib.star",
"compile_build_cmd",
"download_grabpl_step",
"identify_runner_step",
"lint_starlark_step",
)
load(
"scripts/drone/utils/utils.star",
"pipeline",
)
def verify_starlark(trigger, ver_mode):
environment = {"EDITION": "oss"}
steps = [
identify_runner_step(),
download_grabpl_step(),
compile_build_cmd(),
lint_starlark_step(),
]
return pipeline(
name = "{}-verify-starlark".format(ver_mode),
edition = "oss",
trigger = trigger,
services = [],
steps = steps,
environment = environment,
)
scripts/drone/pipelines/windows.star
+32 -125
View File
@@ -1,134 +1,41 @@
load(
'scripts/drone/steps/lib.star',
'grabpl_version',
'wix_image',
'identify_runner_step',
)
"""
This module returns the pipeline used for building Grafana on Windows.
"""
load(
'scripts/drone/utils/utils.star',
'pipeline',
"scripts/drone/utils/utils.star",
"pipeline",
)
load(
"scripts/drone/steps/lib.star",
"get_windows_steps",
)
load('scripts/drone/vault.star', 'from_secret', 'prerelease_bucket', 'github_token')
def windows(trigger, edition, ver_mode):
init_cmds = []
sfx = ''
if edition in ('enterprise', 'enterprise2'):
sfx = '-{}'.format(edition)
else:
init_cmds.extend([
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
grabpl_version),
])
steps = [
{
'name': 'windows-init',
'image': wix_image,
'commands': init_cmds,
},
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release', 'release-branch',
):
bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
if ver_mode == 'release':
ver_part = '${DRONE_TAG}'
dir = 'release'
else:
dir = 'main'
bucket = 'grafana-downloads'
build_no = 'DRONE_BUILD_NUMBER'
ver_part = '--build-id $$env:{}'.format(build_no)
installer_commands = [
'$$gcpKey = $$env:GCP_KEY',
'[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
# gcloud fails to read the file unless converted with dos2unix
'dos2unix gcpkey.json',
'gcloud auth activate-service-account --key-file=gcpkey.json',
'rm gcpkey.json',
'cp C:\\App\\nssm-2.24.zip .',
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release',
):
installer_commands.extend([
'.\\grabpl.exe windows-installer --edition {} {}'.format(edition, ver_part),
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
])
if ver_mode == 'main':
installer_commands.extend([
'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir),
])
else:
installer_commands.extend([
'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
])
steps.append({
'name': 'build-windows-installer',
'image': wix_image,
'depends_on': [
'windows-init',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
'GITHUB_TOKEN': from_secret('github_token')
},
'commands': installer_commands,
})
"""Generates the pipeline used for building Grafana on Windows.
if edition in ('enterprise', 'enterprise2'):
if ver_mode == 'release':
committish = '${DRONE_TAG}'
elif ver_mode == 'release-branch':
committish = '$$env:DRONE_BRANCH'
else:
committish = '$$env:DRONE_COMMIT'
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
download_grabpl_step_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
grabpl_version),
]
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
]
clone_cmds.extend([
'cd grafana-enterprise',
'git checkout {}'.format(committish),
])
steps.insert(0, {
'name': 'clone',
'image': wix_image,
'environment': {
'GITHUB_TOKEN': from_secret(github_token),
},
'commands': download_grabpl_step_cmds + clone_cmds,
})
steps[1]['depends_on'] = [
'clone',
]
steps[1]['commands'].extend([
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
'cp -r grafana-enterprise C:\\App\\grafana-enterprise',
'rm -r -force grafana-enterprise',
'cp grabpl.exe C:\\App\\grabpl.exe',
'rm -force grabpl.exe',
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise',
'cp C:\\App\\grabpl.exe grabpl.exe',
])
if 'environment' in steps[1]:
steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)}
else:
steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)}
Args:
trigger: a Drone trigger for the pipeline.
edition: controls whether enterprise code is included in the pipeline steps.
ver_mode: controls whether a pre-release or actual release pipeline is generated.
Also indirectly controls which version of enterprise code is used.
Returns:
Drone pipeline.
"""
environment = {"EDITION": edition}
return pipeline(
name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=[identify_runner_step('windows')] + steps,
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
name = "main-windows",
edition = edition,
trigger = dict(trigger, repo = ["grafana/grafana"]),
steps = get_windows_steps(edition, ver_mode),
depends_on = [
"main-test-frontend",
"main-test-backend",
"main-build-e2e-publish",
"main-integration-tests",
],
platform = "windows",
environment = environment,
)