CI: Refactor/Reorder *.star files (#52384)
* Refactor test pipelines * Move pr, main and release starlark files under scripts/drone/event * Move cron.star to scripts/drone/events * Further unused code removal * Introducing build.star * Extract verify_drone.star * Extract windows.star * Extract publish.star * Remove unused code * Fix formatting in main.star
This commit is contained in:
Dimitris Sotirakis
@@ -0,0 +1,107 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'download_grabpl_step',
|
||||
'build_image',
|
||||
'identify_runner_step',
|
||||
'gen_version_step',
|
||||
'wire_install_step',
|
||||
'yarn_install_step',
|
||||
'build_backend_step',
|
||||
'build_frontend_step',
|
||||
'build_frontend_package_step',
|
||||
'build_plugins_step',
|
||||
'package_step',
|
||||
'grafana_server_step',
|
||||
'e2e_tests_step',
|
||||
'e2e_tests_artifacts',
|
||||
'build_storybook_step',
|
||||
'copy_packages_for_docker_step',
|
||||
'build_docker_images_step',
|
||||
'publish_images_step',
|
||||
'postgres_integration_tests_step',
|
||||
'mysql_integration_tests_step',
|
||||
'redis_integration_tests_step',
|
||||
'memcached_integration_tests_step',
|
||||
'get_windows_steps',
|
||||
'benchmark_ldap_step',
|
||||
'enterprise_downstream_step',
|
||||
'frontend_metrics_step',
|
||||
'store_storybook_step',
|
||||
'release_canary_npm_packages_step',
|
||||
'upload_packages_step',
|
||||
'store_packages_step',
|
||||
'upload_cdn_step',
|
||||
'verify_gen_cue_step',
|
||||
'test_a11y_frontend_step',
|
||||
'trigger_oss',
|
||||
'betterer_frontend_step',
|
||||
'trigger_test_release'
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
def build_e2e(trigger, ver_mode, edition):
|
||||
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
yarn_install_step(),
|
||||
]
|
||||
build_steps = []
|
||||
if ver_mode == 'main':
|
||||
build_steps.extend([trigger_test_release()])
|
||||
build_steps.extend([
|
||||
enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
|
||||
build_backend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
|
||||
build_plugins_step(edition=edition, ver_mode=ver_mode),
|
||||
])
|
||||
if ver_mode == 'main':
|
||||
build_steps.extend([package_step(edition=edition, ver_mode=ver_mode)])
|
||||
elif ver_mode == 'pr':
|
||||
build_steps.extend([package_step(edition=edition, ver_mode=ver_mode, variants=variants)])
|
||||
|
||||
build_steps.extend([
|
||||
grafana_server_step(edition=edition),
|
||||
e2e_tests_step('dashboards-suite', edition=edition),
|
||||
e2e_tests_step('smoke-tests-suite', edition=edition),
|
||||
e2e_tests_step('panels-suite', edition=edition),
|
||||
e2e_tests_step('various-suite', edition=edition),
|
||||
e2e_tests_artifacts(edition=edition),
|
||||
build_storybook_step(edition=edition, ver_mode=ver_mode),
|
||||
copy_packages_for_docker_step(),
|
||||
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
|
||||
])
|
||||
if ver_mode == 'main':
|
||||
build_steps.extend([
|
||||
store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
|
||||
frontend_metrics_step(edition=edition, trigger=trigger_oss)
|
||||
])
|
||||
|
||||
if ver_mode == 'main':
|
||||
build_steps.extend([
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
|
||||
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
|
||||
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss),
|
||||
release_canary_npm_packages_step(edition, trigger=trigger_oss),
|
||||
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
|
||||
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
|
||||
])
|
||||
elif ver_mode == 'pr':
|
||||
build_steps.extend([build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ])])
|
||||
|
||||
publish_suffix = ''
|
||||
if ver_mode == 'main':
|
||||
publish_suffix = '-publish'
|
||||
|
||||
return pipeline(
|
||||
name='{}-build-e2e{}'.format(ver_mode, publish_suffix), edition="oss", trigger=trigger, services=[], steps=init_steps + build_steps,
|
||||
)
|
||||
@@ -1,75 +0,0 @@
|
||||
load('scripts/drone/vault.star', 'from_secret')
|
||||
|
||||
aquasec_trivy_image = 'aquasec/trivy:0.21.0'
|
||||
|
||||
def cronjobs(edition):
|
||||
return [
|
||||
scan_docker_image_pipeline(edition, 'latest'),
|
||||
scan_docker_image_pipeline(edition, 'main'),
|
||||
scan_docker_image_pipeline(edition, 'latest-ubuntu'),
|
||||
scan_docker_image_pipeline(edition, 'main-ubuntu'),
|
||||
]
|
||||
|
||||
def cron_job_pipeline(name, steps):
|
||||
return {
|
||||
'kind': 'pipeline',
|
||||
'type': 'docker',
|
||||
'platform': {
|
||||
'os': 'linux',
|
||||
'arch': 'amd64',
|
||||
},
|
||||
'name': name,
|
||||
'trigger': {
|
||||
'event': 'cron',
|
||||
'cron': 'nightly',
|
||||
},
|
||||
'steps': steps,
|
||||
}
|
||||
|
||||
def scan_docker_image_pipeline(edition, tag):
|
||||
if edition != 'oss':
|
||||
edition='grafana-enterprise'
|
||||
else:
|
||||
edition='grafana'
|
||||
|
||||
dockerImage='grafana/{}:{}'.format(edition, tag)
|
||||
|
||||
return cron_job_pipeline(
|
||||
name='scan-' + dockerImage + '-image',
|
||||
steps=[
|
||||
scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage),
|
||||
scan_docker_image_high_critical_vulnerabilities_step(dockerImage),
|
||||
slack_job_failed_step('grafana-backend-ops', dockerImage),
|
||||
])
|
||||
|
||||
def scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage):
|
||||
return {
|
||||
'name': 'scan-unkown-low-medium-vulnerabilities',
|
||||
'image': aquasec_trivy_image,
|
||||
'commands': [
|
||||
'trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ' + dockerImage,
|
||||
],
|
||||
}
|
||||
|
||||
def scan_docker_image_high_critical_vulnerabilities_step(dockerImage):
|
||||
return {
|
||||
'name': 'scan-high-critical-vulnerabilities',
|
||||
'image': aquasec_trivy_image,
|
||||
'commands': [
|
||||
'trivy --exit-code 1 --severity HIGH,CRITICAL ' + dockerImage,
|
||||
],
|
||||
}
|
||||
|
||||
def slack_job_failed_step(channel, image):
|
||||
return {
|
||||
'name': 'slack-notify-failure',
|
||||
'image': 'plugins/slack',
|
||||
'settings': {
|
||||
'webhook': from_secret('slack_webhook_backend'),
|
||||
'channel': channel,
|
||||
'template': 'Nightly docker image scan job for ' + image + ' failed: {{build.link}}',
|
||||
},
|
||||
'when': {
|
||||
'status': 'failure'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'identify_runner_step',
|
||||
'download_grabpl_step',
|
||||
'verify_gen_cue_step',
|
||||
'wire_install_step',
|
||||
'postgres_integration_tests_step',
|
||||
'mysql_integration_tests_step',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/services/services.star',
|
||||
'integration_test_services',
|
||||
'integration_test_services_volumes',
|
||||
'ldap_service',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
def integration_tests(trigger, ver_mode, edition):
|
||||
services = integration_test_services(edition)
|
||||
volumes = integration_test_services_volumes()
|
||||
init_steps = [
|
||||
download_grabpl_step(),
|
||||
identify_runner_step(),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
|
||||
return pipeline(
|
||||
name='{}-integration-tests'.format(ver_mode), edition="oss", trigger=trigger, services=services, steps=init_steps + test_steps,
|
||||
volumes=volumes
|
||||
)
|
||||
@@ -1,243 +0,0 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'download_grabpl_step',
|
||||
'build_image',
|
||||
'identify_runner_step',
|
||||
'gen_version_step',
|
||||
'wire_install_step',
|
||||
'yarn_install_step',
|
||||
'lint_drone_step',
|
||||
'lint_backend_step',
|
||||
'lint_frontend_step',
|
||||
'codespell_step',
|
||||
'shellcheck_step',
|
||||
'test_backend_step',
|
||||
'test_backend_integration_step',
|
||||
'test_frontend_step',
|
||||
'build_backend_step',
|
||||
'build_frontend_step',
|
||||
'build_frontend_package_step',
|
||||
'build_plugins_step',
|
||||
'package_step',
|
||||
'grafana_server_step',
|
||||
'e2e_tests_step',
|
||||
'e2e_tests_artifacts',
|
||||
'build_storybook_step',
|
||||
'copy_packages_for_docker_step',
|
||||
'build_docker_images_step',
|
||||
'publish_images_step',
|
||||
'postgres_integration_tests_step',
|
||||
'mysql_integration_tests_step',
|
||||
'redis_integration_tests_step',
|
||||
'memcached_integration_tests_step',
|
||||
'get_windows_steps',
|
||||
'benchmark_ldap_step',
|
||||
'enterprise_downstream_step',
|
||||
'frontend_metrics_step',
|
||||
'store_storybook_step',
|
||||
'release_canary_npm_packages_step',
|
||||
'upload_packages_step',
|
||||
'store_packages_step',
|
||||
'upload_cdn_step',
|
||||
'verify_gen_cue_step',
|
||||
'test_a11y_frontend_step',
|
||||
'trigger_oss',
|
||||
'betterer_frontend_step'
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/services/services.star',
|
||||
'integration_test_services',
|
||||
'integration_test_services_volumes',
|
||||
'ldap_service',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
'notify_pipeline',
|
||||
'failure_template',
|
||||
'drone_change_template',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/docs.star',
|
||||
'docs_pipelines',
|
||||
)
|
||||
|
||||
load('scripts/drone/vault.star', 'from_secret')
|
||||
|
||||
|
||||
ver_mode = 'main'
|
||||
trigger = {
|
||||
'event': ['push',],
|
||||
'branch': 'main',
|
||||
}
|
||||
|
||||
def main_test_frontend():
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
yarn_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
lint_frontend_step(),
|
||||
betterer_frontend_step(),
|
||||
test_frontend_step(),
|
||||
]
|
||||
return pipeline(
|
||||
name='main-test-frontend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
|
||||
)
|
||||
|
||||
def main_test_backend():
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
lint_drone_step(),
|
||||
codespell_step(),
|
||||
shellcheck_step(),
|
||||
lint_backend_step(edition="oss"),
|
||||
test_backend_step(edition="oss"),
|
||||
test_backend_integration_step(edition="oss"),
|
||||
]
|
||||
return pipeline(
|
||||
name='main-test-backend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
|
||||
)
|
||||
|
||||
|
||||
def get_steps(edition):
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
yarn_install_step(),
|
||||
]
|
||||
build_steps = [
|
||||
trigger_test_release(),
|
||||
enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
|
||||
build_backend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
|
||||
build_plugins_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
integration_test_steps = [
|
||||
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
|
||||
# Insert remaining steps
|
||||
build_steps.extend([
|
||||
package_step(edition=edition, ver_mode=ver_mode),
|
||||
grafana_server_step(edition=edition),
|
||||
e2e_tests_step('dashboards-suite', edition=edition),
|
||||
e2e_tests_step('smoke-tests-suite', edition=edition),
|
||||
e2e_tests_step('panels-suite', edition=edition),
|
||||
e2e_tests_step('various-suite', edition=edition),
|
||||
e2e_tests_artifacts(edition=edition),
|
||||
build_storybook_step(edition=edition, ver_mode=ver_mode),
|
||||
store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
|
||||
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
|
||||
frontend_metrics_step(edition=edition, trigger=trigger_oss),
|
||||
copy_packages_for_docker_step(),
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
|
||||
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
|
||||
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss)
|
||||
])
|
||||
|
||||
build_steps.extend([
|
||||
release_canary_npm_packages_step(edition, trigger=trigger_oss),
|
||||
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
|
||||
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
|
||||
])
|
||||
|
||||
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
|
||||
store_steps = [store_packages_step(edition=edition, ver_mode=ver_mode),]
|
||||
|
||||
return init_steps, build_steps, integration_test_steps, windows_steps, store_steps
|
||||
|
||||
def trigger_test_release():
|
||||
return {
|
||||
'name': 'trigger-test-release',
|
||||
'image': build_image,
|
||||
'environment': {
|
||||
'GITHUB_TOKEN': from_secret('github_token'),
|
||||
'DOWNSTREAM_REPO': from_secret('downstream'),
|
||||
'TEST_TAG': 'v0.0.0-test',
|
||||
},
|
||||
'commands': [
|
||||
'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1',
|
||||
'cd grafana-enterprise',
|
||||
'git fetch origin "refs/tags/*:refs/tags/*"',
|
||||
'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}',
|
||||
'cd -',
|
||||
'git fetch origin "refs/tags/*:refs/tags/*"',
|
||||
'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git',
|
||||
'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet',
|
||||
],
|
||||
'failure': 'ignore',
|
||||
'when': {
|
||||
'paths': {
|
||||
'include': [
|
||||
'.drone.yml',
|
||||
]
|
||||
},
|
||||
'repo': [
|
||||
'grafana/grafana',
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
def main_pipelines(edition):
|
||||
services = integration_test_services(edition)
|
||||
volumes = integration_test_services_volumes()
|
||||
drone_change_trigger = {
|
||||
'event': ['push',],
|
||||
'branch': 'main',
|
||||
'repo': [
|
||||
'grafana/grafana',
|
||||
],
|
||||
'paths': {
|
||||
'include': [
|
||||
'.drone.yml',
|
||||
],
|
||||
'exclude': [
|
||||
'exclude',
|
||||
],
|
||||
},
|
||||
}
|
||||
init_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
|
||||
|
||||
pipelines = [docs_pipelines(edition, ver_mode, trigger), main_test_frontend(), main_test_backend(), pipeline(
|
||||
name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
|
||||
steps=init_steps + build_steps,
|
||||
), pipeline(
|
||||
name='main-integration-tests', edition=edition, trigger=trigger, services=services,
|
||||
steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps,
|
||||
volumes=volumes,
|
||||
), pipeline(
|
||||
name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
|
||||
steps=[identify_runner_step('windows')] + windows_steps,
|
||||
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
|
||||
), notify_pipeline(
|
||||
name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
|
||||
template=drone_change_template, secret='drone-changes-webhook',
|
||||
), pipeline(
|
||||
name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
|
||||
steps=[download_grabpl_step(), gen_version_step(ver_mode), identify_runner_step(),] + store_steps,
|
||||
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
|
||||
), notify_pipeline(
|
||||
name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
|
||||
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'],
|
||||
template=failure_template, secret='slack_webhook'
|
||||
)]
|
||||
|
||||
return pipelines
|
||||
@@ -1,192 +0,0 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'download_grabpl_step',
|
||||
'gen_version_step',
|
||||
'yarn_install_step',
|
||||
'wire_install_step',
|
||||
'identify_runner_step',
|
||||
'lint_drone_step',
|
||||
'lint_backend_step',
|
||||
'lint_frontend_step',
|
||||
'codespell_step',
|
||||
'shellcheck_step',
|
||||
'build_backend_step',
|
||||
'build_frontend_step',
|
||||
'build_frontend_package_step',
|
||||
'build_plugins_step',
|
||||
'test_backend_step',
|
||||
'test_backend_integration_step',
|
||||
'test_frontend_step',
|
||||
'package_step',
|
||||
'grafana_server_step',
|
||||
'e2e_tests_step',
|
||||
'e2e_tests_artifacts',
|
||||
'build_storybook_step',
|
||||
'copy_packages_for_docker_step',
|
||||
'build_docker_images_step',
|
||||
'postgres_integration_tests_step',
|
||||
'mysql_integration_tests_step',
|
||||
'redis_integration_tests_step',
|
||||
'memcached_integration_tests_step',
|
||||
'benchmark_ldap_step',
|
||||
'verify_gen_cue_step',
|
||||
'test_a11y_frontend_step',
|
||||
'enterprise_downstream_step',
|
||||
'betterer_frontend_step',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/services/services.star',
|
||||
'integration_test_services',
|
||||
'integration_test_services_volumes',
|
||||
'ldap_service',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'notify_pipeline',
|
||||
'pipeline',
|
||||
'failure_template',
|
||||
'drone_change_template',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/docs.star',
|
||||
'docs_pipelines',
|
||||
'trigger_docs',
|
||||
)
|
||||
|
||||
ver_mode = 'pr'
|
||||
trigger = {
|
||||
'event': [
|
||||
'pull_request',
|
||||
],
|
||||
'paths': {
|
||||
'exclude': [
|
||||
'*.md',
|
||||
'docs/**',
|
||||
'latest.json',
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
def pr_verify_drone():
|
||||
steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
lint_drone_step(),
|
||||
]
|
||||
return pipeline(
|
||||
name='pr-verify-drone', edition="oss", trigger=get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), services=[], steps=steps,
|
||||
)
|
||||
|
||||
|
||||
def pr_test_frontend():
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
yarn_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
lint_frontend_step(),
|
||||
betterer_frontend_step(),
|
||||
test_frontend_step(),
|
||||
]
|
||||
return pipeline(
|
||||
name='pr-test-frontend', edition="oss", trigger=get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), services=[], steps=init_steps + test_steps,
|
||||
)
|
||||
|
||||
|
||||
def pr_test_backend():
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
codespell_step(),
|
||||
shellcheck_step(),
|
||||
lint_backend_step(edition="oss"),
|
||||
test_backend_step(edition="oss"),
|
||||
test_backend_integration_step(edition="oss"),
|
||||
]
|
||||
return pipeline(
|
||||
name='pr-test-backend', edition="oss", trigger=get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), services=[], steps=init_steps + test_steps,
|
||||
)
|
||||
|
||||
|
||||
def pr_pipelines(edition):
|
||||
services = integration_test_services(edition)
|
||||
volumes = integration_test_services_volumes()
|
||||
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',]
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
yarn_install_step(),
|
||||
]
|
||||
build_steps = [
|
||||
enterprise_downstream_step(edition=edition, ver_mode=ver_mode),
|
||||
build_backend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
|
||||
build_plugins_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
integration_test_steps = [
|
||||
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
|
||||
# Insert remaining build_steps
|
||||
build_steps.extend([
|
||||
package_step(edition=edition, ver_mode=ver_mode, variants=variants),
|
||||
grafana_server_step(edition=edition),
|
||||
e2e_tests_step('dashboards-suite', edition=edition),
|
||||
e2e_tests_step('smoke-tests-suite', edition=edition),
|
||||
e2e_tests_step('panels-suite', edition=edition),
|
||||
e2e_tests_step('various-suite', edition=edition),
|
||||
e2e_tests_artifacts(edition=edition),
|
||||
build_storybook_step(edition=edition, ver_mode=ver_mode),
|
||||
test_a11y_frontend_step(ver_mode=ver_mode, edition=edition),
|
||||
copy_packages_for_docker_step(),
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ]),
|
||||
])
|
||||
|
||||
return [
|
||||
pr_verify_drone(),
|
||||
pr_test_frontend(),
|
||||
pr_test_backend(),
|
||||
pipeline(
|
||||
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps,
|
||||
), pipeline(
|
||||
name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
|
||||
steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps,
|
||||
volumes=volumes,
|
||||
), docs_pipelines(edition, ver_mode, trigger_docs())
|
||||
]
|
||||
|
||||
|
||||
def get_pr_trigger(include_paths=None, exclude_paths=None):
|
||||
paths_ex = ['docs/**', '*.md']
|
||||
paths_in = []
|
||||
if include_paths:
|
||||
for path in include_paths:
|
||||
paths_in.extend([path])
|
||||
if exclude_paths:
|
||||
for path in exclude_paths:
|
||||
paths_ex.extend([path])
|
||||
return {
|
||||
'event': [
|
||||
'pull_request',
|
||||
],
|
||||
'paths': {
|
||||
'exclude': paths_ex,
|
||||
'include': paths_in,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'identify_runner_step',
|
||||
'gen_version_step',
|
||||
'download_grabpl_step',
|
||||
'store_packages_step'
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
def publish(trigger, ver_mode, edition):
|
||||
steps = [
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
identify_runner_step(),
|
||||
store_packages_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
return pipeline(
|
||||
name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
|
||||
steps=steps,
|
||||
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ],
|
||||
)
|
||||
@@ -1,470 +0,0 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'disable_tests',
|
||||
'clone_enterprise_step',
|
||||
'download_grabpl_step',
|
||||
'gen_version_step',
|
||||
'yarn_install_step',
|
||||
'wire_install_step',
|
||||
'init_enterprise_step',
|
||||
'lint_drone_step',
|
||||
'build_image',
|
||||
'identify_runner_step',
|
||||
'publish_image',
|
||||
'lint_backend_step',
|
||||
'lint_frontend_step',
|
||||
'codespell_step',
|
||||
'shellcheck_step',
|
||||
'test_backend_step',
|
||||
'test_backend_integration_step',
|
||||
'test_frontend_step',
|
||||
'build_backend_step',
|
||||
'build_frontend_step',
|
||||
'build_frontend_package_step',
|
||||
'build_plugins_step',
|
||||
'package_step',
|
||||
'grafana_server_step',
|
||||
'e2e_tests_step',
|
||||
'e2e_tests_artifacts',
|
||||
'build_storybook_step',
|
||||
'copy_packages_for_docker_step',
|
||||
'build_docker_images_step',
|
||||
'postgres_integration_tests_step',
|
||||
'mysql_integration_tests_step',
|
||||
'redis_integration_tests_step',
|
||||
'memcached_integration_tests_step',
|
||||
'get_windows_steps',
|
||||
'benchmark_ldap_step',
|
||||
'store_storybook_step',
|
||||
'upload_packages_step',
|
||||
'store_packages_step',
|
||||
'upload_cdn_step',
|
||||
'verify_gen_cue_step',
|
||||
'publish_images_step',
|
||||
'trigger_oss',
|
||||
'artifacts_page_step'
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/services/services.star',
|
||||
'integration_test_services',
|
||||
'integration_test_services_volumes',
|
||||
'ldap_service',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
'notify_pipeline',
|
||||
'failure_template',
|
||||
'drone_change_template',
|
||||
)
|
||||
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
|
||||
|
||||
def store_npm_packages_step():
|
||||
return {
|
||||
'name': 'store-npm-packages',
|
||||
'image': publish_image,
|
||||
'depends_on': [
|
||||
'build-frontend-packages',
|
||||
],
|
||||
'environment': {
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
|
||||
},
|
||||
'commands': [
|
||||
'./bin/grabpl artifacts npm store --tag ${DRONE_TAG}'
|
||||
],
|
||||
}
|
||||
|
||||
def retrieve_npm_packages_step():
|
||||
return {
|
||||
'name': 'retrieve-npm-packages',
|
||||
'image': publish_image,
|
||||
'depends_on': [
|
||||
'yarn-install',
|
||||
],
|
||||
'environment': {
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
|
||||
},
|
||||
'commands': [
|
||||
'./bin/grabpl artifacts npm retrieve --tag v${TAG}'
|
||||
],
|
||||
}
|
||||
|
||||
def release_npm_packages_step():
|
||||
return {
|
||||
'name': 'release-npm-packages',
|
||||
'image': build_image,
|
||||
'depends_on': [
|
||||
'retrieve-npm-packages',
|
||||
],
|
||||
'environment': {
|
||||
'NPM_TOKEN': from_secret('npm_token'),
|
||||
},
|
||||
'commands': [
|
||||
'./bin/grabpl artifacts npm release --tag v${TAG}'
|
||||
],
|
||||
}
|
||||
|
||||
def fetch_images_step(edition):
|
||||
return {
|
||||
'name': 'fetch-images-{}'.format(edition),
|
||||
'image': 'google/cloud-sdk',
|
||||
'environment': {
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
'DOCKER_USER': from_secret('docker_username'),
|
||||
'DOCKER_PASSWORD': from_secret('docker_password'),
|
||||
},
|
||||
'commands': ['./bin/grabpl artifacts docker fetch --version-tag ${{TAG}} --edition {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(edition)],
|
||||
'depends_on': ['grabpl'],
|
||||
'volumes': [{
|
||||
'name': 'docker',
|
||||
'path': '/var/run/docker.sock'
|
||||
}],
|
||||
}
|
||||
|
||||
def publish_image_steps(version, mode, docker_repo, additional_docker_repo=""):
|
||||
steps = [
|
||||
download_grabpl_step(),
|
||||
fetch_images_step(version),
|
||||
publish_images_step(version, 'release', mode, docker_repo),
|
||||
]
|
||||
if additional_docker_repo != "":
|
||||
steps.extend([publish_images_step(version, 'release', mode, additional_docker_repo)])
|
||||
|
||||
return steps
|
||||
|
||||
def publish_image_pipelines(mode):
|
||||
trigger = {
|
||||
'event': ['promote'],
|
||||
'target': [mode],
|
||||
}
|
||||
|
||||
return [pipeline(
|
||||
name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='oss', mode=mode, docker_repo='grafana', additional_docker_repo='grafana-oss'), edition=""
|
||||
), pipeline(
|
||||
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(version='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition=""
|
||||
),]
|
||||
|
||||
def get_steps(edition, ver_mode):
|
||||
package_steps = []
|
||||
publish_steps = []
|
||||
should_publish = ver_mode == 'release'
|
||||
should_upload = should_publish or ver_mode in ('release-branch',)
|
||||
include_enterprise2 = edition == 'enterprise'
|
||||
edition2 = 'enterprise2'
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition),
|
||||
wire_install_step(),
|
||||
yarn_install_step(),
|
||||
]
|
||||
|
||||
test_steps = []
|
||||
if edition != 'enterprise':
|
||||
test_steps.extend([shellcheck_step()])
|
||||
|
||||
test_steps.extend([
|
||||
codespell_step(),
|
||||
lint_backend_step(edition=edition),
|
||||
lint_frontend_step(),
|
||||
test_backend_step(edition=edition),
|
||||
test_backend_integration_step(edition=edition),
|
||||
test_frontend_step(),
|
||||
])
|
||||
|
||||
build_steps = [
|
||||
build_backend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_step(edition=edition, ver_mode=ver_mode),
|
||||
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
|
||||
build_plugins_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
|
||||
integration_test_steps = [
|
||||
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
|
||||
]
|
||||
|
||||
if include_enterprise2:
|
||||
test_steps.extend([
|
||||
lint_backend_step(edition=edition2),
|
||||
test_backend_step(edition=edition2),
|
||||
])
|
||||
build_steps.extend([
|
||||
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']),
|
||||
])
|
||||
|
||||
# Insert remaining steps
|
||||
build_steps.extend([
|
||||
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2),
|
||||
copy_packages_for_docker_step(),
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=True),
|
||||
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=True),
|
||||
grafana_server_step(edition=edition),
|
||||
])
|
||||
|
||||
if not disable_tests:
|
||||
build_steps.extend([
|
||||
e2e_tests_step('dashboards-suite', edition=edition, tries=3),
|
||||
e2e_tests_step('smoke-tests-suite', edition=edition, tries=3),
|
||||
e2e_tests_step('panels-suite', edition=edition, tries=3),
|
||||
e2e_tests_step('various-suite', edition=edition, tries=3),
|
||||
e2e_tests_artifacts(edition=edition),
|
||||
])
|
||||
|
||||
build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode)
|
||||
if build_storybook:
|
||||
build_steps.append(build_storybook)
|
||||
|
||||
if should_upload:
|
||||
publish_steps.append(upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss))
|
||||
publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss))
|
||||
if should_publish:
|
||||
publish_step = store_storybook_step(edition=edition, ver_mode=ver_mode)
|
||||
store_npm_step = store_npm_packages_step()
|
||||
if publish_step:
|
||||
publish_steps.append(publish_step)
|
||||
if store_npm_step:
|
||||
publish_steps.append(store_npm_step)
|
||||
windows_package_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
|
||||
|
||||
if include_enterprise2:
|
||||
publish_steps.extend([
|
||||
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64']),
|
||||
upload_cdn_step(edition=edition2, ver_mode=ver_mode),
|
||||
])
|
||||
if should_upload:
|
||||
step = upload_packages_step(edition=edition2, ver_mode=ver_mode)
|
||||
if step:
|
||||
publish_steps.append(step)
|
||||
|
||||
return init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps
|
||||
|
||||
def get_oss_pipelines(trigger, ver_mode):
|
||||
edition = 'oss'
|
||||
services = integration_test_services(edition=edition)
|
||||
volumes = integration_test_services_volumes()
|
||||
init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
|
||||
windows_pipeline = pipeline(
|
||||
name='{}-oss-windows'.format(ver_mode), edition=edition, trigger=trigger,
|
||||
steps=[identify_runner_step('windows')] + windows_package_steps,
|
||||
platform='windows', depends_on=[
|
||||
'oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode),
|
||||
],
|
||||
)
|
||||
pipelines = [
|
||||
pipeline(
|
||||
name='{}-oss-build{}-publish'.format(ver_mode, get_e2e_suffix()), edition=edition, trigger=trigger, services=[],
|
||||
steps=init_steps + build_steps + package_steps + publish_steps,
|
||||
volumes=volumes,
|
||||
),
|
||||
]
|
||||
if not disable_tests:
|
||||
pipelines.extend([
|
||||
pipeline(
|
||||
name='{}-oss-test'.format(ver_mode), edition=edition, trigger=trigger, services=[],
|
||||
steps=init_steps + test_steps,
|
||||
volumes=[],
|
||||
),
|
||||
pipeline(
|
||||
name='{}-oss-integration-tests'.format(ver_mode), edition=edition, trigger=trigger, services=services,
|
||||
steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition), wire_install_step(), ] + integration_test_steps,
|
||||
volumes=volumes,
|
||||
)
|
||||
])
|
||||
deps = {
|
||||
'depends_on': [
|
||||
'{}-oss-build{}-publish'.format(ver_mode, get_e2e_suffix()),
|
||||
'{}-oss-test'.format(ver_mode),
|
||||
'{}-oss-integration-tests'.format(ver_mode)
|
||||
]
|
||||
}
|
||||
windows_pipeline.update(deps)
|
||||
|
||||
pipelines.extend([windows_pipeline])
|
||||
return pipelines
|
||||
|
||||
def get_enterprise_pipelines(trigger, ver_mode):
|
||||
edition = 'enterprise'
|
||||
services = integration_test_services(edition=edition)
|
||||
volumes = integration_test_services_volumes()
|
||||
deps_on_clone_enterprise_step = {
|
||||
'depends_on': [
|
||||
'init-enterprise',
|
||||
]
|
||||
}
|
||||
_, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
|
||||
init_steps = [
|
||||
download_grabpl_step(),
|
||||
identify_runner_step(),
|
||||
clone_enterprise_step(ver_mode),
|
||||
init_enterprise_step(ver_mode)
|
||||
]
|
||||
for step in [wire_install_step(), yarn_install_step(), gen_version_step(ver_mode), verify_gen_cue_step(edition)]:
|
||||
step.update(deps_on_clone_enterprise_step)
|
||||
init_steps.extend([step])
|
||||
|
||||
windows_pipeline = pipeline(
|
||||
name='{}-enterprise-windows'.format(ver_mode), edition=edition, trigger=trigger,
|
||||
steps=[identify_runner_step('windows')] + windows_package_steps,
|
||||
platform='windows', depends_on=[
|
||||
'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode),
|
||||
],
|
||||
)
|
||||
pipelines = [
|
||||
pipeline(
|
||||
name='{}-enterprise-build{}-publish'.format(ver_mode, get_e2e_suffix()), edition=edition, trigger=trigger, services=[],
|
||||
steps=init_steps + build_steps + package_steps + publish_steps,
|
||||
volumes=volumes,
|
||||
),
|
||||
]
|
||||
if not disable_tests:
|
||||
pipelines.extend([
|
||||
pipeline(
|
||||
name='{}-enterprise-test'.format(ver_mode), edition=edition, trigger=trigger, services=[],
|
||||
steps=init_steps + test_steps,
|
||||
volumes=[],
|
||||
),
|
||||
pipeline(
|
||||
name='{}-enterprise-integration-tests'.format(ver_mode), edition=edition, trigger=trigger, services=services,
|
||||
steps=[download_grabpl_step(), identify_runner_step(), clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode), verify_gen_cue_step(edition), wire_install_step()] + integration_test_steps + [redis_integration_tests_step(), memcached_integration_tests_step()],
|
||||
volumes=volumes,
|
||||
),
|
||||
])
|
||||
deps = {
|
||||
'depends_on': [
|
||||
'{}-enterprise-build{}-publish'.format(ver_mode, get_e2e_suffix()),
|
||||
'{}-enterprise-test'.format(ver_mode),
|
||||
'{}-enterprise-integration-tests'.format(ver_mode)
|
||||
]
|
||||
}
|
||||
windows_pipeline.update(deps)
|
||||
|
||||
pipelines.extend([windows_pipeline])
|
||||
|
||||
return pipelines
|
||||
|
||||
def publish_artifacts_step(mode):
|
||||
security = ''
|
||||
if mode == 'security':
|
||||
security = '--security '
|
||||
return {
|
||||
'name': 'publish-artifacts',
|
||||
'image': publish_image,
|
||||
'environment': {
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
'PRERELEASE_BUCKET': from_secret('prerelease_bucket'),
|
||||
},
|
||||
'commands': ['./bin/grabpl artifacts publish {}--tag ${{TAG}} --src-bucket $${{PRERELEASE_BUCKET}}'.format(security)],
|
||||
'depends_on': ['grabpl'],
|
||||
}
|
||||
|
||||
def publish_packages_step(edition):
|
||||
return {
|
||||
'name': 'publish-packages-{}'.format(edition),
|
||||
'image': publish_image,
|
||||
'environment': {
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
},
|
||||
'commands': ['./bin/grabpl store-packages {}'.format(edition)],
|
||||
'depends_on': ['grabpl'],
|
||||
}
|
||||
|
||||
def publish_artifacts_pipelines(mode):
|
||||
trigger = {
|
||||
'event': ['promote'],
|
||||
'target': [mode],
|
||||
}
|
||||
steps = [
|
||||
download_grabpl_step(),
|
||||
publish_artifacts_step(mode),
|
||||
]
|
||||
|
||||
return [pipeline(
|
||||
name='publish-artifacts-{}'.format(mode), trigger=trigger, steps=steps, edition="all"
|
||||
)]
|
||||
|
||||
def publish_packages_pipeline():
|
||||
trigger = {
|
||||
'event': ['promote'],
|
||||
'target': ['public'],
|
||||
}
|
||||
oss_steps = [
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode='release'),
|
||||
store_packages_step(edition='oss', ver_mode='release'),
|
||||
]
|
||||
|
||||
enterprise_steps = [
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode='release'),
|
||||
store_packages_step(edition='enterprise', ver_mode='release'),
|
||||
]
|
||||
deps = [
|
||||
'publish-artifacts-public',
|
||||
'publish-docker-oss-public',
|
||||
'publish-docker-enterprise-public'
|
||||
]
|
||||
|
||||
return [pipeline(
|
||||
name='publish-packages-oss', trigger=trigger, steps=oss_steps, edition="all", depends_on=deps
|
||||
), pipeline(
|
||||
name='publish-packages-enterprise', trigger=trigger, steps=enterprise_steps, edition="all", depends_on=deps
|
||||
)]
|
||||
|
||||
def publish_npm_pipelines(mode):
|
||||
trigger = {
|
||||
'event': ['promote'],
|
||||
'target': [mode],
|
||||
}
|
||||
steps = [
|
||||
download_grabpl_step(),
|
||||
yarn_install_step(),
|
||||
retrieve_npm_packages_step(),
|
||||
release_npm_packages_step()
|
||||
]
|
||||
|
||||
return [pipeline(
|
||||
name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = steps, edition="all"
|
||||
)]
|
||||
|
||||
def artifacts_page_pipeline():
|
||||
trigger = {
|
||||
'event': ['promote'],
|
||||
'target': 'security',
|
||||
}
|
||||
return [pipeline(name='publish-artifacts-page', trigger=trigger, steps = [download_grabpl_step(), artifacts_page_step()], edition="all")]
|
||||
|
||||
def release_pipelines(ver_mode='release', trigger=None):
|
||||
# 'enterprise' edition services contain both OSS and enterprise services
|
||||
if not trigger:
|
||||
trigger = {
|
||||
'event': {
|
||||
'exclude': [
|
||||
'promote'
|
||||
]
|
||||
},
|
||||
'ref': ['refs/tags/v*',],
|
||||
'repo': {
|
||||
'exclude': ['grafana/grafana'],
|
||||
},
|
||||
}
|
||||
|
||||
# The release pipelines include also enterprise ones, so both editions are built for a release.
|
||||
# We could also solve this by triggering a downstream build for the enterprise repo, but by including enterprise
|
||||
# in OSS release builds, we simplify the UX for the release engineer.
|
||||
oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger)
|
||||
enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
|
||||
|
||||
pipelines = oss_pipelines + enterprise_pipelines
|
||||
|
||||
return pipelines
|
||||
|
||||
def get_e2e_suffix():
|
||||
if not disable_tests:
|
||||
return '-e2e'
|
||||
return ''
|
||||
@@ -0,0 +1,41 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'identify_runner_step',
|
||||
'download_grabpl_step',
|
||||
'gen_version_step',
|
||||
'wire_install_step',
|
||||
'codespell_step',
|
||||
'shellcheck_step',
|
||||
'lint_backend_step',
|
||||
'lint_drone_step',
|
||||
'test_backend_step',
|
||||
'test_backend_integration_step',
|
||||
'verify_gen_cue_step',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
def test_backend(trigger, ver_mode):
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
verify_gen_cue_step(edition="oss"),
|
||||
wire_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
codespell_step(),
|
||||
shellcheck_step(),
|
||||
lint_backend_step(edition="oss"),
|
||||
test_backend_step(edition="oss"),
|
||||
test_backend_integration_step(edition="oss"),
|
||||
]
|
||||
if ver_mode == 'main':
|
||||
test_steps.extend([lint_drone_step()])
|
||||
|
||||
return pipeline(
|
||||
name='{}-test-backend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'identify_runner_step',
|
||||
'download_grabpl_step',
|
||||
'gen_version_step',
|
||||
'yarn_install_step',
|
||||
'lint_frontend_step',
|
||||
'betterer_frontend_step',
|
||||
'test_frontend_step',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
def test_frontend(trigger, ver_mode):
|
||||
init_steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
gen_version_step(ver_mode),
|
||||
yarn_install_step(),
|
||||
]
|
||||
test_steps = [
|
||||
lint_frontend_step(),
|
||||
betterer_frontend_step(),
|
||||
test_frontend_step(),
|
||||
]
|
||||
return pipeline(
|
||||
name='{}-test-frontend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps,
|
||||
)
|
||||
@@ -0,0 +1,21 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'identify_runner_step',
|
||||
'download_grabpl_step',
|
||||
'lint_drone_step',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
def verify_drone(trigger, ver_mode):
|
||||
steps = [
|
||||
identify_runner_step(),
|
||||
download_grabpl_step(),
|
||||
lint_drone_step(),
|
||||
]
|
||||
return pipeline(
|
||||
name='{}-verify-drone'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=steps,
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'grabpl_version',
|
||||
'wix_image',
|
||||
'identify_runner_step',
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
|
||||
load('scripts/drone/vault.star', 'from_secret', 'prerelease_bucket', 'github_token')
|
||||
|
||||
def windows(trigger, edition, ver_mode):
|
||||
init_cmds = []
|
||||
sfx = ''
|
||||
if edition in ('enterprise', 'enterprise2'):
|
||||
sfx = '-{}'.format(edition)
|
||||
else:
|
||||
init_cmds.extend([
|
||||
'$$ProgressPreference = "SilentlyContinue"',
|
||||
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
|
||||
grabpl_version),
|
||||
])
|
||||
steps = [
|
||||
{
|
||||
'name': 'windows-init',
|
||||
'image': wix_image,
|
||||
'commands': init_cmds,
|
||||
},
|
||||
]
|
||||
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
|
||||
'release', 'release-branch',
|
||||
):
|
||||
bucket_part = ''
|
||||
bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
|
||||
if ver_mode == 'release':
|
||||
ver_part = '${DRONE_TAG}'
|
||||
dir = 'release'
|
||||
else:
|
||||
dir = 'main'
|
||||
bucket = 'grafana-downloads'
|
||||
bucket_part = ' --packages-bucket {}'.format(bucket)
|
||||
build_no = 'DRONE_BUILD_NUMBER'
|
||||
ver_part = '--build-id $$env:{}'.format(build_no)
|
||||
installer_commands = [
|
||||
'$$gcpKey = $$env:GCP_KEY',
|
||||
'[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
|
||||
# gcloud fails to read the file unless converted with dos2unix
|
||||
'dos2unix gcpkey.json',
|
||||
'gcloud auth activate-service-account --key-file=gcpkey.json',
|
||||
'rm gcpkey.json',
|
||||
'cp C:\\App\\nssm-2.24.zip .',
|
||||
]
|
||||
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
|
||||
'release',
|
||||
):
|
||||
installer_commands.extend([
|
||||
'.\\grabpl.exe gen-version {}'.format(ver_part),
|
||||
'.\\grabpl.exe windows-installer --edition {}{} {}'.format(edition, bucket_part, ver_part),
|
||||
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
|
||||
])
|
||||
if ver_mode == 'main':
|
||||
installer_commands.extend([
|
||||
'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir),
|
||||
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir),
|
||||
])
|
||||
else:
|
||||
installer_commands.extend([
|
||||
'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
|
||||
'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir),
|
||||
])
|
||||
steps.append({
|
||||
'name': 'build-windows-installer',
|
||||
'image': wix_image,
|
||||
'depends_on': [
|
||||
'windows-init',
|
||||
],
|
||||
'environment': {
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
|
||||
'GITHUB_TOKEN': from_secret('github_token')
|
||||
},
|
||||
'commands': installer_commands,
|
||||
})
|
||||
|
||||
if edition in ('enterprise', 'enterprise2'):
|
||||
if ver_mode == 'release':
|
||||
committish = '${DRONE_TAG}'
|
||||
elif ver_mode == 'release-branch':
|
||||
committish = '$$env:DRONE_BRANCH'
|
||||
else:
|
||||
committish = '$$env:DRONE_COMMIT'
|
||||
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
|
||||
download_grabpl_step_cmds = [
|
||||
'$$ProgressPreference = "SilentlyContinue"',
|
||||
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
|
||||
grabpl_version),
|
||||
]
|
||||
clone_cmds = [
|
||||
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
|
||||
]
|
||||
clone_cmds.extend([
|
||||
'cd grafana-enterprise',
|
||||
'git checkout {}'.format(committish),
|
||||
])
|
||||
steps.insert(0, {
|
||||
'name': 'clone',
|
||||
'image': wix_image,
|
||||
'environment': {
|
||||
'GITHUB_TOKEN': from_secret(github_token),
|
||||
},
|
||||
'commands': download_grabpl_step_cmds + clone_cmds,
|
||||
})
|
||||
steps[1]['depends_on'] = [
|
||||
'clone',
|
||||
]
|
||||
steps[1]['commands'].extend([
|
||||
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
|
||||
'cp -r grafana-enterprise C:\\App\\grafana-enterprise',
|
||||
'rm -r -force grafana-enterprise',
|
||||
'cp grabpl.exe C:\\App\\grabpl.exe',
|
||||
'rm -force grabpl.exe',
|
||||
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise',
|
||||
'cp C:\\App\\grabpl.exe grabpl.exe',
|
||||
])
|
||||
if 'environment' in steps[1]:
|
||||
steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)}
|
||||
else:
|
||||
steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)}
|
||||
|
||||
return pipeline(
|
||||
name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
|
||||
steps=[identify_runner_step('windows')] + steps,
|
||||
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
|
||||
)
|
||||
Reference in New Issue
Block a user