public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Remove initialize step (#47622)

Browse Source 
* Remove initialize step from pr.star

* Remove initialize step from main.star

* Remove initialize step from release.star

* Reorder dependencies

* Re-add identify runner step

* Add enterprise steps

* Remove is_downstream variable from starlark logic (#47715)

* Make init steps depend on init-enterprise for enterprise pipelines

* Exclude shellcheck from enterprise pipelines

* Add init steps to enterprise integration tests pipeline

* Add dependency for windows enterprise step

* Update grabpl version
This commit is contained in:
Dimitris Sotirakis
2022-04-14 14:56:14 +03:00
committed by GitHub
parent e50bd5cac8
commit 90892050d0
6 changed files with 680 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/pipelines/docs.star
+10 -7
View File
@@ -1,7 +1,9 @@
load(
'scripts/drone/steps/lib.star',
'build_image',
'initialize_step',
'yarn_install_step',
'identify_runner_step',
'gen_version_step',
'download_grabpl_step',
'lint_frontend_step',
'codespell_step',
@@ -25,16 +27,17 @@ load(
def docs_pipelines(edition, ver_mode, trigger):
steps = [download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
# Insert remaining steps
steps.extend([
steps = [
download_grabpl_step(),
identify_runner_step(),
gen_version_step(ver_mode),
yarn_install_step(),
codespell_step(),
lint_docs(),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_frontend_docs_step(edition=edition),
build_docs_website_step(),
])
]
return pipeline(
name='{}-docs'.format(ver_mode), edition=edition, trigger=trigger, services=[], steps=steps,
@@ -45,7 +48,7 @@ def lint_docs():
'name': 'lint-docs',
'image': build_image,
'depends_on': [
'initialize',
'yarn-install',
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',
scripts/drone/pipelines/main.star
+30 -22
View File
@@ -2,7 +2,10 @@ load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'build_image',
'initialize_step',
'identify_runner_step',
'gen_version_step',
'wire_install_step',
'yarn_install_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
@@ -67,10 +70,16 @@ load('scripts/drone/vault.star', 'from_secret')
ver_mode = 'main'
def get_steps(edition, is_downstream=False):
def get_steps(edition):
services = integration_test_services(edition)
publish = edition != 'enterprise' or is_downstream
include_enterprise2 = edition == 'enterprise'
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
wire_install_step(),
yarn_install_step(),
]
test_steps = [
lint_drone_step(),
codespell_step(),
@@ -84,9 +93,9 @@ def get_steps(edition, is_downstream=False):
build_steps = [
trigger_test_release(),
enterprise_downstream_step(edition=edition),
build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_package_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, sign=True),
validate_scuemata_step(),
ensure_cuetsified_step(),
@@ -106,12 +115,12 @@ def get_steps(edition, is_downstream=False):
test_backend_integration_step(edition=edition2),
])
build_steps.extend([
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64'], is_downstream=is_downstream),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']),
])
# Insert remaining steps
build_steps.extend([
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream),
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2),
grafana_server_step(edition=edition),
e2e_tests_step('dashboards-suite', edition=edition),
e2e_tests_step('smoke-tests-suite', edition=edition),
@@ -134,27 +143,27 @@ def get_steps(edition, is_downstream=False):
build_steps.extend([
release_canary_npm_packages_step(edition, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
])
if include_enterprise2:
edition2 = 'enterprise2'
build_steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64'], is_downstream=is_downstream),
upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-amd64']),
upload_packages_step(edition=edition2, ver_mode=ver_mode),
upload_cdn_step(edition=edition2, ver_mode=ver_mode)
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream)
if edition == 'enterprise' and not is_downstream:
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if edition == 'enterprise':
store_steps = []
else:
store_steps = [
store_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
store_packages_step(edition=edition, ver_mode=ver_mode),
]
return test_steps, build_steps, integration_test_steps, windows_steps, store_steps
return init_steps, test_steps, build_steps, integration_test_steps, windows_steps, store_steps
def trigger_test_release():
return {
@@ -210,7 +219,7 @@ def main_pipelines(edition):
],
},
}
test_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
init_steps, test_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition)
if edition == 'enterprise':
services.append(ldap_service())
@@ -218,27 +227,26 @@ def main_pipelines(edition):
pipelines = [docs_pipelines(edition, ver_mode, trigger), pipeline(
name='main-test', edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps,
steps=init_steps + test_steps,
volumes=[],
), pipeline(
name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps,
steps=init_steps + build_steps,
volumes=volumes,
), pipeline(
name='main-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
volumes=volumes,
), pipeline(
name='windows-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
steps=[identify_runner_step('windows')] + windows_steps,
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
), notify_pipeline(
name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
template=drone_change_template, secret='drone-changes-webhook',
), pipeline(
name='publish-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode,
install_deps=False) + store_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + store_steps,
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', ],
), notify_pipeline(
name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
scripts/drone/pipelines/pr.star
+14 -6
View File
@@ -1,7 +1,10 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'initialize_step',
'gen_version_step',
'yarn_install_step',
'wire_install_step',
'identify_runner_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
@@ -59,6 +62,13 @@ def pr_pipelines(edition):
volumes = integration_test_services_volumes()
variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64', 'armv6',]
include_enterprise2 = edition == 'enterprise'
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
wire_install_step(),
yarn_install_step(),
]
test_steps = [
lint_drone_step(),
codespell_step(),
@@ -132,14 +142,12 @@ def pr_pipelines(edition):
return [
pipeline(
name='pr-test', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ test_steps,
name='pr-test', edition=edition, trigger=trigger, services=[], steps=init_steps + test_steps,
), pipeline(
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ build_steps,
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps,
), pipeline(
name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
volumes=volumes,
), docs_pipelines(edition, ver_mode, trigger_docs())
]
scripts/drone/pipelines/release.star
+50 -23
View File
@@ -1,10 +1,15 @@
load(
'scripts/drone/steps/lib.star',
'disable_tests',
'clone_enterprise_step',
'download_grabpl_step',
'initialize_step',
'gen_version_step',
'yarn_install_step',
'wire_install_step',
'init_enterprise_step',
'lint_drone_step',
'build_image',
'identify_runner_step',
'publish_image',
'lint_backend_step',
'lint_frontend_step',
@@ -77,7 +82,7 @@ def retrieve_npm_packages_step():
'name': 'retrieve-npm-packages',
'image': publish_image,
'depends_on': [
'initialize',
'grabpl',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
@@ -150,16 +155,26 @@ def get_steps(edition, ver_mode):
should_upload = should_publish or ver_mode in ('release-branch',)
include_enterprise2 = edition == 'enterprise'
edition2 = 'enterprise2'
init_steps = [
identify_runner_step(),
download_grabpl_step(),
gen_version_step(ver_mode),
wire_install_step(),
yarn_install_step(),
]
test_steps = [
test_steps = []
if edition != 'enterprise':
test_steps.extend([shellcheck_step()])
test_steps.extend([
codespell_step(),
shellcheck_step(),
lint_backend_step(edition=edition),
lint_frontend_step(),
test_backend_step(edition=edition),
test_backend_integration_step(edition=edition),
test_frontend_step(),
]
])
build_steps = [
build_backend_step(edition=edition, ver_mode=ver_mode),
@@ -233,16 +248,16 @@ def get_steps(edition, ver_mode):
if step:
publish_steps.append(step)
return test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps
return init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps
def get_oss_pipelines(trigger, ver_mode):
edition = 'oss'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
init_steps, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
windows_pipeline = pipeline(
name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps,
steps=[identify_runner_step('windows')] + windows_package_steps,
platform='windows', depends_on=[
'oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode),
],
@@ -250,8 +265,7 @@ def get_oss_pipelines(trigger, ver_mode):
pipelines = [
pipeline(
name='oss-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
steps=init_steps + build_steps + package_steps + publish_steps,
volumes=volumes,
),
]
@@ -259,14 +273,12 @@ def get_oss_pipelines(trigger, ver_mode):
pipelines.extend([
pipeline(
name='oss-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
test_steps,
steps=init_steps + test_steps,
volumes=[],
),
pipeline(
name='oss-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
volumes=volumes,
)
])
@@ -286,10 +298,28 @@ def get_enterprise_pipelines(trigger, ver_mode):
edition = 'enterprise'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
deps_on_clone_enterprise_step = {
'depends_on': [
'init-enterprise',
]
}
_, test_steps, build_steps, integration_test_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(ver_mode),
init_enterprise_step(ver_mode)
]
for step in [wire_install_step(), yarn_install_step(), gen_version_step(ver_mode)]:
step.update(deps_on_clone_enterprise_step)
init_steps.extend([step])
for step in integration_test_steps:
step.update(deps_on_clone_enterprise_step)
windows_pipeline = pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_package_steps,
steps=[identify_runner_step('windows')] + windows_package_steps,
platform='windows', depends_on=[
'enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode),
],
@@ -297,8 +327,7 @@ def get_enterprise_pipelines(trigger, ver_mode):
pipelines = [
pipeline(
name='enterprise-build{}-publish-{}'.format(get_e2e_suffix(), ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
steps=init_steps + build_steps + package_steps + publish_steps,
volumes=volumes,
),
]
@@ -306,14 +335,12 @@ def get_enterprise_pipelines(trigger, ver_mode):
pipelines.extend([
pipeline(
name='enterprise-test-{}'.format(ver_mode), edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
test_steps,
steps=init_steps + test_steps,
volumes=[],
),
pipeline(
name='enterprise-integration-tests-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
integration_test_steps,
steps=[download_grabpl_step(), identify_runner_step(), clone_enterprise_step(ver_mode), init_enterprise_step(ver_mode),] + integration_test_steps,
volumes=volumes,
),
])
@@ -401,7 +428,7 @@ def publish_npm_pipelines(mode):
]
return [pipeline(
name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = initialize_step(edition='oss', platform='linux', ver_mode='release') + steps, edition="all"
name='publish-npm-packages-{}'.format(mode), trigger=trigger, steps = steps, edition="all"
)]
def release_pipelines(ver_mode='release', trigger=None, environment=None):