public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Make build, test and integration tests steps run in parallel (#40794)

Browse Source 
* Make build and test steps run in parallel

* Make tests and builds run on different pipelines

* Make test-backend depend on initialize

* Rename

* Create standalone integration tests pipeline (#41730)

* Remove unused variables
This commit is contained in:
Dimitris Sotirakis
2021-11-17 11:25:55 +02:00
committed by GitHub
parent 818b8739c0
commit b8dd9fdd4a
6 changed files with 292 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/pipelines/main.star
+18 -13
View File
@@ -1,5 +1,7 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'initialize_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
@@ -139,18 +141,20 @@ def main_pipelines(edition):
pipelines = [
pipeline(
name='build-main', edition=edition, trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
name='build-main', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
),
pipeline(
name='windows-main', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
depends_on=['build-main'], ver_mode=ver_mode,
name='windows-main', edition=edition, trigger=trigger,
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
depends_on=['build-main'], platform='windows',
),
]
if edition != 'enterprise':
pipelines.append(pipeline(
name='publish-main', edition=edition, trigger=trigger, steps=publish_steps,
depends_on=['build-main', 'windows-main',], install_deps=False, ver_mode=ver_mode,
name='publish-main', edition=edition, trigger=trigger,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + publish_steps,
depends_on=['build-main', 'windows-main',],
))
pipelines.append(notify_pipeline(
@@ -164,17 +168,18 @@ def main_pipelines(edition):
}
steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
pipelines.append(pipeline(
name='build-main-downstream', edition=edition, trigger=trigger, services=services, steps=steps,
is_downstream=True, ver_mode=ver_mode,
name='build-main-downstream', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
))
pipelines.append(pipeline(
name='windows-main-downstream', edition=edition, trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['build-main-downstream'], is_downstream=True, ver_mode=ver_mode,
name='windows-main-downstream', edition=edition, trigger=trigger,
steps=[download_grabpl_step()] + initialize_step(edition, platform='windows', ver_mode=ver_mode, is_downstream=True) + windows_steps,
platform='windows', depends_on=['build-main-downstream'],
))
pipelines.append(pipeline(
name='publish-main-downstream', edition=edition, trigger=trigger, steps=publish_steps,
depends_on=['build-main-downstream', 'windows-main-downstream'], is_downstream=True, install_deps=False,
ver_mode=ver_mode,
name='publish-main-downstream', edition=edition, trigger=trigger,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True, install_deps=False) + publish_steps,
depends_on=['build-main-downstream', 'windows-main-downstream'],
))
pipelines.append(notify_pipeline(
scripts/drone/pipelines/pr.star
+26 -10
View File
@@ -1,5 +1,7 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'initialize_step',
'lint_drone_step',
'lint_backend_step',
'lint_frontend_step',
@@ -46,7 +48,7 @@ def pr_pipelines(edition):
services = integration_test_services(edition)
variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64', 'armv6',]
include_enterprise2 = edition == 'enterprise'
steps = [
test_steps = [
lint_drone_step(),
codespell_step(),
shellcheck_step(),
@@ -55,28 +57,34 @@ def pr_pipelines(edition):
test_backend_step(edition=edition),
test_backend_integration_step(edition=edition),
test_frontend_step(),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
build_steps = [
build_backend_step(edition=edition, ver_mode=ver_mode, variants=variants),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition),
validate_scuemata_step(),
ensure_cuetsified_step(),
]
integration_test_steps = [
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
if include_enterprise2:
edition2 = 'enterprise2'
steps.append(benchmark_ldap_step())
build_steps.append(benchmark_ldap_step())
services.append(ldap_service())
steps.extend([
test_steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
test_backend_integration_step(edition=edition2),
])
build_steps.extend([
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
])
# Insert remaining steps
steps.extend([
# Insert remaining build_steps
build_steps.extend([
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=variants),
e2e_tests_server_step(edition=edition),
e2e_tests_step(edition=edition),
@@ -89,9 +97,11 @@ def pr_pipelines(edition):
])
if include_enterprise2:
steps.extend([
integration_test_steps.extend([
redis_integration_tests_step(),
memcached_integration_tests_step(),
])
build_steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64']),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002),
@@ -100,9 +110,15 @@ def pr_pipelines(edition):
trigger = {
'event': ['pull_request',],
}
return [
pipeline(
name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
name='pr-test', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ test_steps,
), pipeline(
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ build_steps,
), pipeline(
name='pr-integration-tests', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + integration_test_steps,
),
]
scripts/drone/pipelines/release.star
+33 -22
View File
@@ -1,5 +1,7 @@
load(
'scripts/drone/steps/lib.star',
'download_grabpl_step',
'initialize_step',
'lint_drone_step',
'test_release_ver',
'build_image',
@@ -154,30 +156,34 @@ def get_steps(edition, ver_mode):
return steps, windows_steps
def get_oss_pipelines(trigger, ver_mode):
services = integration_test_services(edition='oss')
steps, windows_steps = get_steps(edition='oss', ver_mode=ver_mode)
edition = 'oss'
services = integration_test_services(edition=edition)
steps, windows_steps = get_steps(edition=edition, ver_mode=ver_mode)
return [
pipeline(
name='oss-build-{}'.format(ver_mode), edition='oss', trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
name='oss-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
),
pipeline(
name='oss-windows-{}'.format(ver_mode), edition='oss', trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['oss-build-{}'.format(ver_mode)], ver_mode=ver_mode,
name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
steps=[download_grabpl_step()] + initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
platform='windows', depends_on=['oss-build-{}'.format(ver_mode)],
),
]
def get_enterprise_pipelines(trigger, ver_mode):
services = integration_test_services(edition='enterprise')
steps, windows_steps = get_steps(edition='enterprise', ver_mode=ver_mode)
edition = 'enterprise'
services = integration_test_services(edition=edition)
steps, windows_steps = get_steps(edition=edition, ver_mode=ver_mode)
return [
pipeline(
name='enterprise-build-{}'.format(ver_mode), edition='enterprise', trigger=trigger, services=services,
steps=steps, ver_mode=ver_mode,
name='enterprise-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
),
pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition='enterprise', trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['enterprise-build-{}'.format(ver_mode)], ver_mode=ver_mode,
name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
steps=[download_grabpl_step()] + initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
platform='windows', depends_on=['enterprise-build-{}'.format(ver_mode)],
),
]
@@ -199,12 +205,14 @@ def release_pipelines(ver_mode='release', trigger=None):
pipelines = oss_pipelines + enterprise_pipelines
if should_publish:
steps = [
publish_packages_step(edition='oss', ver_mode=ver_mode),
publish_packages_step(edition='enterprise', ver_mode=ver_mode),
]
publish_pipeline = pipeline(
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
publish_packages_step(edition='oss', ver_mode=ver_mode),
publish_packages_step(edition='enterprise', ver_mode=ver_mode),
], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
ver_mode=ver_mode,
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss',
steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
)
pipelines.append(publish_pipeline)
@@ -228,12 +236,15 @@ def test_release_pipelines():
publish_cmd = './bin/grabpl publish-packages --edition {{}} --dry-run {}'.format(test_release_ver)
steps = [
publish_packages_step(edition='oss', ver_mode=ver_mode),
publish_packages_step(edition='enterprise', ver_mode=ver_mode),
]
publish_pipeline = pipeline(
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
publish_packages_step(edition='oss', ver_mode=ver_mode),
publish_packages_step(edition='enterprise', ver_mode=ver_mode),
], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
ver_mode=ver_mode,
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss',
steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
)
pipelines = oss_pipelines + enterprise_pipelines + [publish_pipeline,]
scripts/drone/steps/lib.star
+12 -14
View File
@@ -124,7 +124,7 @@ def initialize_step(edition, platform, ver_mode, is_downstream=False, install_de
return steps
def download_grabpl():
def download_grabpl_step():
return {
'name': 'grabpl',
'image': curl_image,
@@ -312,7 +312,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
'name': 'build-backend' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'test-backend' + enterprise2_suffix(edition),
'initialize',
],
'environment': env,
'commands': cmds,
@@ -345,7 +345,7 @@ def build_frontend_step(edition, ver_mode, is_downstream=False):
'name': 'build-frontend',
'image': build_image,
'depends_on': [
'test-frontend',
'initialize',
],
'commands': cmds,
}
@@ -375,7 +375,7 @@ def build_plugins_step(edition, sign=False):
'name': 'build-plugins',
'image': build_image,
'depends_on': [
'lint-backend',
'initialize',
],
'environment': env,
'commands': [
@@ -389,7 +389,7 @@ def test_backend_step(edition):
'name': 'test-backend' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'lint-backend',
'initialize',
],
'commands': [
'./bin/grabpl test-backend --edition {}'.format(edition),
@@ -401,7 +401,7 @@ def test_backend_integration_step(edition):
'name': 'test-backend-integration' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'lint-backend',
'initialize',
],
'commands': [
'./bin/grabpl integration-tests --edition {}'.format(edition),
@@ -688,7 +688,7 @@ def postgres_integration_tests_step():
'name': 'postgres-integration-tests',
'image': build_image,
'depends_on': [
'initialize',
'grabpl',
],
'environment': {
'PGPASSWORD': 'grafanatest',
@@ -712,7 +712,7 @@ def mysql_integration_tests_step():
'name': 'mysql-integration-tests',
'image': build_image,
'depends_on': [
'initialize',
'grabpl',
],
'environment': {
'GRAFANA_TEST_DB': 'mysql',
@@ -734,8 +734,7 @@ def redis_integration_tests_step():
'name': 'redis-integration-tests',
'image': build_image,
'depends_on': [
'test-backend',
'test-frontend',
'grabpl',
],
'environment': {
'REDIS_URL': 'redis://redis:6379/0',
@@ -751,8 +750,7 @@ def memcached_integration_tests_step():
'name': 'memcached-integration-tests',
'image': build_image,
'depends_on': [
'test-backend',
'test-frontend',
'grabpl',
],
'environment': {
'MEMCACHED_HOSTS': 'memcached:11211',
@@ -939,7 +937,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
else:
committish = '$$env:DRONE_COMMIT'
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
download_grabpl_cmds = [
download_grabpl_step_cmds = [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
]
@@ -957,7 +955,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'environment': {
'GITHUB_TOKEN': from_secret(github_token),
},
'commands': download_grabpl_cmds + clone_cmds,
'commands': download_grabpl_step_cmds + clone_cmds,
})
steps[1]['depends_on'] = [
'clone',
scripts/drone/utils/utils.star
+3 -9
View File
@@ -1,18 +1,15 @@
load(
'scripts/drone/steps/lib.star',
'initialize_step',
'download_grabpl',
'download_grabpl_step',
'slack_step',
)
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
def pipeline(
name, edition, trigger, steps, ver_mode, services=[], platform='linux', depends_on=[],
is_downstream=False, install_deps=True,
name, edition, trigger, steps, services=[], platform='linux', depends_on=[],
):
if platform != 'windows':
grabpl_step = [download_grabpl()]
platform_conf = {
'platform': {
'os': 'linux',
@@ -25,7 +22,6 @@ def pipeline(
}
}
else:
grabpl_step = []
platform_conf = {
'platform': {
'os': 'windows',
@@ -40,9 +36,7 @@ def pipeline(
'name': name,
'trigger': trigger,
'services': services,
'steps': grabpl_step + initialize_step(
edition, platform, is_downstream=is_downstream, install_deps=install_deps, ver_mode=ver_mode,
) + steps,
'steps': steps,
'depends_on': depends_on,
}
pipeline.update(platform_conf)