CI: Make build, test and integration tests steps run in parallel (#40794)

* Make build and test steps run in parallel

* Make tests and builds run on different pipelines

* Make test-backend depend on initialize

* Rename

* Create standalone integration tests pipeline (#41730)

* Remove unused variables

scripts/drone/pipelines/main.star

@@ -1,5 +1,7 @@
 load(
     'scripts/drone/steps/lib.star',
+    'download_grabpl_step',
+    'initialize_step',
     'lint_drone_step',
     'lint_backend_step',
     'lint_frontend_step',
@@ -139,18 +141,20 @@ def main_pipelines(edition):
 
     pipelines = [
         pipeline(
-            name='build-main', edition=edition, trigger=trigger, services=services, steps=steps,
-            ver_mode=ver_mode,
+            name='build-main', edition=edition, trigger=trigger, services=services,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
         ),
         pipeline(
-            name='windows-main', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
-            depends_on=['build-main'], ver_mode=ver_mode,
+            name='windows-main', edition=edition, trigger=trigger,
+            steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
+            depends_on=['build-main'], platform='windows',
         ),
     ]
     if edition != 'enterprise':
         pipelines.append(pipeline(
-            name='publish-main', edition=edition, trigger=trigger, steps=publish_steps,
-            depends_on=['build-main', 'windows-main',], install_deps=False, ver_mode=ver_mode,
+            name='publish-main', edition=edition, trigger=trigger,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + publish_steps,
+            depends_on=['build-main', 'windows-main',],
         ))
 
         pipelines.append(notify_pipeline(
@@ -164,17 +168,18 @@ def main_pipelines(edition):
         }
         steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
         pipelines.append(pipeline(
-            name='build-main-downstream', edition=edition, trigger=trigger, services=services, steps=steps,
-            is_downstream=True, ver_mode=ver_mode,
+            name='build-main-downstream', edition=edition, trigger=trigger, services=services,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
         ))
         pipelines.append(pipeline(
-            name='windows-main-downstream', edition=edition, trigger=trigger, steps=windows_steps,
-            platform='windows', depends_on=['build-main-downstream'], is_downstream=True, ver_mode=ver_mode,
+            name='windows-main-downstream', edition=edition, trigger=trigger,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='windows', ver_mode=ver_mode, is_downstream=True) + windows_steps,
+            platform='windows', depends_on=['build-main-downstream'],
         ))
         pipelines.append(pipeline(
-            name='publish-main-downstream', edition=edition, trigger=trigger, steps=publish_steps,
-            depends_on=['build-main-downstream', 'windows-main-downstream'], is_downstream=True, install_deps=False,
-            ver_mode=ver_mode,
+            name='publish-main-downstream', edition=edition, trigger=trigger,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True, install_deps=False) + publish_steps,
+            depends_on=['build-main-downstream', 'windows-main-downstream'],
         ))
 
         pipelines.append(notify_pipeline(

scripts/drone/pipelines/pr.star

@@ -1,5 +1,7 @@
 load(
     'scripts/drone/steps/lib.star',
+    'download_grabpl_step',
+    'initialize_step',
     'lint_drone_step',
     'lint_backend_step',
     'lint_frontend_step',
@@ -46,7 +48,7 @@ def pr_pipelines(edition):
     services = integration_test_services(edition)
     variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64', 'armv6',]
     include_enterprise2 = edition == 'enterprise'
-    steps = [
+    test_steps = [
         lint_drone_step(),
         codespell_step(),
         shellcheck_step(),
@@ -55,28 +57,34 @@ def pr_pipelines(edition):
         test_backend_step(edition=edition),
         test_backend_integration_step(edition=edition),
         test_frontend_step(),
-        postgres_integration_tests_step(),
-        mysql_integration_tests_step(),
+    ]
+    build_steps = [
         build_backend_step(edition=edition, ver_mode=ver_mode, variants=variants),
         build_frontend_step(edition=edition, ver_mode=ver_mode),
         build_plugins_step(edition=edition),
         validate_scuemata_step(),
         ensure_cuetsified_step(),
     ]
+    integration_test_steps = [
+        postgres_integration_tests_step(),
+        mysql_integration_tests_step(),
+    ]
 
     if include_enterprise2:
         edition2 = 'enterprise2'
-        steps.append(benchmark_ldap_step())
+        build_steps.append(benchmark_ldap_step())
         services.append(ldap_service())
-        steps.extend([
+        test_steps.extend([
             lint_backend_step(edition=edition2),
             test_backend_step(edition=edition2),
             test_backend_integration_step(edition=edition2),
+        ])
+        build_steps.extend([
             build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
         ])
 
-    # Insert remaining steps
-    steps.extend([
+    # Insert remaining build_steps
+    build_steps.extend([
         package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=variants),
         e2e_tests_server_step(edition=edition),
         e2e_tests_step(edition=edition),
@@ -89,9 +97,11 @@ def pr_pipelines(edition):
     ])
 
     if include_enterprise2:
-        steps.extend([
+        integration_test_steps.extend([
             redis_integration_tests_step(),
             memcached_integration_tests_step(),
+        ])
+        build_steps.extend([
             package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64']),
             e2e_tests_server_step(edition=edition2, port=3002),
             e2e_tests_step(edition=edition2, port=3002),
@@ -100,9 +110,15 @@ def pr_pipelines(edition):
     trigger = {
         'event': ['pull_request',],
     }
+
     return [
         pipeline(
-            name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps,
-            ver_mode=ver_mode,
+            name='pr-test', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+                + test_steps,
+        ), pipeline(
+            name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+                + build_steps,
+        ), pipeline(
+            name='pr-integration-tests', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + integration_test_steps,
         ),
     ]

scripts/drone/pipelines/release.star

@@ -1,5 +1,7 @@
 load(
     'scripts/drone/steps/lib.star',
+    'download_grabpl_step',
+    'initialize_step',
     'lint_drone_step',
     'test_release_ver',
     'build_image',
@@ -154,30 +156,34 @@ def get_steps(edition, ver_mode):
     return steps, windows_steps
 
 def get_oss_pipelines(trigger, ver_mode):
-    services = integration_test_services(edition='oss')
-    steps, windows_steps = get_steps(edition='oss', ver_mode=ver_mode)
+    edition = 'oss'
+    services = integration_test_services(edition=edition)
+    steps, windows_steps = get_steps(edition=edition, ver_mode=ver_mode)
     return [
         pipeline(
-            name='oss-build-{}'.format(ver_mode), edition='oss', trigger=trigger, services=services, steps=steps,
-            ver_mode=ver_mode,
+            name='oss-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
         ),
         pipeline(
-            name='oss-windows-{}'.format(ver_mode), edition='oss', trigger=trigger, steps=windows_steps,
-            platform='windows', depends_on=['oss-build-{}'.format(ver_mode)], ver_mode=ver_mode,
+            name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
+            platform='windows', depends_on=['oss-build-{}'.format(ver_mode)],
         ),
     ]
 
 def get_enterprise_pipelines(trigger, ver_mode):
-    services = integration_test_services(edition='enterprise')
-    steps, windows_steps = get_steps(edition='enterprise', ver_mode=ver_mode)
+    edition = 'enterprise'
+    services = integration_test_services(edition=edition)
+    steps, windows_steps = get_steps(edition=edition, ver_mode=ver_mode)
     return [
         pipeline(
-            name='enterprise-build-{}'.format(ver_mode), edition='enterprise', trigger=trigger, services=services,
-            steps=steps, ver_mode=ver_mode,
+            name='enterprise-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
         ),
         pipeline(
-            name='enterprise-windows-{}'.format(ver_mode), edition='enterprise', trigger=trigger, steps=windows_steps,
-            platform='windows', depends_on=['enterprise-build-{}'.format(ver_mode)], ver_mode=ver_mode,
+            name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
+            steps=[download_grabpl_step()] + initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
+            platform='windows', depends_on=['enterprise-build-{}'.format(ver_mode)],
         ),
     ]
 
@@ -199,12 +205,14 @@ def release_pipelines(ver_mode='release', trigger=None):
 
     pipelines = oss_pipelines + enterprise_pipelines
     if should_publish:
+        steps = [
+            publish_packages_step(edition='oss', ver_mode=ver_mode),
+            publish_packages_step(edition='enterprise', ver_mode=ver_mode),
+        ]
         publish_pipeline = pipeline(
-            name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
-                publish_packages_step(edition='oss', ver_mode=ver_mode),
-                publish_packages_step(edition='enterprise', ver_mode=ver_mode),
-            ], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
-            ver_mode=ver_mode,
+            name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss',
+            steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
+            depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
         )
         pipelines.append(publish_pipeline)
 
@@ -228,12 +236,15 @@ def test_release_pipelines():
 
     publish_cmd = './bin/grabpl publish-packages --edition {{}} --dry-run {}'.format(test_release_ver)
 
+    steps = [
+        publish_packages_step(edition='oss', ver_mode=ver_mode),
+        publish_packages_step(edition='enterprise', ver_mode=ver_mode),
+    ]
+
     publish_pipeline = pipeline(
-        name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
-            publish_packages_step(edition='oss', ver_mode=ver_mode),
-            publish_packages_step(edition='enterprise', ver_mode=ver_mode),
-        ], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
-        ver_mode=ver_mode,
+        name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss',
+        steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
+        depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
     )
 
     pipelines = oss_pipelines + enterprise_pipelines + [publish_pipeline,]