public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v8.3.x] CI: Split pipelines on main builds (#43327)

Browse Source 
* Split pipelines on main builds

* Add services to main-integration-tests
This commit is contained in:
Dimitris Sotirakis
2021-12-20 09:54:27 +01:00
committed by GitHub
parent 0d390c6c12
commit 1359a44c11
2 changed files with 190 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+152 -64
View File
@@ -407,33 +407,13 @@ volumes:
---
depends_on: []
kind: pipeline
name: build-main
name: main-test
node:
type: no-parallel
platform:
arch: amd64
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
MYSQL_ROOT_PASSWORD: rootpass
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
services: []
steps:
- commands:
- mkdir -p bin
@@ -457,17 +437,6 @@ steps:
- grabpl
image: byrnedo/alpine-curl:0.1.8
name: lint-drone
- image: grafana/drone-downstream
name: trigger-enterprise-downstream
settings:
params:
- SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
- SOURCE_COMMIT=${DRONE_COMMIT}
repositories:
- grafana/grafana-enterprise@main
server: https://drone.grafana.net
token:
from_secret: drone_token
- commands:
- |-
echo -e "unknwon
@@ -526,36 +495,55 @@ steps:
TEST_MAX_WORKERS: 50%
image: grafana/build-container:1.4.8
name: test-frontend
trigger:
branch: main
event:
- push
type: docker
volumes:
- name: cypress_cache
temp: {}
- host:
path: /var/run/docker.sock
name: docker
---
depends_on: []
kind: pipeline
name: main-build-e2e-publish
node:
type: no-parallel
platform:
arch: amd64
os: linux
services: []
steps:
- commands:
- apt-get update
- apt-get install -yq postgresql-client
- dockerize -wait tcp://postgres:5432 -timeout 120s
- psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
- go clean -testcache
- ./bin/grabpl integration-tests --database postgres
depends_on:
- grabpl
environment:
GRAFANA_TEST_DB: postgres
PGPASSWORD: grafanatest
POSTGRES_HOST: postgres
image: grafana/build-container:1.4.8
name: postgres-integration-tests
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- apt-get update
- apt-get install -yq default-mysql-client
- dockerize -wait tcp://mysql:3306 -timeout 120s
- cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
-prootpass
- go clean -testcache
- ./bin/grabpl integration-tests --database mysql
depends_on:
- grabpl
environment:
GRAFANA_TEST_DB: mysql
MYSQL_HOST: mysql
- echo $DRONE_RUNNER_NAME
image: alpine:3.14.3
name: identify-runner
- commands:
- make gen-go
- ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER}
- yarn install --immutable
image: grafana/build-container:1.4.8
name: mysql-integration-tests
name: initialize
- image: grafana/drone-downstream
name: trigger-enterprise-downstream
settings:
params:
- SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}
- SOURCE_COMMIT=${DRONE_COMMIT}
repositories:
- grafana/grafana-enterprise@main
server: https://drone.grafana.net
token:
from_secret: drone_token
- commands:
- ./bin/grabpl build-backend --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER}
--no-pull-enterprise
@@ -832,8 +820,104 @@ volumes:
temp:
medium: memory
---
depends_on: []
kind: pipeline
name: main-integration-tests
node:
type: no-parallel
platform:
arch: amd64
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
MYSQL_ROOT_PASSWORD: rootpass
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
- curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.7.5/grabpl
- chmod +x bin/grabpl
image: byrnedo/alpine-curl:0.1.8
name: grabpl
- commands:
- echo $DRONE_RUNNER_NAME
image: alpine:3.14.3
name: identify-runner
- commands:
- make gen-go
- ./bin/grabpl gen-version --build-id ${DRONE_BUILD_NUMBER}
- yarn install --immutable
image: grafana/build-container:1.4.8
name: initialize
- commands:
- apt-get update
- apt-get install -yq postgresql-client
- dockerize -wait tcp://postgres:5432 -timeout 120s
- psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
- go clean -testcache
- ./bin/grabpl integration-tests --database postgres
depends_on:
- grabpl
environment:
GRAFANA_TEST_DB: postgres
PGPASSWORD: grafanatest
POSTGRES_HOST: postgres
image: grafana/build-container:1.4.8
name: postgres-integration-tests
- commands:
- apt-get update
- apt-get install -yq default-mysql-client
- dockerize -wait tcp://mysql:3306 -timeout 120s
- cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
-prootpass
- go clean -testcache
- ./bin/grabpl integration-tests --database mysql
depends_on:
- grabpl
environment:
GRAFANA_TEST_DB: mysql
MYSQL_HOST: mysql
image: grafana/build-container:1.4.8
name: mysql-integration-tests
trigger:
branch: main
event:
- push
type: docker
volumes:
- name: cypress_cache
temp: {}
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- build-main
- main-test
- main-build-e2e-publish
- main-integration-tests
kind: pipeline
name: windows-main
platform:
@@ -888,7 +972,9 @@ volumes:
name: docker
---
depends_on:
- build-main
- main-test
- main-build-e2e-publish
- main-integration-tests
- windows-main
kind: pipeline
name: publish-main
@@ -945,7 +1031,9 @@ volumes:
name: docker
---
depends_on:
- build-main
- main-test
- main-build-e2e-publish
- main-integration-tests
- windows-main
- publish-main
kind: pipeline
@@ -4105,6 +4193,6 @@ kind: secret
name: prerelease_bucket
---
kind: signature
hmac: e161a5f2487865089e097e8b0ccf5749f225f0c5b29b217008f1577917a5938f
hmac: 0f338f340292dfccc4f8dea56edba284a1ca92417681ecc02e7494e8c003aa45
...
scripts/drone/pipelines/main.star
+38 -19
View File
@@ -55,11 +55,11 @@ load(
ver_mode = 'main'
def get_steps(edition, is_downstream=False):
services = integration_test_services(edition)
publish = edition != 'enterprise' or is_downstream
include_enterprise2 = edition == 'enterprise'
steps = [
test_steps = [
lint_drone_step(),
enterprise_downstream_step(edition=edition),
codespell_step(),
shellcheck_step(),
lint_backend_step(edition=edition),
@@ -67,26 +67,35 @@ def get_steps(edition, is_downstream=False):
test_backend_step(edition=edition),
test_backend_integration_step(edition=edition),
test_frontend_step(),
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
]
build_steps = [
enterprise_downstream_step(edition=edition),
build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_plugins_step(edition=edition, sign=True),
validate_scuemata_step(),
ensure_cuetsified_step(),
]
integration_test_steps = [
postgres_integration_tests_step(edition=edition, ver_mode=ver_mode),
mysql_integration_tests_step(edition=edition, ver_mode=ver_mode),
]
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
build_steps.append(benchmark_ldap_step())
services.append(ldap_service())
test_steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
test_backend_integration_step(edition=edition2),
])
build_steps.extend([
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
])
# Insert remaining steps
steps.extend([
build_steps.extend([
package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream),
install_cypress_step(),
e2e_tests_server_step(edition=edition),
@@ -105,9 +114,9 @@ def get_steps(edition, is_downstream=False):
])
if include_enterprise2:
steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)])
integration_test_steps.extend([redis_integration_tests_step(edition=edition2, ver_mode=ver_mode), memcached_integration_tests_step(edition=edition2, ver_mode=ver_mode)])
steps.extend([
build_steps.extend([
release_canary_npm_packages_step(edition),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
upload_cdn_step(edition=edition, ver_mode=ver_mode)
@@ -115,7 +124,7 @@ def get_steps(edition, is_downstream=False):
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
build_steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=['linux-x64'], is_downstream=is_downstream),
upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
upload_cdn_step(edition=edition2, ver_mode=ver_mode)
@@ -129,7 +138,7 @@ def get_steps(edition, is_downstream=False):
publish_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
]
return steps, windows_steps, publish_steps
return test_steps, build_steps, integration_test_steps, windows_steps, publish_steps
def main_pipelines(edition):
services = integration_test_services(edition)
@@ -138,44 +147,54 @@ def main_pipelines(edition):
'event': ['push',],
'branch': 'main',
}
steps, windows_steps, publish_steps = get_steps(edition=edition)
test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition)
if edition == 'enterprise':
steps.append(benchmark_ldap_step())
services.append(ldap_service())
integration_test_steps.append(benchmark_ldap_step())
pipelines = [
pipeline(
name='build-main', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
name='main-test', edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + test_steps,
volumes=[],
),
pipeline(
name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[],
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps,
volumes=volumes,
),
pipeline(
name='main-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
volumes=volumes,
),
pipeline(
name='windows-main', edition=edition, trigger=trigger,
steps=initialize_step(edition, platform='windows', ver_mode=ver_mode) + windows_steps,
depends_on=['build-main'], platform='windows',
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows',
),
]
if edition != 'enterprise':
pipelines.append(pipeline(
name='publish-main', edition=edition, trigger=trigger,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, install_deps=False) + publish_steps,
depends_on=['build-main', 'windows-main',],
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main',],
))
pipelines.append(notify_pipeline(
name='notify-main', slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=['build-main', 'windows-main', 'publish-main'],
depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', 'publish-main'],
))
else:
# Add downstream enterprise pipelines triggerable from OSS builds
trigger = {
'event': ['custom',],
}
steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
test_steps, build_steps, integration_test_steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
pipelines.append(pipeline(
name='build-main-downstream', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + test_steps + build_steps + integration_test_steps,
volumes=volumes,
))
pipelines.append(pipeline(