public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build: Speed up integration tests (#43045) (#43169)

Browse Source 
Changes so that postgres/mysql services in Drone maps their data volumes
to tmpfs volumes (memory) to improve performance.

(cherry picked from commit 5bec46ce23)
This commit is contained in:
Marcus Efraimsson
2021-12-15 18:20:00 +01:00
committed by GitHub
parent 20222bada0
commit e7680a0d65
6 changed files with 136 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+105 -1
View File
@@ -322,11 +322,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -334,6 +338,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@@ -391,6 +398,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on: []
kind: pipeline
@@ -402,11 +415,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -414,6 +431,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@@ -805,6 +825,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- build-main
@@ -956,11 +982,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -968,6 +998,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@@ -1328,6 +1361,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- oss-build-release
@@ -1400,11 +1439,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -1412,6 +1455,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
- environment: {}
image: redis:6.2.1-alpine
name: redis
@@ -1854,6 +1900,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
clone:
disable: true
@@ -1946,11 +1998,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -1958,6 +2014,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@@ -2295,6 +2354,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- oss-build-test-release
@@ -2365,11 +2430,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -2377,6 +2446,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
- environment: {}
image: redis:6.2.1-alpine
name: redis
@@ -2815,6 +2887,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
clone:
disable: true
@@ -3016,11 +3094,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -3028,6 +3110,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
steps:
- commands:
- mkdir -p bin
@@ -3347,6 +3432,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
depends_on:
- oss-build-release-branch
@@ -3412,11 +3503,15 @@ platform:
os: linux
services:
- environment:
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_DB: grafanatest
POSTGRES_PASSWORD: grafanatest
POSTGRES_USER: grafanatest
image: postgres:12.3-alpine
name: postgres
volumes:
- name: postgres
path: /var/lib/postgresql/data/pgdata
- environment:
MYSQL_DATABASE: grafana_tests
MYSQL_PASSWORD: password
@@ -3424,6 +3519,9 @@ services:
MYSQL_USER: grafana
image: mysql:5.6.48
name: mysql
volumes:
- name: mysql
path: /var/lib/mysql
- environment: {}
image: redis:6.2.1-alpine
name: redis
@@ -3866,6 +3964,12 @@ volumes:
- host:
path: /var/run/docker.sock
name: docker
- name: postgres
temp:
medium: memory
- name: mysql
temp:
medium: memory
---
clone:
disable: true
@@ -4001,6 +4105,6 @@ kind: secret
name: prerelease_bucket
---
kind: signature
hmac: 18aa4bedb40fcb3c91faf357a5ecba3b9903bc9e4fb04a29fbbd0e1c0ae7a349
hmac: e161a5f2487865089e097e8b0ccf5749f225f0c5b29b217008f1577917a5938f
...
scripts/drone/pipelines/main.star
+4
View File
@@ -42,6 +42,7 @@ load(
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
@@ -132,6 +133,7 @@ def get_steps(edition, is_downstream=False):
def main_pipelines(edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
trigger = {
'event': ['push',],
'branch': 'main',
@@ -146,6 +148,7 @@ def main_pipelines(edition):
pipeline(
name='build-main', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps,
volumes=volumes,
),
pipeline(
name='windows-main', edition=edition, trigger=trigger,
@@ -173,6 +176,7 @@ def main_pipelines(edition):
pipelines.append(pipeline(
name='build-main-downstream', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps,
volumes=volumes,
))
pipelines.append(pipeline(
name='windows-main-downstream', edition=edition, trigger=trigger,
scripts/drone/pipelines/pr.star
+5 -1
View File
@@ -35,6 +35,7 @@ load(
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
@@ -48,6 +49,7 @@ ver_mode = 'pr'
def pr_pipelines(edition):
services = integration_test_services(edition)
volumes = integration_test_services_volumes()
variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64', 'armv6',]
include_enterprise2 = edition == 'enterprise'
test_steps = [
@@ -123,6 +125,8 @@ def pr_pipelines(edition):
name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode)
+ build_steps,
), pipeline(
name='pr-integration-tests', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps,
volumes=volumes,
),
]
scripts/drone/pipelines/release.star
+5
View File
@@ -41,6 +41,7 @@ load(
load(
'scripts/drone/services/services.star',
'integration_test_services',
'integration_test_services_volumes',
'ldap_service',
)
@@ -203,12 +204,14 @@ def get_steps(edition, ver_mode):
def get_oss_pipelines(trigger, ver_mode):
edition = 'oss'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
return [
pipeline(
name='oss-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
volumes=volumes,
),
pipeline(
name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
@@ -220,12 +223,14 @@ def get_oss_pipelines(trigger, ver_mode):
def get_enterprise_pipelines(trigger, ver_mode):
edition = 'enterprise'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode)
return [
pipeline(
name='enterprise-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services,
steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) +
build_steps + package_steps + publish_steps,
volumes=volumes,
),
pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger,
scripts/drone/services/services.star
+15
View File
@@ -1,3 +1,9 @@
def integration_test_services_volumes():
return [
{ 'name': 'postgres', 'temp': { 'medium': 'memory' } },
{ 'name': 'mysql', 'temp': { 'medium': 'memory' }
}]
def integration_test_services(edition):
services = [
{
@@ -7,7 +13,12 @@ def integration_test_services(edition):
'POSTGRES_USER': 'grafanatest',
'POSTGRES_PASSWORD': 'grafanatest',
'POSTGRES_DB': 'grafanatest',
'PGDATA': '/var/lib/postgresql/data/pgdata',
},
'volumes': [{
'name': 'postgres',
'path': '/var/lib/postgresql/data/pgdata'
}],
},
{
'name': 'mysql',
@@ -18,6 +29,10 @@ def integration_test_services(edition):
'MYSQL_USER': 'grafana',
'MYSQL_PASSWORD': 'password',
},
'volumes': [{
'name': 'mysql',
'path': '/var/lib/mysql'
}]
},
]
scripts/drone/utils/utils.star
+2 -1
View File
@@ -7,7 +7,7 @@ load(
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
def pipeline(
name, edition, trigger, steps, services=[], platform='linux', depends_on=[],
name, edition, trigger, steps, services=[], platform='linux', depends_on=[], volumes=[],
):
if platform != 'windows':
platform_conf = {
@@ -48,6 +48,7 @@ def pipeline(
}],
'depends_on': depends_on,
}
pipeline['volumes'].extend(volumes)
pipeline.update(platform_conf)
if edition in ('enterprise', 'enterprise2'):