From e7680a0d65f01025832dcfd2e06cb0598574f9a6 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 15 Dec 2021 18:20:00 +0100 Subject: [PATCH] Build: Speed up integration tests (#43045) (#43169) Changes so that postgres/mysql services in Drone maps their data volumes to tmpfs volumes (memory) to improve performance. (cherry picked from commit 5bec46ce237547c41c3a46fe16404679145e8fc7) --- .drone.yml | 106 ++++++++++++++++++++++++++- scripts/drone/pipelines/main.star | 4 + scripts/drone/pipelines/pr.star | 6 +- scripts/drone/pipelines/release.star | 5 ++ scripts/drone/services/services.star | 15 ++++ scripts/drone/utils/utils.star | 3 +- 6 files changed, 136 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6707d2223b1..6428b9b0adf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -322,11 +322,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -334,6 +338,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql steps: - commands: - mkdir -p bin @@ -391,6 +398,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- depends_on: [] kind: pipeline @@ -402,11 +415,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -414,6 +431,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql steps: - commands: - mkdir -p bin @@ -805,6 +825,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- depends_on: - build-main @@ -956,11 +982,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -968,6 +998,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql steps: - commands: - mkdir -p bin @@ -1328,6 +1361,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- depends_on: - oss-build-release @@ -1400,11 +1439,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -1412,6 +1455,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql - environment: {} image: redis:6.2.1-alpine name: redis @@ -1854,6 +1900,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- clone: disable: true @@ -1946,11 +1998,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -1958,6 +2014,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql steps: - commands: - mkdir -p bin @@ -2295,6 +2354,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- depends_on: - oss-build-test-release @@ -2365,11 +2430,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -2377,6 +2446,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql - environment: {} image: redis:6.2.1-alpine name: redis @@ -2815,6 +2887,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- clone: disable: true @@ -3016,11 +3094,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -3028,6 +3110,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql steps: - commands: - mkdir -p bin @@ -3347,6 +3432,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- depends_on: - oss-build-release-branch @@ -3412,11 +3503,15 @@ platform: os: linux services: - environment: + PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_DB: grafanatest POSTGRES_PASSWORD: grafanatest POSTGRES_USER: grafanatest image: postgres:12.3-alpine name: postgres + volumes: + - name: postgres + path: /var/lib/postgresql/data/pgdata - environment: MYSQL_DATABASE: grafana_tests MYSQL_PASSWORD: password @@ -3424,6 +3519,9 @@ services: MYSQL_USER: grafana image: mysql:5.6.48 name: mysql + volumes: + - name: mysql + path: /var/lib/mysql - environment: {} image: redis:6.2.1-alpine name: redis @@ -3866,6 +3964,12 @@ volumes: - host: path: /var/run/docker.sock name: docker +- name: postgres + temp: + medium: memory +- name: mysql + temp: + medium: memory --- clone: disable: true @@ -4001,6 +4105,6 @@ kind: secret name: prerelease_bucket --- kind: signature -hmac: 18aa4bedb40fcb3c91faf357a5ecba3b9903bc9e4fb04a29fbbd0e1c0ae7a349 +hmac: e161a5f2487865089e097e8b0ccf5749f225f0c5b29b217008f1577917a5938f ... diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index eb259e266de..04a32c422c3 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -42,6 +42,7 @@ load( load( 'scripts/drone/services/services.star', 'integration_test_services', + 'integration_test_services_volumes', 'ldap_service', ) @@ -132,6 +133,7 @@ def get_steps(edition, is_downstream=False): def main_pipelines(edition): services = integration_test_services(edition) + volumes = integration_test_services_volumes() trigger = { 'event': ['push',], 'branch': 'main', @@ -146,6 +148,7 @@ def main_pipelines(edition): pipeline( name='build-main', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + steps, + volumes=volumes, ), pipeline( name='windows-main', edition=edition, trigger=trigger, @@ -173,6 +176,7 @@ def main_pipelines(edition): pipelines.append(pipeline( name='build-main-downstream', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode, is_downstream=True) + steps, + volumes=volumes, )) pipelines.append(pipeline( name='windows-main-downstream', edition=edition, trigger=trigger, diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star index 91a55d04b39..bc4876b78c8 100644 --- a/scripts/drone/pipelines/pr.star +++ b/scripts/drone/pipelines/pr.star @@ -35,6 +35,7 @@ load( load( 'scripts/drone/services/services.star', 'integration_test_services', + 'integration_test_services_volumes', 'ldap_service', ) @@ -48,6 +49,7 @@ ver_mode = 'pr' def pr_pipelines(edition): services = integration_test_services(edition) + volumes = integration_test_services_volumes() variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64', 'armv6',] include_enterprise2 = edition == 'enterprise' test_steps = [ @@ -123,6 +125,8 @@ def pr_pipelines(edition): name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps, ), pipeline( - name='pr-integration-tests', edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps, + name='pr-integration-tests', edition=edition, trigger=trigger, services=services, + steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + integration_test_steps, + volumes=volumes, ), ] diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index 56ba6af31a3..83b4fe05195 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -41,6 +41,7 @@ load( load( 'scripts/drone/services/services.star', 'integration_test_services', + 'integration_test_services_volumes', 'ldap_service', ) @@ -203,12 +204,14 @@ def get_steps(edition, ver_mode): def get_oss_pipelines(trigger, ver_mode): edition = 'oss' services = integration_test_services(edition=edition) + volumes = integration_test_services_volumes() build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) return [ pipeline( name='oss-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps + package_steps + publish_steps, + volumes=volumes, ), pipeline( name='oss-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, @@ -220,12 +223,14 @@ def get_oss_pipelines(trigger, ver_mode): def get_enterprise_pipelines(trigger, ver_mode): edition = 'enterprise' services = integration_test_services(edition=edition) + volumes = integration_test_services_volumes() build_steps, package_steps, windows_package_steps, publish_steps = get_steps(edition=edition, ver_mode=ver_mode) return [ pipeline( name='enterprise-build-{}'.format(ver_mode), edition=edition, trigger=trigger, services=services, steps=[download_grabpl_step()] + initialize_step(edition, platform='linux', ver_mode=ver_mode) + build_steps + package_steps + publish_steps, + volumes=volumes, ), pipeline( name='enterprise-windows-{}'.format(ver_mode), edition=edition, trigger=trigger, diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index 4c3e4f5c016..5170e15c060 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -1,3 +1,9 @@ +def integration_test_services_volumes(): + return [ + { 'name': 'postgres', 'temp': { 'medium': 'memory' } }, + { 'name': 'mysql', 'temp': { 'medium': 'memory' } + }] + def integration_test_services(edition): services = [ { @@ -7,7 +13,12 @@ def integration_test_services(edition): 'POSTGRES_USER': 'grafanatest', 'POSTGRES_PASSWORD': 'grafanatest', 'POSTGRES_DB': 'grafanatest', + 'PGDATA': '/var/lib/postgresql/data/pgdata', }, + 'volumes': [{ + 'name': 'postgres', + 'path': '/var/lib/postgresql/data/pgdata' + }], }, { 'name': 'mysql', @@ -18,6 +29,10 @@ def integration_test_services(edition): 'MYSQL_USER': 'grafana', 'MYSQL_PASSWORD': 'password', }, + 'volumes': [{ + 'name': 'mysql', + 'path': '/var/lib/mysql' + }] }, ] diff --git a/scripts/drone/utils/utils.star b/scripts/drone/utils/utils.star index c449f11ebbd..4edb9183e2d 100644 --- a/scripts/drone/utils/utils.star +++ b/scripts/drone/utils/utils.star @@ -7,7 +7,7 @@ load( load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token') def pipeline( - name, edition, trigger, steps, services=[], platform='linux', depends_on=[], + name, edition, trigger, steps, services=[], platform='linux', depends_on=[], volumes=[], ): if platform != 'windows': platform_conf = { @@ -48,6 +48,7 @@ def pipeline( }], 'depends_on': depends_on, } + pipeline['volumes'].extend(volumes) pipeline.update(platform_conf) if edition in ('enterprise', 'enterprise2'):