From 7bb5a5b3188004b14c67cef671662052b8fbe1d4 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Wed, 2 Feb 2022 12:40:22 +0200 Subject: [PATCH] CI: Make e2e tests depend on binary builds (#44647) * Make e2e test depend on binary builds * Search for binary in the correct folders * Remove package file var * Add ARCH var --- .drone.yml | 82 +++++++++++++++++----------- scripts/drone/pipelines/main.star | 4 +- scripts/drone/pipelines/pr.star | 4 +- scripts/drone/pipelines/release.star | 4 +- scripts/drone/steps/lib.star | 10 ++-- scripts/grafana-server/start-server | 50 ++++++++--------- scripts/grafana-server/variables | 2 +- 7 files changed, 84 insertions(+), 72 deletions(-) diff --git a/.drone.yml b/.drone.yml index f85642fef3e..f8d744be837 100644 --- a/.drone.yml +++ b/.drone.yml @@ -189,9 +189,12 @@ steps: - commands: - ./scripts/grafana-server/start-server depends_on: - - package + - build-plugins + - build-backend + - build-frontend detach: true environment: + ARCH: linux-amd64 PORT: 3001 image: grafana/build-container:1.4.9 name: grafana-server @@ -199,7 +202,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -208,7 +211,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -217,7 +220,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -226,7 +229,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite various-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -619,9 +622,12 @@ steps: - commands: - ./scripts/grafana-server/start-server depends_on: - - package + - build-plugins + - build-backend + - build-frontend detach: true environment: + ARCH: linux-amd64 PORT: 3001 image: grafana/build-container:1.4.9 name: grafana-server @@ -629,7 +635,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -638,7 +644,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -647,7 +653,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -656,7 +662,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite various-suite depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1226,9 +1232,12 @@ steps: - commands: - ./scripts/grafana-server/start-server depends_on: - - package + - build-plugins + - build-backend + - build-frontend detach: true environment: + ARCH: linux-amd64 PORT: 3001 image: grafana/build-container:1.4.9 name: grafana-server @@ -1236,7 +1245,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1245,7 +1254,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1254,7 +1263,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1263,7 +1272,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1828,10 +1837,12 @@ steps: - commands: - ./scripts/grafana-server/start-server depends_on: - - package + - build-plugins + - build-backend + - build-frontend detach: true environment: - PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz + ARCH: linux-amd64 PORT: 3001 RUNDIR: scripts/grafana-server/tmp-grafana-enterprise image: grafana/build-container:1.4.9 @@ -1840,7 +1851,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1849,7 +1860,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1858,7 +1869,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -1867,7 +1878,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -2982,9 +2993,12 @@ steps: - commands: - ./scripts/grafana-server/start-server depends_on: - - package + - build-plugins + - build-backend + - build-frontend detach: true environment: + ARCH: linux-amd64 PORT: 3001 image: grafana/build-container:1.4.9 name: grafana-server @@ -2992,7 +3006,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3001,7 +3015,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3010,7 +3024,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3019,7 +3033,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3512,10 +3526,12 @@ steps: - commands: - ./scripts/grafana-server/start-server depends_on: - - package + - build-plugins + - build-backend + - build-frontend detach: true environment: - PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz + ARCH: linux-amd64 PORT: 3001 RUNDIR: scripts/grafana-server/tmp-grafana-enterprise image: grafana/build-container:1.4.9 @@ -3524,7 +3540,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3533,7 +3549,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3542,7 +3558,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -3551,7 +3567,7 @@ steps: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 depends_on: - - package + - grafana-server environment: HOST: grafana-server image: cypress/included:9.3.1 @@ -4175,6 +4191,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 19229313440a0d96416f34c4c7ac5d8d00f30478e71a0a6072ae106145e1794c +hmac: 4d1a5696bf1e510fb51a021c07e240c50cb913724ce08ed52cce037ff02dd8de ... diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index 2f338448043..9c65e7a0bc9 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -14,7 +14,7 @@ load( 'build_frontend_step', 'build_plugins_step', 'package_step', - 'e2e_tests_server_step', + 'grafana_server_step', 'e2e_tests_step', 'e2e_tests_artifacts', 'build_storybook_step', @@ -99,7 +99,7 @@ def get_steps(edition, is_downstream=False): # Insert remaining steps build_steps.extend([ package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, is_downstream=is_downstream), - e2e_tests_server_step(edition=edition), + grafana_server_step(edition=edition), e2e_tests_step('dashboards-suite', edition=edition), e2e_tests_step('smoke-tests-suite', edition=edition), e2e_tests_step('panels-suite', edition=edition), diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star index c3c4828dfd0..2054098e899 100644 --- a/scripts/drone/pipelines/pr.star +++ b/scripts/drone/pipelines/pr.star @@ -14,7 +14,7 @@ load( 'test_backend_integration_step', 'test_frontend_step', 'package_step', - 'e2e_tests_server_step', + 'grafana_server_step', 'e2e_tests_step', 'e2e_tests_artifacts', 'build_storybook_step', @@ -92,7 +92,7 @@ def pr_pipelines(edition): # Insert remaining build_steps build_steps.extend([ package_step(edition=edition, ver_mode=ver_mode, include_enterprise2=include_enterprise2, variants=variants), - e2e_tests_server_step(edition=edition), + grafana_server_step(edition=edition), e2e_tests_step('dashboards-suite', edition=edition), e2e_tests_step('smoke-tests-suite', edition=edition), e2e_tests_step('panels-suite', edition=edition), diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index 1c0b967f317..1c592e9d730 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -18,7 +18,7 @@ load( 'build_frontend_step', 'build_plugins_step', 'package_step', - 'e2e_tests_server_step', + 'grafana_server_step', 'e2e_tests_step', 'e2e_tests_artifacts', 'build_storybook_step', @@ -226,7 +226,7 @@ def get_steps(edition, ver_mode): copy_packages_for_docker_step(), package_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish), package_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish), - e2e_tests_server_step(edition=edition), + grafana_server_step(edition=edition), ]) if not disable_tests: diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index b71c1ed6712..9c900fa4c64 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -650,7 +650,7 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is } -def e2e_tests_server_step(edition, port=3001): +def grafana_server_step(edition, port=3001): package_file_pfx = '' if edition == 'enterprise2': package_file_pfx = 'grafana' + enterprise2_suffix(edition) @@ -659,9 +659,9 @@ def e2e_tests_server_step(edition, port=3001): environment = { 'PORT': port, + 'ARCH': 'linux-amd64' } if package_file_pfx: - environment['PACKAGE_FILE'] = 'dist/{}-*linux-amd64.tar.gz'.format(package_file_pfx) environment['RUNDIR'] = 'scripts/grafana-server/tmp-{}'.format(package_file_pfx) return { @@ -669,7 +669,9 @@ def e2e_tests_server_step(edition, port=3001): 'image': build_image, 'detach': True, 'depends_on': [ - 'package' + enterprise2_suffix(edition), + 'build-plugins', + 'build-backend', + 'build-frontend', ], 'environment': environment, 'commands': [ @@ -685,7 +687,7 @@ def e2e_tests_step(suite, edition, port=3001, tries=None): 'name': 'end-to-end-tests-{}'.format(suite) + enterprise2_suffix(edition), 'image': 'cypress/included:9.3.1', 'depends_on': [ - 'package', + 'grafana-server', ], 'environment': { 'HOST': 'grafana-server' + enterprise2_suffix(edition), diff --git a/scripts/grafana-server/start-server b/scripts/grafana-server/start-server index fdf9c20f8e6..7af0f4933ca 100755 --- a/scripts/grafana-server/start-server +++ b/scripts/grafana-server/start-server @@ -4,7 +4,11 @@ set -eo pipefail . scripts/grafana-server/variables PORT=${PORT:-$DEFAULT_PORT} -PACKAGE_FILE=${PACKAGE_FILE:-$DEFAULT_PACKAGE_FILE} +ARCH=${ARCH:-$DEFAULT_ARCH} + +if [ "$ARCH" ]; then + ARCH+="/" +fi ./scripts/grafana-server/kill-server @@ -12,34 +16,24 @@ mkdir $RUNDIR echo -e "Copying grafana backend files to temp dir..." -# Expand any wildcards -pkgs=(${PACKAGE_FILE}) -pkg=${pkgs[0]} -if [[ -f ${pkg} ]]; then - echo "Found package tar file ${pkg}, extracting..." - tar zxf ${pkg} -C $RUNDIR - mv $RUNDIR/grafana-*/* $RUNDIR -else - echo "Couldn't find package ${PACKAGE_FILE} - copying local dev files" - - if [[ ! -f bin/grafana-server ]]; then - echo bin/grafana-server missing - exit 1 - fi - - cp -r ./bin $RUNDIR - cp -r ./public $RUNDIR - cp -r ./tools $RUNDIR - - mkdir $RUNDIR/conf - mkdir $PROV_DIR - mkdir $PROV_DIR/datasources - mkdir $PROV_DIR/dashboards - - cp ./scripts/grafana-server/custom.ini $RUNDIR/conf/custom.ini - cp ./conf/defaults.ini $RUNDIR/conf/defaults.ini +if [[ ! -f bin/"$ARCH"grafana-server ]]; then + echo "bin/linux-amd64/grafana-server missing, trying local grafana-server binary" fi +echo starting server + +cp -r ./bin $RUNDIR +cp -r ./public $RUNDIR +cp -r ./tools $RUNDIR + +mkdir $RUNDIR/conf +mkdir $PROV_DIR +mkdir $PROV_DIR/datasources +mkdir $PROV_DIR/dashboards + +cp ./scripts/grafana-server/custom.ini $RUNDIR/conf/custom.ini +cp ./conf/defaults.ini $RUNDIR/conf/defaults.ini + echo -e "Copy provisioning setup from devenv" cp devenv/datasources.yaml $PROV_DIR/datasources @@ -49,7 +43,7 @@ cp -r devenv $RUNDIR echo -e "Starting Grafana Server port $PORT" -$RUNDIR/bin/grafana-server \ +$RUNDIR/bin/"$ARCH"grafana-server \ --homepath=$HOME_PATH \ --pidfile=$RUNDIR/pid \ cfg:server.http_port=$PORT \ diff --git a/scripts/grafana-server/variables b/scripts/grafana-server/variables index 3b1cdac4ebf..541c4f21f69 100644 --- a/scripts/grafana-server/variables +++ b/scripts/grafana-server/variables @@ -2,9 +2,9 @@ DEFAULT_RUNDIR=scripts/grafana-server/tmp RUNDIR=${RUNDIR:-$DEFAULT_RUNDIR} +DEFAULT_ARCH= HOME_PATH=$PWD/$RUNDIR PIDFILE=$RUNDIR/pid -DEFAULT_PACKAGE_FILE=dist/grafana-*linux-amd64.tar.gz PROV_DIR=$RUNDIR/conf/provisioning DEFAULT_HOST=localhost DEFAULT_PORT=3001