Fix integration tests for mysql and postgres (#50867)

* Fix integration tests for mysql and postgres

* Package by package testing

* Adding make gen-go to the postgres and mysql integration step

* Rewrite command for backend integration tests

* Diffentiate command for OSS and enterprise
* Add wire-install as dependant for OSS integration tests
* set GRAFANA_TEST_DB variable in the go test command

* No verbose output

* Remove redundant env var

* Try fixing the tests

* Trying stg else

* Remove verbose

Co-authored-by: Sofia Papagiannaki <sofia@grafana.com>

.drone.yml

@@ -454,15 +454,30 @@ steps:
   - echo $DRONE_RUNNER_NAME
   image: alpine:3.15
   name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - CODEGEN_VERIFY=1 make gen-cue
+  image: grafana/build-container:1.5.5
+  name: verify-gen-cue
+- commands:
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: grafana/build-container:1.5.5
+  name: wire-install
 - commands:
   - apt-get update
   - apt-get install -yq postgresql-client
   - dockerize -wait tcp://postgres:5432 -timeout 120s
   - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database postgres
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: postgres
     PGPASSWORD: grafanatest
@@ -476,9 +491,10 @@ steps:
   - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
     -prootpass
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database mysql
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: mysql
     MYSQL_HOST: mysql
@@ -1281,15 +1297,30 @@ steps:
   - echo $DRONE_RUNNER_NAME
   image: alpine:3.15
   name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - CODEGEN_VERIFY=1 make gen-cue
+  image: grafana/build-container:1.5.5
+  name: verify-gen-cue
+- commands:
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: grafana/build-container:1.5.5
+  name: wire-install
 - commands:
   - apt-get update
   - apt-get install -yq postgresql-client
   - dockerize -wait tcp://postgres:5432 -timeout 120s
   - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database postgres
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: postgres
     PGPASSWORD: grafanatest
@@ -1303,9 +1334,10 @@ steps:
   - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
     -prootpass
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database mysql
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: mysql
     MYSQL_HOST: mysql
@@ -1958,15 +1990,30 @@ steps:
   - echo $DRONE_RUNNER_NAME
   image: alpine:3.15
   name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - CODEGEN_VERIFY=1 make gen-cue
+  image: grafana/build-container:1.5.5
+  name: verify-gen-cue
+- commands:
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: grafana/build-container:1.5.5
+  name: wire-install
 - commands:
   - apt-get update
   - apt-get install -yq postgresql-client
   - dockerize -wait tcp://postgres:5432 -timeout 120s
   - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database postgres
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: postgres
     PGPASSWORD: grafanatest
@@ -1980,9 +2027,10 @@ steps:
   - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
     -prootpass
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database mysql
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: mysql
     MYSQL_HOST: mysql
@@ -3734,15 +3782,30 @@ steps:
   - echo $DRONE_RUNNER_NAME
   image: alpine:3.15
   name: identify-runner
+- commands:
+  - '# It is required that code generated from Thema/CUE be committed and in sync
+    with its inputs.'
+  - '# The following command will fail if running code generators produces any diff
+    in output.'
+  - CODEGEN_VERIFY=1 make gen-cue
+  image: grafana/build-container:1.5.5
+  name: verify-gen-cue
+- commands:
+  - make gen-go
+  depends_on:
+  - verify-gen-cue
+  image: grafana/build-container:1.5.5
+  name: wire-install
 - commands:
   - apt-get update
   - apt-get install -yq postgresql-client
   - dockerize -wait tcp://postgres:5432 -timeout 120s
   - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database postgres
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: postgres
     PGPASSWORD: grafanatest
@@ -3756,9 +3819,10 @@ steps:
   - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
     -prootpass
   - go clean -testcache
-  - ./bin/grabpl integration-tests --database mysql
+  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
+    -timeout=30m {}'
   depends_on:
-  - grabpl
+  - wire-install
   environment:
     GRAFANA_TEST_DB: mysql
     MYSQL_HOST: mysql
@@ -4683,6 +4747,6 @@ kind: secret
 name: gcp_upload_artifacts_key
 ---
 kind: signature
-hmac: a3bda50a9e18232abceea55b0449c46945cc4f320ddb4524e110128297823369
+hmac: e0b93430bf690b093bde6979dda9eb9cb2846c7d5a71feae8c2ea81072aecf19
 
 ...

scripts/drone/pipelines/main.star

@@ -220,7 +220,7 @@ def main_pipelines(edition):
         volumes=volumes,
     ), pipeline(
         name='main-integration-tests', edition=edition, trigger=trigger, services=services,
-        steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
+        steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(), wire_install_step(), ] + integration_test_steps,
         volumes=volumes,
     ), pipeline(
         name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']),

scripts/drone/pipelines/pr.star

@@ -153,7 +153,7 @@ def pr_pipelines(edition):
             name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps,
         ), pipeline(
             name='pr-integration-tests', edition=edition, trigger=trigger, services=services,
-            steps=[download_grabpl_step(), identify_runner_step(), ] + integration_test_steps,
+            steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(), wire_install_step(), ] + integration_test_steps,
             volumes=volumes,
         ), docs_pipelines(edition, ver_mode, trigger_docs())
     ]

scripts/drone/pipelines/release.star

@@ -275,7 +275,7 @@ def get_oss_pipelines(trigger, ver_mode):
             ),
             pipeline(
                 name='{}-oss-integration-tests'.format(ver_mode), edition=edition, trigger=trigger, services=services,
-                steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps,
+                steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(), wire_install_step(), ] + integration_test_steps,
                 volumes=volumes,
             )
         ])

scripts/drone/steps/lib.star

@@ -870,7 +870,21 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
 
 def postgres_integration_tests_step(edition, ver_mode):
     deps = []
-    deps.extend(['grabpl'])
+    cmds = [
+            'apt-get update',
+            'apt-get install -yq postgresql-client',
+            'dockerize -wait tcp://postgres:5432 -timeout 120s',
+            'psql -p 5432 -h postgres -U grafanatest -d grafanatest -f ' +
+            'devenv/docker/blocks/postgres_tests/setup.sql',
+            # Make sure that we don't use cached results for another database
+            'go clean -testcache',
+        ]
+    if edition == 'oss':
+        deps.extend(['wire-install'])
+        cmds.extend(["go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=30m {}'"])
+    else:
+        deps.extend(['grabpl'])
+        cmds.extend(['./bin/grabpl integration-tests --database postgres'])
     return {
         'name': 'postgres-integration-tests',
         'image': build_image,
@@ -880,22 +894,26 @@ def postgres_integration_tests_step(edition, ver_mode):
             'GRAFANA_TEST_DB': 'postgres',
             'POSTGRES_HOST': 'postgres',
         },
-        'commands': [
+        'commands': cmds,
-            'apt-get update',
-            'apt-get install -yq postgresql-client',
-            'dockerize -wait tcp://postgres:5432 -timeout 120s',
-            'psql -p 5432 -h postgres -U grafanatest -d grafanatest -f ' +
-            'devenv/docker/blocks/postgres_tests/setup.sql',
-            # Make sure that we don't use cached results for another database
-            'go clean -testcache',
-            './bin/grabpl integration-tests --database postgres',
-        ],
     }
 
 
 def mysql_integration_tests_step(edition, ver_mode):
     deps = []
-    deps.extend(['grabpl'])
+    cmds = [
+            'apt-get update',
+            'apt-get install -yq default-mysql-client',
+            'dockerize -wait tcp://mysql:3306 -timeout 120s',
+            'cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass',
+            # Make sure that we don't use cached results for another database
+            'go clean -testcache',
+        ]
+    if edition == 'oss':
+        deps.extend(['wire-install'])
+        cmds.extend(["go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=30m {}'"])
+    else:
+        deps.extend(['grabpl'])
+        cmds.extend(['./bin/grabpl integration-tests --database mysql'])
     return {
         'name': 'mysql-integration-tests',
         'image': build_image,
@@ -904,15 +922,7 @@ def mysql_integration_tests_step(edition, ver_mode):
             'GRAFANA_TEST_DB': 'mysql',
             'MYSQL_HOST': 'mysql',
         },
-        'commands': [
+        'commands': cmds,
-            'apt-get update',
-            'apt-get install -yq default-mysql-client',
-            'dockerize -wait tcp://mysql:3306 -timeout 120s',
-            'cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass',
-            # Make sure that we don't use cached results for another database
-            'go clean -testcache',
-            './bin/grabpl integration-tests --database mysql',
-        ],
     }