E2E: Add plugin-e2e scenario verification tests (#79969)

* add playwright test and plugin-e2e * run tests in ci * add ds config tests * add panel edit tests * add annotation test * add variable edit page tests * add explore page tests * add panel plugin tests * add readme * remove comments * fix broken test * remove user.json * remove newline in starlark * fix lint issue * ignore failure of playwright tests * update code owners * add detailed error messages in every expect * update message frame * fix link * upload report to gcp * echo url * add playwright developer guide * bump plugin-e2e * add custom provisioning dir * update plugin-e2e * remove not used imports * fix typo * minor fixes * use latest version of plugin-e2e * fix broken link * use latest plugin-e2e * add feature toggle scenario verification tests * bump version * use auth file from package * fix type error * add panel data assertions * rename parent dir and bump version * fix codeowners * reset files * remove not used file * update plugin-e2e * separate tests per role * pass prov dir * skip using provisioning fixture * wip * fix permission test * move to e2e dir * fix path to readme * post comment with report url * format starlark * post comment with report url * post comment with report url * fix token * make test fail * fix exit code * bump version * bump to latest plugin-e2e * revert reporting message * remove comments * readding report comment * change exit code * format starlark * force test to fail * add new step that posts comment * fix link * use latest playwright image * fix failing test * format starlark * remove unused fixture Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com> --------- Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>
2024-02-23 12:39:30 +01:00
parent b25667223c
commit 3e456127cb
22 changed files with 2347 additions and 212 deletions
@@ -360,6 +360,65 @@ def e2e_tests_artifacts():
        ],
    }

+def playwright_e2e_report_upload():
+    return {
+        "name": "playwright-e2e-report-upload",
+        "image": images["cloudsdk"],
+        "depends_on": [
+            "playwright-plugin-e2e",
+        ],
+        "failure": "ignore",
+        "when": {
+            "status": [
+                "success",
+                "failure",
+            ],
+        },
+        "environment": {
+            "GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY": from_secret(gcp_upload_artifacts_key),
+        },
+        "commands": [
+            "apt-get update",
+            "apt-get install -yq zip",
+            "printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json",
+            "gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json",
+            "gsutil cp -r ./playwright-report/. gs://releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report",
+            "export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html",
+            'echo "E2E Playwright report uploaded to: \n $${E2E_PLAYWRIGHT_REPORT_URL}"',
+        ],
+    }
+
+def playwright_e2e_report_post_link():
+    return {
+        "name": "playwright-e2e-report-post-link",
+        "image": images["curl"],
+        "depends_on": [
+            "playwright-e2e-report-upload",
+        ],
+        "failure": "ignore",
+        "when": {
+            "status": [
+                "success",
+                "failure",
+            ],
+        },
+        "environment": {
+            "GITHUB_TOKEN": from_secret("github_token"),
+        },
+        "commands": [
+            # if the trace doesn't folder exists, it means that there are no failed tests.
+            "if [ ! -d ./playwright-report/trace ]; then echo 'all tests passed'; exit 0; fi",
+            # if it exists, we will post a comment on the PR with the link to the report
+            "export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html",
+            "curl -L " +
+            "-X POST https://api.github.com/repos/grafana/grafana/issues/${DRONE_PULL_REQUEST}/comments " +
+            '-H "Accept: application/vnd.github+json" ' +
+            '-H "Authorization: Bearer $${GITHUB_TOKEN}" ' +
+            '-H "X-GitHub-Api-Version: 2022-11-28" -d ' +
+            '"{\\"body\\":\\"❌ Failed to run Playwright plugin e2e tests. <br /> <br /> Click [here]($${E2E_PLAYWRIGHT_REPORT_URL}) to browse the Playwright report and trace viewer. <br /> For information on how to run Playwright tests locally, refer to the [Developer guide](https://github.com/grafana/grafana/blob/main/contribute/developer-guide.md#to-run-the-playwright-tests). \\"}"',
+        ],
+    }
+
 def upload_cdn_step(ver_mode, trigger = None):
    """Uploads CDN assets using the Grafana build tool.

@@ -786,6 +845,25 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None):
    step = dict(step, when = when)
    return step

+def playwright_e2e_tests_step():
+    return {
+        "environment": {
+            "PORT": "3001",
+            "HOST": "grafana-server",
+            "PROV_DIR": "/grafana/scripts/grafana-server/tmp/conf/provisioning",
+        },
+        "name": "playwright-plugin-e2e",
+        "image": images["playwright"],
+        "failure": "ignore",
+        "depends_on": [
+            "grafana-server",
+        ],
+        "commands": [
+            "sleep 10s",  # it seems sometimes that grafana-server is not actually ready when the step starts, so waiting for a few seconds before running the tests
+            "yarn e2e:playwright",
+        ],
+    }
+
 def build_docs_website_step():
    return {
        "name": "build-docs-website",