E2E: Add plugin-e2e scenario verification tests (#79969)

* add playwright test and plugin-e2e * run tests in ci * add ds config tests * add panel edit tests * add annotation test * add variable edit page tests * add explore page tests * add panel plugin tests * add readme * remove comments * fix broken test * remove user.json * remove newline in starlark * fix lint issue * ignore failure of playwright tests * update code owners * add detailed error messages in every expect * update message frame * fix link * upload report to gcp * echo url * add playwright developer guide * bump plugin-e2e * add custom provisioning dir * update plugin-e2e * remove not used imports * fix typo * minor fixes * use latest version of plugin-e2e * fix broken link * use latest plugin-e2e * add feature toggle scenario verification tests * bump version * use auth file from package * fix type error * add panel data assertions * rename parent dir and bump version * fix codeowners * reset files * remove not used file * update plugin-e2e * separate tests per role * pass prov dir * skip using provisioning fixture * wip * fix permission test * move to e2e dir * fix path to readme * post comment with report url * format starlark * post comment with report url * post comment with report url * fix token * make test fail * fix exit code * bump version * bump to latest plugin-e2e * revert reporting message * remove comments * readding report comment * change exit code * format starlark * force test to fail * add new step that posts comment * fix link * use latest playwright image * fix failing test * format starlark * remove unused fixture Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com> --------- Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>
2024-02-23 12:39:30 +01:00
parent b25667223c
commit 3e456127cb
22 changed files with 2347 additions and 212 deletions
@@ -0,0 +1,15 @@
+import { expect, test } from '@grafana/plugin-e2e';
+
+import { formatExpectError } from '../errors';
+import { successfulAnnotationQuery } from '../mocks/queries';
+
+test('annotation query data with mocked response', async ({ annotationEditPage, page }) => {
+  annotationEditPage.mockQueryDataResponse(successfulAnnotationQuery);
+  await annotationEditPage.datasource.set('gdev-testdata');
+  await page.getByLabel('Scenario').last().fill('CSV Content');
+  await page.keyboard.press('Tab');
+  await expect(
+    annotationEditPage.runQuery(),
+    formatExpectError('Expected annotation query to execute successfully')
+  ).toBeOK();
+});
@@ -0,0 +1,35 @@
+import { expect, test } from '@grafana/plugin-e2e';
+
+import { formatExpectError } from '../errors';
+
+test.describe('test createDataSourceConfigPage fixture, saveAndTest and toBeOK matcher', () => {
+  test('invalid credentials should return an error', async ({ createDataSourceConfigPage, page }) => {
+    const configPage = await createDataSourceConfigPage({ type: 'prometheus' });
+    await page.getByPlaceholder('http://localhost:9090').fill('http://localhost:9090');
+    await expect(
+      configPage.saveAndTest(),
+      formatExpectError('Expected save data source config to fail when Prometheus server is not running')
+    ).not.toBeOK();
+  });
+
+  test('valid credentials should return a 200 status code', async ({ createDataSourceConfigPage, page }) => {
+    const configPage = await createDataSourceConfigPage({ type: 'prometheus' });
+    configPage.mockHealthCheckResponse({ status: 200 });
+    await page.getByPlaceholder('http://localhost:9090').fill('http://localhost:9090');
+    await expect(
+      configPage.saveAndTest(),
+      formatExpectError('Expected data source config to be successfully saved')
+    ).toBeOK();
+  });
+});
+
+test.describe('test data source with frontend only health check', () => {
+  test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage }) => {
+    const configPage = await createDataSourceConfigPage({ type: 'testdata' });
+    await configPage.saveAndTest({ skipWaitForResponse: true });
+    await expect(
+      configPage,
+      formatExpectError('Expected data source config to display success alert after save')
+    ).toHaveAlert('success', { hasNotText: 'Datasource updated' });
+  });
+});
@@ -0,0 +1,15 @@
+import { expect, test } from '@grafana/plugin-e2e';
+
+import { formatExpectError } from '../errors';
+
+test('query data response should be OK when query is valid', async ({ explorePage }) => {
+  await explorePage.datasource.set('gdev-testdata');
+  await expect(explorePage.runQuery(), formatExpectError('Expected Explore query to execute successfully')).toBeOK();
+});
+
+test('query data response should not be OK when query is invalid', async ({ explorePage }) => {
+  await explorePage.datasource.set('gdev-testdata');
+  const queryEditorRow = await explorePage.getQueryEditorRow('A');
+  await queryEditorRow.getByLabel('Labels').fill('invalid-label-format');
+  await expect(explorePage.runQuery(), formatExpectError('Expected Explore query to fail')).not.toBeOK();
+});
@@ -0,0 +1,17 @@
+import { expect, test } from '@grafana/plugin-e2e';
+
+const TRUTHY_CUSTOM_TOGGLE = 'custom_toggle1';
+const FALSY_CUSTOM_TOGGLE = 'custom_toggle2';
+
+// override the feature toggles defined in playwright.config.ts only for tests in this file
+test.use({
+  featureToggles: {
+    [TRUTHY_CUSTOM_TOGGLE]: true,
+    [FALSY_CUSTOM_TOGGLE]: false,
+  },
+});
+
+test('should set and check feature toggles correctly', async ({ isFeatureToggleEnabled }) => {
+  expect(await isFeatureToggleEnabled(TRUTHY_CUSTOM_TOGGLE)).toBeTruthy();
+  expect(await isFeatureToggleEnabled(FALSY_CUSTOM_TOGGLE)).toBeFalsy();
+});
@@ -0,0 +1,110 @@
+import { expect, test, PanelEditPage, DashboardPage } from '@grafana/plugin-e2e';
+
+import { formatExpectError } from '../errors';
+import { successfulDataQuery } from '../mocks/queries';
+
+const REACT_TABLE_DASHBOARD = { uid: 'U_bZIMRMk' };
+
+test.describe('panel edit page', () => {
+  test('table panel data assertions with provisioned dashboard', async ({
+    page,
+    selectors,
+    grafanaVersion,
+    request,
+  }) => {
+    const panelEditPage = new PanelEditPage(
+      { page, selectors, grafanaVersion, request },
+      { dashboard: REACT_TABLE_DASHBOARD, id: '4' }
+    );
+    await panelEditPage.goto();
+    await expect(
+      panelEditPage.panel.locator,
+      formatExpectError('Could not locate panel in panel edit page')
+    ).toBeVisible();
+    await expect(
+      panelEditPage.panel.fieldNames,
+      formatExpectError('Could not locate header elements in table panel')
+    ).toContainText(['Field', 'Max', 'Mean', 'Last']);
+  });
+
+  test('table panel data assertions', async ({ panelEditPage }) => {
+    await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200);
+    await panelEditPage.datasource.set('gdev-testdata');
+    await panelEditPage.setVisualization('Table');
+    await panelEditPage.refreshPanel();
+    await expect(
+      panelEditPage.panel.locator,
+      formatExpectError('Could not locate panel in panel edit page')
+    ).toBeVisible();
+    await expect(
+      panelEditPage.panel.fieldNames,
+      formatExpectError('Could not locate header elements in table panel')
+    ).toContainText(['col1', 'col2']);
+    await expect(panelEditPage.panel.data, formatExpectError('Could not locate headers in table panel')).toContainText([
+      'val1',
+      'val2',
+      'val3',
+      'val4',
+    ]);
+  });
+
+  test('timeseries panel - table view assertions', async ({ panelEditPage }) => {
+    await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200);
+    await panelEditPage.datasource.set('gdev-testdata');
+    await panelEditPage.setVisualization('Time series');
+    await panelEditPage.refreshPanel();
+    await panelEditPage.toggleTableView();
+    await expect(
+      panelEditPage.panel.locator,
+      formatExpectError('Could not locate panel in panel edit page')
+    ).toBeVisible();
+    await expect(
+      panelEditPage.panel.fieldNames,
+      formatExpectError('Could not locate header elements in table panel')
+    ).toContainText(['col1', 'col2']);
+    await expect(
+      panelEditPage.panel.data,
+      formatExpectError('Could not locate data elements in table panel')
+    ).toContainText(['val1', 'val2', 'val3', 'val4']);
+  });
+});
+
+test.describe('dashboard page', () => {
+  test('getting panel by title', async ({ page, selectors, grafanaVersion, request }) => {
+    const dashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, REACT_TABLE_DASHBOARD);
+    await dashboardPage.goto();
+    const panel = await dashboardPage.getPanelByTitle('Colored background');
+    await expect(panel.fieldNames).toContainText(['Field', 'Max', 'Mean', 'Last']);
+  });
+
+  test('getting panel by id', async ({ page, selectors, grafanaVersion, request }) => {
+    const dashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, REACT_TABLE_DASHBOARD);
+    await dashboardPage.goto();
+    const panel = await dashboardPage.getPanelById('4');
+    await expect(panel.fieldNames, formatExpectError('Could not locate header elements in table panel')).toContainText([
+      'Field',
+      'Max',
+      'Mean',
+      'Last',
+    ]);
+  });
+});
+
+test.describe('explore page', () => {
+  test('table panel', async ({ explorePage }) => {
+    const url =
+      'left=%7B"datasource":"grafana","queries":%5B%7B"queryType":"randomWalk","refId":"A","datasource":%7B"type":"datasource","uid":"grafana"%7D%7D%5D,"range":%7B"from":"1547161200000","to":"1576364400000"%7D%7D&orgId=1';
+    await explorePage.goto({
+      queryParams: new URLSearchParams(url),
+    });
+    await expect(
+      explorePage.timeSeriesPanel.locator,
+      formatExpectError('Could not locate time series panel in explore page')
+    ).toBeVisible();
+    await expect(
+      explorePage.tablePanel.locator,
+      formatExpectError('Could not locate table panel in explore page')
+    ).toBeVisible();
+    await expect(explorePage.tablePanel.fieldNames).toContainText(['time', 'A-series']);
+  });
+});
@@ -0,0 +1,85 @@
+import { expect, test } from '@grafana/plugin-e2e';
+
+import { formatExpectError } from '../errors';
+import { successfulDataQuery } from '../mocks/queries';
+import { scenarios } from '../mocks/resources';
+
+const PANEL_TITLE = 'Table panel E2E test';
+const TABLE_VIZ_NAME = 'Table';
+const STANDARD_OTIONS_CATEGORY = 'Standard options';
+const DISPLAY_NAME_LABEL = 'Display name';
+
+test.describe('query editor query data', () => {
+  test('query data response should be OK when query is valid', async ({ panelEditPage }) => {
+    await panelEditPage.datasource.set('gdev-testdata');
+    await expect(
+      panelEditPage.refreshPanel(),
+      formatExpectError('Expected panel query to execute successfully')
+    ).toBeOK();
+  });
+
+  test('query data response should not be OK and panel error should be displayed when query is invalid', async ({
+    panelEditPage,
+  }) => {
+    await panelEditPage.datasource.set('gdev-testdata');
+    const queryEditorRow = await panelEditPage.getQueryEditorRow('A');
+    await queryEditorRow.getByLabel('Labels').fill('invalid-label-format');
+    await expect(panelEditPage.refreshPanel(), formatExpectError('Expected panel query to fail')).not.toBeOK();
+    await expect(
+      panelEditPage.panel.getErrorIcon(),
+      formatExpectError('Expected panel error to be displayed after query execution')
+    ).toBeVisible();
+  });
+});
+
+test.describe('query editor with mocked responses', () => {
+  test('and resource `scenarios` is mocked', async ({ panelEditPage, selectors }) => {
+    await panelEditPage.mockResourceResponse('scenarios', scenarios);
+    await panelEditPage.datasource.set('gdev-testdata');
+    const queryEditorRow = await panelEditPage.getQueryEditorRow('A');
+    await queryEditorRow.getByLabel('Scenario').last().click();
+    await expect(
+      panelEditPage.getByTestIdOrAriaLabel(selectors.components.Select.option),
+      formatExpectError('Expected certain select options to be displayed after clicking on the select input')
+    ).toHaveText(scenarios.map((s) => s.name));
+  });
+
+  test('mocked query data response', async ({ panelEditPage, selectors }) => {
+    await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200);
+    await panelEditPage.datasource.set('gdev-testdata');
+    await panelEditPage.setVisualization(TABLE_VIZ_NAME);
+    await panelEditPage.refreshPanel();
+    await expect(
+      panelEditPage.panel.getErrorIcon(),
+      formatExpectError('Did not expect panel error to be displayed after query execution')
+    ).not.toBeVisible();
+    await expect(
+      panelEditPage.getByTestIdOrAriaLabel(selectors.components.Panels.Visualization.Table.body),
+      formatExpectError('Expected certain select options to be displayed after clicking on the select input')
+    ).toHaveText('val1val2val3val4');
+  });
+});
+
+test.describe('edit panel plugin settings', () => {
+  test('change viz to table panel, set panel title and collapse section', async ({
+    panelEditPage,
+    selectors,
+    page,
+  }) => {
+    await panelEditPage.setVisualization(TABLE_VIZ_NAME);
+    await expect(
+      panelEditPage.getByTestIdOrAriaLabel(selectors.components.PanelEditor.toggleVizPicker),
+      formatExpectError('Expected panel visualization to be set to table')
+    ).toHaveText(TABLE_VIZ_NAME);
+    await panelEditPage.setPanelTitle(PANEL_TITLE);
+    await expect(
+      panelEditPage.getByTestIdOrAriaLabel(selectors.components.Panels.Panel.title(PANEL_TITLE)),
+      formatExpectError('Expected panel title to be updated')
+    ).toBeVisible();
+    await panelEditPage.collapseSection(STANDARD_OTIONS_CATEGORY);
+    await expect(
+      page.getByText(DISPLAY_NAME_LABEL),
+      formatExpectError('Expected section to be collapsed')
+    ).toBeVisible();
+  });
+});
@@ -0,0 +1,16 @@
+import { expect, test } from '@grafana/plugin-e2e';
+
+import { formatExpectError } from '../errors';
+import { prometheusLabels } from '../mocks/resources';
+
+test('variable query with mocked response', async ({ variableEditPage, page }) => {
+  variableEditPage.mockResourceResponse('api/v1/labels?*', prometheusLabels);
+  await variableEditPage.datasource.set('gdev-prometheus');
+  await variableEditPage.getByTestIdOrAriaLabel('Query type').fill('Label names');
+  await page.keyboard.press('Tab');
+  await variableEditPage.runQuery();
+  await expect(
+    variableEditPage,
+    formatExpectError('Expected variable edit page to display certain label names after query execution')
+  ).toDisplayPreviews(prometheusLabels.data);
+});