Dashboards: Support list of dashboards for tests to run against (#111084)

* Allow specifying a list of dashboards, both normal and reloabable, that the tests can run against

* refactor

* refactor

* lint

e2e-playwright/dashboard-cujs/README.md

@@ -22,7 +22,15 @@ Configures the path to the API mocking configuration file. This enables dynamic
 
 If the `API_CONFIG_PATH` is not set, the test suite will use mocked responses for API calls using the default configuration, which has settings for the default testing environment. If set, the test suite will make real API calls instead of using mocks, based on the configuration provided in the specified file. This configuration would be used only in live data scenarios where the endpoints might differ due to testing on different dashboards that might use different DataSources which furthermore might have different API endpoints.
 
-The config file should contain endpoint glob patterns for labels and values APIs. These endpoints are used to fetch labels (keys) and values for the AdHocFilters and the GroupBy variables. The pattern should be a string glob pattern, e.g.: '\*\*/resources/\*\*/labels\*'
+The config file should contain endpoint glob patterns for labels and values APIs. These endpoints are used to fetch labels (keys) and values for the AdHocFilters and the GroupBy variables. The pattern should be a string glob pattern, e.g.: `\*\*/resources/\*\*/labels\*`
+
+Alongside the endpoint patterns, the config file can also specify a list of dashboard UIDs that would be used to run navigation and view tests against each of the dashboards. This is useful to verify that static dashboards, alongside reloadable dashboards are viewed and navigated correctly. To use this property, simply add
+
+```json
+  "dashboards": ["cuj-static-dashboard", "cuj-reloadable-dashboard"]
+```
+
+in the config file. The respective tests will then run against the specified dashboards.
 
 ```bash
 API_CONFIG_PATH=/path/to/custom-config.json yarn e2e:playwright

e2e-playwright/dashboard-cujs/adhoc-filters-cujs.spec.ts

@@ -13,7 +13,7 @@ import {
 } from './cuj-selectors';
 import { prepareAPIMocks } from './utils';
 
-export const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
+const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
 
 test.use({
   featureToggles: {

e2e-playwright/dashboard-cujs/dashboard-navigation.spec.ts

@@ -11,6 +11,7 @@ import {
   getScopesDashboardsSearchInput,
   getScopesSelectorInput,
 } from './cuj-selectors';
+import { getConfigDashboards } from './utils';
 
 test.use({
   featureToggles: {
@@ -20,8 +21,9 @@ test.use({
   },
 });
 
-export const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
-export const NAVIGATE_TO = 'cuj-dashboard-2';
+const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
+const DASHBOARD_UNDER_TEST_2 = 'cuj-dashboard-2';
+const NAVIGATE_TO = 'cuj-dashboard-3';
 
 test.describe(
   'Dashboard navigation CUJs',
@@ -79,41 +81,51 @@ test.describe(
         await expect(markdownContent).toContainText(`now-12h`);
       });
 
-      await test.step('3.See filter/groupby selection persisting when navigating from dashboard to dashboard', async () => {
-        const dashboardPage = await gotoDashboardPage({ uid: NAVIGATE_TO });
+      const dashboards = await getConfigDashboards();
+      if (dashboards.length === 0) {
+        dashboards.push(DASHBOARD_UNDER_TEST_2);
+      }
 
-        await setScopes(page, { title: 'CUJ Dashboard 3', uid: 'cuj-dashboard-3' });
+      for (const db of dashboards) {
+        await test.step(
+          '3.See filter/groupby selection persisting when navigating from dashboard to dashboard - ' + db,
+          async () => {
+            const dashboardPage = await gotoDashboardPage({ uid: db });
 
-        await expect(scopeSelectorInput).toHaveValue(/.+/);
+            await setScopes(page, { title: 'CUJ Dashboard 3', uid: NAVIGATE_TO });
 
-        const pills = await adhocFilterPills.allTextContents();
-        const processedPills = pills
-          .map((p) => {
-            const parts = p.split(' ');
-            return `${parts[0]}${parts[1]}"${parts[2]}"`;
-          })
-          .join(',');
+            await expect(scopeSelectorInput).toHaveValue(/.+/);
 
-        // assert the panel is visible and has the correct value
-        const markdownContent = await getMarkdownHTMLContent(dashboardPage, selectors);
-        // no groupBy value
-        await expect(markdownContent).toContainText(`GroupByVar: \n\nAdHocVar: ${processedPills}`);
+            const pills = await adhocFilterPills.allTextContents();
+            const processedPills = pills
+              .map((p) => {
+                const parts = p.split(' ');
+                return `${parts[0]}${parts[1]}"${parts[2]}"`;
+              })
+              .join(',');
 
-        const groupByVariable = getGroupByInput(dashboardPage, selectors);
+            // assert the panel is visible and has the correct value
+            const markdownContent = await getMarkdownHTMLContent(dashboardPage, selectors);
+            // no groupBy value
+            await expect(markdownContent).toContainText(`GroupByVar: \n\nAdHocVar: ${processedPills}`);
 
-        // add a custom groupBy value
-        await groupByVariable.click();
-        await groupByVariable.fill('dev');
-        await groupByVariable.press('Enter');
-        await groupByVariable.press('Escape');
+            const groupByVariable = getGroupByInput(dashboardPage, selectors);
 
-        await expect(scopesDashboards.first()).toBeVisible();
-        await scopesDashboards.first().click();
-        await page.waitForURL('**/d/**');
+            // add a custom groupBy value
+            await groupByVariable.click();
+            await groupByVariable.fill('dev');
+            await groupByVariable.press('Enter');
+            await groupByVariable.press('Escape');
 
-        //all values are set after dashboard switch
-        await expect(markdownContent).toContainText(`GroupByVar: dev\n\nAdHocVar: ${processedPills}`);
-      });
+            await expect(scopesDashboards.first()).toBeVisible();
+            await scopesDashboards.first().click();
+            await page.waitForURL('**/d/**');
+
+            //all values are set after dashboard switch
+            await expect(markdownContent).toContainText(`GroupByVar: dev\n\nAdHocVar: ${processedPills}`);
+          }
+        );
+      }
 
       await test.step('4.Unmodified default filters and groupBy keys are not propagated to a different dashboard', async () => {
         const dashboardPage = await gotoDashboardPage({ uid: DASHBOARD_UNDER_TEST });

e2e-playwright/dashboard-cujs/dashboard-view.spec.ts

@@ -1,5 +1,7 @@
 import { test, expect } from '@grafana/plugin-e2e';
 
+import { getConfigDashboards } from './utils';
+
 test.use({
   featureToggles: {
     scopeFilters: true,
@@ -8,8 +10,8 @@ test.use({
   },
 });
 
-export const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
-export const PANEL_UNDER_TEST = 'Panel Title';
+const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
+const PANEL_UNDER_TEST = 'Panel Title';
 
 test.describe(
   'Dashboard view CUJs',
@@ -18,7 +20,23 @@ test.describe(
   },
   () => {
     test('View a dashboard', async ({ page, gotoDashboardPage, selectors }) => {
-      await test.step('1.Top level selectors', async () => {
+      const dashboards = await getConfigDashboards();
+      if (dashboards.length === 0) {
+        dashboards.push(DASHBOARD_UNDER_TEST);
+      }
+
+      for (const db of dashboards) {
+        await test.step('1.Loads dashboard successfully - ' + db, async () => {
+          const dashboardPage = await gotoDashboardPage({ uid: db });
+
+          const panelTitle = dashboardPage.getByGrafanaSelector(
+            selectors.components.Panels.Panel.title(PANEL_UNDER_TEST)
+          );
+          await expect(panelTitle).toBeVisible();
+        });
+      }
+
+      await test.step('2.Top level selectors', async () => {
         const dashboardPage = await gotoDashboardPage({ uid: DASHBOARD_UNDER_TEST });
 
         const groupByVariable = dashboardPage.getByGrafanaSelector(
@@ -33,7 +51,7 @@ test.describe(
         expect(adHocVariable).toBeVisible();
       });
 
-      await test.step('2.View individual panel', async () => {
+      await test.step('3.View individual panel', async () => {
         const dashboardPage = await gotoDashboardPage({ uid: DASHBOARD_UNDER_TEST });
 
         const viewPanelBreadcrumb = dashboardPage.getByGrafanaSelector(
@@ -60,12 +78,12 @@ test.describe(
         await expect(viewPanelBreadcrumb).not.toBeVisible();
       });
 
-      await test.step('3.Set time range for the dashboard', async () => {
+      await test.step('4.Set time range for the dashboard', async () => {
         const dashboardPage = await gotoDashboardPage({ uid: DASHBOARD_UNDER_TEST });
 
         const timePickerButton = dashboardPage.getByGrafanaSelector(selectors.components.TimePicker.openButton);
 
-        await test.step('3.1.Click on Quick range', async () => {
+        await test.step('4.1.Click on Quick range', async () => {
           await page.mouse.move(0, 0);
           await expect.soft(timePickerButton).toContainText('Last 6 hours');
           await timePickerButton.click();
@@ -76,7 +94,7 @@ test.describe(
           expect.soft(await timePickerButton.textContent()).toContain('Last 5 minutes');
         });
 
-        await test.step('3.2.Set absolute time range', async () => {
+        await test.step('4.2.Set absolute time range', async () => {
           await timePickerButton.click();
           await dashboardPage
             .getByGrafanaSelector(selectors.components.TimePicker.fromField)
@@ -87,7 +105,7 @@ test.describe(
           expect.soft(await timePickerButton.textContent()).toContain('2024-01-01 00:00:00 to 2024-01-01 23:59:59');
         });
 
-        await test.step('3.3.Change time zone', async () => {
+        await test.step('4.3.Change time zone', async () => {
           await timePickerButton.click();
           await dashboardPage
             .getByGrafanaSelector(selectors.components.TimeZonePicker.changeTimeSettingsButton)
@@ -105,7 +123,7 @@ test.describe(
           await timePickerButton.click();
         });
 
-        await test.step('3.4.Navigate time range', async () => {
+        await test.step('4.4.Navigate time range', async () => {
           await timePickerButton.click();
 
           await dashboardPage
@@ -130,7 +148,7 @@ test.describe(
         });
       });
 
-      await test.step('4.Force refresh', async () => {
+      await test.step('5.Force refresh', async () => {
         const dashboardPage = await gotoDashboardPage({ uid: DASHBOARD_UNDER_TEST });
 
         const refreshBtn = dashboardPage.getByGrafanaSelector(selectors.components.RefreshPicker.runButtonV2);
@@ -146,7 +164,7 @@ test.describe(
         expect(await panelContent.textContent()).not.toBe(panelContents);
       });
 
-      await test.step('5.Turn off refresh', async () => {
+      await test.step('6.Turn off refresh', async () => {
         const dashboardPage = await gotoDashboardPage({ uid: DASHBOARD_UNDER_TEST });
 
         const intervalRefreshBtn = dashboardPage.getByGrafanaSelector(

e2e-playwright/dashboard-cujs/global-teardown.spec.ts

@@ -6,7 +6,7 @@ test.describe('Dashboard CUJS Global Teardown', () => {
   test('cleanup test dashboards', async ({ request }) => {
     const dashboardUIDs = getDashboardUIDs();
 
-    if (!dashboardUIDs) {
+    if (!dashboardUIDs.length) {
       return;
     }
 

e2e-playwright/dashboard-cujs/group-by-cujs.spec.ts

@@ -17,7 +17,7 @@ test.use({
   },
 });
 
-export const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
+const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
 
 test.describe(
   'GroupBy CUJs',

e2e-playwright/dashboard-cujs/scope-cujs.spec.ts

@@ -25,7 +25,7 @@ test.use({
 
 const USE_LIVE_DATA = Boolean(process.env.API_CONFIG_PATH);
 
-export const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
+const DASHBOARD_UNDER_TEST = 'cuj-dashboard-1';
 
 test.describe(
   'Scope CUJs',

e2e-playwright/dashboard-cujs/utils.ts

@@ -17,6 +17,11 @@ async function loadApiConfig() {
   }
 }
 
+export async function getConfigDashboards() {
+  const config = await loadApiConfig();
+  return config.dashboards || [];
+}
+
 export async function prepareAPIMocks(page: Page) {
   const apiConfig = await loadApiConfig();