AzureMonitor: Add health check retry logic (#68120) (#68198)

AzureMonitor: Bump credential wait time (#68120)

Bump credential wait time as 15s seems too short

(cherry picked from commit 0d3a78d86d)

# Conflicts:
#	e2e/cloud-plugins-suite/azure-monitor.spec.ts

e2e/cloud-plugins-suite/azure-monitor.spec.ts

@@ -1,3 +1,4 @@
+import { Interception } from 'cypress/types/net-stubbing';
 import { load } from 'js-yaml';
 import { v4 as uuidv4 } from 'uuid';
 
@@ -22,12 +23,29 @@ type AzureMonitorProvision = { datasources: AzureMonitorConfig[] };
 
 const dataSourceName = `Azure Monitor E2E Tests - ${uuidv4()}`;
 
+const maxRetryCount = 3;
+
+Cypress.Commands.add('checkHealthRetryable', function (fn: Function, retryCount: number) {
+  cy.then(() => {
+    const result = fn(++retryCount);
+    result.then((res: Interception) => {
+      if (retryCount < maxRetryCount && res.response.statusCode !== 200) {
+        cy.wait(20000);
+        cy.checkHealthRetryable(fn, retryCount);
+      }
+    });
+  });
+});
+
 function provisionAzureMonitorDatasources(datasources: AzureMonitorProvision[]) {
   const datasource = datasources[0].datasources[0];
 
   e2e()
     .intercept(/subscriptions/)
     .as('subscriptions');
+  e2e()
+    .intercept(/health/)
+    .as('health');
 
   e2e.flows.addDataSource({
     type: 'Azure Monitor',
@@ -43,8 +61,11 @@ function provisionAzureMonitorDatasources(datasources: AzureMonitorProvision[])
         .type(datasource.secureJsonData.clientSecret, { log: false });
       e2eSelectors.configEditor.loadSubscriptions.button().click().wait('@subscriptions').wait(500);
       e2eSelectors.configEditor.defaultSubscription.input().find('input').type('datasources{enter}');
-      // Wait for 15s so that credentials are ready. 5s has been tested locally before and seemed insufficient.
-      e2e().wait(30000);
+
+      // We can do this because awaitHealth is set to true so @health is defined
+      cy.checkHealthRetryable(() => {
+        return e2e.pages.DataSource.saveAndTest().click().wait('@health');
+      }, 0);
     },
     expectedAlertMessage: 'Successfully connected to all Azure Monitor endpoints',
     // Reduce the timeout from 30s to error faster when an invalid alert message is presented