Alerting: Add counts for firing and pending alert rules (#113309)

* add counts for firing and pending alert rules * resolve Pr comments part 1 * resolve PR comments part 2 * resolve PR comments part 3
2025-11-04 10:17:07 +00:00
parent df816d00e4
commit c75a451b13
3 changed files with 426 additions and 29 deletions
@@ -0,0 +1,227 @@
+import { DataFrameView } from '@grafana/data';
+import { PromAlertingRuleState } from 'app/types/unified-alerting-dto';
+
+import { RuleFrame, countRules, parseAlertstateFilter } from './SummaryStats';
+
+describe('parseAlertstateFilter', () => {
+  it('should return "firing" when filter contains alertstate="firing"', () => {
+    expect(parseAlertstateFilter('alertstate="firing"')).toBe(PromAlertingRuleState.Firing);
+  });
+
+  it('should return "pending" when filter contains alertstate="pending"', () => {
+    expect(parseAlertstateFilter('alertstate="pending"')).toBe(PromAlertingRuleState.Pending);
+  });
+
+  it('should return null when filter contains both firing and pending', () => {
+    expect(parseAlertstateFilter('alertstate=~"firing|pending"')).toBe(null);
+  });
+
+  it('should return null when no alertstate filter', () => {
+    expect(parseAlertstateFilter('')).toBe(null);
+    expect(parseAlertstateFilter('namespace="default"')).toBe(null);
+  });
+
+  it('should handle regex operator =~', () => {
+    expect(parseAlertstateFilter('alertstate=~"firing"')).toBe(PromAlertingRuleState.Firing);
+    expect(parseAlertstateFilter('alertstate=~"pending"')).toBe(PromAlertingRuleState.Pending);
+  });
+
+  it('should handle whitespace', () => {
+    expect(parseAlertstateFilter('alertstate = "firing"')).toBe(PromAlertingRuleState.Firing);
+    expect(parseAlertstateFilter('alertstate =~ "pending"')).toBe(PromAlertingRuleState.Pending);
+  });
+});
+
+describe('countRules', () => {
+  // Helper to create mock DataFrameView
+  function createMockRuleDfv(
+    data: Array<{ ruleUID: string; alertstate: PromAlertingRuleState.Firing | PromAlertingRuleState.Pending }>
+  ): DataFrameView<RuleFrame> {
+    return {
+      length: data.length,
+      fields: {
+        grafana_rule_uid: {
+          values: data.map((d) => d.ruleUID),
+        },
+        alertstate: {
+          values: data.map((d) => d.alertstate),
+        },
+      },
+    } as unknown as DataFrameView<RuleFrame>;
+  }
+
+  describe('when no alertstate filter applied', () => {
+    it('should count rules with ANY firing instances', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule3', alertstate: PromAlertingRuleState.Pending },
+      ]);
+
+      const result = countRules(ruleDfv, null);
+
+      expect(result.firing).toBe(2);
+      expect(result.pending).toBe(1);
+    });
+
+    it('should count rules with ONLY pending instances (no firing)', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'rule3', alertstate: PromAlertingRuleState.Firing },
+      ]);
+
+      const result = countRules(ruleDfv, null);
+
+      expect(result.firing).toBe(1);
+      expect(result.pending).toBe(2);
+    });
+
+    it('should count rules with BOTH firing and pending as firing (firing takes precedence)', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Pending }, // Same rule, both states
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Firing }, // Only firing
+        { ruleUID: 'rule3', alertstate: PromAlertingRuleState.Pending }, // Only pending
+      ]);
+
+      const result = countRules(ruleDfv, null);
+
+      // rule1 should be counted as firing (firing takes precedence)
+      // rule2 should be counted as firing
+      // rule3 should be counted as pending
+      expect(result.firing).toBe(2); // rule1 and rule2
+      expect(result.pending).toBe(1); // only rule3
+    });
+
+    it('should handle multiple instances of the same rule with same state', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing }, // Same rule, duplicate entry
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Pending }, // Same rule, duplicate entry
+      ]);
+
+      const result = countRules(ruleDfv, null);
+
+      // Each rule should only be counted once despite multiple entries
+      expect(result.firing).toBe(1);
+      expect(result.pending).toBe(1);
+    });
+
+    it('should return 0 for both counts when no rules', () => {
+      const ruleDfv = createMockRuleDfv([]);
+
+      const result = countRules(ruleDfv, null);
+
+      expect(result.firing).toBe(0);
+      expect(result.pending).toBe(0);
+    });
+  });
+
+  describe('when filtering by alertstate=pending', () => {
+    it('should count ALL rules with any pending instances', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Pending }, // Has both
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Pending }, // Only pending
+        { ruleUID: 'rule3', alertstate: PromAlertingRuleState.Firing }, // Only firing
+      ]);
+
+      const result = countRules(ruleDfv, PromAlertingRuleState.Pending);
+
+      // Should count rule1 and rule2 (both have pending instances)
+      expect(result.pending).toBe(2);
+      expect(result.firing).toBe(0);
+    });
+
+    it('should count rules with BOTH states as pending', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'rule3', alertstate: PromAlertingRuleState.Pending },
+      ]);
+
+      const result = countRules(ruleDfv, PromAlertingRuleState.Pending);
+
+      // Should count all three rules (all have pending instances)
+      expect(result.pending).toBe(3);
+      expect(result.firing).toBe(0);
+    });
+
+    it('should be >= pending count from no filter scenario', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Pending },
+      ]);
+
+      const noFilterResult = countRules(ruleDfv, null);
+      const pendingFilterResult = countRules(ruleDfv, PromAlertingRuleState.Pending);
+
+      // With pending filter: should count rule1 and rule2 = 2
+      // Without filter: should count only rule2 = 1 (rule1 is counted as firing, not pending)
+      expect(pendingFilterResult.pending).toBeGreaterThanOrEqual(noFilterResult.pending);
+      expect(pendingFilterResult.pending).toBe(2);
+      expect(noFilterResult.pending).toBe(1); // Only rule2 (rule1 is firing)
+      expect(noFilterResult.firing).toBe(1); // rule1
+    });
+  });
+
+  describe('when filtering by alertstate=firing', () => {
+    it('should count ALL rules with any firing instances', () => {
+      const ruleDfv = createMockRuleDfv([
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'rule1', alertstate: PromAlertingRuleState.Pending }, // Has both
+        { ruleUID: 'rule2', alertstate: PromAlertingRuleState.Firing }, // Only firing
+        { ruleUID: 'rule3', alertstate: PromAlertingRuleState.Pending }, // Only pending
+      ]);
+
+      const result = countRules(ruleDfv, PromAlertingRuleState.Firing);
+
+      // Should count rule1 and rule2 (both have firing instances)
+      expect(result.firing).toBe(2);
+      expect(result.pending).toBe(0);
+    });
+  });
+
+  describe('real-world scenarios', () => {
+    it('should handle complex scenario with many rules', () => {
+      const ruleDfv = createMockRuleDfv([
+        // Rule A: Only firing (3 instances)
+        { ruleUID: 'ruleA', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'ruleA', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'ruleA', alertstate: PromAlertingRuleState.Firing },
+        // Rule B: Only pending (2 instances)
+        { ruleUID: 'ruleB', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'ruleB', alertstate: PromAlertingRuleState.Pending },
+        // Rule C: Both firing and pending (4 instances)
+        { ruleUID: 'ruleC', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'ruleC', alertstate: PromAlertingRuleState.Firing },
+        { ruleUID: 'ruleC', alertstate: PromAlertingRuleState.Pending },
+        { ruleUID: 'ruleC', alertstate: PromAlertingRuleState.Pending },
+        // Rule D: Only firing (1 instance)
+        { ruleUID: 'ruleD', alertstate: PromAlertingRuleState.Firing },
+        // Rule E: Only pending (1 instance)
+        { ruleUID: 'ruleE', alertstate: PromAlertingRuleState.Pending },
+      ]);
+
+      // No filter: Firing takes precedence
+      const noFilter = countRules(ruleDfv, null);
+      expect(noFilter.firing).toBe(3); // Rule A, C, and D (C has firing so it's counted as firing)
+      expect(noFilter.pending).toBe(2); // Rule B and E (only those with NO firing)
+
+      // With pending filter: Count all rules with ANY pending instances
+      const pendingFilter = countRules(ruleDfv, PromAlertingRuleState.Pending);
+      expect(pendingFilter.pending).toBe(3); // Rule B, C, and E
+      expect(pendingFilter.firing).toBe(0);
+
+      // With firing filter: Count all rules with ANY firing instances
+      const firingFilter = countRules(ruleDfv, PromAlertingRuleState.Firing);
+      expect(firingFilter.firing).toBe(3); // Rule A, C, and D
+      expect(firingFilter.pending).toBe(0);
+    });
+  });
+});
@@ -2,63 +2,235 @@ import { DataFrameView } from '@grafana/data';
 import { Trans } from '@grafana/i18n';
 import { SceneObjectBase, SceneObjectState } from '@grafana/scenes';
 import { useQueryRunner } from '@grafana/scenes-react';
-import { Stack, Text } from '@grafana/ui';
+import { ErrorBoundaryAlert, Stack, Text } from '@grafana/ui';
+import { PromAlertingRuleState } from 'app/types/unified-alerting-dto';

 import { Spacer } from '../../components/Spacer';
 import { METRIC_NAME } from '../constants';

 import { getDataQuery, useQueryFilter } from './utils';

+type AlertState = PromAlertingRuleState.Firing | PromAlertingRuleState.Pending;
+
 interface Frame {
-  alertstate: 'firing' | 'pending';
+  alertstate: AlertState;
  Value: number;
 }

-export function SummaryStatsReact() {
-  const filter = useQueryFilter();
+export interface RuleFrame {
+  alertstate: AlertState;
+  alertname: string;
+  grafana_folder: string;
+  grafana_rule_uid: string;
+  Value: number;
+}

-  const dataProvider = useQueryRunner({
+export function parseAlertstateFilter(filter: string): AlertState | null {
+  const firingMatch = filter.match(/alertstate\s*=~?\s*"firing"/);
+  const pendingMatch = filter.match(/alertstate\s*=~?\s*"pending"/);
+
+  if (firingMatch && pendingMatch) {
+    return null;
+  }
+
+  if (firingMatch) {
+    return PromAlertingRuleState.Firing;
+  }
+
+  if (pendingMatch) {
+    return PromAlertingRuleState.Pending;
+  }
+
+  return null;
+}
+
+export function countRules(ruleDfv: DataFrameView<RuleFrame>, alertstateFilter: AlertState | null) {
+  const rulesWithFiring = new Set<string>();
+  const rulesWithPending = new Set<string>();
+
+  ruleDfv.fields.grafana_rule_uid.values.forEach((ruleUID, i) => {
+    const alertstate = ruleDfv.fields.alertstate.values[i];
+    if (alertstate === PromAlertingRuleState.Firing) {
+      rulesWithFiring.add(ruleUID);
+    }
+    if (alertstate === PromAlertingRuleState.Pending) {
+      rulesWithPending.add(ruleUID);
+    }
+  });
+
+  // When filtering by pending, count all rules with pending instances (may also have firing)
+  if (alertstateFilter === PromAlertingRuleState.Pending) {
+    return {
+      firing: 0,
+      pending: rulesWithPending.size,
+    };
+  }
+
+  // When filtering by firing, count all rules with firing instances (may also have pending)
+  if (alertstateFilter === PromAlertingRuleState.Firing) {
+    return {
+      firing: rulesWithFiring.size,
+      pending: 0,
+    };
+  }
+
+  // When no filter: firing takes precedence
+  // A rule is "firing" if it has ANY firing instances (even if it also has pending)
+  // A rule is "pending" ONLY if it has pending instances but NO firing instances
+  const onlyPending = new Set([...rulesWithPending].filter((uid) => !rulesWithFiring.has(uid)));
+
+  return {
+    firing: rulesWithFiring.size, // ALL rules with firing instances
+    pending: onlyPending.size, // ONLY rules with no firing instances
+  };
+}
+
+function countInstances(instanceDfv: DataFrameView<Frame>) {
+  const getValue = (state: AlertState) => {
+    const index = instanceDfv.fields.alertstate.values.findIndex((s) => s === state);
+    return instanceDfv.fields.Value.values[index] ?? 0;
+  };
+  return { firing: getValue(PromAlertingRuleState.Firing), pending: getValue(PromAlertingRuleState.Pending) };
+}
+
+interface StatRowProps {
+  i18nKey: string;
+  color: 'error' | 'warning';
+  values: Record<string, number>;
+  children: React.ReactNode;
+}
+
+function StatRow({ i18nKey, color, values, children }: StatRowProps) {
+  return (
+    <Text color={color}>
+      <Trans i18nKey={i18nKey} values={values}>
+        {children}
+      </Trans>
+    </Text>
+  );
+}
+
+function SummaryStatsContent() {
+  const filter = useQueryFilter();
+  const alertstateFilter = parseAlertstateFilter(filter);
+
+  const instanceDataProvider = useQueryRunner({
+    queries: [getDataQuery(`count by (alertstate) (${METRIC_NAME}{${filter}})`, { instant: true, format: 'table' })],
+  });
+
+  // Always remove alertstate filter from rule query to get accurate counts across both states
+  // This ensures we can count rules that have instances in either state
+  const ruleFilter = filter
+    .replace(/alertstate\s*=~?\s*"(firing|pending)"[,\s]*/, '')
+    .replace(/,\s*$/, '')
+    .replace(/^\s*,/, '');
+  const ruleDataProvider = useQueryRunner({
    queries: [
-      getDataQuery(`count by (alertstate) (${METRIC_NAME}{${filter}})`, {
-        instant: true,
-        exemplar: false,
-        format: 'table',
-      }),
+      getDataQuery(
+        `count by (alertname, grafana_folder, grafana_rule_uid, alertstate) (${METRIC_NAME}{${ruleFilter}})`,
+        {
+          instant: true,
+          format: 'table',
+        }
+      ),
    ],
  });

-  const isLoading = !dataProvider.isDataReadyToDisplay;
-  const data = dataProvider.useState().data;
-  const firstFrame = data?.series?.at(0);
+  const { data: instanceData } = instanceDataProvider.useState();
+  const { data: ruleData } = ruleDataProvider.useState();
+  const instanceFrame = instanceData?.series?.at(0);
+  const ruleFrame = ruleData?.series?.at(0);

-  if (isLoading || !firstFrame) {
+  if (
+    !instanceDataProvider.isDataReadyToDisplay() ||
+    !ruleDataProvider.isDataReadyToDisplay() ||
+    !instanceFrame ||
+    !ruleFrame
+  ) {
    return <div />;
  }

-  const dfv = new DataFrameView<Frame>(firstFrame);
-  if (dfv.length === 0) {
+  const instanceDfv = new DataFrameView<Frame>(instanceFrame);
+  const ruleDfv = new DataFrameView<RuleFrame>(ruleFrame);
+
+  if (instanceDfv.length === 0 && ruleDfv.length === 0) {
    return <div />;
  }

-  const firingIndex = dfv.fields.alertstate.values.findIndex((state) => state === 'firing');
-  const firingCount = dfv.fields.Value.values[firingIndex] ?? 0;
-
-  const pendingIndex = dfv.fields.alertstate.values.findIndex((state) => state === 'pending');
-  const pendingCount = dfv.fields.Value.values[pendingIndex] ?? 0;
+  const instances = countInstances(instanceDfv);
+  const rules = countRules(ruleDfv, alertstateFilter);

  return (
    <Stack direction="column" alignItems="flex-end" gap={0}>
      <Spacer />
-      <Text color="error">
-        <Trans i18nKey="alerting.triage.firing-instances-count">{{ firingCount }} firing instances</Trans>
-      </Text>
-      <Text color="warning">
-        <Trans i18nKey="alerting.triage.pending-instances-count">{{ pendingCount }} pending instances</Trans>
-      </Text>
+      {alertstateFilter === PromAlertingRuleState.Firing && (
+        <>
+          <StatRow i18nKey="alerting.triage.firing-rules-count" color="error" values={{ count: rules.firing }}>
+            {'{{count}} firing alert rules'}
+          </StatRow>
+          <StatRow
+            i18nKey="alerting.triage.firing-instances-count"
+            color="error"
+            values={{ firingCount: instances.firing }}
+          >
+            {'{{firingCount}} firing instances'}
+          </StatRow>
+        </>
+      )}
+      {alertstateFilter === PromAlertingRuleState.Pending && (
+        <>
+          <StatRow
+            i18nKey="alerting.triage.rules-with-pending-instances"
+            color="warning"
+            values={{ count: rules.pending }}
+          >
+            {'{{count}} rules with pending instances'}
+          </StatRow>
+          <StatRow
+            i18nKey="alerting.triage.pending-instances-count"
+            color="warning"
+            values={{ pendingCount: instances.pending }}
+          >
+            {'{{pendingCount}} pending instances'}
+          </StatRow>
+        </>
+      )}
+      {!alertstateFilter && (
+        <>
+          <StatRow i18nKey="alerting.triage.firing-rules-count" color="error" values={{ count: rules.firing }}>
+            {'{{count}} firing alert rules'}
+          </StatRow>
+          <StatRow
+            i18nKey="alerting.triage.firing-instances-count"
+            color="error"
+            values={{ firingCount: instances.firing }}
+          >
+            {'{{firingCount}} firing instances'}
+          </StatRow>
+          <StatRow i18nKey="alerting.triage.pending-rules-count" color="warning" values={{ count: rules.pending }}>
+            {'{{count}} pending alert rules'}
+          </StatRow>
+          <StatRow
+            i18nKey="alerting.triage.pending-instances-count"
+            color="warning"
+            values={{ pendingCount: instances.pending }}
+          >
+            {'{{pendingCount}} pending instances'}
+          </StatRow>
+        </>
+      )}
    </Stack>
  );
 }

+export function SummaryStatsReact() {
+  return (
+    <ErrorBoundaryAlert>
+      <SummaryStatsContent />
+    </ErrorBoundaryAlert>
+  );
+}
+
 // simple wrapper so we can render the Chart using a Scene parent
 export class SummaryStatsScene extends SceneObjectBase<SceneObjectState> {
  static Component = SummaryStatsReact;
@@ -2945,7 +2945,6 @@
    "triage": {
      "alert-instances": "Alert instances",
      "error-loading-rule": "Error loading rule",
-      "firing-instances-count": "{{firingCount}} firing instances",
      "instance-details-drawer": {
        "instance-details": "Instance details"
      },
@@ -2953,7 +2952,6 @@
      "no-labels": "No labels",
      "open-in-sidebar": "Open in sidebar",
      "open-rule-details": "Open rule details",
-      "pending-instances-count": "{{pendingCount}} pending instances",
      "rule-details": {
        "subtitle": "Rule details and conditions",
        "title": "Rule Details"