PromQail: use metric type when available else use fallback heuristic (#77702)

* PromQail: query suggester uses metric type if available * If metadata missing, use heuristic to guess metric type * Fix linter, fix histogram,summary case * Fix bug confusing metric with metric family name * Better method to fetch just metadata from datasource provider
2023-11-07 22:26:39 -05:00
parent 963251b520
commit 08f6abe4ac
2 changed files with 178 additions and 2 deletions
@@ -0,0 +1,31 @@
+import { guessMetricType } from './helpers';
+
+const metricListWithType = [
+  // below is summary metric family
+  ['go_gc_duration_seconds', 'summary'],
+  ['go_gc_duration_seconds_count', 'summary'],
+  ['go_gc_duration_seconds_sum', 'summary'],
+  // below is histogram metric family
+  ['go_gc_heap_allocs_by_size_bytes_total_bucket', 'histogram'],
+  ['go_gc_heap_allocs_by_size_bytes_total_count', 'histogram'],
+  ['go_gc_heap_allocs_by_size_bytes_total_sum', 'histogram'],
+  // below are counters
+  ['go_gc_heap_allocs_bytes_total', 'counter'],
+  ['scrape_samples_post_metric_relabeling', 'counter'],
+  // below are gauges
+  ['go_gc_heap_goal_bytes', 'gauge'],
+  ['nounderscorename', 'gauge'],
+  // below is both a histogram & summary
+  ['alertmanager_http_response_size_bytes', 'histogram,summary'],
+  ['alertmanager_http_response_size_bytes_bucket', 'histogram,summary'],
+  ['alertmanager_http_response_size_bytes_count', 'histogram,summary'],
+  ['alertmanager_http_response_size_bytes_sum', 'histogram,summary'],
+];
+
+const metricList = metricListWithType.map((item) => item[0]);
+
+describe('guessMetricType', () => {
+  it.each(metricListWithType)("where input is '%s'", (metric: string, metricType: string) => {
+    expect(guessMetricType(metric, metricList)).toBe(metricType);
+  });
+});
@@ -137,6 +137,129 @@ export async function promQailExplain(
    });
 }

+/**
+ * Check if sublist is fully contained in the superlist
+ *
+ * @param sublist
+ * @param superlist
+ * @returns true if fully contained, else false
+ */
+function isContainedIn(sublist: string[], superlist: string[]): boolean {
+  for (const item of sublist) {
+    if (!superlist.includes(item)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Guess the type of a metric, based on its name and its relation to other metrics available
+ *
+ * @param metric     - name of metric whose type to guess
+ * @param allMetrics - list of all available metrics
+ * @returns          - the guess of the type (string): counter,gauge,summary,histogram,'histogram,summary'
+ */
+export function guessMetricType(metric: string, allMetrics: string[]): string {
+  const synthetic_metrics = new Set<string>([
+    'up',
+    'scrape_duration_seconds',
+    'scrape_samples_post_metric_relabeling',
+    'scrape_series_added',
+    'scrape_samples_scraped',
+    'ALERTS',
+    'ALERTS_FOR_STATE',
+  ]);
+
+  if (synthetic_metrics.has(metric)) {
+    // these are all known to be counters
+    return 'counter';
+  }
+  if (metric.startsWith(':')) {
+    // probably recording rule
+    return 'gauge';
+  }
+  if (metric.endsWith('_info')) {
+    // typically series of 1s only, the labels are the useful part. TODO: add 'info' type
+    return 'counter';
+  }
+
+  if (metric.endsWith('_created') || metric.endsWith('_total')) {
+    // prometheus naming style recommends counters to have these suffixes.
+    return 'counter';
+  }
+
+  const underscoreIndex = metric.lastIndexOf('_');
+  if (underscoreIndex < 0) {
+    // No underscores in the name at all, very little info to go on. Guess
+    return 'gauge';
+  }
+
+  // See if the suffix is histogram-y or summary-y
+  const [root, suffix] = [metric.slice(0, underscoreIndex), metric.slice(underscoreIndex + 1)];
+
+  if (['bucket', 'count', 'sum'].includes(suffix)) {
+    // Might be histogram + summary
+    let familyMetrics = [`${root}_bucket`, `${root}_count`, `${root}_sum`, root];
+    if (isContainedIn(familyMetrics, allMetrics)) {
+      return 'histogram,summary';
+    }
+
+    // Might be a histogram, if so all these metrics should exist too:
+    familyMetrics = [`${root}_bucket`, `${root}_count`, `${root}_sum`];
+    if (isContainedIn(familyMetrics, allMetrics)) {
+      return 'histogram';
+    }
+
+    // Or might be a summary
+    familyMetrics = [`${root}_sum`, `${root}_count`, root];
+    if (isContainedIn(familyMetrics, allMetrics)) {
+      return 'summary';
+    }
+
+    // Otherwise it's probably just a counter!
+    return 'counter';
+  }
+
+  // One case above doesn't catch: summary or histogram,summary where the non-suffixed metric is chosen
+  const familyMetrics = [`${metric}_sum`, `${metric}_count`, metric];
+  if (isContainedIn(familyMetrics, allMetrics)) {
+    if (allMetrics.includes(`${metric}_bucket`)) {
+      return 'histogram,summary';
+    } else {
+      return 'summary';
+    }
+  }
+
+  // All else fails, guess gauge
+  return 'gauge';
+}
+
+/**
+ * Generate a suitable filter structure for the VectorDB call
+ * @param types: list of metric types to include in the result
+ * @returns the structure to pass to the vectorDB call.
+ */
+function generateMetricTypeFilters(types: string[]) {
+  return types.map((type) => ({
+    metric_type: {
+      $eq: type,
+    },
+  }));
+}
+
+/**
+ * Taking in a metric name, try to guess its corresponding metric _family_ name
+ * @param metric name
+ * @returns metric family name
+ */
+function guessMetricFamily(metric: string): string {
+  if (metric.endsWith('_bucket') || metric.endsWith('_count') || metric.endsWith('_sum')) {
+    return metric.slice(0, metric.lastIndexOf('_'));
+  }
+  return metric;
+}
+
 /**
 * Calls the API and adds suggestions to the interaction
 *
@@ -159,10 +282,30 @@ export async function promQailSuggest(

  const interactionToUpdate = interaction ? interaction : createInteraction(SuggestionType.Historical);

+  // Decide metric type
+  let metricType = '';
+  // Makes sure we loaded the metadata for metrics. Usually this is done in the start() method of the
+  // provider but we only need the metadata here.
+  if (!datasource.languageProvider.metricsMetadata) {
+    await datasource.languageProvider.loadMetricsMetadata();
+  }
+  if (datasource.languageProvider.metricsMetadata) {
+    // `datasource.languageProvider.metricsMetadata` is a list of metric family names (with desired type)
+    // from the datasource metadata endoint, but unfortunately the expanded _sum, _count, _bucket raw
+    // metric names are also generated and populating this list (all of type counter). We want the metric
+    // family type, so need to guess the metric family name from the chosen metric name, and test if that
+    // metric family has a type specified.
+    const metricFamilyGuess = guessMetricFamily(query.metric);
+    metricType = getMetadataType(metricFamilyGuess, datasource.languageProvider.metricsMetadata) ?? '';
+  }
+  if (metricType === '') {
+    // fallback to heuristic guess
+    metricType = guessMetricType(query.metric, datasource.languageProvider.metrics);
+  }
+
  if (!check || interactionToUpdate.suggestionType === SuggestionType.Historical) {
    return new Promise<void>((resolve) => {
      return setTimeout(() => {
-        let metricType = getMetadataType(query.metric, datasource.languageProvider.metricsMetadata!) ?? '';
        const suggestions = getTemplateSuggestions(
          query.metric,
          metricType,
@@ -194,12 +337,14 @@ export async function promQailSuggest(
    if (interaction?.suggestionType === SuggestionType.AI) {
      feedTheAI = { ...feedTheAI, prompt: interaction.prompt };

-      // TODO: filter by metric type
      // @ts-ignore llms types issue
      results = await llms.vector.search<TemplateSearchResult>({
        query: interaction.prompt,
        collection: promQLTemplatesCollection,
        topK: 5,
+        filter: {
+          $or: generateMetricTypeFilters(metricType.split(',').concat(['*'])),
+        },
      });
      reportInteraction('grafana_prometheus_promqail_vector_results', {
        metric: query.metric,