Loki: Implement hints for query builder

.betterer.results

@@ -17096,15 +17096,15 @@ exports[`no undocumented stories`] = {
       [868, 15, 27, "Do not use any type assertions.", "2478144973"],
       [870, 9, 3, "Unexpected any. Specify a different type.", "193409811"]
     ],
-    "public/app/plugins/datasource/loki/datasource.ts:979697429": [
-      [219, 23, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [355, 30, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [359, 30, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [359, 45, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [373, 40, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [560, 33, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [685, 41, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [692, 46, 3, "Unexpected any. Specify a different type.", "193409811"]
+    "public/app/plugins/datasource/loki/datasource.ts:1928151504": [
+      [221, 23, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [374, 30, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [378, 30, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [378, 45, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [392, 40, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [587, 33, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [716, 41, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [723, 46, 3, "Unexpected any. Specify a different type.", "193409811"]
     ],
     "public/app/plugins/datasource/loki/getDerivedFields.ts:1557842937": [
       [37, 87, 3, "Unexpected any. Specify a different type.", "193409811"],
@@ -17157,16 +17157,16 @@ exports[`no undocumented stories`] = {
       [43, 18, 3, "Unexpected any. Specify a different type.", "193409811"],
       [44, 9, 3, "Unexpected any. Specify a different type.", "193409811"]
     ],
-    "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx:1944597108": [
-      [34, 86, 3, "Unexpected any. Specify a different type.", "193409811"],
-      [98, 22, 27, "Do not use any type assertions.", "2133479311"]
+    "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx:1440796281": [
+      [36, 86, 3, "Unexpected any. Specify a different type.", "193409811"],
+      [100, 22, 27, "Do not use any type assertions.", "2133479311"]
     ],
     "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.test.tsx:3009382449": [
       [25, 16, 9, "Do not use any type assertions.", "3692209159"],
       [25, 22, 3, "Unexpected any. Specify a different type.", "193409811"]
     ],
-    "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx:4118866003": [
-      [72, 16, 21, "Do not use any type assertions.", "3630142339"]
+    "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx:2715012229": [
+      [75, 16, 21, "Do not use any type assertions.", "3630142339"]
     ],
     "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.test.tsx:410698953": [
       [25, 41, 3, "Unexpected any. Specify a different type.", "193409811"],

public/app/plugins/datasource/loki/add_label_to_query.ts

@@ -4,8 +4,8 @@ import { QueryBuilderLabelFilter } from '../prometheus/querybuilder/shared/types
 
 import { LokiQueryModeller } from './querybuilder/LokiQueryModeller';
 import { buildVisualQueryFromString } from './querybuilder/parsing';
-import { LokiVisualQuery } from './querybuilder/types';
 
+type Position = { from: number; to: number };
 /**
  * Adds label filter to existing query. Useful for query modification for example for ad hoc filters.
  *
@@ -39,22 +39,36 @@ export function addLabelToQuery(query: string, key: string, operator: string, va
   }
 }
 
-type StreamSelectorPosition = { from: number; to: number; query: LokiVisualQuery };
-type PipelineStagePosition = { from: number; to: number };
+/**
+ * Adds parser to existing query. Useful for query modification for hints.
+ * It uses LogQL parser to find instances of stream selectors or line filters and adds parser after them.
+ *
+ * @param query
+ * @param parser
+ */
+export function addParserToQuery(query: string, parser: string): string {
+  const lineFilterPositions = getLineFiltersPositions(query);
+
+  if (lineFilterPositions.length) {
+    return addParser(query, lineFilterPositions, parser);
+  } else {
+    const streamSelectorPositions = getStreamSelectorPositions(query);
+    return addParser(query, streamSelectorPositions, parser);
+  }
+}
 
 /**
  * Parse the string and get all Selector positions in the query together with parsed representation of the
  * selector.
  * @param query
  */
-function getStreamSelectorPositions(query: string): StreamSelectorPosition[] {
+function getStreamSelectorPositions(query: string): Position[] {
   const tree = parser.parse(query);
-  const positions: StreamSelectorPosition[] = [];
+  const positions: Position[] = [];
   tree.iterate({
     enter: (type, from, to, get): false | void => {
       if (type.name === 'Selector') {
-        const visQuery = buildVisualQueryFromString(query.substring(from, to));
-        positions.push({ query: visQuery.query, from, to });
+        positions.push({ from, to });
         return false;
       }
     },
@@ -66,9 +80,9 @@ function getStreamSelectorPositions(query: string): StreamSelectorPosition[] {
  * Parse the string and get all LabelParser positions in the query.
  * @param query
  */
-function getParserPositions(query: string): PipelineStagePosition[] {
+export function getParserPositions(query: string): Position[] {
   const tree = parser.parse(query);
-  const positions: PipelineStagePosition[] = [];
+  const positions: Position[] = [];
   tree.iterate({
     enter: (type, from, to, get): false | void => {
       if (type.name === 'LabelParser') {
@@ -80,6 +94,24 @@ function getParserPositions(query: string): PipelineStagePosition[] {
   return positions;
 }
 
+/**
+ * Parse the string and get all LabelParser positions in the query.
+ * @param query
+ */
+function getLineFiltersPositions(query: string): Position[] {
+  const tree = parser.parse(query);
+  const positions: Position[] = [];
+  tree.iterate({
+    enter: (type, from, to, get): false | void => {
+      if (type.name === 'LineFilters') {
+        positions.push({ from, to });
+        return false;
+      }
+    },
+  });
+  return positions;
+}
+
 function toLabelFilter(key: string, value: string, operator: string): QueryBuilderLabelFilter {
   // We need to make sure that we convert the value back to string because it may be a number
   return { label: key, op: operator, value };
@@ -93,7 +125,7 @@ function toLabelFilter(key: string, value: string, operator: string): QueryBuild
  */
 function addFilterToStreamSelector(
   query: string,
-  vectorSelectorPositions: StreamSelectorPosition[],
+  vectorSelectorPositions: Position[],
   filter: QueryBuilderLabelFilter
 ): string {
   const modeller = new LokiQueryModeller();
@@ -108,12 +140,13 @@ function addFilterToStreamSelector(
 
     const start = query.substring(prev, match.from);
     const end = isLast ? query.substring(match.to) : '';
+    const matchVisQuery = buildVisualQueryFromString(query.substring(match.from, match.to));
 
-    if (!labelExists(match.query.labels, filter)) {
+    if (!labelExists(matchVisQuery.query.labels, filter)) {
       // We don't want to add duplicate labels.
-      match.query.labels.push(filter);
+      matchVisQuery.query.labels.push(filter);
     }
-    const newLabels = modeller.renderQuery(match.query);
+    const newLabels = modeller.renderQuery(matchVisQuery.query);
     newQuery += start + newLabels + end;
     prev = match.to;
   }
@@ -126,11 +159,7 @@ function addFilterToStreamSelector(
  * @param parserPositions
  * @param filter
  */
-function addFilterAsLabelFilter(
-  query: string,
-  parserPositions: PipelineStagePosition[],
-  filter: QueryBuilderLabelFilter
-): string {
+function addFilterAsLabelFilter(query: string, parserPositions: Position[], filter: QueryBuilderLabelFilter): string {
   let newQuery = '';
   let prev = 0;
 
@@ -149,6 +178,31 @@ function addFilterAsLabelFilter(
   return newQuery;
 }
 
+/**
+ * Add parser after line filter or stream selector
+ * @param query
+ * @param queryPartPositions
+ * @param parser
+ */
+function addParser(query: string, queryPartPositions: Position[], parser: string): string {
+  let newQuery = '';
+  let prev = 0;
+
+  for (let i = 0; i < queryPartPositions.length; i++) {
+    // Splice on a string for each matched vector selector
+    const match = queryPartPositions[i];
+    const isLast = i === queryPartPositions.length - 1;
+
+    const start = query.substring(prev, match.to);
+    const end = isLast ? query.substring(match.to) : '';
+
+    // Add parser
+    newQuery += start + ` | ${parser}` + end;
+    prev = match.to;
+  }
+  return newQuery;
+}
+
 /**
  * Check if label exists in the list of labels but ignore the operator.
  * @param labels

public/app/plugins/datasource/loki/datasource.ts

@@ -33,6 +33,7 @@ import {
   TimeRange,
   rangeUtil,
   toUtc,
+  QueryHint,
 } from '@grafana/data';
 import { FetchError, config, DataSourceWithBackend } from '@grafana/runtime';
 import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
@@ -44,13 +45,14 @@ import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_sr
 import { serializeParams } from '../../../core/utils/fetch';
 import { renderLegendFormat } from '../prometheus/legend';
 
-import { addLabelToQuery } from './add_label_to_query';
+import { addLabelToQuery, addParserToQuery } from './add_label_to_query';
 import { transformBackendResult } from './backendResultTransformer';
 import { LokiAnnotationsQueryEditor } from './components/AnnotationsQueryEditor';
 import LanguageProvider from './language_provider';
 import { escapeLabelValueInSelector } from './language_utils';
 import { LiveStreams, LokiLiveTarget } from './live_streams';
-import { getNormalizedLokiQuery } from './query_utils';
+import { getQueryHints } from './queryHints';
+import { getNormalizedLokiQuery, isLogsQuery, isValidQuery } from './query_utils';
 import { sortDataFrameByTime } from './sortDataFrame';
 import { doLokiChannelStream } from './streaming';
 import syntax from './syntax';
@@ -348,7 +350,24 @@ export class LokiDatasource
     return Array.from(streams);
   }
 
-  // By implementing getTagKeys and getTagValues we add ad-hoc filtters functionality
+  async getDataSamples(query: LokiQuery): Promise<DataFrame[]> {
+    // Currently works only for log samples
+    if (!isValidQuery(query.expr) || !isLogsQuery(query.expr)) {
+      return [];
+    }
+
+    const lokiLogsQuery: LokiQuery = {
+      expr: query.expr,
+      queryType: LokiQueryType.Range,
+      refId: 'log-samples',
+      maxLines: 10,
+    };
+
+    const request = makeRequest(lokiLogsQuery, this.timeSrv.timeRange(), CoreApp.Explore, 'log-samples');
+    return await lastValueFrom(this.query(request).pipe(switchMap((res) => of(res.data))));
+  }
+
+  // By implementing getTagKeys and getTagValues we add ad-hoc filters functionality
   async getTagKeys() {
     return await this.labelNamesQuery();
   }
@@ -382,6 +401,14 @@ export class LokiDatasource
         expression = this.addLabelToQuery(expression, action.key, '!=', action.value);
         break;
       }
+      case 'ADD_LOGFMT_PARSER': {
+        expression = addParserToQuery(expression, 'logfmt');
+        break;
+      }
+      case 'ADD_JSON_PARSER': {
+        expression = addParserToQuery(expression, 'json');
+        break;
+      }
       default:
         break;
     }
@@ -681,6 +708,10 @@ export class LokiDatasource
   getVariables(): string[] {
     return this.templateSrv.getVariables().map((v) => `$${v.name}`);
   }
+
+  getQueryHints(query: LokiQuery, result: DataFrame[]): QueryHint[] {
+    return getQueryHints(query.expr, result);
+  }
 }
 
 export function lokiRegularEscape(value: any) {

public/app/plugins/datasource/loki/queryHints.ts

@@ -0,0 +1,42 @@
+import { DataFrame, QueryHint } from '@grafana/data';
+
+import { isQueryWithParser } from './query_utils';
+import { extractLogParserFromDataFrame } from './responseUtils';
+
+export function getQueryHints(query: string, series: DataFrame[]): QueryHint[] {
+  const hints: QueryHint[] = [];
+  if (series.length > 0) {
+    const { hasLogfmt, hasJSON } = extractLogParserFromDataFrame(series[0]);
+    const queryWithParser = isQueryWithParser(query);
+
+    if (hasJSON && !queryWithParser) {
+      hints.push({
+        type: 'ADD_JSON_PARSER',
+        label: 'Selected log stream selector has JSON formatted logs. ',
+        fix: {
+          label: 'Consider using JSON parser.',
+          action: {
+            type: 'ADD_JSON_PARSER',
+            query,
+          },
+        },
+      });
+    }
+
+    if (hasLogfmt && !queryWithParser) {
+      hints.push({
+        type: 'ADD_LOGFMT_PARSER',
+        label: 'Selected log stream selector has logfmt formatted logs. ',
+        fix: {
+          label: 'Consider using logfmt parser.',
+          action: {
+            type: 'ADD_LOGFMT_PARSER',
+            query,
+          },
+        },
+      });
+    }
+  }
+
+  return hints;
+}

public/app/plugins/datasource/loki/query_utils.ts

@@ -1,5 +1,9 @@
 import { escapeRegExp } from 'lodash';
 
+import { parser } from '@grafana/lezer-logql';
+
+import { ErrorName } from '../prometheus/querybuilder/shared/parsingUtils';
+
 import { LokiQuery, LokiQueryType } from './types';
 
 export function formatQuery(selector: string | undefined): string {
@@ -90,3 +94,42 @@ export function getNormalizedLokiQuery(query: LokiQuery): LokiQuery {
   const { instant, range, ...rest } = query;
   return { ...rest, queryType: LokiQueryType.Range };
 }
+
+export function isValidQuery(query: string): boolean {
+  let isValid = true;
+  const tree = parser.parse(query);
+  tree.iterate({
+    enter: (type): false | void => {
+      if (type.name === ErrorName) {
+        isValid = false;
+      }
+    },
+  });
+  return isValid;
+}
+
+export function isLogsQuery(query: string): boolean {
+  let isLogsQuery = true;
+  const tree = parser.parse(query);
+  tree.iterate({
+    enter: (type): false | void => {
+      if (type.name === 'MetricExpr') {
+        isLogsQuery = false;
+      }
+    },
+  });
+  return isLogsQuery;
+}
+
+export function isQueryWithParser(query: string): boolean {
+  let hasParser = false;
+  const tree = parser.parse(query);
+  tree.iterate({
+    enter: (type): false | void => {
+      if (type.name === 'LabelParser') {
+        hasParser = true;
+      }
+    },
+  });
+  return hasParser;
+}

public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx

@@ -1,6 +1,6 @@
 import React, { useMemo } from 'react';
 
-import { DataSourceApi, SelectableValue } from '@grafana/data';
+import { DataSourceApi, PanelData, SelectableValue } from '@grafana/data';
 import { EditorRow } from '@grafana/experimental';
 import { LabelFilters } from 'app/plugins/datasource/prometheus/querybuilder/shared/LabelFilters';
 import { OperationList } from 'app/plugins/datasource/prometheus/querybuilder/shared/OperationList';
@@ -12,6 +12,7 @@ import { escapeLabelValueInSelector } from '../../language_utils';
 import { lokiQueryModeller } from '../LokiQueryModeller';
 import { LokiOperationId, LokiVisualQuery } from '../types';
 
+import { LokiQueryBuilderHints } from './LokiQueryBuilderHints';
 import { NestedQueryList } from './NestedQueryList';
 
 export interface Props {
@@ -19,9 +20,10 @@ export interface Props {
   datasource: LokiDatasource;
   onChange: (update: LokiVisualQuery) => void;
   onRunQuery: () => void;
+  data?: PanelData;
 }
 
-export const LokiQueryBuilder = React.memo<Props>(({ datasource, query, onChange, onRunQuery }) => {
+export const LokiQueryBuilder = React.memo<Props>(({ datasource, query, onChange, onRunQuery, data }) => {
   const onChangeLabels = (labels: QueryBuilderLabelFilter[]) => {
     onChange({ ...query, labels });
   };
@@ -97,6 +99,7 @@ export const LokiQueryBuilder = React.memo<Props>(({ datasource, query, onChange
           onRunQuery={onRunQuery}
           datasource={datasource as DataSourceApi}
         />
+        <LokiQueryBuilderHints datasource={datasource} query={query} onChange={onChange} data={data} />
       </OperationsEditorRow>
       {query.binaryQueries && query.binaryQueries.length > 0 && (
         <NestedQueryList query={query} datasource={datasource} onChange={onChange} onRunQuery={onRunQuery} />

public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx

@@ -1,6 +1,8 @@
 import { createSlice, PayloadAction } from '@reduxjs/toolkit';
 import React, { useEffect, useReducer } from 'react';
 
+import { PanelData } from '@grafana/data';
+
 import { LokiDatasource } from '../../datasource';
 import { LokiQuery } from '../../types';
 import { lokiQueryModeller } from '../LokiQueryModeller';
@@ -16,6 +18,7 @@ export interface Props {
   onChange: (update: LokiQuery) => void;
   onRunQuery: () => void;
   showRawQuery: boolean;
+  data?: PanelData;
 }
 
 export interface State {
@@ -27,7 +30,7 @@ export interface State {
  * This component is here just to contain the translation logic between string query and the visual query builder model.
  */
 export function LokiQueryBuilderContainer(props: Props) {
-  const { query, onChange, onRunQuery, datasource, showRawQuery } = props;
+  const { query, onChange, onRunQuery, datasource, showRawQuery, data } = props;
   const [state, dispatch] = useReducer(stateSlice.reducer, {
     expr: query.expr,
     // Use initial visual query only if query.expr is empty string
@@ -62,6 +65,7 @@ export function LokiQueryBuilderContainer(props: Props) {
         datasource={datasource}
         onChange={onVisQueryChange}
         onRunQuery={onRunQuery}
+        data={data}
       />
       {showRawQuery && <QueryPreview query={query.expr} />}
     </>

public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderHints.tsx

@@ -0,0 +1,84 @@
+import { css } from '@emotion/css';
+import React, { useState, useEffect, useRef } from 'react';
+
+import { GrafanaTheme2, PanelData, QueryHint } from '@grafana/data';
+import { Button, Tooltip, useStyles2 } from '@grafana/ui';
+
+import { LokiDatasource } from '../../datasource';
+import { lokiQueryModeller } from '../LokiQueryModeller';
+import { buildVisualQueryFromString } from '../parsing';
+import { LokiVisualQuery } from '../types';
+
+export interface Props {
+  query: LokiVisualQuery;
+  datasource: LokiDatasource;
+  onChange: (update: LokiVisualQuery) => void;
+  data?: PanelData;
+}
+
+export const LokiQueryBuilderHints = React.memo<Props>(({ datasource, query, onChange, data }) => {
+  const [hints, setHints] = useState<QueryHint[]>([]);
+  const styles = useStyles2(getStyles);
+  const prevQuery = useRef('');
+
+  useEffect(() => {
+    const expr = lokiQueryModeller.renderQuery(query);
+
+    const getHints = async () => {
+      // Run only if query changed
+      if (prevQuery.current === expr) {
+        return;
+      } else {
+        const lokiQuery = { expr, refId: 'data-samples' };
+        prevQuery.current = expr;
+        const sampleData = await datasource.getDataSamples(lokiQuery);
+        const hints = datasource.getQueryHints(lokiQuery, sampleData).filter((hint) => hint.fix?.action);
+        setHints(hints);
+      }
+    };
+
+    getHints().catch(console.error);
+  }, [datasource, query, onChange, data, styles.hint]);
+
+  return (
+    <>
+      {hints.length > 0 && (
+        <div className={styles.container}>
+          {hints.map((hint) => {
+            return (
+              <Tooltip content={`${hint.label} ${hint.fix?.label}`} key={hint.type}>
+                <Button
+                  onClick={() => {
+                    const lokiQuery = { expr: lokiQueryModeller.renderQuery(query), refId: '' };
+                    const newLokiQuery = datasource.modifyQuery(lokiQuery, hint!.fix!.action);
+                    const visualQuery = buildVisualQueryFromString(newLokiQuery.expr);
+                    return onChange(visualQuery.query);
+                  }}
+                  fill="outline"
+                  size="sm"
+                  className={styles.hint}
+                >
+                  {'hint: ' + hint.fix?.action?.type.toLowerCase().replace(/_/g, ' ')}
+                </Button>
+              </Tooltip>
+            );
+          })}
+        </div>
+      )}
+    </>
+  );
+});
+
+LokiQueryBuilderHints.displayName = 'LokiQueryBuilderHints';
+
+const getStyles = (theme: GrafanaTheme2) => {
+  return {
+    container: css`
+      display: flex;
+      align-items: start;
+    `,
+    hint: css`
+      margin-right: ${theme.spacing(1)};
+    `,
+  };
+};

public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.tsx

@@ -126,6 +126,7 @@ export const LokiQueryEditorSelector = React.memo<LokiQueryEditorProps>((props)
             onChange={onChangeInternal}
             onRunQuery={props.onRunQuery}
             showRawQuery={rawQuery}
+            data={data}
           />
         )}
         {editorMode === QueryEditorMode.Explain && <LokiQueryBuilderExplained query={query.expr} />}

public/app/plugins/datasource/loki/responseUtils.ts

@@ -1,6 +1,29 @@
-import { DataFrame, Labels } from '@grafana/data';
+import { DataFrame, FieldType, getParser, Labels, LogsParsers } from '@grafana/data';
 
 export function dataFrameHasLokiError(frame: DataFrame): boolean {
   const labelSets: Labels[] = frame.fields.find((f) => f.name === 'labels')?.values.toArray() ?? [];
   return labelSets.some((labels) => labels.__error__ !== undefined);
 }
+export function extractLogParserFromDataFrame(frame: DataFrame): { hasLogfmt: boolean; hasJSON: boolean } {
+  const lineField = frame.fields.find((field) => field.type === FieldType.string);
+  if (lineField == null) {
+    return { hasJSON: false, hasLogfmt: false };
+  }
+
+  const logLines: string[] = lineField.values.toArray();
+
+  let hasJSON = false;
+  let hasLogfmt = false;
+
+  logLines.forEach((line) => {
+    const parser = getParser(line);
+    if (parser === LogsParsers.JSON) {
+      hasJSON = true;
+    }
+    if (parser === LogsParsers.logfmt) {
+      hasLogfmt = true;
+    }
+  });
+
+  return { hasLogfmt, hasJSON };
+}