From 92e17e4b344a17217647a44615a5166530c0eb98 Mon Sep 17 00:00:00 2001 From: Ivana Huckova Date: Wed, 29 Jun 2022 15:35:23 +0200 Subject: [PATCH] Loki: Implement hints for query builder --- .betterer.results | 28 +++--- .../datasource/loki/add_label_to_query.ts | 90 +++++++++++++++---- .../app/plugins/datasource/loki/datasource.ts | 37 +++++++- .../app/plugins/datasource/loki/queryHints.ts | 42 +++++++++ .../plugins/datasource/loki/query_utils.ts | 43 +++++++++ .../components/LokiQueryBuilder.tsx | 7 +- .../components/LokiQueryBuilderContainer.tsx | 6 +- .../components/LokiQueryBuilderHints.tsx | 84 +++++++++++++++++ .../components/LokiQueryEditorSelector.tsx | 1 + .../plugins/datasource/loki/responseUtils.ts | 25 +++++- 10 files changed, 324 insertions(+), 39 deletions(-) create mode 100644 public/app/plugins/datasource/loki/queryHints.ts create mode 100644 public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderHints.tsx diff --git a/.betterer.results b/.betterer.results index d2598bb0bf6..37232f4d477 100644 --- a/.betterer.results +++ b/.betterer.results @@ -17096,15 +17096,15 @@ exports[`no undocumented stories`] = { [868, 15, 27, "Do not use any type assertions.", "2478144973"], [870, 9, 3, "Unexpected any. Specify a different type.", "193409811"] ], - "public/app/plugins/datasource/loki/datasource.ts:979697429": [ - [219, 23, 3, "Unexpected any. Specify a different type.", "193409811"], - [355, 30, 3, "Unexpected any. Specify a different type.", "193409811"], - [359, 30, 3, "Unexpected any. Specify a different type.", "193409811"], - [359, 45, 3, "Unexpected any. Specify a different type.", "193409811"], - [373, 40, 3, "Unexpected any. Specify a different type.", "193409811"], - [560, 33, 3, "Unexpected any. Specify a different type.", "193409811"], - [685, 41, 3, "Unexpected any. Specify a different type.", "193409811"], - [692, 46, 3, "Unexpected any. Specify a different type.", "193409811"] + "public/app/plugins/datasource/loki/datasource.ts:1928151504": [ + [221, 23, 3, "Unexpected any. Specify a different type.", "193409811"], + [374, 30, 3, "Unexpected any. Specify a different type.", "193409811"], + [378, 30, 3, "Unexpected any. Specify a different type.", "193409811"], + [378, 45, 3, "Unexpected any. Specify a different type.", "193409811"], + [392, 40, 3, "Unexpected any. Specify a different type.", "193409811"], + [587, 33, 3, "Unexpected any. Specify a different type.", "193409811"], + [716, 41, 3, "Unexpected any. Specify a different type.", "193409811"], + [723, 46, 3, "Unexpected any. Specify a different type.", "193409811"] ], "public/app/plugins/datasource/loki/getDerivedFields.ts:1557842937": [ [37, 87, 3, "Unexpected any. Specify a different type.", "193409811"], @@ -17157,16 +17157,16 @@ exports[`no undocumented stories`] = { [43, 18, 3, "Unexpected any. Specify a different type.", "193409811"], [44, 9, 3, "Unexpected any. Specify a different type.", "193409811"] ], - "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx:1944597108": [ - [34, 86, 3, "Unexpected any. Specify a different type.", "193409811"], - [98, 22, 27, "Do not use any type assertions.", "2133479311"] + "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx:1440796281": [ + [36, 86, 3, "Unexpected any. Specify a different type.", "193409811"], + [100, 22, 27, "Do not use any type assertions.", "2133479311"] ], "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.test.tsx:3009382449": [ [25, 16, 9, "Do not use any type assertions.", "3692209159"], [25, 22, 3, "Unexpected any. Specify a different type.", "193409811"] ], - "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx:4118866003": [ - [72, 16, 21, "Do not use any type assertions.", "3630142339"] + "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx:2715012229": [ + [75, 16, 21, "Do not use any type assertions.", "3630142339"] ], "public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.test.tsx:410698953": [ [25, 41, 3, "Unexpected any. Specify a different type.", "193409811"], diff --git a/public/app/plugins/datasource/loki/add_label_to_query.ts b/public/app/plugins/datasource/loki/add_label_to_query.ts index 799b545da6d..84900fc25ff 100644 --- a/public/app/plugins/datasource/loki/add_label_to_query.ts +++ b/public/app/plugins/datasource/loki/add_label_to_query.ts @@ -4,8 +4,8 @@ import { QueryBuilderLabelFilter } from '../prometheus/querybuilder/shared/types import { LokiQueryModeller } from './querybuilder/LokiQueryModeller'; import { buildVisualQueryFromString } from './querybuilder/parsing'; -import { LokiVisualQuery } from './querybuilder/types'; +type Position = { from: number; to: number }; /** * Adds label filter to existing query. Useful for query modification for example for ad hoc filters. * @@ -39,22 +39,36 @@ export function addLabelToQuery(query: string, key: string, operator: string, va } } -type StreamSelectorPosition = { from: number; to: number; query: LokiVisualQuery }; -type PipelineStagePosition = { from: number; to: number }; +/** + * Adds parser to existing query. Useful for query modification for hints. + * It uses LogQL parser to find instances of stream selectors or line filters and adds parser after them. + * + * @param query + * @param parser + */ +export function addParserToQuery(query: string, parser: string): string { + const lineFilterPositions = getLineFiltersPositions(query); + + if (lineFilterPositions.length) { + return addParser(query, lineFilterPositions, parser); + } else { + const streamSelectorPositions = getStreamSelectorPositions(query); + return addParser(query, streamSelectorPositions, parser); + } +} /** * Parse the string and get all Selector positions in the query together with parsed representation of the * selector. * @param query */ -function getStreamSelectorPositions(query: string): StreamSelectorPosition[] { +function getStreamSelectorPositions(query: string): Position[] { const tree = parser.parse(query); - const positions: StreamSelectorPosition[] = []; + const positions: Position[] = []; tree.iterate({ enter: (type, from, to, get): false | void => { if (type.name === 'Selector') { - const visQuery = buildVisualQueryFromString(query.substring(from, to)); - positions.push({ query: visQuery.query, from, to }); + positions.push({ from, to }); return false; } }, @@ -66,9 +80,9 @@ function getStreamSelectorPositions(query: string): StreamSelectorPosition[] { * Parse the string and get all LabelParser positions in the query. * @param query */ -function getParserPositions(query: string): PipelineStagePosition[] { +export function getParserPositions(query: string): Position[] { const tree = parser.parse(query); - const positions: PipelineStagePosition[] = []; + const positions: Position[] = []; tree.iterate({ enter: (type, from, to, get): false | void => { if (type.name === 'LabelParser') { @@ -80,6 +94,24 @@ function getParserPositions(query: string): PipelineStagePosition[] { return positions; } +/** + * Parse the string and get all LabelParser positions in the query. + * @param query + */ +function getLineFiltersPositions(query: string): Position[] { + const tree = parser.parse(query); + const positions: Position[] = []; + tree.iterate({ + enter: (type, from, to, get): false | void => { + if (type.name === 'LineFilters') { + positions.push({ from, to }); + return false; + } + }, + }); + return positions; +} + function toLabelFilter(key: string, value: string, operator: string): QueryBuilderLabelFilter { // We need to make sure that we convert the value back to string because it may be a number return { label: key, op: operator, value }; @@ -93,7 +125,7 @@ function toLabelFilter(key: string, value: string, operator: string): QueryBuild */ function addFilterToStreamSelector( query: string, - vectorSelectorPositions: StreamSelectorPosition[], + vectorSelectorPositions: Position[], filter: QueryBuilderLabelFilter ): string { const modeller = new LokiQueryModeller(); @@ -108,12 +140,13 @@ function addFilterToStreamSelector( const start = query.substring(prev, match.from); const end = isLast ? query.substring(match.to) : ''; + const matchVisQuery = buildVisualQueryFromString(query.substring(match.from, match.to)); - if (!labelExists(match.query.labels, filter)) { + if (!labelExists(matchVisQuery.query.labels, filter)) { // We don't want to add duplicate labels. - match.query.labels.push(filter); + matchVisQuery.query.labels.push(filter); } - const newLabels = modeller.renderQuery(match.query); + const newLabels = modeller.renderQuery(matchVisQuery.query); newQuery += start + newLabels + end; prev = match.to; } @@ -126,11 +159,7 @@ function addFilterToStreamSelector( * @param parserPositions * @param filter */ -function addFilterAsLabelFilter( - query: string, - parserPositions: PipelineStagePosition[], - filter: QueryBuilderLabelFilter -): string { +function addFilterAsLabelFilter(query: string, parserPositions: Position[], filter: QueryBuilderLabelFilter): string { let newQuery = ''; let prev = 0; @@ -149,6 +178,31 @@ function addFilterAsLabelFilter( return newQuery; } +/** + * Add parser after line filter or stream selector + * @param query + * @param queryPartPositions + * @param parser + */ +function addParser(query: string, queryPartPositions: Position[], parser: string): string { + let newQuery = ''; + let prev = 0; + + for (let i = 0; i < queryPartPositions.length; i++) { + // Splice on a string for each matched vector selector + const match = queryPartPositions[i]; + const isLast = i === queryPartPositions.length - 1; + + const start = query.substring(prev, match.to); + const end = isLast ? query.substring(match.to) : ''; + + // Add parser + newQuery += start + ` | ${parser}` + end; + prev = match.to; + } + return newQuery; +} + /** * Check if label exists in the list of labels but ignore the operator. * @param labels diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 4f2f2af7612..ef5d60b17d1 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -33,6 +33,7 @@ import { TimeRange, rangeUtil, toUtc, + QueryHint, } from '@grafana/data'; import { FetchError, config, DataSourceWithBackend } from '@grafana/runtime'; import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider'; @@ -44,13 +45,14 @@ import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_sr import { serializeParams } from '../../../core/utils/fetch'; import { renderLegendFormat } from '../prometheus/legend'; -import { addLabelToQuery } from './add_label_to_query'; +import { addLabelToQuery, addParserToQuery } from './add_label_to_query'; import { transformBackendResult } from './backendResultTransformer'; import { LokiAnnotationsQueryEditor } from './components/AnnotationsQueryEditor'; import LanguageProvider from './language_provider'; import { escapeLabelValueInSelector } from './language_utils'; import { LiveStreams, LokiLiveTarget } from './live_streams'; -import { getNormalizedLokiQuery } from './query_utils'; +import { getQueryHints } from './queryHints'; +import { getNormalizedLokiQuery, isLogsQuery, isValidQuery } from './query_utils'; import { sortDataFrameByTime } from './sortDataFrame'; import { doLokiChannelStream } from './streaming'; import syntax from './syntax'; @@ -348,7 +350,24 @@ export class LokiDatasource return Array.from(streams); } - // By implementing getTagKeys and getTagValues we add ad-hoc filtters functionality + async getDataSamples(query: LokiQuery): Promise { + // Currently works only for log samples + if (!isValidQuery(query.expr) || !isLogsQuery(query.expr)) { + return []; + } + + const lokiLogsQuery: LokiQuery = { + expr: query.expr, + queryType: LokiQueryType.Range, + refId: 'log-samples', + maxLines: 10, + }; + + const request = makeRequest(lokiLogsQuery, this.timeSrv.timeRange(), CoreApp.Explore, 'log-samples'); + return await lastValueFrom(this.query(request).pipe(switchMap((res) => of(res.data)))); + } + + // By implementing getTagKeys and getTagValues we add ad-hoc filters functionality async getTagKeys() { return await this.labelNamesQuery(); } @@ -382,6 +401,14 @@ export class LokiDatasource expression = this.addLabelToQuery(expression, action.key, '!=', action.value); break; } + case 'ADD_LOGFMT_PARSER': { + expression = addParserToQuery(expression, 'logfmt'); + break; + } + case 'ADD_JSON_PARSER': { + expression = addParserToQuery(expression, 'json'); + break; + } default: break; } @@ -681,6 +708,10 @@ export class LokiDatasource getVariables(): string[] { return this.templateSrv.getVariables().map((v) => `$${v.name}`); } + + getQueryHints(query: LokiQuery, result: DataFrame[]): QueryHint[] { + return getQueryHints(query.expr, result); + } } export function lokiRegularEscape(value: any) { diff --git a/public/app/plugins/datasource/loki/queryHints.ts b/public/app/plugins/datasource/loki/queryHints.ts new file mode 100644 index 00000000000..e559780e711 --- /dev/null +++ b/public/app/plugins/datasource/loki/queryHints.ts @@ -0,0 +1,42 @@ +import { DataFrame, QueryHint } from '@grafana/data'; + +import { isQueryWithParser } from './query_utils'; +import { extractLogParserFromDataFrame } from './responseUtils'; + +export function getQueryHints(query: string, series: DataFrame[]): QueryHint[] { + const hints: QueryHint[] = []; + if (series.length > 0) { + const { hasLogfmt, hasJSON } = extractLogParserFromDataFrame(series[0]); + const queryWithParser = isQueryWithParser(query); + + if (hasJSON && !queryWithParser) { + hints.push({ + type: 'ADD_JSON_PARSER', + label: 'Selected log stream selector has JSON formatted logs. ', + fix: { + label: 'Consider using JSON parser.', + action: { + type: 'ADD_JSON_PARSER', + query, + }, + }, + }); + } + + if (hasLogfmt && !queryWithParser) { + hints.push({ + type: 'ADD_LOGFMT_PARSER', + label: 'Selected log stream selector has logfmt formatted logs. ', + fix: { + label: 'Consider using logfmt parser.', + action: { + type: 'ADD_LOGFMT_PARSER', + query, + }, + }, + }); + } + } + + return hints; +} diff --git a/public/app/plugins/datasource/loki/query_utils.ts b/public/app/plugins/datasource/loki/query_utils.ts index 53d88814dd5..5717053f391 100644 --- a/public/app/plugins/datasource/loki/query_utils.ts +++ b/public/app/plugins/datasource/loki/query_utils.ts @@ -1,5 +1,9 @@ import { escapeRegExp } from 'lodash'; +import { parser } from '@grafana/lezer-logql'; + +import { ErrorName } from '../prometheus/querybuilder/shared/parsingUtils'; + import { LokiQuery, LokiQueryType } from './types'; export function formatQuery(selector: string | undefined): string { @@ -90,3 +94,42 @@ export function getNormalizedLokiQuery(query: LokiQuery): LokiQuery { const { instant, range, ...rest } = query; return { ...rest, queryType: LokiQueryType.Range }; } + +export function isValidQuery(query: string): boolean { + let isValid = true; + const tree = parser.parse(query); + tree.iterate({ + enter: (type): false | void => { + if (type.name === ErrorName) { + isValid = false; + } + }, + }); + return isValid; +} + +export function isLogsQuery(query: string): boolean { + let isLogsQuery = true; + const tree = parser.parse(query); + tree.iterate({ + enter: (type): false | void => { + if (type.name === 'MetricExpr') { + isLogsQuery = false; + } + }, + }); + return isLogsQuery; +} + +export function isQueryWithParser(query: string): boolean { + let hasParser = false; + const tree = parser.parse(query); + tree.iterate({ + enter: (type): false | void => { + if (type.name === 'LabelParser') { + hasParser = true; + } + }, + }); + return hasParser; +} diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx index ed8f544e0b5..09e6d50fe82 100644 --- a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx @@ -1,6 +1,6 @@ import React, { useMemo } from 'react'; -import { DataSourceApi, SelectableValue } from '@grafana/data'; +import { DataSourceApi, PanelData, SelectableValue } from '@grafana/data'; import { EditorRow } from '@grafana/experimental'; import { LabelFilters } from 'app/plugins/datasource/prometheus/querybuilder/shared/LabelFilters'; import { OperationList } from 'app/plugins/datasource/prometheus/querybuilder/shared/OperationList'; @@ -12,6 +12,7 @@ import { escapeLabelValueInSelector } from '../../language_utils'; import { lokiQueryModeller } from '../LokiQueryModeller'; import { LokiOperationId, LokiVisualQuery } from '../types'; +import { LokiQueryBuilderHints } from './LokiQueryBuilderHints'; import { NestedQueryList } from './NestedQueryList'; export interface Props { @@ -19,9 +20,10 @@ export interface Props { datasource: LokiDatasource; onChange: (update: LokiVisualQuery) => void; onRunQuery: () => void; + data?: PanelData; } -export const LokiQueryBuilder = React.memo(({ datasource, query, onChange, onRunQuery }) => { +export const LokiQueryBuilder = React.memo(({ datasource, query, onChange, onRunQuery, data }) => { const onChangeLabels = (labels: QueryBuilderLabelFilter[]) => { onChange({ ...query, labels }); }; @@ -97,6 +99,7 @@ export const LokiQueryBuilder = React.memo(({ datasource, query, onChange onRunQuery={onRunQuery} datasource={datasource as DataSourceApi} /> + {query.binaryQueries && query.binaryQueries.length > 0 && ( diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx index 09a6fed59c2..810ba935528 100644 --- a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderContainer.tsx @@ -1,6 +1,8 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import React, { useEffect, useReducer } from 'react'; +import { PanelData } from '@grafana/data'; + import { LokiDatasource } from '../../datasource'; import { LokiQuery } from '../../types'; import { lokiQueryModeller } from '../LokiQueryModeller'; @@ -16,6 +18,7 @@ export interface Props { onChange: (update: LokiQuery) => void; onRunQuery: () => void; showRawQuery: boolean; + data?: PanelData; } export interface State { @@ -27,7 +30,7 @@ export interface State { * This component is here just to contain the translation logic between string query and the visual query builder model. */ export function LokiQueryBuilderContainer(props: Props) { - const { query, onChange, onRunQuery, datasource, showRawQuery } = props; + const { query, onChange, onRunQuery, datasource, showRawQuery, data } = props; const [state, dispatch] = useReducer(stateSlice.reducer, { expr: query.expr, // Use initial visual query only if query.expr is empty string @@ -62,6 +65,7 @@ export function LokiQueryBuilderContainer(props: Props) { datasource={datasource} onChange={onVisQueryChange} onRunQuery={onRunQuery} + data={data} /> {showRawQuery && } diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderHints.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderHints.tsx new file mode 100644 index 00000000000..92ed2bc14ee --- /dev/null +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilderHints.tsx @@ -0,0 +1,84 @@ +import { css } from '@emotion/css'; +import React, { useState, useEffect, useRef } from 'react'; + +import { GrafanaTheme2, PanelData, QueryHint } from '@grafana/data'; +import { Button, Tooltip, useStyles2 } from '@grafana/ui'; + +import { LokiDatasource } from '../../datasource'; +import { lokiQueryModeller } from '../LokiQueryModeller'; +import { buildVisualQueryFromString } from '../parsing'; +import { LokiVisualQuery } from '../types'; + +export interface Props { + query: LokiVisualQuery; + datasource: LokiDatasource; + onChange: (update: LokiVisualQuery) => void; + data?: PanelData; +} + +export const LokiQueryBuilderHints = React.memo(({ datasource, query, onChange, data }) => { + const [hints, setHints] = useState([]); + const styles = useStyles2(getStyles); + const prevQuery = useRef(''); + + useEffect(() => { + const expr = lokiQueryModeller.renderQuery(query); + + const getHints = async () => { + // Run only if query changed + if (prevQuery.current === expr) { + return; + } else { + const lokiQuery = { expr, refId: 'data-samples' }; + prevQuery.current = expr; + const sampleData = await datasource.getDataSamples(lokiQuery); + const hints = datasource.getQueryHints(lokiQuery, sampleData).filter((hint) => hint.fix?.action); + setHints(hints); + } + }; + + getHints().catch(console.error); + }, [datasource, query, onChange, data, styles.hint]); + + return ( + <> + {hints.length > 0 && ( +
+ {hints.map((hint) => { + return ( + + + + ); + })} +
+ )} + + ); +}); + +LokiQueryBuilderHints.displayName = 'LokiQueryBuilderHints'; + +const getStyles = (theme: GrafanaTheme2) => { + return { + container: css` + display: flex; + align-items: start; + `, + hint: css` + margin-right: ${theme.spacing(1)}; + `, + }; +}; diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.tsx index 45a3d2b5fbe..e9b46b8beb7 100644 --- a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.tsx +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryEditorSelector.tsx @@ -126,6 +126,7 @@ export const LokiQueryEditorSelector = React.memo((props) onChange={onChangeInternal} onRunQuery={props.onRunQuery} showRawQuery={rawQuery} + data={data} /> )} {editorMode === QueryEditorMode.Explain && } diff --git a/public/app/plugins/datasource/loki/responseUtils.ts b/public/app/plugins/datasource/loki/responseUtils.ts index 033d174d525..e0eb15665af 100644 --- a/public/app/plugins/datasource/loki/responseUtils.ts +++ b/public/app/plugins/datasource/loki/responseUtils.ts @@ -1,6 +1,29 @@ -import { DataFrame, Labels } from '@grafana/data'; +import { DataFrame, FieldType, getParser, Labels, LogsParsers } from '@grafana/data'; export function dataFrameHasLokiError(frame: DataFrame): boolean { const labelSets: Labels[] = frame.fields.find((f) => f.name === 'labels')?.values.toArray() ?? []; return labelSets.some((labels) => labels.__error__ !== undefined); } +export function extractLogParserFromDataFrame(frame: DataFrame): { hasLogfmt: boolean; hasJSON: boolean } { + const lineField = frame.fields.find((field) => field.type === FieldType.string); + if (lineField == null) { + return { hasJSON: false, hasLogfmt: false }; + } + + const logLines: string[] = lineField.values.toArray(); + + let hasJSON = false; + let hasLogfmt = false; + + logLines.forEach((line) => { + const parser = getParser(line); + if (parser === LogsParsers.JSON) { + hasJSON = true; + } + if (parser === LogsParsers.logfmt) { + hasLogfmt = true; + } + }); + + return { hasLogfmt, hasJSON }; +}