From 46f0672215011e6c233b9df4ccef5e5c5af6e271 Mon Sep 17 00:00:00 2001 From: GitStart <1501599+gitstart@users.noreply.github.com> Date: Fri, 16 Sep 2022 19:05:20 +0530 Subject: [PATCH] Loki: Remove hardcoded values for parsed query parts (#54755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nitesh Singh Co-authored-by: gitstart Co-authored-by: Rafael Toledo <87545086+Toledodev@users.noreply.github.com> Co-authored-by: Murilo Amaral <87545137+MuriloAmarals@users.noreply.github.com> Co-authored-by: gitstart Co-authored-by: Matheus Benini Ferreira <88898100+MatheusBeniniF@users.noreply.github.com> Co-authored-by: Matheus Benini Co-authored-by: Rubens Rafael <70234898+RubensRafael@users.noreply.github.com> Co-authored-by: Júlio Piubello da Silva Cabral Co-authored-by: gitstart Co-authored-by: Nitesh Singh Co-authored-by: Rafael Toledo <87545086+Toledodev@users.noreply.github.com> Co-authored-by: Murilo Amaral <87545137+MuriloAmarals@users.noreply.github.com> Co-authored-by: gitstart Co-authored-by: Matheus Benini Ferreira <88898100+MatheusBeniniF@users.noreply.github.com> Co-authored-by: Matheus Benini Co-authored-by: Rubens Rafael <70234898+RubensRafael@users.noreply.github.com> Co-authored-by: Júlio Piubello da Silva Cabral --- .../plugins/datasource/loki/modifyQuery.ts | 38 +++-- .../app/plugins/datasource/loki/queryUtils.ts | 50 ++++-- .../datasource/loki/querybuilder/parsing.ts | 146 +++++++++++------- .../prometheus/querybuilder/parsing.ts | 7 +- .../querybuilder/shared/parsingUtils.ts | 4 +- 5 files changed, 155 insertions(+), 90 deletions(-) diff --git a/public/app/plugins/datasource/loki/modifyQuery.ts b/public/app/plugins/datasource/loki/modifyQuery.ts index 8c37891c130..d6e05838cf9 100644 --- a/public/app/plugins/datasource/loki/modifyQuery.ts +++ b/public/app/plugins/datasource/loki/modifyQuery.ts @@ -1,6 +1,18 @@ import { sortBy } from 'lodash'; -import { LineComment, parser } from '@grafana/lezer-logql'; +import { + JsonExpressionParser, + LabelFilter, + LabelParser, + LineComment, + LineFilters, + LogExpr, + LogRangeExpr, + parser, + PipelineExpr, + Selector, + UnwrapExpr, +} from '@grafana/lezer-logql'; import { QueryBuilderLabelFilter } from '../prometheus/querybuilder/shared/types'; @@ -124,7 +136,7 @@ function getStreamSelectorPositions(query: string): Position[] { const positions: Position[] = []; tree.iterate({ enter: ({ type, from, to }): false | void => { - if (type.name === 'Selector') { + if (type.id === Selector) { positions.push({ from, to }); return false; } @@ -142,7 +154,7 @@ export function getParserPositions(query: string): Position[] { const positions: Position[] = []; tree.iterate({ enter: ({ type, from, to }): false | void => { - if (type.name === 'LabelParser' || type.name === 'JsonExpressionParser') { + if (type.id === LabelParser || type.id === JsonExpressionParser) { positions.push({ from, to }); return false; } @@ -159,8 +171,8 @@ export function getLabelFilterPositions(query: string): Position[] { const tree = parser.parse(query); const positions: Position[] = []; tree.iterate({ - enter: ({ name, from, to }): false | void => { - if (name === 'LabelFilter') { + enter: ({ type, from, to }): false | void => { + if (type.id === LabelFilter) { positions.push({ from, to }); return false; } @@ -177,8 +189,8 @@ function getLineFiltersPositions(query: string): Position[] { const tree = parser.parse(query); const positions: Position[] = []; tree.iterate({ - enter: ({ node }): false | void => { - if (node.name === 'LineFilters') { + enter: ({ type, node }): false | void => { + if (type.id === LineFilters) { positions.push({ from: node.from, to: node.to }); return false; } @@ -195,28 +207,28 @@ function getLogQueryPositions(query: string): Position[] { const tree = parser.parse(query); const positions: Position[] = []; tree.iterate({ - enter: ({ name, from, to, node }): false | void => { - if (name === 'LogExpr') { + enter: ({ type, from, to, node }): false | void => { + if (type.id === LogExpr) { positions.push({ from, to }); return false; } // This is a case in metrics query - if (name === 'LogRangeExpr') { + if (type.id === LogRangeExpr) { // Unfortunately, LogRangeExpr includes both log and non-log (e.g. Duration/Range/...) parts of query. // We get position of all log-parts within LogRangeExpr: Selector, PipelineExpr and UnwrapExpr. const logPartsPositions: Position[] = []; - const selector = node.getChild('Selector'); + const selector = node.getChild(Selector); if (selector) { logPartsPositions.push({ from: selector.from, to: selector.to }); } - const pipeline = node.getChild('PipelineExpr'); + const pipeline = node.getChild(PipelineExpr); if (pipeline) { logPartsPositions.push({ from: pipeline.from, to: pipeline.to }); } - const unwrap = node.getChild('UnwrapExpr'); + const unwrap = node.getChild(UnwrapExpr); if (unwrap) { logPartsPositions.push({ from: unwrap.from, to: unwrap.to }); } diff --git a/public/app/plugins/datasource/loki/queryUtils.ts b/public/app/plugins/datasource/loki/queryUtils.ts index a13c4b392e3..93613f3f2df 100644 --- a/public/app/plugins/datasource/loki/queryUtils.ts +++ b/public/app/plugins/datasource/loki/queryUtils.ts @@ -1,9 +1,25 @@ import { SyntaxNode } from '@lezer/common'; import { escapeRegExp } from 'lodash'; -import { parser, LineFilter, PipeExact, PipeMatch, Filter, String } from '@grafana/lezer-logql'; +import { + parser, + LineFilter, + PipeExact, + PipeMatch, + Filter, + String, + LabelFormatExpr, + Selector, + PipelineExpr, + LabelParser, + JsonExpressionParser, + LabelFilter, + MetricExpr, + Matcher, + Identifier, +} from '@grafana/lezer-logql'; -import { ErrorName } from '../prometheus/querybuilder/shared/parsingUtils'; +import { ErrorId } from '../prometheus/querybuilder/shared/parsingUtils'; import { LokiQuery, LokiQueryType } from './types'; @@ -96,8 +112,8 @@ export function isValidQuery(query: string): boolean { let isValid = true; const tree = parser.parse(query); tree.iterate({ - enter: (type): false | void => { - if (type.name === ErrorName) { + enter: ({ type }): false | void => { + if (type.id === ErrorId) { isValid = false; } }, @@ -109,8 +125,8 @@ export function isLogsQuery(query: string): boolean { let isLogsQuery = true; const tree = parser.parse(query); tree.iterate({ - enter: (type): false | void => { - if (type.name === 'MetricExpr') { + enter: ({ type }): false | void => { + if (type.id === MetricExpr) { isLogsQuery = false; } }, @@ -122,8 +138,8 @@ export function isQueryWithParser(query: string): { queryWithParser: boolean; pa let parserCount = 0; const tree = parser.parse(query); tree.iterate({ - enter: (type): false | void => { - if (type.name === 'LabelParser' || type.name === 'JsonExpressionParser') { + enter: ({ type }): false | void => { + if (type.id === LabelParser || type.id === JsonExpressionParser) { parserCount++; } }, @@ -135,9 +151,9 @@ export function isQueryPipelineErrorFiltering(query: string): boolean { let isQueryPipelineErrorFiltering = false; const tree = parser.parse(query); tree.iterate({ - enter: ({ name, node }): false | void => { - if (name === 'LabelFilter') { - const label = node.getChild('Matcher')?.getChild('Identifier'); + enter: ({ type, node }): false | void => { + if (type.id === LabelFilter) { + const label = node.getChild(Matcher)?.getChild(Identifier); if (label) { const labelName = query.substring(label.from, label.to); if (labelName === '__error__') { @@ -155,8 +171,8 @@ export function isQueryWithLabelFormat(query: string): boolean { let queryWithLabelFormat = false; const tree = parser.parse(query); tree.iterate({ - enter: (type): false | void => { - if (type.name === 'LabelFormatExpr') { + enter: ({ type }): false | void => { + if (type.id === LabelFormatExpr) { queryWithLabelFormat = true; } }, @@ -174,8 +190,8 @@ export function getLogQueryFromMetricsQuery(query: string): string { // Log query in metrics query composes of Selector & PipelineExpr let selector = ''; tree.iterate({ - enter: ({ name, from, to }): false | void => { - if (name === 'Selector') { + enter: ({ type, from, to }): false | void => { + if (type.id === Selector) { selector = query.substring(from, to); return false; } @@ -184,8 +200,8 @@ export function getLogQueryFromMetricsQuery(query: string): string { let pipelineExpr = ''; tree.iterate({ - enter: ({ name, from, to }): false | void => { - if (name === 'PipelineExpr') { + enter: ({ type, from, to }): false | void => { + if (type.id === PipelineExpr) { pipelineExpr = query.substring(from, to); return false; } diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.ts index 54c248aba44..2128bb8761d 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.ts @@ -1,9 +1,49 @@ import { SyntaxNode } from '@lezer/common'; - -import { parser } from '@grafana/lezer-logql'; +import { BinModifiers, OnOrIgnoring } from '@prometheus-io/lezer-promql'; import { - ErrorName, + And, + BinOpExpr, + Bool, + By, + ConvOp, + Filter, + FilterOp, + Grouping, + GroupingLabelList, + GroupingLabels, + Identifier, + Ip, + IpLabelFilter, + Json, + JsonExpression, + JsonExpressionParser, + LabelFilter, + LabelFormatMatcher, + LabelParser, + LineFilter, + LineFormatExpr, + LogRangeExpr, + Matcher, + MetricExpr, + Number as NumberLezer, + On, + Or, + parser, + Range, + RangeAggregationExpr, + RangeOp, + String, + UnitFilter, + Unwrap, + UnwrapExpr, + VectorAggregationExpr, + VectorOp, + Without, +} from '@grafana/lezer-logql'; + +import { + ErrorId, getAllByType, getLeftMostChild, getString, @@ -65,17 +105,17 @@ export function buildVisualQueryFromString(expr: string): Context { export function handleExpression(expr: string, node: SyntaxNode, context: Context) { const visQuery = context.query; - switch (node.name) { - case 'Matcher': { + switch (node.type.id) { + case Matcher: { visQuery.labels.push(getLabel(expr, node)); - const err = node.getChild(ErrorName); + const err = node.getChild(ErrorId); if (err) { context.errors.push(makeError(expr, err)); } break; } - case 'LineFilter': { + case LineFilter: { const { operation, error } = getLineFilter(expr, node); if (operation) { visQuery.operations.push(operation); @@ -87,12 +127,12 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex break; } - case 'LabelParser': { + case LabelParser: { visQuery.operations.push(getLabelParser(expr, node)); break; } - case 'LabelFilter': { + case LabelFilter: { const { operation, error } = getLabelFilter(expr, node); if (operation) { visQuery.operations.push(operation); @@ -103,22 +143,22 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex } break; } - case 'JsonExpressionParser': { + case JsonExpressionParser: { visQuery.operations.push(getJsonExpressionParser(expr, node)); break; } - case 'LineFormatExpr': { + case LineFormatExpr: { visQuery.operations.push(getLineFormat(expr, node)); break; } - case 'LabelFormatMatcher': { + case LabelFormatMatcher: { visQuery.operations.push(getLabelFormat(expr, node)); break; } - case 'UnwrapExpr': { + case UnwrapExpr: { const { operation, error } = handleUnwrapExpr(expr, node, context); if (operation) { visQuery.operations.push(operation); @@ -131,22 +171,22 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex break; } - case 'RangeAggregationExpr': { + case RangeAggregationExpr: { visQuery.operations.push(handleRangeAggregation(expr, node, context)); break; } - case 'VectorAggregationExpr': { + case VectorAggregationExpr: { visQuery.operations.push(handleVectorAggregation(expr, node, context)); break; } - case 'BinOpExpr': { + case BinOpExpr: { handleBinary(expr, node, context); break; } - case ErrorName: { + case ErrorId: { if (isIntervalVariableError(node)) { break; } @@ -169,10 +209,10 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex } function getLabel(expr: string, node: SyntaxNode): QueryBuilderLabelFilter { - const labelNode = node.getChild('Identifier'); + const labelNode = node.getChild(Identifier); const label = getString(expr, labelNode); const op = getString(expr, labelNode!.nextSibling); - const value = getString(expr, node.getChild('String')).replace(/"/g, ''); + const value = getString(expr, node.getChild(String)).replace(/"/g, ''); return { label, @@ -182,9 +222,9 @@ function getLabel(expr: string, node: SyntaxNode): QueryBuilderLabelFilter { } function getLineFilter(expr: string, node: SyntaxNode): { operation?: QueryBuilderOperation; error?: string } { - const filter = getString(expr, node.getChild('Filter')); - const filterExpr = handleQuotes(getString(expr, node.getChild('String'))); - const ipLineFilter = node.getChild('FilterOp')?.getChild('Ip'); + const filter = getString(expr, node.getChild(Filter)); + const filterExpr = handleQuotes(getString(expr, node.getChild(String))); + const ipLineFilter = node.getChild(FilterOp)?.getChild(Ip); if (ipLineFilter) { return { @@ -213,7 +253,7 @@ function getLabelParser(expr: string, node: SyntaxNode): QueryBuilderOperation { const parserNode = node.firstChild; const parser = getString(expr, parserNode); - const string = handleQuotes(getString(expr, node.getChild('String'))); + const string = handleQuotes(getString(expr, node.getChild(String))); const params = !!string ? [string] : []; return { id: parser, @@ -222,10 +262,10 @@ function getLabelParser(expr: string, node: SyntaxNode): QueryBuilderOperation { } function getJsonExpressionParser(expr: string, node: SyntaxNode): QueryBuilderOperation { - const parserNode = node.getChild('Json'); + const parserNode = node.getChild(Json); const parser = getString(expr, parserNode); - const params = [...getAllByType(expr, node, 'JsonExpression')]; + const params = [...getAllByType(expr, node, JsonExpression)]; return { id: parser, params, @@ -234,16 +274,16 @@ function getJsonExpressionParser(expr: string, node: SyntaxNode): QueryBuilderOp function getLabelFilter(expr: string, node: SyntaxNode): { operation?: QueryBuilderOperation; error?: string } { // Check for nodes not supported in visual builder and return error - if (node.getChild('Or') || node.getChild('And') || node.getChild('Comma')) { + if (node.getChild(Or) || node.getChild(And) || node.getChild('Comma')) { return { error: 'Label filter with comma, "and", "or" not supported in query builder', }; } - if (node.firstChild!.name === 'IpLabelFilter') { + if (node.firstChild!.type.id === IpLabelFilter) { const ipLabelFilter = node.firstChild; - const label = ipLabelFilter?.getChild('Identifier'); + const label = ipLabelFilter?.getChild(Identifier); const op = label?.nextSibling; - const value = ipLabelFilter?.getChild('String'); + const value = ipLabelFilter?.getChild(String); const valueString = handleQuotes(getString(expr, value)); return { @@ -255,7 +295,7 @@ function getLabelFilter(expr: string, node: SyntaxNode): { operation?: QueryBuil } const id = LokiOperationId.LabelFilter; - if (node.firstChild!.name === 'UnitFilter') { + if (node.firstChild!.type.id === UnitFilter) { const filter = node.firstChild!.firstChild; const label = filter!.firstChild; const op = label!.nextSibling; @@ -296,7 +336,7 @@ function getLabelFilter(expr: string, node: SyntaxNode): { operation?: QueryBuil function getLineFormat(expr: string, node: SyntaxNode): QueryBuilderOperation { const id = LokiOperationId.LineFormat; - const string = handleQuotes(getString(expr, node.getChild('String'))); + const string = handleQuotes(getString(expr, node.getChild(String))); return { id, @@ -306,7 +346,7 @@ function getLineFormat(expr: string, node: SyntaxNode): QueryBuilderOperation { function getLabelFormat(expr: string, node: SyntaxNode): QueryBuilderOperation { const id = LokiOperationId.LabelFormat; - const renameTo = node.getChild('Identifier'); + const renameTo = node.getChild(Identifier); const op = renameTo!.nextSibling; const originalLabel = op!.nextSibling; @@ -321,9 +361,9 @@ function handleUnwrapExpr( node: SyntaxNode, context: Context ): { operation?: QueryBuilderOperation; error?: string } { - const unwrapExprChild = node.getChild('UnwrapExpr'); - const labelFilterChild = node.getChild('LabelFilter'); - const unwrapChild = node.getChild('Unwrap'); + const unwrapExprChild = node.getChild(UnwrapExpr); + const labelFilterChild = node.getChild(LabelFilter); + const unwrapChild = node.getChild(Unwrap); if (unwrapExprChild) { handleExpression(expr, unwrapExprChild, context); @@ -334,7 +374,7 @@ function handleUnwrapExpr( } if (unwrapChild) { - if (unwrapChild.nextSibling?.type.name === 'ConvOp') { + if (unwrapChild.nextSibling?.type.id === ConvOp) { const convOp = unwrapChild.nextSibling; const identifier = convOp.nextSibling; return { @@ -356,10 +396,10 @@ function handleUnwrapExpr( return {}; } function handleRangeAggregation(expr: string, node: SyntaxNode, context: Context) { - const nameNode = node.getChild('RangeOp'); + const nameNode = node.getChild(RangeOp); const funcName = getString(expr, nameNode); - const number = node.getChild('Number'); - const logExpr = node.getChild('LogRangeExpr'); + const number = node.getChild(NumberLezer); + const logExpr = node.getChild(LogRangeExpr); const params = number !== null && number !== undefined ? [getString(expr, number)] : []; let match = getString(expr, node).match(/\[(.+)\]/); @@ -380,33 +420,33 @@ function handleRangeAggregation(expr: string, node: SyntaxNode, context: Context } function handleVectorAggregation(expr: string, node: SyntaxNode, context: Context) { - const nameNode = node.getChild('VectorOp'); + const nameNode = node.getChild(VectorOp); let funcName = getString(expr, nameNode); - const grouping = node.getChild('Grouping'); + const grouping = node.getChild(Grouping); const params = []; - const numberNode = node.getChild('Number'); + const numberNode = node.getChild(NumberLezer); if (numberNode) { params.push(Number(getString(expr, numberNode))); } if (grouping) { - const byModifier = grouping.getChild(`By`); + const byModifier = grouping.getChild(By); if (byModifier && funcName) { funcName = `__${funcName}_by`; } - const withoutModifier = grouping.getChild(`Without`); + const withoutModifier = grouping.getChild(Without); if (withoutModifier) { funcName = `__${funcName}_without`; } - params.push(...getAllByType(expr, grouping, 'Identifier')); + params.push(...getAllByType(expr, grouping, Identifier)); } - const metricExpr = node.getChild('MetricExpr'); + const metricExpr = node.getChild(MetricExpr); const op: QueryBuilderOperation = { id: funcName, params }; if (metricExpr) { @@ -435,7 +475,7 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) { const visQuery = context.query; const left = node.firstChild!; const op = getString(expr, left.nextSibling); - const binModifier = getBinaryModifier(expr, node.getChild('BinModifiers')); + const binModifier = getBinaryModifier(expr, node.getChild(BinModifiers)); const right = node.lastChild!; @@ -444,7 +484,7 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) { const leftNumber = getLastChildWithSelector(left, 'MetricExpr.LiteralExpr.Number'); const rightNumber = getLastChildWithSelector(right, 'MetricExpr.LiteralExpr.Number'); - const rightBinary = right.getChild('BinOpExpr'); + const rightBinary = right.getChild(BinOpExpr); if (leftNumber) { // TODO: this should be already handled in case parent is binary expression as it has to be added to parent @@ -499,26 +539,26 @@ function getBinaryModifier( if (!node) { return undefined; } - if (node.getChild('Bool')) { + if (node.getChild(Bool)) { return { isBool: true, isMatcher: false }; } else { - const matcher = node.getChild('OnOrIgnoring'); + const matcher = node.getChild(OnOrIgnoring); if (!matcher) { // Not sure what this could be, maybe should be an error. return undefined; } - const labels = getString(expr, matcher.getChild('GroupingLabels')?.getChild('GroupingLabelList')); + const labels = getString(expr, matcher.getChild(GroupingLabels)?.getChild(GroupingLabelList)); return { isMatcher: true, isBool: false, matches: labels, - matchType: matcher.getChild('On') ? 'on' : 'ignoring', + matchType: matcher.getChild(On) ? 'on' : 'ignoring', }; } } function isIntervalVariableError(node: SyntaxNode) { - return node?.parent?.name === 'Range'; + return node?.parent?.type.id === Range; } function handleQuotes(string: string) { diff --git a/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts b/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts index 0edfe3543f4..b7e3a052a25 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts @@ -29,7 +29,7 @@ import { import { binaryScalarOperatorToOperatorName } from './binaryScalarOperations'; import { - ErrorName, + ErrorId, getAllByType, getLeftMostChild, getString, @@ -95,9 +95,6 @@ interface Context { errors: ParsingError[]; } -// Although 0 isn't explicitly provided in the lezer-promql library as the error node ID, it does appear to be the ID of error nodes within lezer. -const ErrorId = 0; - /** * Handler for default state. It will traverse the tree and call the appropriate handler for each node. The node * handled here does not necessarily need to be of type == Expr. @@ -118,7 +115,7 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex case LabelMatcher: { // Same as MetricIdentifier should be just one per query. visQuery.labels.push(getLabel(expr, node)); - const err = node.getChild(ErrorName); + const err = node.getChild(ErrorId); if (err) { context.errors.push(makeError(expr, err)); } diff --git a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts b/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts index 21d0d1e268e..fd3bc4061ab 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts +++ b/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts @@ -2,8 +2,8 @@ import { SyntaxNode, TreeCursor } from '@lezer/common'; import { QueryBuilderOperation } from './types'; -// This is used for error type for some reason -export const ErrorName = '⚠'; +// Although 0 isn't explicitly provided in the lezer-promql & @grafana/lezer-logql library as the error node ID, it does appear to be the ID of error nodes within lezer. +export const ErrorId = 0; export function getLeftMostChild(cur: SyntaxNode): SyntaxNode { return cur.firstChild ? getLeftMostChild(cur.firstChild) : cur;