From e269445d79ff4db740b8955a7251bbeb3d1e8775 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 14 Oct 2020 10:22:39 +0200 Subject: [PATCH] Loki: LogQL v2 support (#27884) * Add new logql v2 functions * Fix autocompletion if missing ending } * Refactor operators regex, add pipe operator and tests * Add parsers * Update tests * Add parsers to suggestions, add test * Add operators to syntax * Create pipe operator autocomplete + highlighting + add tests * Add to documentation that pipe operations are available in in Loki 2.0+ * Update snapshot test * Update operators list, add regex quotes and move cursor * Fix spelling * Update documentation * Update --- .../LokiExploreQueryEditor.test.tsx.snap | 1 + .../datasource/loki/language_provider.test.ts | 10 ++ .../datasource/loki/language_provider.ts | 22 +++- .../plugins/datasource/loki/syntax.test.ts | 27 +++++ public/app/plugins/datasource/loki/syntax.ts | 112 +++++++++++++++++- 5 files changed, 168 insertions(+), 4 deletions(-) diff --git a/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap b/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap index 7ae42ecbffd..0a46681c9b3 100644 --- a/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap +++ b/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap @@ -54,6 +54,7 @@ exports[`LokiExploreQueryEditor should render component 1`] = ` "datasource": [Circular], "fetchSeriesLabels": [Function], "getBeginningCompletionItems": [Function], + "getPipeCompletionItem": [Function], "getTermCompletionItems": [Function], "labelKeys": Array [], "labelsCache": LRUCache { diff --git a/public/app/plugins/datasource/loki/language_provider.test.ts b/public/app/plugins/datasource/loki/language_provider.test.ts index 0d42433e492..24e5ff19510 100644 --- a/public/app/plugins/datasource/loki/language_provider.test.ts +++ b/public/app/plugins/datasource/loki/language_provider.test.ts @@ -82,6 +82,16 @@ describe('Language completion provider', () => { expect(result.suggestions[0].label).toEqual('History'); expect(result.suggestions[1].label).toEqual('Functions'); }); + + it('returns pipe operations on pipe context', async () => { + const instance = new LanguageProvider(datasource); + const input = createTypeaheadInput('{app="test"} | ', ' ', '', 15, ['context-pipe']); + const result = await instance.provideCompletionItems(input, { absoluteRange: rangeMock }); + expect(result.context).toBeUndefined(); + expect(result.suggestions.length).toEqual(2); + expect(result.suggestions[0].label).toEqual('Operators'); + expect(result.suggestions[1].label).toEqual('Parsers'); + }); }); describe('label key suggestions', () => { diff --git a/public/app/plugins/datasource/loki/language_provider.ts b/public/app/plugins/datasource/loki/language_provider.ts index 57985d8bf76..671fc7f86aa 100644 --- a/public/app/plugins/datasource/loki/language_provider.ts +++ b/public/app/plugins/datasource/loki/language_provider.ts @@ -9,7 +9,7 @@ import { selectorRegexp, processLabels, } from 'app/plugins/datasource/prometheus/language_utils'; -import syntax, { FUNCTIONS } from './syntax'; +import syntax, { FUNCTIONS, PIPE_PARSERS, PIPE_OPERATORS } from './syntax'; // Types import { LokiQuery } from './types'; @@ -84,7 +84,7 @@ export default class LokiLanguageProvider extends LanguageProvider { } // Strip syntax chars - cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%]/g, '').trim(); + cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%\|]/g, '').trim(); getSyntax(): Grammar { return syntax; @@ -165,6 +165,8 @@ export default class LokiLanguageProvider extends LanguageProvider { } else if (wrapperClasses.includes('context-labels')) { // Suggestions for {|} and {foo=|} return await this.getLabelCompletionItems(input, context); + } else if (wrapperClasses.includes('context-pipe')) { + return this.getPipeCompletionItem(); } else if (empty) { // Suggestions for empty query field return this.getEmptyCompletionItems(context); @@ -222,6 +224,22 @@ export default class LokiLanguageProvider extends LanguageProvider { return { suggestions }; }; + getPipeCompletionItem = (): TypeaheadOutput => { + const suggestions = []; + + suggestions.push({ + label: 'Operators', + items: PIPE_OPERATORS.map(suggestion => ({ ...suggestion, kind: 'operators' })), + }); + + suggestions.push({ + label: 'Parsers', + items: PIPE_PARSERS.map(suggestion => ({ ...suggestion, kind: 'parsers' })), + }); + + return { suggestions }; + }; + getRangeCompletionItems(): TypeaheadOutput { return { context: 'context-range', diff --git a/public/app/plugins/datasource/loki/syntax.test.ts b/public/app/plugins/datasource/loki/syntax.test.ts index 7a5fcf18568..3b96c52841e 100644 --- a/public/app/plugins/datasource/loki/syntax.test.ts +++ b/public/app/plugins/datasource/loki/syntax.test.ts @@ -18,5 +18,32 @@ describe('Loki syntax', () => { expect(Prism.highlight('{key="value"}#test', syntax, 'loki')).toBe( '{key="value"}#test' ); + expect(Prism.highlight('{key="value"', syntax, 'loki')).toBe( + '{key="value"' + ); + }); + it('should highlight functions in Loki query correctly', () => { + expect(Prism.highlight('rate({key="value"}[5m])', syntax, 'loki')).toContain( + 'rate' + ); + expect(Prism.highlight('avg_over_time({key="value"}[5m])', syntax, 'loki')).toContain( + 'avg_over_time' + ); + }); + it('should highlight operators in Loki query correctly', () => { + expect(Prism.highlight('{key="value"} |= "test"', syntax, 'loki')).toContain( + ' |= ' + ); + expect(Prism.highlight('{key="value"} |~"test"', syntax, 'loki')).toContain( + ' |~' + ); + }); + it('should highlight pipe operations in Loki query correctly', () => { + expect(Prism.highlight('{key="value"} |= "test" | logfmt', syntax, 'loki')).toContain( + '| logfmt' + ); + expect(Prism.highlight('{key="value"} |= "test" | label_format', syntax, 'loki')).toContain( + ' | label_format' + ); }); }); diff --git a/public/app/plugins/datasource/loki/syntax.ts b/public/app/plugins/datasource/loki/syntax.ts index fc2406479cf..b7c05e2e7b3 100644 --- a/public/app/plugins/datasource/loki/syntax.ts +++ b/public/app/plugins/datasource/loki/syntax.ts @@ -49,13 +49,108 @@ const AGGREGATION_OPERATORS: CompletionItem[] = [ }, ]; +export const PIPE_PARSERS: CompletionItem[] = [ + { + label: 'json', + insertText: 'json', + documentation: 'Extracting labels from the log line using json parser. Only available in Loki 2.0+.', + }, + { + label: 'regexp', + insertText: 'regexp ""', + documentation: 'Extracting labels from the log line using regexp parser. Only available in Loki 2.0+.', + move: -1, + }, + { + label: 'logfmt', + insertText: 'logfmt', + documentation: 'Extracting labels from the log line using logfmt parser. Only available in Loki 2.0+.', + }, +]; + +export const PIPE_OPERATORS: CompletionItem[] = [ + { + label: 'unwrap', + insertText: 'unwrap', + detail: 'unwrap identifier', + documentation: + 'Take labels and use the values as sample data for metric aggregations. Only available in Loki 2.0+.', + }, + { + label: 'label_format', + insertText: 'label_format', + documentation: 'Only available in Loki 2.0+.', + }, + { + label: 'line_format', + insertText: 'line_format', + documentation: 'Only available in Loki 2.0+.', + }, +]; + export const RANGE_VEC_FUNCTIONS = [ + { + insertText: 'avg_over_time', + label: 'avg_over_time', + detail: 'avg_over_time(range-vector)', + documentation: 'The average of all values in the specified interval. Only available in Loki 2.0+.', + }, + { + insertText: 'min_over_time', + label: 'min_over_time', + detail: 'min_over_time(range-vector)', + documentation: 'The minimum of all values in the specified interval. Only available in Loki 2.0+.', + }, + { + insertText: 'max_over_time', + label: 'max_over_time', + detail: 'max_over_time(range-vector)', + documentation: 'The maximum of all values in the specified interval. Only available in Loki 2.0+.', + }, + { + insertText: 'sum_over_time', + label: 'sum_over_time', + detail: 'sum_over_time(range-vector)', + documentation: 'The sum of all values in the specified interval. Only available in Loki 2.0+.', + }, { insertText: 'count_over_time', label: 'count_over_time', detail: 'count_over_time(range-vector)', documentation: 'The count of all values in the specified interval.', }, + { + insertText: 'stdvar_over_time', + label: 'stdvar_over_time', + detail: 'stdvar_over_time(range-vector)', + documentation: + 'The population standard variance of the values in the specified interval. Only available in Loki 2.0+.', + }, + { + insertText: 'stddev_over_time', + label: 'stddev_over_time', + detail: 'stddev_over_time(range-vector)', + documentation: + 'The population standard deviation of the values in the specified interval. Only available in Loki 2.0+.', + }, + { + insertText: 'quantile_over_time', + label: 'quantile_over_time', + detail: 'quantile_over_time(scalar, range-vector)', + documentation: 'The φ-quantile (0 ≤ φ ≤ 1) of the values in the specified interval. Only available in Loki 2.0+.', + }, + { + insertText: 'bytes_over_time', + label: 'bytes_over_time', + detail: 'bytes_over_time(range-vector)', + documentation: 'Counts the amount of bytes used by each log stream for a given range', + }, + { + insertText: 'bytes_rate', + label: 'bytes_rate', + detail: 'bytes_rate(range-vector)', + documentation: 'Calculates the number of bytes per second for each stream.', + }, { insertText: 'rate', label: 'rate', @@ -83,7 +178,7 @@ const tokenizer: Grammar = { }, }, 'context-labels': { - pattern: /\{[^}]*(?=})/, + pattern: /\{[^}]*(?=}?)/, greedy: true, inside: { comment: { @@ -102,6 +197,19 @@ const tokenizer: Grammar = { punctuation: /[{]/, }, }, + 'context-pipe': { + pattern: /\s\|[^=~]\s?\w*/i, + inside: { + 'pipe-operator': { + pattern: /\|/i, + alias: 'operator', + }, + 'pipe-operations': { + pattern: new RegExp(`${[...PIPE_PARSERS, ...PIPE_OPERATORS].map(f => f.label).join('|')}`, 'i'), + alias: 'keyword', + }, + }, + }, function: new RegExp(`\\b(?:${FUNCTIONS.map(f => f.label).join('|')})(?=\\s*\\()`, 'i'), 'context-range': [ { @@ -125,7 +233,7 @@ const tokenizer: Grammar = { }, ], number: /\b-?\d+((\.\d*)?([eE][+-]?\d+)?)?\b/, - operator: new RegExp(`/&&?|\\|?\\||!=?|<(?:=>?|<|>)?|>[>=]?`, 'i'), + operator: /\s?(\|[=~]?|!=?|<(?:=>?|<|>)?|>[>=]?)\s?/i, punctuation: /[{}()`,.]/, };