From 71429d721059f1a317b8342c7412837570c33779 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Mon, 10 Dec 2018 10:05:57 -0800 Subject: [PATCH] Explore: Improved line parsing for logging - changed log parser API to be more semantic: getting fields, getting values - regexps become an implementation detail of parser and are no longer part of the API - improved JSON parser to not be tripped by nested quotation marks - improved JSON parser to support numbers - added more tests --- public/app/core/logs_model.ts | 52 ++++++++++++++++++++---- public/app/core/specs/logs_model.test.ts | 45 +++++++++++++++----- public/app/features/explore/Logs.tsx | 29 +++++-------- 3 files changed, 90 insertions(+), 36 deletions(-) diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 934b950be72..650edfaca4f 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -108,11 +108,21 @@ export interface LogsParser { * Used to filter rows, and first capture group contains the value. */ buildMatcher: (label: string) => RegExp; + /** - * Regex to find a field in the log line. - * First capture group contains the label value, second capture group the value. + * Returns all parsable substrings from a line, used for highlighting */ - fieldRegex: RegExp; + getFields: (line: string) => string[]; + + /** + * Gets the label name from a parsable substring of a line + */ + getLabelFromField: (field: string) => string; + + /** + * Gets the label value from a parsable substring of a line + */ + getValueFromField: (field: string) => string; /** * Function to verify if this is a valid parser for the given line. * The parser accepts the line unless it returns undefined. @@ -120,20 +130,48 @@ export interface LogsParser { test: (line: string) => any; } +const LOGFMT_REGEXP = /(?:^|\s)(\w+)=("[^"]*"|\S+)/; + export const LogsParsers: { [name: string]: LogsParser } = { JSON: { - buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`), - fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/, + buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"?([\\d\\.]+|[^"]*)"?`), + getFields: line => { + const fields = []; + try { + const parsed = JSON.parse(line); + _.map(parsed, (value, key) => { + const fieldMatcher = new RegExp(`"${key}"\\s*:\\s*"?${_.escapeRegExp(JSON.stringify(value))}"?`); + + const match = line.match(fieldMatcher); + if (match) { + fields.push(match[0]); + } + }); + } catch {} + return fields; + }, + getLabelFromField: field => (field.match(/^"(\w+)"\s*:/) || [])[1], + getValueFromField: field => (field.match(/:\s*(.*)$/) || [])[1], test: line => { try { return JSON.parse(line); } catch (error) {} }, }, + logfmt: { buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`), - fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/, - test: line => LogsParsers.logfmt.fieldRegex.test(line), + getFields: line => { + const fields = []; + line.replace(new RegExp(LOGFMT_REGEXP, 'g'), substring => { + fields.push(substring.trim()); + return ''; + }); + return fields; + }, + getLabelFromField: field => (field.match(LOGFMT_REGEXP) || [])[1], + getValueFromField: field => (field.match(LOGFMT_REGEXP) || [])[2], + test: line => LOGFMT_REGEXP.test(line), }, }; diff --git a/public/app/core/specs/logs_model.test.ts b/public/app/core/specs/logs_model.test.ts index 85f75b50ed0..c59d18e8b7d 100644 --- a/public/app/core/specs/logs_model.test.ts +++ b/public/app/core/specs/logs_model.test.ts @@ -240,11 +240,16 @@ describe('LogsParsers', () => { expect(parser.test('foo=bar')).toBeTruthy(); }); - test('should have a valid fieldRegex', () => { - const match = 'foo=bar'.match(parser.fieldRegex); - expect(match).toBeDefined(); - expect(match[1]).toBe('foo'); - expect(match[2]).toBe('bar'); + test('should return parsed fields', () => { + expect(parser.getFields('foo=bar baz="42 + 1"')).toEqual(['foo=bar', 'baz="42 + 1"']); + }); + + test('should return label for field', () => { + expect(parser.getLabelFromField('foo=bar')).toBe('foo'); + }); + + test('should return value for field', () => { + expect(parser.getValueFromField('foo=bar')).toBe('bar'); }); test('should build a valid value matcher', () => { @@ -263,18 +268,36 @@ describe('LogsParsers', () => { expect(parser.test('{"foo":"bar"}')).toBeTruthy(); }); - test('should have a valid fieldRegex', () => { - const match = '{"foo":"bar"}'.match(parser.fieldRegex); - expect(match).toBeDefined(); - expect(match[1]).toBe('foo'); - expect(match[2]).toBe('bar'); + test('should return parsed fields', () => { + expect(parser.getFields('{ "foo" : "bar", "baz" : 42 }')).toEqual(['"foo" : "bar"', '"baz" : 42']); }); - test('should build a valid value matcher', () => { + test('should return parsed fields for nested quotes', () => { + expect(parser.getFields(`{"foo":"bar: '[value=\\"42\\"]'"}`)).toEqual([`"foo":"bar: '[value=\\"42\\"]'"`]); + }); + + test('should return label for field', () => { + expect(parser.getLabelFromField('"foo" : "bar"')).toBe('foo'); + }); + + test('should return value for field', () => { + expect(parser.getValueFromField('"foo" : "bar"')).toBe('"bar"'); + expect(parser.getValueFromField('"foo" : 42')).toBe('42'); + expect(parser.getValueFromField('"foo" : 42.1')).toBe('42.1'); + }); + + test('should build a valid value matcher for strings', () => { const matcher = parser.buildMatcher('foo'); const match = '{"foo":"bar"}'.match(matcher); expect(match).toBeDefined(); expect(match[1]).toBe('bar'); }); + + test('should build a valid value matcher for integers', () => { + const matcher = parser.buildMatcher('foo'); + const match = '{"foo":42.1}'.match(matcher); + expect(match).toBeDefined(); + expect(match[1]).toBe('42.1'); + }); }); }); diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 96d49bf8da4..4d2962fa275 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -73,7 +73,7 @@ interface RowState { fieldStats: LogsLabelStat[]; fieldValue: string; parsed: boolean; - parser: LogsParser; + parser?: LogsParser; parsedFieldHighlights: string[]; showFieldStats: boolean; } @@ -94,7 +94,7 @@ class Row extends PureComponent { fieldStats: null, fieldValue: null, parsed: false, - parser: null, + parser: undefined, parsedFieldHighlights: [], showFieldStats: false, }; @@ -110,19 +110,16 @@ class Row extends PureComponent { onClickHighlight = (fieldText: string) => { const { getRows } = this.props; const { parser } = this.state; + const allRows = getRows(); - const fieldMatch = fieldText.match(parser.fieldRegex); - if (fieldMatch) { - const allRows = getRows(); - // Build value-agnostic row matcher based on the field label - const fieldLabel = fieldMatch[1]; - const fieldValue = fieldMatch[2]; - const matcher = parser.buildMatcher(fieldLabel); - const fieldStats = calculateFieldStats(allRows, matcher); - const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0); + // Build value-agnostic row matcher based on the field label + const fieldLabel = parser.getLabelFromField(fieldText); + const fieldValue = parser.getValueFromField(fieldText); + const matcher = parser.buildMatcher(fieldLabel); + const fieldStats = calculateFieldStats(allRows, matcher); + const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0); - this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true }); - } + this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true }); }; onMouseOverMessage = () => { @@ -141,11 +138,7 @@ class Row extends PureComponent { const parser = getParser(row.entry); if (parser) { // Use parser to highlight detected fields - const parsedFieldHighlights = []; - this.props.row.entry.replace(new RegExp(parser.fieldRegex, 'g'), substring => { - parsedFieldHighlights.push(substring.trim()); - return ''; - }); + const parsedFieldHighlights = parser.getFields(this.props.row.entry); this.setState({ parsedFieldHighlights, parsed: true, parser }); } }