From 83289065f6591b2a456457e00438d6d636e18f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Thu, 13 Apr 2023 13:32:09 +0200 Subject: [PATCH] GrafanaData: Remove obsolete logs exports (#66271) * grafana-data: removed obsolete logs exports * updated betterer checksums --- .betterer.results | 4 - packages/grafana-data/src/types/logs.ts | 29 -- packages/grafana-data/src/utils/index.ts | 1 - packages/grafana-data/src/utils/logs.test.ts | 372 ------------------- packages/grafana-data/src/utils/logs.ts | 246 ------------ 5 files changed, 652 deletions(-) delete mode 100644 packages/grafana-data/src/utils/logs.test.ts delete mode 100644 packages/grafana-data/src/utils/logs.ts diff --git a/.betterer.results b/.betterer.results index 9d49a248460..c904362ac2d 100644 --- a/.betterer.results +++ b/.betterer.results @@ -613,10 +613,6 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "2"], [0, 0, 0, "Unexpected any. Specify a different type.", "3"] ], - "packages/grafana-data/src/utils/logs.ts:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"], - [0, 0, 0, "Do not use any type assertions.", "1"] - ], "packages/grafana-data/src/utils/testdata/testTheme.ts:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], [0, 0, 0, "Do not use any type assertions.", "1"] diff --git a/packages/grafana-data/src/types/logs.ts b/packages/grafana-data/src/types/logs.ts index 5e0d078a4b2..9ad6d49ef04 100644 --- a/packages/grafana-data/src/types/logs.ts +++ b/packages/grafana-data/src/types/logs.ts @@ -103,35 +103,6 @@ export interface LogLabelStatsModel { value: string; } -/** @deprecated will be removed in the next major version */ -export interface LogsParser { - /** - * Value-agnostic matcher for a field label. - * Used to filter rows, and first capture group contains the value. - */ - buildMatcher: (label: string) => RegExp; - - /** - * Returns all parsable substrings from a line, used for highlighting - */ - getFields: (line: string) => string[]; - - /** - * Gets the label name from a parsable substring of a line - */ - getLabelFromField: (field: string) => string; - - /** - * Gets the label value from a parsable substring of a line - */ - getValueFromField: (field: string) => string; - /** - * Function to verify if this is a valid parser for the given line. - * The parser accepts the line if it returns true. - */ - test: (line: string) => boolean; -} - export enum LogsDedupDescription { none = 'No de-duplication', exact = 'De-duplication of successive lines that are identical, ignoring ISO datetimes.', diff --git a/packages/grafana-data/src/utils/index.ts b/packages/grafana-data/src/utils/index.ts index 5df29eb910b..53b6c632d5f 100644 --- a/packages/grafana-data/src/utils/index.ts +++ b/packages/grafana-data/src/utils/index.ts @@ -4,7 +4,6 @@ export * from './Registry'; export * from './datasource'; export * from './deprecationWarning'; export * from './csv'; -export * from './logs'; export * from './labels'; export * from './numbers'; export * from './object'; diff --git a/packages/grafana-data/src/utils/logs.test.ts b/packages/grafana-data/src/utils/logs.test.ts deleted file mode 100644 index f4c9966e981..00000000000 --- a/packages/grafana-data/src/utils/logs.test.ts +++ /dev/null @@ -1,372 +0,0 @@ -import { MutableDataFrame } from '../dataframe/MutableDataFrame'; -import { Labels } from '../types/data'; -import { LogLevel, LogsModel, LogRowModel, LogsSortOrder } from '../types/logs'; - -import { - getLogLevel, - calculateLogsLabelStats, - calculateFieldStats, - getParser, - LogsParsers, - calculateStats, - getLogLevelFromKey, - sortLogsResult, - checkLogsError, -} from './logs'; - -describe('getLoglevel()', () => { - it('returns no log level on empty line', () => { - expect(getLogLevel('')).toBe(LogLevel.unknown); - }); - - it('returns no log level on when level is part of a word', () => { - expect(getLogLevel('who warns us')).toBe(LogLevel.unknown); - }); - - it('returns same log level for long and short version', () => { - expect(getLogLevel('[Warn]')).toBe(LogLevel.warning); - expect(getLogLevel('[Warning]')).toBe(LogLevel.warning); - expect(getLogLevel('[Warn]')).toBe('warning'); - }); - - it('returns correct log level when level is capitalized', () => { - expect(getLogLevel('WARN')).toBe(LogLevel.warn); - }); - - it('returns log level on line contains a log level', () => { - expect(getLogLevel('warn: it is looking bad')).toBe(LogLevel.warn); - expect(getLogLevel('2007-12-12 12:12:12 [WARN]: it is looking bad')).toBe(LogLevel.warn); - }); - - it('returns first log level found', () => { - expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn); - expect(getLogLevel('WARN this is a non-critical message')).toBe(LogLevel.warn); - }); -}); - -describe('getLogLevelFromKey()', () => { - it('returns correct log level', () => { - expect(getLogLevelFromKey('info')).toBe(LogLevel.info); - }); - it('returns correct log level when level is capitalized', () => { - expect(getLogLevelFromKey('INFO')).toBe(LogLevel.info); - }); - it('returns unknown log level when level is integer', () => { - expect(getLogLevelFromKey(1)).toBe(LogLevel.unknown); - }); -}); - -describe('calculateLogsLabelStats()', () => { - test('should return no stats for empty rows', () => { - expect(calculateLogsLabelStats([], '')).toEqual([]); - }); - - test('should return no stats of label is not found', () => { - const rows = [ - { - entry: 'foo 1', - labels: { - foo: 'bar', - } as Labels, - }, - ] as LogRowModel[]; - - expect(calculateLogsLabelStats(rows, 'baz')).toEqual([]); - }); - - test('should return stats for found labels', () => { - const rows = [ - { - entry: 'foo 1', - labels: { - foo: 'bar', - } as Labels, - }, - { - entry: 'foo 0', - labels: { - foo: 'xxx', - } as Labels, - }, - { - entry: 'foo 2', - labels: { - foo: 'bar', - } as Labels, - }, - ] as LogRowModel[]; - - expect(calculateLogsLabelStats(rows, 'foo')).toMatchObject([ - { - value: 'bar', - count: 2, - }, - { - value: 'xxx', - count: 1, - }, - ]); - }); -}); - -describe('LogsParsers', () => { - describe('logfmt', () => { - const parser = LogsParsers.logfmt; - - test('should detect format', () => { - expect(parser.test('foo')).toBeFalsy(); - expect(parser.test('foo=bar')).toBeTruthy(); - }); - - test('should return detected fields', () => { - expect( - parser.getFields( - 'foo=bar baz="42 + 1" msg="[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1" time(ms)=50 label{foo}=bar' - ) - ).toEqual([ - 'foo=bar', - 'baz="42 + 1"', - 'msg="[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1"', - 'time(ms)=50', - 'label{foo}=bar', - ]); - }); - - test('should return label for field', () => { - expect(parser.getLabelFromField('foo=bar')).toBe('foo'); - expect(parser.getLabelFromField('time(ms)=50')).toBe('time(ms)'); - }); - - test('should return value for field', () => { - expect(parser.getValueFromField('foo=bar')).toBe('bar'); - expect(parser.getValueFromField('time(ms)=50')).toBe('50'); - expect( - parser.getValueFromField( - 'msg="[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1"' - ) - ).toBe('"[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1"'); - }); - - test('should build a valid value matcher', () => { - const matcher = parser.buildMatcher('foo'); - const match = 'foo=bar'.match(matcher); - expect(match).toBeDefined(); - expect(match![1]).toBe('bar'); - }); - - test('should build a valid complex value matcher', () => { - const matcher = parser.buildMatcher('time(ms)'); - const match = 'time(ms)=50'.match(matcher); - expect(match).toBeDefined(); - expect(match![1]).toBe('50'); - }); - }); - - describe('JSON', () => { - const parser = LogsParsers.JSON; - - test('should detect format', () => { - expect(parser.test('foo')).toBeFalsy(); - expect(parser.test('"foo"')).toBeFalsy(); - expect(parser.test('{"foo":"bar"}')).toBeTruthy(); - }); - - test('should return detected fields', () => { - expect(parser.getFields('{ "foo" : "bar", "baz" : 42 }')).toEqual(['"foo":"bar"', '"baz":42']); - }); - - test('should return detected fields for nested quotes', () => { - expect(parser.getFields(`{"foo":"bar: '[value=\\"42\\"]'"}`)).toEqual([`"foo":"bar: '[value=\\"42\\"]'"`]); - }); - - test('should return label for field', () => { - expect(parser.getLabelFromField('"foo" : "bar"')).toBe('foo'); - expect(parser.getLabelFromField('"docker.memory.fail.count":0')).toBe('docker.memory.fail.count'); - }); - - test('should return value for field', () => { - expect(parser.getValueFromField('"foo" : "bar"')).toBe('"bar"'); - expect(parser.getValueFromField('"foo" : 42')).toBe('42'); - expect(parser.getValueFromField('"foo" : 42.1')).toBe('42.1'); - }); - - test('should build a valid value matcher for strings', () => { - const matcher = parser.buildMatcher('foo'); - const match = '{"foo":"bar"}'.match(matcher); - expect(match).toBeDefined(); - expect(match![1]).toBe('bar'); - }); - - test('should build a valid value matcher for integers', () => { - const matcher = parser.buildMatcher('foo'); - const match = '{"foo":42.1}'.match(matcher); - expect(match).toBeDefined(); - expect(match![1]).toBe('42.1'); - }); - }); -}); - -describe('calculateFieldStats()', () => { - test('should return no stats for empty rows', () => { - expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]); - }); - - test('should return no stats if extractor does not match', () => { - const rows = [ - { - entry: 'foo=bar', - }, - ] as LogRowModel[]; - - expect(calculateFieldStats(rows, /baz=(.*)/)).toEqual([]); - }); - - test('should return stats for found field', () => { - const rows = [ - { - entry: 'foo="42 + 1"', - }, - { - entry: 'foo=503 baz=foo', - }, - { - entry: 'foo="42 + 1"', - }, - { - entry: 't=2018-12-05T07:44:59+0000 foo=503', - }, - ] as LogRowModel[]; - - expect(calculateFieldStats(rows, /foo=("[^"]*"|\S+)/)).toMatchObject([ - { - value: '"42 + 1"', - count: 2, - }, - { - value: '503', - count: 2, - }, - ]); - }); -}); - -describe('calculateStats()', () => { - test('should return no stats for empty array', () => { - expect(calculateStats([])).toEqual([]); - }); - - test('should return correct stats', () => { - const values = ['one', 'one', null, undefined, 'two']; - expect(calculateStats(values)).toMatchObject([ - { - value: 'one', - count: 2, - proportion: 2 / 3, - }, - { - value: 'two', - count: 1, - proportion: 1 / 3, - }, - ]); - }); -}); - -describe('getParser()', () => { - test('should return no parser on empty line', () => { - expect(getParser('')).toBeUndefined(); - }); - - test('should return no parser on unknown line pattern', () => { - expect(getParser('To Be or not to be')).toBeUndefined(); - }); - - test('should return logfmt parser on key value patterns', () => { - expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt); - }); - - test('should return JSON parser on JSON log lines', () => { - // TODO implement other JSON value types than string - expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON); - }); -}); - -describe('sortLogsResult', () => { - const firstRow: LogRowModel = { - rowIndex: 0, - entryFieldIndex: 0, - dataFrame: new MutableDataFrame(), - entry: '', - hasAnsi: false, - hasUnescapedContent: false, - labels: {}, - logLevel: LogLevel.info, - raw: '', - timeEpochMs: 0, - timeEpochNs: '0', - timeFromNow: '', - timeLocal: '', - timeUtc: '', - uid: '1', - }; - const sameAsFirstRow = firstRow; - const secondRow: LogRowModel = { - rowIndex: 1, - entryFieldIndex: 0, - dataFrame: new MutableDataFrame(), - entry: '', - hasAnsi: false, - hasUnescapedContent: false, - labels: {}, - logLevel: LogLevel.info, - raw: '', - timeEpochMs: 10, - timeEpochNs: '10000000', - timeFromNow: '', - timeLocal: '', - timeUtc: '', - uid: '2', - }; - - describe('when called with LogsSortOrder.Descending', () => { - it('then it should sort descending', () => { - const logsResult: LogsModel = { - rows: [firstRow, sameAsFirstRow, secondRow], - hasUniqueLabels: false, - }; - const result = sortLogsResult(logsResult, LogsSortOrder.Descending); - - expect(result).toEqual({ - rows: [secondRow, firstRow, sameAsFirstRow], - hasUniqueLabels: false, - }); - }); - }); - - describe('when called with LogsSortOrder.Ascending', () => { - it('then it should sort ascending', () => { - const logsResult: LogsModel = { - rows: [secondRow, firstRow, sameAsFirstRow], - hasUniqueLabels: false, - }; - const result = sortLogsResult(logsResult, LogsSortOrder.Ascending); - - expect(result).toEqual({ - rows: [firstRow, sameAsFirstRow, secondRow], - hasUniqueLabels: false, - }); - }); - }); -}); - -describe('checkLogsError()', () => { - const log = { - labels: { - __error__: 'Error Message', - foo: 'boo', - } as Labels, - } as LogRowModel; - test('should return correct error if error is present', () => { - expect(checkLogsError(log)).toStrictEqual({ hasError: true, errorMessage: 'Error Message' }); - }); -}); diff --git a/packages/grafana-data/src/utils/logs.ts b/packages/grafana-data/src/utils/logs.ts deleted file mode 100644 index e9ec99fb66c..00000000000 --- a/packages/grafana-data/src/utils/logs.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { countBy, chain, escapeRegExp } from 'lodash'; - -import { DataFrame, FieldType } from '../types/index'; -import { LogLevel, LogRowModel, LogLabelStatsModel, LogsParser, LogsModel, LogsSortOrder } from '../types/logs'; -import { ArrayVector } from '../vector/ArrayVector'; - -// This matches: -// first a label from start of the string or first white space, then any word chars until "=" -// second either an empty quotes, or anything that starts with quote and ends with unescaped quote, -// or any non whitespace chars that do not start with quote -const LOGFMT_REGEXP = /(?:^|\s)([\w\(\)\[\]\{\}]+)=(""|(?:".*?[^\\]"|[^"\s]\S*))/; - -/** - * Returns the log level of a log line. - * Parse the line for level words. If no level is found, it returns `LogLevel.unknown`. - * - * Example: `getLogLevel('WARN 1999-12-31 this is great') // LogLevel.warn` - */ -/** @deprecated will be removed in the next major version */ -export function getLogLevel(line: string): LogLevel { - if (!line) { - return LogLevel.unknown; - } - let level = LogLevel.unknown; - let currentIndex: number | undefined = undefined; - - for (const key of Object.keys(LogLevel)) { - const regexp = new RegExp(`\\b${key}\\b`, 'i'); - const result = regexp.exec(line); - - if (result) { - if (currentIndex === undefined || result.index < currentIndex) { - level = LogLevel[key as keyof typeof LogLevel]; - currentIndex = result.index; - } - } - } - return level; -} - -/** @deprecated will be removed in the next major version */ -export function getLogLevelFromKey(key: string | number): LogLevel { - const level = LogLevel[key.toString().toLowerCase() as keyof typeof LogLevel]; - if (level) { - return level; - } - - return LogLevel.unknown; -} - -/** @deprecated will be removed in the next major version */ -export function addLogLevelToSeries(series: DataFrame, lineIndex: number): DataFrame { - const levels = new ArrayVector(); - const lines = series.fields[lineIndex]; - for (let i = 0; i < lines.values.length; i++) { - const line = lines.values.get(lineIndex); - levels.buffer.push(getLogLevel(line)); - } - - return { - ...series, // Keeps Tags, RefID etc - fields: [ - ...series.fields, - { - name: 'LogLevel', - type: FieldType.string, - values: levels, - config: {}, - }, - ], - }; -} - -/** @deprecated will be removed in the next major version */ -export const LogsParsers: { [name: string]: LogsParser } = { - JSON: { - buildMatcher: (label) => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"?([\\d\\.]+|[^"]*)"?`), - getFields: (line) => { - try { - const parsed = JSON.parse(line); - return Object.keys(parsed).map((key) => { - return `"${key}":${JSON.stringify(parsed[key])}`; - }); - } catch {} - return []; - }, - getLabelFromField: (field) => (field.match(/^"([^"]+)"\s*:/) || [])[1], - getValueFromField: (field) => (field.match(/:\s*(.*)$/) || [])[1], - test: (line) => { - let parsed; - try { - parsed = JSON.parse(line); - } catch (error) {} - // The JSON parser should only be used for log lines that are valid serialized JSON objects. - // If it would be used for a string, detected fields would include each letter as a separate field. - return typeof parsed === 'object'; - }, - }, - - logfmt: { - buildMatcher: (label) => new RegExp(`(?:^|\\s)${escapeRegExp(label)}=("[^"]*"|\\S+)`), - getFields: (line) => { - const fields: string[] = []; - line.replace(new RegExp(LOGFMT_REGEXP, 'g'), (substring) => { - fields.push(substring.trim()); - return ''; - }); - return fields; - }, - getLabelFromField: (field) => (field.match(LOGFMT_REGEXP) || [])[1], - getValueFromField: (field) => (field.match(LOGFMT_REGEXP) || [])[2], - test: (line) => LOGFMT_REGEXP.test(line), - }, -}; - -/** @deprecated will be removed in the next major version */ -export function calculateFieldStats(rows: LogRowModel[], extractor: RegExp): LogLabelStatsModel[] { - // Consider only rows that satisfy the matcher - const rowsWithField = rows.filter((row) => extractor.test(row.entry)); - const rowCount = rowsWithField.length; - - // Get field value counts for eligible rows - const countsByValue = countBy(rowsWithField, (r) => { - const row: LogRowModel = r; - const match = row.entry.match(extractor); - - return match ? match[1] : null; - }); - return getSortedCounts(countsByValue, rowCount); -} - -/** @deprecated will be removed in the next major version */ -export function calculateLogsLabelStats(rows: LogRowModel[], label: string): LogLabelStatsModel[] { - // Consider only rows that have the given label - const rowsWithLabel = rows.filter((row) => row.labels[label] !== undefined); - const rowCount = rowsWithLabel.length; - - // Get label value counts for eligible rows - const countsByValue = countBy(rowsWithLabel, (row) => row.labels[label]); - return getSortedCounts(countsByValue, rowCount); -} - -/** @deprecated will be removed in the next major version */ -export function calculateStats(values: unknown[]): LogLabelStatsModel[] { - const nonEmptyValues = values.filter((value) => value !== undefined && value !== null); - const countsByValue = countBy(nonEmptyValues); - return getSortedCounts(countsByValue, nonEmptyValues.length); -} - -const getSortedCounts = (countsByValue: { [value: string]: number }, rowCount: number) => { - return chain(countsByValue) - .map((count, value) => ({ count, value, proportion: count / rowCount })) - .sortBy('count') - .reverse() - .value(); -}; - -/** @deprecated will be removed in the next major version */ -export function getParser(line: string): LogsParser | undefined { - let parser; - try { - if (LogsParsers.JSON.test(line)) { - parser = LogsParsers.JSON; - } - } catch (error) {} - - if (!parser && LogsParsers.logfmt.test(line)) { - parser = LogsParsers.logfmt; - } - - return parser; -} - -/** @deprecated will be removed in the next major version */ -export const sortInAscendingOrder = (a: LogRowModel, b: LogRowModel) => { - // compare milliseconds - if (a.timeEpochMs < b.timeEpochMs) { - return -1; - } - - if (a.timeEpochMs > b.timeEpochMs) { - return 1; - } - - // if milliseconds are equal, compare nanoseconds - if (a.timeEpochNs < b.timeEpochNs) { - return -1; - } - - if (a.timeEpochNs > b.timeEpochNs) { - return 1; - } - - return 0; -}; - -/** @deprecated will be removed in the next major version */ -export const sortInDescendingOrder = (a: LogRowModel, b: LogRowModel) => { - // compare milliseconds - if (a.timeEpochMs > b.timeEpochMs) { - return -1; - } - - if (a.timeEpochMs < b.timeEpochMs) { - return 1; - } - - // if milliseconds are equal, compare nanoseconds - if (a.timeEpochNs > b.timeEpochNs) { - return -1; - } - - if (a.timeEpochNs < b.timeEpochNs) { - return 1; - } - - return 0; -}; - -/** @deprecated will be removed in the next major version */ -export const sortLogsResult = (logsResult: LogsModel | null, sortOrder: LogsSortOrder): LogsModel => { - const rows = logsResult ? sortLogRows(logsResult.rows, sortOrder) : []; - return logsResult ? { ...logsResult, rows } : { hasUniqueLabels: false, rows }; -}; - -/** @deprecated will be removed in the next major version */ -export const sortLogRows = (logRows: LogRowModel[], sortOrder: LogsSortOrder) => - sortOrder === LogsSortOrder.Ascending ? logRows.sort(sortInAscendingOrder) : logRows.sort(sortInDescendingOrder); - -// Currently supports only error condition in Loki logs -/** @deprecated will be removed in the next major version */ -export const checkLogsError = (logRow: LogRowModel): { hasError: boolean; errorMessage?: string } => { - if (logRow.labels.__error__) { - return { - hasError: true, - errorMessage: logRow.labels.__error__, - }; - } - return { - hasError: false, - }; -}; - -/** @deprecated will be removed in the next major version */ -export const escapeUnescapedString = (string: string) => - string.replace(/\\r\\n|\\n|\\t|\\r/g, (match: string) => (match.slice(1) === 't' ? '\t' : '\n'));