From 53758ad7642e9d3646bbad3698e55774bdf0623c Mon Sep 17 00:00:00 2001 From: Zodan Jodan Date: Tue, 14 Nov 2023 22:06:02 +0800 Subject: [PATCH] Loki: Option to add derived fields based on labels (#76162) * Plugin: Deriving fields by name from parsed logs Loki only derives fields by a regex matcher, this limits its usage when functions such as `line_formatter` is used on top of the logs. Some users already have logs parsed in json or logfmt structure which are detected as fields in loki. This pull request allows the mapping between detected fields values and derived values by matching the fields' names. Currently the feature is behind `lokiEnableNameMatcherOption` feature toggle. * improve settings page to have a `fieldType` * improve derived fields getter to use `matcherRegex` * fix failing test * rename feature toggle to `lokiDerivedFieldsFromLabels` * added suggestions from review * add empty config object * remove feature flag * fix width of select * default to `regex` derived field * fix failing test --------- Co-authored-by: Sven Grossmann --- .../loki/configuration/DerivedField.tsx | 48 +++++++++++++++++-- .../loki/configuration/DerivedFields.tsx | 9 +++- .../datasource/loki/getDerivedFields.test.ts | 43 +++++++++++++++++ .../datasource/loki/getDerivedFields.ts | 34 +++++++++++-- public/app/plugins/datasource/loki/types.ts | 1 + 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/public/app/plugins/datasource/loki/configuration/DerivedField.tsx b/public/app/plugins/datasource/loki/configuration/DerivedField.tsx index b6cff64c3ca..5c7a4dd1a22 100644 --- a/public/app/plugins/datasource/loki/configuration/DerivedField.tsx +++ b/public/app/plugins/datasource/loki/configuration/DerivedField.tsx @@ -3,11 +3,13 @@ import React, { ChangeEvent, useEffect, useState } from 'react'; import { usePrevious } from 'react-use'; import { GrafanaTheme2, DataSourceInstanceSettings, VariableSuggestion } from '@grafana/data'; -import { Button, DataLinkInput, Field, Icon, Input, Label, Tooltip, useStyles2, Switch } from '@grafana/ui'; +import { Button, DataLinkInput, Field, Icon, Input, Label, Tooltip, useStyles2, Select, Switch } from '@grafana/ui'; import { DataSourcePicker } from 'app/features/datasources/components/picker/DataSourcePicker'; import { DerivedFieldConfig } from '../types'; +type MatcherType = 'label' | 'regex'; + const getStyles = (theme: GrafanaTheme2) => ({ row: css` display: flex; @@ -32,6 +34,10 @@ const getStyles = (theme: GrafanaTheme2) => ({ margin-right: ${theme.spacing(1)}; `, dataSource: css``, + nameMatcherField: css({ + width: theme.spacing(20), + marginRight: theme.spacing(0.5), + }), }); type Props = { @@ -47,6 +53,7 @@ export const DerivedField = (props: Props) => { const styles = useStyles2(getStyles); const [showInternalLink, setShowInternalLink] = useState(!!value.datasourceUid); const previousUid = usePrevious(value.datasourceUid); + const [fieldType, setFieldType] = useState(value.matcherType ?? 'regex'); // Force internal link visibility change if uid changed outside of this component. useEffect(() => { @@ -74,13 +81,46 @@ export const DerivedField = (props: Props) => { } + > + diff --git a/public/app/plugins/datasource/loki/configuration/DerivedFields.tsx b/public/app/plugins/datasource/loki/configuration/DerivedFields.tsx index 2c7840b9951..77019368db1 100644 --- a/public/app/plugins/datasource/loki/configuration/DerivedFields.tsx +++ b/public/app/plugins/datasource/loki/configuration/DerivedFields.tsx @@ -91,7 +91,14 @@ export const DerivedFields = ({ fields = [], onChange }: Props) => { icon="plus" onClick={(event) => { event.preventDefault(); - const newDerivedFields = [...fields, { name: '', matcherRegex: '', urlDisplayLabel: '', url: '' }]; + const emptyConfig: DerivedFieldConfig = { + name: '', + matcherRegex: '', + urlDisplayLabel: '', + url: '', + matcherType: 'regex', + }; + const newDerivedFields = [...fields, emptyConfig]; onChange(newDerivedFields); }} > diff --git a/public/app/plugins/datasource/loki/getDerivedFields.test.ts b/public/app/plugins/datasource/loki/getDerivedFields.test.ts index 78e14d002e7..3c08cf4dd01 100644 --- a/public/app/plugins/datasource/loki/getDerivedFields.test.ts +++ b/public/app/plugins/datasource/loki/getDerivedFields.test.ts @@ -105,4 +105,47 @@ describe('getDerivedFields', () => { url: '', }); }); + it('adds links to fields with labels', () => { + const df = createDataFrame({ + fields: [ + { name: 'labels', values: [{ trace3: 'bar', trace4: 'blank' }, { trace3: 'tar' }, {}, null] }, + { name: 'line', values: ['nothing', 'trace1=1234', 'trace2=aa', ''] }, + ], + }); + const newFields = getDerivedFields(df, [ + { + matcherRegex: 'trace1=(\\w+)', + name: 'trace1', + url: 'http://localhost/${__value.raw}', + }, + { + matcherRegex: 'trace3', + name: 'trace3Name', + url: 'http://localhost:8080/${__value.raw}', + matcherType: 'label', + }, + { + matcherRegex: 'trace4', + name: 'trace4Name', + matcherType: 'regex', + }, + ]); + expect(newFields.length).toBe(3); + const trace1 = newFields.find((f) => f.name === 'trace1'); + expect(trace1!.values).toEqual([null, '1234', null, null]); + expect(trace1!.config.links![0]).toEqual({ + url: 'http://localhost/${__value.raw}', + title: '', + }); + + const trace3 = newFields.find((f) => f.name === 'trace3Name'); + expect(trace3!.values).toEqual(['bar', 'tar']); + expect(trace3!.config.links![0]).toEqual({ + url: 'http://localhost:8080/${__value.raw}', + title: '', + }); + + const trace4 = newFields.find((f) => f.name === 'trace4Name'); + expect(trace4!.values).toEqual([]); + }); }); diff --git a/public/app/plugins/datasource/loki/getDerivedFields.ts b/public/app/plugins/datasource/loki/getDerivedFields.ts index e2434c23fd8..8b3ebf9bfed 100644 --- a/public/app/plugins/datasource/loki/getDerivedFields.ts +++ b/public/app/plugins/datasource/loki/getDerivedFields.ts @@ -22,12 +22,38 @@ export function getDerivedFields(dataFrame: DataFrame, derivedFieldConfigs: Deri throw new Error('invalid logs-dataframe, string-field missing'); } - lineField.values.forEach((line) => { + const labelFields = dataFrame.fields.find((f) => f.type === FieldType.other && f.name === 'labels'); + + for (let i = 0; i < lineField.values.length; i++) { for (const field of newFields) { - const logMatch = line.match(derivedFieldsGrouped[field.name][0].matcherRegex); - field.values.push(logMatch && logMatch[1]); + // `matcherRegex` can be either a RegExp that is used to extract the value from the log line, or it can be a label key to derive the field from the labels + if (derivedFieldsGrouped[field.name][0].matcherType === 'label' && labelFields) { + const label = labelFields.values[i]; + if (label) { + // Find the key that matches both, the `matcherRegex` and the label key + const intersectingKey = Object.keys(label).find( + (key) => derivedFieldsGrouped[field.name][0].matcherRegex === key + ); + + if (intersectingKey) { + field.values.push(label[intersectingKey]); + continue; + } + } + } else if (derivedFieldsGrouped[field.name][0].matcherType !== 'regex') { + // `matcherRegex` will actually be used as a RegExp here + const line = lineField.values[i]; + const logMatch = line.match(derivedFieldsGrouped[field.name][0].matcherRegex); + + if (logMatch && logMatch[1]) { + field.values.push(logMatch[1]); + continue; + } + + field.values.push(null); + } } - }); + } return newFields; } diff --git a/public/app/plugins/datasource/loki/types.ts b/public/app/plugins/datasource/loki/types.ts index 5ba790263b6..e7ee90d6182 100644 --- a/public/app/plugins/datasource/loki/types.ts +++ b/public/app/plugins/datasource/loki/types.ts @@ -54,6 +54,7 @@ export type DerivedFieldConfig = { url?: string; urlDisplayLabel?: string; datasourceUid?: string; + matcherType?: 'label' | 'regex'; }; export enum LokiVariableQueryType {