public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transformations: Add RegExp option to Extract fields transformer (#96593)

Browse Source
This commit is contained in:
Leon Sorokin
2024-11-19 22:04:31 +02:00
committed by GitHub
parent 51efde972b
commit 33bf94f4d2
7 changed files with 92 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/sources/panels-visualizations/query-transform-data/transform-data/index.md
+1
View File
@@ -338,6 +338,7 @@ Use this transformation to select a source of data and extract content from it i
- **Format** - Choose one of the following:
- **JSON** - Parse JSON content from the source.
- **Key+value pairs** - Parse content in the format 'a=b' or 'c:d' from the source.
- **RegExp** - Parse content using a regular expression with [named capturing group(s)](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Named_capturing_group).
- **Auto** - Discover fields automatically.
- **Replace All Fields** - (Optional) Select this option to hide all other fields and display only your calculated field in the visualization.
- **Keep Time** - (Optional) Available only if **Replace All Fields** is true. Keeps the time field in the output.
public/app/features/transformers/docs/content.ts
+1
View File
@@ -240,6 +240,7 @@ Use this transformation to select a source of data and extract content from it i
- **Format** - Choose one of the following:
- **JSON** - Parse JSON content from the source.
- **Key+value pairs** - Parse content in the format 'a=b' or 'c:d' from the source.
- **RegExp** - Parse content using a regular expression with [named capturing group(s)](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Named_capturing_group).
- **Auto** - Discover fields automatically.
- **Replace All Fields** - (Optional) Select this option to hide all other fields and display only your calculated field in the visualization.
- **Keep Time** - (Optional) Available only if **Replace All Fields** is true. Keeps the time field in the output.
public/app/features/transformers/extractFields/ExtractFieldsTransformerEditor.tsx
+20 -2
View File
@@ -1,3 +1,5 @@
import { ChangeEvent } from 'react';
import {
DataTransformerID,
TransformerRegistryItem,
@@ -7,7 +9,7 @@ import {
StandardEditorsRegistryItem,
TransformerCategory,
} from '@grafana/data';
import { InlineField, InlineFieldRow, Select, InlineSwitch } from '@grafana/ui';
import { InlineField, InlineFieldRow, Select, InlineSwitch, Input } from '@grafana/ui';
import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
import { getTransformationContent } from '../docs/getTransformationContent';
@@ -53,6 +55,13 @@ export const extractFieldsTransformerEditor = ({
});
};
const onRegexpChange = (e: ChangeEvent<HTMLInputElement>) => {
onChange({
...options,
regExp: e.target.value,
});
};
const onToggleReplace = () => {
if (options.replace) {
options.keepTime = false;
@@ -96,7 +105,16 @@ export const extractFieldsTransformerEditor = ({
/>
</InlineField>
</InlineFieldRow>
{options.format === 'json' && <JSONPathEditor options={options.jsonPaths ?? []} onChange={onJSONPathsChange} />}
{options.format === FieldExtractorID.RegExp && (
<InlineFieldRow>
<InlineField label="RegExp" labelWidth={16} interactive={true} tooltip="Example: /(?<NewField>.*)/">
<Input placeholder="/(?<NewField>.*)/" value={options.regExp} onChange={onRegexpChange} />
</InlineField>
</InlineFieldRow>
)}
{options.format === FieldExtractorID.JSON && (
<JSONPathEditor options={options.jsonPaths ?? []} onChange={onJSONPathsChange} />
)}
<InlineFieldRow>
<InlineField label={'Replace all fields'} labelWidth={16}>
<InlineSwitch value={options.replace ?? false} onChange={onToggleReplace} />
public/app/features/transformers/extractFields/extractFields.ts
+2 -1
View File
@@ -52,13 +52,14 @@ export function addExtractedFields(frame: DataFrame, options: ExtractFieldsOptio
const count = frame.length;
const names: string[] = []; // keep order
const values = new Map<string, any[]>();
const parse = ext.getParser(options);
for (let i = 0; i < count; i++) {
let obj = source.values[i];
if (isString(obj)) {
try {
obj = ext.parse(obj);
obj = parse(obj);
} catch {
obj = {}; // empty
}
public/app/features/transformers/extractFields/fieldExtractor.test.ts
+23 -9
View File
@@ -3,12 +3,12 @@ import { config } from '@grafana/runtime';
import { addExtractedFields } from './extractFields';
import { fieldExtractors } from './fieldExtractors';
import { FieldExtractorID } from './types';
import { ExtractFieldsOptions, FieldExtractorID } from './types';
describe('Extract fields from text', () => {
it('JSON extractor', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.JSON);
const out = extractor.parse('{"a":"148.1672","av":41923755,"c":148.25}');
const out = extractor.getParser({})('{"a":"148.1672","av":41923755,"c":148.25}');
expect(out).toMatchInlineSnapshot(`
{
@@ -21,7 +21,7 @@ describe('Extract fields from text', () => {
it('Test key-values with single/double quotes', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse('a="1", "b"=\'2\',c=3 x:y ;\r\nz="d and 4"');
const out = extractor.getParser({})('a="1", "b"=\'2\',c=3 x:y ;\r\nz="d and 4"');
expect(out).toMatchInlineSnapshot(`
{
"a": "1",
@@ -35,7 +35,7 @@ describe('Extract fields from text', () => {
it('Test key-values with nested single/double quotes', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse(
const out = extractor.getParser({})(
`a="1", "b"=\'2\',c=3 x:y ;\r\nz="dbl_quotes=\\"Double Quotes\\" sgl_quotes='Single Quotes'"`
);
@@ -52,7 +52,7 @@ describe('Extract fields from text', () => {
it('Test key-values with nested separator characters', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse(`a="1", "b"=\'2\',c=3 x:y ;\r\nz="This is; testing& validating, 1=:2"`);
const out = extractor.getParser({})(`a="1", "b"=\'2\',c=3 x:y ;\r\nz="This is; testing& validating, 1=:2"`);
expect(out).toMatchInlineSnapshot(`
{
@@ -67,7 +67,7 @@ describe('Extract fields from text', () => {
it('Test key-values where some values are null', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse(`a=, "b"=\'2\',c=3 x: `);
const out = extractor.getParser({})(`a=, "b"=\'2\',c=3 x: `);
expect(out).toMatchInlineSnapshot(`
{
@@ -81,7 +81,7 @@ describe('Extract fields from text', () => {
it('Split key+values', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse('a="1", "b"=\'2\',c=3 x:y ;\r\nz="7"');
const out = extractor.getParser({})('a="1", "b"=\'2\',c=3 x:y ;\r\nz="7"');
expect(out).toMatchInlineSnapshot(`
{
"a": "1",
@@ -95,7 +95,7 @@ describe('Extract fields from text', () => {
it('Split URL style parameters', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse('a=b&c=d&x=123');
const out = extractor.getParser({})('a=b&c=d&x=123');
expect(out).toMatchInlineSnapshot(`
{
"a": "b",
@@ -107,7 +107,7 @@ describe('Extract fields from text', () => {
it('Prometheus labels style (not really supported)', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
const out = extractor.parse('{foo="bar", baz="42"}');
const out = extractor.getParser({})('{foo="bar", baz="42"}');
expect(out).toMatchInlineSnapshot(`
{
"baz": "42",
@@ -137,4 +137,18 @@ describe('Extract fields from text', () => {
expect(newFrame.fields.length).toBe(2);
expect(newFrame.fields[1].name).toBe('bar');
});
it('splits by regexp', async () => {
const extractor = fieldExtractors.get(FieldExtractorID.RegExp);
const opts: ExtractFieldsOptions = { regExp: '/^(?<FieldA>\\w+)[^\\w]+(?<FieldB>\\w+)$/' };
const parse = extractor.getParser(opts);
const out = parse('abc - re30z');
expect(out).toMatchInlineSnapshot(`
{
"FieldA": "abc",
"FieldB": "re30z",
}
`);
});
});
public/app/features/transformers/extractFields/fieldExtractors.ts
+43 -16
View File
@@ -1,20 +1,43 @@
import { Registry, RegistryItem } from '@grafana/data';
import { Registry, RegistryItem, stringStartsAsRegEx, stringToJsRegex } from '@grafana/data';
import { FieldExtractorID } from './types';
import { ExtractFieldsOptions, FieldExtractorID } from './types';
type Parser = (v: string) => Record<string, any> | undefined;
export interface FieldExtractor extends RegistryItem {
parse: (v: string) => Record<string, any> | undefined;
getParser: (opts: ExtractFieldsOptions) => Parser;
}
const extJSON: FieldExtractor = {
id: FieldExtractorID.JSON,
name: 'JSON',
description: 'Parse JSON string',
parse: (v: string) => {
getParser: (options) => (v: string) => {
return JSON.parse(v);
},
};
const extRegExp: FieldExtractor = {
id: FieldExtractorID.RegExp,
name: 'RegExp',
description: 'Parse with RegExp',
getParser: (options) => {
let regex: RegExp | null = /(?<NewField>.*)/;
if (stringStartsAsRegEx(options.regExp!)) {
try {
regex = stringToJsRegex(options.regExp!);
} catch (error) {
if (error instanceof Error) {
console.warn(error.message);
}
}
}
return (v: string) => v.match(regex)?.groups;
},
};
function parseKeyValuePairs(raw: string): Record<string, string> {
const buff: string[] = []; // array of characters
let esc = '';
@@ -107,25 +130,29 @@ const extLabels: FieldExtractor = {
id: FieldExtractorID.KeyValues,
name: 'Key+value pairs',
description: 'Look for a=b, c: d values in the line',
parse: parseKeyValuePairs,
getParser: (options) => parseKeyValuePairs,
};
const fmts = [extJSON, extLabels];
const fmts = [extJSON, extLabels, extRegExp];
const extAuto: FieldExtractor = {
id: FieldExtractorID.Auto,
name: 'Auto',
description: 'parse new fields automatically',
parse: (v: string) => {
for (const f of fmts) {
try {
const r = f.parse(v);
if (r != null) {
return r;
}
} catch {} // ignore errors
}
return undefined;
getParser: (options) => {
const parsers = fmts.map((fmt) => fmt.getParser(options));
return (v: string) => {
for (const parse of parsers) {
try {
const r = parse(v);
if (r != null) {
return r;
}
} catch {} // ignore errors
}
return undefined;
};
},
};
public/app/features/transformers/extractFields/types.ts
+2
View File
@@ -2,6 +2,7 @@ export enum FieldExtractorID {
JSON = 'json',
KeyValues = 'kvp',
Auto = 'auto',
RegExp = 'regexp',
}
export interface JSONPath {
@@ -11,6 +12,7 @@ export interface JSONPath {
export interface ExtractFieldsOptions {
source?: string;
jsonPaths?: JSONPath[];
regExp?: string;
format?: FieldExtractorID;
replace?: boolean;
keepTime?: boolean;