From 270e38cfcf335add73d0ff709c87943e822c3082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Wed, 11 May 2022 09:29:04 +0200 Subject: [PATCH] loki: backend-mode: handle derived fields (#48873) --- .../loki/backendResultTransformer.test.ts | 67 ++++++++++++++-- .../loki/backendResultTransformer.ts | 32 ++++++-- .../app/plugins/datasource/loki/datasource.ts | 6 +- .../datasource/loki/getDerivedFields.test.ts | 62 +++++++++++++++ .../datasource/loki/getDerivedFields.ts | 77 +++++++++++++++++++ 5 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 public/app/plugins/datasource/loki/getDerivedFields.test.ts create mode 100644 public/app/plugins/datasource/loki/getDerivedFields.ts diff --git a/public/app/plugins/datasource/loki/backendResultTransformer.test.ts b/public/app/plugins/datasource/loki/backendResultTransformer.test.ts index 89ee35f8dfd..1b713005141 100644 --- a/public/app/plugins/datasource/loki/backendResultTransformer.test.ts +++ b/public/app/plugins/datasource/loki/backendResultTransformer.test.ts @@ -1,9 +1,22 @@ import { cloneDeep } from 'lodash'; -import { ArrayVector, DataFrame, DataQueryResponse, FieldType } from '@grafana/data'; +import { ArrayVector, DataFrame, DataQueryResponse, Field, FieldType } from '@grafana/data'; import { transformBackendResult } from './backendResultTransformer'; +// needed because the derived-fields functionality calls it +jest.mock('@grafana/runtime', () => ({ + // @ts-ignore + ...jest.requireActual('@grafana/runtime'), + getDataSourceSrv: () => { + return { + getInstanceSettings: () => { + return { name: 'Loki1' }; + }, + }; + }, +})); + const LOKI_EXPR = '{level="info"} |= "thing1"'; const inputFrame: DataFrame = { refId: 'A', @@ -65,12 +78,52 @@ describe('loki backendResultTransformer', () => { const expected: DataQueryResponse = { data: [expectedFrame] }; - const result = transformBackendResult(response, [ - { - refId: 'A', - expr: LOKI_EXPR, - }, - ]); + const result = transformBackendResult( + response, + [ + { + refId: 'A', + expr: LOKI_EXPR, + }, + ], + [] + ); expect(result).toEqual(expected); }); + + it('processed derived fields correctly', () => { + const input: DataFrame = { + length: 1, + fields: [ + { + name: 'time', + config: {}, + values: new ArrayVector([1]), + type: FieldType.time, + }, + { + name: 'line', + config: {}, + values: new ArrayVector(['line1']), + type: FieldType.string, + }, + ], + }; + const response: DataQueryResponse = { data: [input] }; + const result = transformBackendResult( + response, + [{ refId: 'A', expr: '' }], + [ + { + matcherRegex: 'trace=(w+)', + name: 'derived1', + url: 'example.com', + }, + ] + ); + + expect( + result.data[0].fields.filter((field: Field) => field.name === 'derived1' && field.type === 'string').length + ).toBe(1); + }); }); diff --git a/public/app/plugins/datasource/loki/backendResultTransformer.ts b/public/app/plugins/datasource/loki/backendResultTransformer.ts index d2d8004f6f2..93504c1caef 100644 --- a/public/app/plugins/datasource/loki/backendResultTransformer.ts +++ b/public/app/plugins/datasource/loki/backendResultTransformer.ts @@ -1,8 +1,9 @@ import { DataQueryResponse, DataFrame, isDataFrame, FieldType, QueryResultMeta } from '@grafana/data'; +import { getDerivedFields } from './getDerivedFields'; import { makeTableFrames } from './makeTableFrames'; import { formatQuery, getHighlighterExpressionsFromQuery } from './query_utils'; -import { LokiQuery, LokiQueryType } from './types'; +import { DerivedFieldConfig, LokiQuery, LokiQueryType } from './types'; function isMetricFrame(frame: DataFrame): boolean { return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number); @@ -19,7 +20,11 @@ function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame { }; } -function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): DataFrame { +function processStreamFrame( + frame: DataFrame, + query: LokiQuery | undefined, + derivedFieldConfigs: DerivedFieldConfig[] +): DataFrame { const meta: QueryResultMeta = { preferredVisualisationType: 'logs', searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(formatQuery(query.expr)) : undefined, @@ -29,13 +34,22 @@ function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): Dat }, }; - return setFrameMeta(frame, meta); + const newFrame = setFrameMeta(frame, meta); + const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs); + return { + ...newFrame, + fields: [...newFrame.fields, ...derivedFields], + }; } -function processStreamsFrames(frames: DataFrame[], queryMap: Map): DataFrame[] { +function processStreamsFrames( + frames: DataFrame[], + queryMap: Map, + derivedFieldConfigs: DerivedFieldConfig[] +): DataFrame[] { return frames.map((frame) => { const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined; - return processStreamFrame(frame, query); + return processStreamFrame(frame, query, derivedFieldConfigs); }); } @@ -78,7 +92,11 @@ function groupFrames( return { streamsFrames, metricInstantFrames, metricRangeFrames }; } -export function transformBackendResult(response: DataQueryResponse, queries: LokiQuery[]): DataQueryResponse { +export function transformBackendResult( + response: DataQueryResponse, + queries: LokiQuery[], + derivedFieldConfigs: DerivedFieldConfig[] +): DataQueryResponse { const { data, ...rest } = response; // in the typescript type, data is an array of basically anything. @@ -100,7 +118,7 @@ export function transformBackendResult(response: DataQueryResponse, queries: Lok data: [ ...processMetricRangeFrames(metricRangeFrames), ...processMetricInstantFrames(metricInstantFrames), - ...processStreamsFrames(streamsFrames, queryMap), + ...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs), ], }; } diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index a1d220b9394..07265aa5c90 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -180,7 +180,11 @@ export class LokiDatasource } else { return super .query(fixedRequest) - .pipe(map((response) => transformBackendResult(response, fixedRequest.targets))); + .pipe( + map((response) => + transformBackendResult(response, fixedRequest.targets, this.instanceSettings.jsonData.derivedFields ?? []) + ) + ); } } diff --git a/public/app/plugins/datasource/loki/getDerivedFields.test.ts b/public/app/plugins/datasource/loki/getDerivedFields.test.ts new file mode 100644 index 00000000000..6853c097c38 --- /dev/null +++ b/public/app/plugins/datasource/loki/getDerivedFields.test.ts @@ -0,0 +1,62 @@ +import { MutableDataFrame } from '@grafana/data'; + +import { getDerivedFields } from './getDerivedFields'; + +jest.mock('@grafana/runtime', () => ({ + // @ts-ignore + ...jest.requireActual('@grafana/runtime'), + getDataSourceSrv: () => { + return { + getInstanceSettings: () => { + return { name: 'Loki1' }; + }, + }; + }, +})); + +describe('getDerivedFields', () => { + it('adds links to fields', () => { + const df = new MutableDataFrame({ fields: [{ name: 'line', values: ['nothing', 'trace1=1234', 'trace2=foo'] }] }); + const newFields = getDerivedFields(df, [ + { + matcherRegex: 'trace1=(\\w+)', + name: 'trace1', + url: 'http://localhost/${__value.raw}', + }, + { + matcherRegex: 'trace2=(\\w+)', + name: 'trace2', + url: 'test', + datasourceUid: 'uid', + }, + { + matcherRegex: 'trace2=(\\w+)', + name: 'trace2', + url: 'test', + datasourceUid: 'uid2', + urlDisplayLabel: 'Custom Label', + }, + ]); + expect(newFields.length).toBe(2); + const trace1 = newFields.find((f) => f.name === 'trace1'); + expect(trace1!.values.toArray()).toEqual([null, '1234', null]); + expect(trace1!.config.links![0]).toEqual({ + url: 'http://localhost/${__value.raw}', + title: '', + }); + + const trace2 = newFields.find((f) => f.name === 'trace2'); + expect(trace2!.values.toArray()).toEqual([null, null, 'foo']); + expect(trace2!.config.links!.length).toBe(2); + expect(trace2!.config.links![0]).toEqual({ + title: '', + internal: { datasourceName: 'Loki1', datasourceUid: 'uid', query: { query: 'test' } }, + url: '', + }); + expect(trace2!.config.links![1]).toEqual({ + title: 'Custom Label', + internal: { datasourceName: 'Loki1', datasourceUid: 'uid2', query: { query: 'test' } }, + url: '', + }); + }); +}); diff --git a/public/app/plugins/datasource/loki/getDerivedFields.ts b/public/app/plugins/datasource/loki/getDerivedFields.ts new file mode 100644 index 00000000000..5c0a465c051 --- /dev/null +++ b/public/app/plugins/datasource/loki/getDerivedFields.ts @@ -0,0 +1,77 @@ +import { groupBy } from 'lodash'; + +import { FieldType, DataFrame, ArrayVector, DataLink, Field } from '@grafana/data'; +import { getDataSourceSrv } from '@grafana/runtime'; + +import { DerivedFieldConfig } from './types'; + +export function getDerivedFields(dataFrame: DataFrame, derivedFieldConfigs: DerivedFieldConfig[]): Field[] { + if (!derivedFieldConfigs.length) { + return []; + } + const derivedFieldsGrouped = groupBy(derivedFieldConfigs, 'name'); + + const newFields = Object.values(derivedFieldsGrouped).map(fieldFromDerivedFieldConfig); + + // line-field is the first string-field + // NOTE: we should create some common log-frame-extra-string-field code somewhere + const lineField = dataFrame.fields.find((f) => f.type === FieldType.string); + + if (lineField === undefined) { + // if this is happening, something went wrong, let's raise an error + throw new Error('invalid logs-dataframe, string-field missing'); + } + + lineField.values.toArray().forEach((line) => { + for (const field of newFields) { + const logMatch = line.match(derivedFieldsGrouped[field.name][0].matcherRegex); + field.values.add(logMatch && logMatch[1]); + } + }); + + return newFields; +} + +/** + * Transform derivedField config into dataframe field with config that contains link. + */ +function fieldFromDerivedFieldConfig(derivedFieldConfigs: DerivedFieldConfig[]): Field { + const dataSourceSrv = getDataSourceSrv(); + + const dataLinks = derivedFieldConfigs.reduce((acc, derivedFieldConfig) => { + // Having field.datasourceUid means it is an internal link. + if (derivedFieldConfig.datasourceUid) { + const dsSettings = dataSourceSrv.getInstanceSettings(derivedFieldConfig.datasourceUid); + + acc.push({ + // Will be filled out later + title: derivedFieldConfig.urlDisplayLabel || '', + url: '', + // This is hardcoded for Jaeger or Zipkin not way right now to specify datasource specific query object + internal: { + query: { query: derivedFieldConfig.url }, + datasourceUid: derivedFieldConfig.datasourceUid, + datasourceName: dsSettings?.name ?? 'Data source not found', + }, + }); + } else if (derivedFieldConfig.url) { + acc.push({ + // We do not know what title to give here so we count on presentation layer to create a title from metadata. + title: derivedFieldConfig.urlDisplayLabel || '', + // This is hardcoded for Jaeger or Zipkin not way right now to specify datasource specific query object + url: derivedFieldConfig.url, + }); + } + return acc; + }, [] as DataLink[]); + + return { + name: derivedFieldConfigs[0].name, + type: FieldType.string, + config: { + links: dataLinks, + }, + // We are adding values later on + values: new ArrayVector([]), + }; +}