From 59c754823f9113e8b6056f8246bea666030c3c20 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 10 May 2021 17:12:19 +0200 Subject: [PATCH] Tempo: Search for Traces by querying Loki directly from Tempo (#33308) * Loki query from Tempo UI - add query type selector to tempo - introduce linkedDatasource concept that runs queries on behalf of another datasource - Tempo uses Loki's query field and Loki's derived fields to find a trace matcher - Tempo uses the trace-to-logs mechanism to determine which dataource is linked Loki data loads successfully via tempo Extracted result transformers Skip null values Show trace on list id click Query type selector Use linked field trace regexp * Review feedback --- .../loki/components/LokiQueryField.tsx | 2 +- .../plugins/datasource/tempo/QueryField.tsx | 139 +++++++++++++--- .../plugins/datasource/tempo/datasource.ts | 143 ++++++++--------- .../tempo/resultTransformer.test.ts | 37 +++++ .../datasource/tempo/resultTransformer.ts | 150 ++++++++++++++++++ 5 files changed, 374 insertions(+), 97 deletions(-) create mode 100644 public/app/plugins/datasource/tempo/resultTransformer.test.ts create mode 100644 public/app/plugins/datasource/tempo/resultTransformer.ts diff --git a/public/app/plugins/datasource/loki/components/LokiQueryField.tsx b/public/app/plugins/datasource/loki/components/LokiQueryField.tsx index bfce16b58d1..f981f96c384 100644 --- a/public/app/plugins/datasource/loki/components/LokiQueryField.tsx +++ b/public/app/plugins/datasource/loki/components/LokiQueryField.tsx @@ -92,7 +92,7 @@ export class LokiQueryField extends React.PureComponent; -export class TempoQueryField extends React.PureComponent { - render() { +const DEFAULT_QUERY_TYPE: TempoQueryType = 'traceId'; +interface State { + linkedDatasource?: DataSourceApi; +} +export class TempoQueryField extends React.PureComponent { + state = { + linkedDatasource: undefined, + }; + linkedQuery: DataQuery; + constructor(props: Props) { + super(props); + this.linkedQuery = { refId: 'linked' }; + } + + async componentDidMount() { + const { datasource } = this.props; + // Find query field from linked datasource + const tracesToLogsOptions: TraceToLogsOptions = datasource.tracesToLogs || {}; + const linkedDatasourceUid = tracesToLogsOptions.datasourceUid; + if (linkedDatasourceUid) { + const dsSrv = getDataSourceSrv(); + const linkedDatasource = await dsSrv.get(linkedDatasourceUid); + this.setState({ + linkedDatasource, + }); + } + } + + onChangeLinkedQuery = (value: DataQuery) => { const { query, onChange } = this.props; + this.linkedQuery = value; + onChange({ + ...query, + linkedQuery: this.linkedQuery, + }); + }; + + onRunLinkedQuery = () => { + this.props.onRunQuery(); + }; + + render() { + const { query, onChange, range } = this.props; + const { linkedDatasource } = this.state; + + const absoluteTimeRange = { from: range!.from!.valueOf(), to: range!.to!.valueOf() }; // Range here is never optional return ( - -
- - onChange({ - ...query, - query: e.currentTarget.value, - }) - } - /> -
- - } - /> + <> + + + + options={[ + { value: 'search', label: 'Search' }, + { value: 'traceId', label: 'TraceID' }, + ]} + value={query.queryType || DEFAULT_QUERY_TYPE} + onChange={(v) => + onChange({ + ...query, + queryType: v, + }) + } + size="md" + /> + + + {query.queryType === 'search' && linkedDatasource && ( + <> + + Tempo uses {((linkedDatasource as unknown) as DataSourceApi).name} to find traces. + + + + + )} + {query.queryType === 'search' && !linkedDatasource && ( +
Please set up a Traces-to-logs datasource in the datasource settings.
+ )} + {query.queryType !== 'search' && ( + +
+ + onChange({ + ...query, + query: e.currentTarget.value, + queryType: 'traceId', + linkedQuery: undefined, + }) + } + /> +
+ + } + /> + )} + ); } } diff --git a/public/app/plugins/datasource/tempo/datasource.ts b/public/app/plugins/datasource/tempo/datasource.ts index 30905380449..4fd53c63f5d 100644 --- a/public/app/plugins/datasource/tempo/datasource.ts +++ b/public/app/plugins/datasource/tempo/datasource.ts @@ -1,51 +1,93 @@ import { - ArrayVector, - DataFrame, DataQuery, DataQueryRequest, DataQueryResponse, + DataSourceApi, DataSourceInstanceSettings, - Field, - FieldType, - MutableDataFrame, } from '@grafana/data'; import { DataSourceWithBackend } from '@grafana/runtime'; -import { Observable } from 'rxjs'; +import { TraceToLogsData, TraceToLogsOptions } from 'app/core/components/TraceToLogsSettings'; +import { getDatasourceSrv } from 'app/features/plugins/datasource_srv'; +import { merge, Observable, throwError } from 'rxjs'; import { map } from 'rxjs/operators'; -import { createGraphFrames } from './graphTransform'; +import { LokiOptions } from '../loki/types'; +import { transformTrace, transformTraceList } from './resultTransformer'; + +export type TempoQueryType = 'search' | 'traceId'; export type TempoQuery = { query: string; + // Query to find list of traces, e.g., via Loki + linkedQuery?: DataQuery; + queryType: TempoQueryType; } & DataQuery; -export class TempoDatasource extends DataSourceWithBackend { - constructor(instanceSettings: DataSourceInstanceSettings) { +export class TempoDatasource extends DataSourceWithBackend { + tracesToLogs: TraceToLogsOptions; + linkedDatasource: DataSourceApi; + constructor(instanceSettings: DataSourceInstanceSettings) { super(instanceSettings); + this.tracesToLogs = instanceSettings.jsonData.tracesToLogs || {}; + if (this.tracesToLogs.datasourceUid) { + this.linkDatasource(); + } + } + + async linkDatasource() { + const dsSrv = getDatasourceSrv(); + this.linkedDatasource = await dsSrv.get(this.tracesToLogs.datasourceUid); } query(options: DataQueryRequest): Observable { - return super.query(options).pipe( - map((response) => { - if (response.error) { - return response; - } - // We need to parse some of the fields which contain stringified json. - // Seems like we can't just map the values as the frame we got from backend has some default processing - // and will stringify the json back when we try to set it. So we create a new field and swap it instead. - const frame: DataFrame = response.data[0]; - - if (!frame) { - return emptyDataQueryResponse; - } - - parseJsonFields(frame); - - return { - ...response, - data: [...response.data, ...createGraphFrames(frame)], - }; - }) + const subQueries: Array> = []; + const filteredTargets = options.targets.filter((target) => !target.hide); + const searchTargets = filteredTargets.filter((target) => target.queryType === 'search'); + const traceTargets = filteredTargets.filter( + (target) => target.queryType === 'traceId' || target.queryType === undefined ); + + // Run search queries on linked datasource + if (this.linkedDatasource && searchTargets.length > 0) { + // Wrap linked query into a data request based on original request + const linkedRequest: DataQueryRequest = { ...options, targets: searchTargets.map((t) => t.linkedQuery!) }; + // Find trace matchers in derived fields of the linked datasource that's identical to this datasource + const settings: DataSourceInstanceSettings = (this.linkedDatasource as any).instanceSettings; + const traceLinkMatcher: string[] = + settings.jsonData.derivedFields + ?.filter((field) => field.datasourceUid === this.uid && field.matcherRegex) + .map((field) => field.matcherRegex) || []; + if (!traceLinkMatcher || traceLinkMatcher.length === 0) { + subQueries.push( + throwError( + 'No Loki datasource configured for search. Set up Derived Fields for traces in a Loki datasource settings and link it to this Tempo datasource.' + ) + ); + } else { + subQueries.push( + (this.linkedDatasource.query(linkedRequest) as Observable).pipe( + map((response) => + response.error ? response : transformTraceList(response, this.uid, this.name, traceLinkMatcher) + ) + ) + ); + } + } + + if (traceTargets.length > 0) { + const traceRequest: DataQueryRequest = { ...options, targets: traceTargets }; + subQueries.push( + super.query(traceRequest).pipe( + map((response) => { + if (response.error) { + return response; + } + return transformTrace(response); + }) + ) + ); + } + + return merge(...subQueries); } async testDatasource(): Promise { @@ -62,44 +104,3 @@ export class TempoDatasource extends DataSourceWithBackend { return query.query; } } - -/** - * Change fields which are json string into JS objects. Modifies the frame in place. - */ -function parseJsonFields(frame: DataFrame) { - for (const fieldName of ['serviceTags', 'logs', 'tags']) { - const field = frame.fields.find((f) => f.name === fieldName); - if (field) { - const fieldIndex = frame.fields.indexOf(field); - const values = new ArrayVector(); - const newField: Field = { - ...field, - values, - type: FieldType.other, - }; - - for (let i = 0; i < field.values.length; i++) { - const value = field.values.get(i); - values.set(i, value === '' ? undefined : JSON.parse(value)); - } - frame.fields[fieldIndex] = newField; - } - } -} - -const emptyDataQueryResponse = { - data: [ - new MutableDataFrame({ - fields: [ - { - name: 'trace', - type: FieldType.trace, - values: [], - }, - ], - meta: { - preferredVisualisationType: 'trace', - }, - }), - ], -}; diff --git a/public/app/plugins/datasource/tempo/resultTransformer.test.ts b/public/app/plugins/datasource/tempo/resultTransformer.test.ts new file mode 100644 index 00000000000..c73b794f18c --- /dev/null +++ b/public/app/plugins/datasource/tempo/resultTransformer.test.ts @@ -0,0 +1,37 @@ +import { FieldType, MutableDataFrame } from '@grafana/data'; +import { createTableFrame } from './resultTransformer'; + +describe('transformTraceList()', () => { + const lokiDataFrame = new MutableDataFrame({ + fields: [ + { + name: 'ts', + type: FieldType.time, + values: ['2020-02-12T15:05:14.265Z', '2020-02-12T15:05:15.265Z', '2020-02-12T15:05:16.265Z'], + }, + { + name: 'line', + type: FieldType.string, + values: [ + 't=2020-02-12T15:04:51+0000 lvl=info msg="Starting Grafana" logger=server', + 't=2020-02-12T15:04:52+0000 lvl=info msg="Starting Grafana" logger=server traceID=asdfa1234', + 't=2020-02-12T15:04:53+0000 lvl=info msg="Starting Grafana" logger=server traceID=asdf88', + ], + }, + ], + meta: { + preferredVisualisationType: 'table', + }, + }); + + test('extracts traceIDs from log lines', () => { + const frame = createTableFrame(lokiDataFrame, 't1', 'tempo', ['traceID=(\\w+)', 'traceID=(\\w\\w)']); + expect(frame.fields[0].name).toBe('Time'); + expect(frame.fields[0].values.get(0)).toBe('2020-02-12T15:05:15.265Z'); + expect(frame.fields[1].name).toBe('traceID'); + expect(frame.fields[1].values.get(0)).toBe('asdfa1234'); + // Second match in new line + expect(frame.fields[0].values.get(1)).toBe('2020-02-12T15:05:15.265Z'); + expect(frame.fields[1].values.get(1)).toBe('as'); + }); +}); diff --git a/public/app/plugins/datasource/tempo/resultTransformer.ts b/public/app/plugins/datasource/tempo/resultTransformer.ts new file mode 100644 index 00000000000..9a0b7e62013 --- /dev/null +++ b/public/app/plugins/datasource/tempo/resultTransformer.ts @@ -0,0 +1,150 @@ +import { DataQueryResponse, ArrayVector, DataFrame, Field, FieldType, MutableDataFrame } from '@grafana/data'; +import { createGraphFrames } from './graphTransform'; + +export function createTableFrame( + logsFrame: DataFrame, + datasourceUid: string, + datasourceName: string, + traceRegexs: string[] +): DataFrame { + const tableFrame = new MutableDataFrame({ + fields: [ + { + name: 'Time', + type: FieldType.time, + }, + { + name: 'traceID', + type: FieldType.string, + config: { + displayNameFromDS: 'Trace ID', + links: [ + { + title: 'Click to open trace ${__value.raw}', + url: '', + internal: { + datasourceUid, + datasourceName, + query: { + query: '${__value.raw}', + }, + }, + }, + ], + }, + }, + { + name: 'Message', + type: FieldType.string, + }, + ], + meta: { + preferredVisualisationType: 'table', + }, + }); + + if (!logsFrame || traceRegexs.length === 0) { + return tableFrame; + } + + const timeField = logsFrame.fields.find((f) => f.type === FieldType.time); + + // Going through all string fields to look for trace IDs + for (let field of logsFrame.fields) { + let hasMatch = false; + if (field.type === FieldType.string) { + const values = field.values.toArray(); + for (let i = 0; i < values.length; i++) { + const line = values[i]; + if (line) { + for (let traceRegex of traceRegexs) { + const match = (line as string).match(traceRegex); + if (match) { + const traceId = match[1]; + const time = timeField ? timeField.values.get(i) : null; + tableFrame.fields[0].values.add(time); + tableFrame.fields[1].values.add(traceId); + tableFrame.fields[2].values.add(line); + hasMatch = true; + } + } + } + } + } + if (hasMatch) { + break; + } + } + + return tableFrame; +} + +export function transformTraceList( + response: DataQueryResponse, + datasourceId: string, + datasourceName: string, + traceRegexs: string[] +): DataQueryResponse { + const frame = createTableFrame(response.data[0], datasourceId, datasourceName, traceRegexs); + response.data[0] = frame; + return response; +} + +export function transformTrace(response: DataQueryResponse): DataQueryResponse { + // We need to parse some of the fields which contain stringified json. + // Seems like we can't just map the values as the frame we got from backend has some default processing + // and will stringify the json back when we try to set it. So we create a new field and swap it instead. + const frame: DataFrame = response.data[0]; + + if (!frame) { + return emptyDataQueryResponse; + } + + parseJsonFields(frame); + + return { + ...response, + data: [...response.data, ...createGraphFrames(frame)], + }; +} + +/** + * Change fields which are json string into JS objects. Modifies the frame in place. + */ +function parseJsonFields(frame: DataFrame) { + for (const fieldName of ['serviceTags', 'logs', 'tags']) { + const field = frame.fields.find((f) => f.name === fieldName); + if (field) { + const fieldIndex = frame.fields.indexOf(field); + const values = new ArrayVector(); + const newField: Field = { + ...field, + values, + type: FieldType.other, + }; + + for (let i = 0; i < field.values.length; i++) { + const value = field.values.get(i); + values.set(i, value === '' ? undefined : JSON.parse(value)); + } + frame.fields[fieldIndex] = newField; + } + } +} + +const emptyDataQueryResponse = { + data: [ + new MutableDataFrame({ + fields: [ + { + name: 'trace', + type: FieldType.trace, + values: [], + }, + ], + meta: { + preferredVisualisationType: 'trace', + }, + }), + ], +};