From 2712686a368a18ec788e601d38c9235ac6e658bd Mon Sep 17 00:00:00 2001 From: Alex Bikfalvi Date: Tue, 11 Mar 2025 13:45:26 +0100 Subject: [PATCH] feat(datasource/Tempo): Instrument Tempo query latency measurements (#101285) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Instrument Tempo query latency measurements Add comprehensive latency tracking and reporting for Tempo queries using reportInteraction: - Add latency measurements for TraceQL metrics queries - Add latency measurements for TraceID queries - Add latency measurements for TraceQL search queries - Track both streaming and non-streaming query performance - Include success/error states and relevant metadata in reports - Measure latency in milliseconds for more precise tracking This instrumentation will help monitor query performance and identify potential bottlenecks in trace queries. Signed-off-by: Alex Bikfalvi * fixup! feat: Instrument Tempo query latency measurements Signed-off-by: Alex Bikfalvi * prettier fix --------- Signed-off-by: Alex Bikfalvi Co-authored-by: André Pereira --- .../plugins/datasource/tempo/datasource.ts | 216 +++++++++++++++++- 1 file changed, 210 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/tempo/datasource.ts b/public/app/plugins/datasource/tempo/datasource.ts index ee8727ce74f..de07cc54e45 100644 --- a/public/app/plugins/datasource/tempo/datasource.ts +++ b/public/app/plugins/datasource/tempo/datasource.ts @@ -1,6 +1,6 @@ import { groupBy } from 'lodash'; import { EMPTY, forkJoin, from, lastValueFrom, merge, Observable, of } from 'rxjs'; -import { catchError, concatMap, map, mergeMap, toArray } from 'rxjs/operators'; +import { catchError, concatMap, finalize, map, mergeMap, toArray } from 'rxjs/operators'; import semver from 'semver'; import { @@ -97,6 +97,16 @@ interface ServiceMapQueryResponseWithRates { edges: DataFrame; } +interface TempoQueryMetrics { + success: boolean; + streaming?: boolean; + latencyMs: number; + query?: string; + error?: string; + statusCode?: number; + statusText?: string; +} + export class TempoDatasource extends DataSourceWithBackend { tracesToLogs?: TraceToLogsOptions; serviceMap?: { @@ -363,8 +373,7 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_traceql_response', options, { + success: true, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: queryFromFilters ?? '', + }); return { data: formatTraceQLResponse( response.data.traces, @@ -442,6 +458,15 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_traceql_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: queryFromFilters ?? '', + error: getErrorMessage(err.message), + statusCode: err.status, + statusText: err.statusText, + }); return of({ error: { message: getErrorMessage(err.data.message) }, data: [] }); }) ) @@ -569,7 +594,11 @@ export class TempoDatasource extends DataSourceWithBackend, targets: TempoQuery[]): Observable { + handleTraceIdQuery( + options: DataQueryRequest, + targets: TempoQuery[], + query: string + ): Observable { const validTargets = targets .filter((t) => t.query) .map((t): TempoQuery => ({ ...t, query: t.query?.trim(), queryType: 'traceId' })); @@ -577,13 +606,41 @@ export class TempoDatasource extends DataSourceWithBackend { if (response.error) { + reportTempoQueryMetrics('grafana_traces_traceID_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(response.error.message), + statusCode: response.error.status, + statusText: response.error.statusText, + }); return response; } + reportTempoQueryMetrics('grafana_traces_traceID_response', options, { + success: true, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + }); return transformTrace(response, this.instanceSettings, this.nodeGraph?.enabled); + }), + catchError((error) => { + reportTempoQueryMetrics('grafana_traces_traceID_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(error.message), + statusCode: error.status, + statusText: error.statusText, + }); + throw error; }) ); } @@ -595,6 +652,7 @@ export class TempoDatasource extends DataSourceWithBackend => { + const startTime = performance.now(); if (this.isStreamingSearchEnabled()) { return this.handleStreamingQuery(options, targets.traceql, queryValue); } else { @@ -606,11 +664,26 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_traceql_response', options, { + success: true, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: queryValue ?? '', + }); return { data: formatTraceQLResponse(response.data.traces, this.instanceSettings, targets.traceql[0].tableType), }; }), catchError((err) => { + reportTempoQueryMetrics('grafana_traces_traceql_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: queryValue ?? '', + error: getErrorMessage(err.message), + statusCode: err.status, + statusText: err.statusText, + }); return of({ error: { message: getErrorMessage(err.data.message) }, data: [] }); }) ); @@ -619,7 +692,8 @@ export class TempoDatasource extends DataSourceWithBackend, - targets: TempoQuery[] + targets: TempoQuery[], + query: string ): Observable { const validTargets = targets .filter((t) => t.query) @@ -630,12 +704,28 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_traceql_metrics_response', options, { + success: true, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + }); return enhanceTraceQlMetricsResponse(response, this.instanceSettings); }), catchError((err) => { + reportTempoQueryMetrics('grafana_traces_traceql_metrics_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(err.data.message), + statusCode: err.status, + statusText: err.statusText, + }); return of({ error: { message: getErrorMessage(err.data.message) }, data: [] }); }) ); @@ -659,6 +749,7 @@ export class TempoDatasource extends DataSourceWithBackend { if (!response.data.summaries) { + reportTempoQueryMetrics('grafana_traces_metrics_summary_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(`No summary data for '${groupBy}'.`), + }); return { error: { message: getErrorMessage(`No summary data for '${groupBy}'.`), @@ -678,6 +776,13 @@ export class TempoDatasource extends DataSourceWithBackend summary.series.length > 0); if (!hasSeries) { + reportTempoQueryMetrics('grafana_traces_metrics_summary_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(`No series data. Ensure you are using an up to date version of Tempo`), + }); return { error: { message: getErrorMessage(`No series data. Ensure you are using an up to date version of Tempo`), @@ -685,11 +790,26 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_metrics_summary_response', options, { + success: false, + streaming: false, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(error.data.message), + statusCode: error.status, + statusText: error.statusText, + }); return of({ error: { message: getErrorMessage(error.data.message) }, data: emptyResponse, @@ -709,6 +829,7 @@ export class TempoDatasource extends DataSourceWithBackend doTempoSearchStreaming( @@ -718,6 +839,28 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_traceql_response', options, { + success: false, + streaming: true, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(error.data.message), + statusCode: error.status, + statusText: error.statusText, + }); + // Re-throw the error to maintain the error chain + throw error; + }), + finalize(() => { + reportTempoQueryMetrics('grafana_traces_traceql_response', options, { + success: true, + streaming: true, + query: query ?? '', + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + }); + }) ); } @@ -732,6 +875,7 @@ export class TempoDatasource extends DataSourceWithBackend doTempoMetricsStreaming( @@ -740,6 +884,28 @@ export class TempoDatasource extends DataSourceWithBackend { + reportTempoQueryMetrics('grafana_traces_traceql_metrics_response', options, { + success: false, + streaming: true, + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + query: query ?? '', + error: getErrorMessage(error.data.message), + statusCode: error.status, + statusText: error.statusText, + }); + // Re-throw the error to maintain the error chain + throw error; + }), + finalize(() => { + reportTempoQueryMetrics('grafana_traces_traceql_metrics_response', options, { + success: true, + streaming: true, + query: query ?? '', + latencyMs: Math.round(performance.now() - startTime), // rounded to nearest millisecond + }); + }) ); } @@ -1442,6 +1608,44 @@ function getServiceGraphViewDataFrames( return df; } +/** + * Reports metrics for Tempo query interactions. + * + * @param options - The data query request options containing app and other context + * @param metrics - Object containing metrics to report: + * - success: Whether the query was successful + * - streaming: (optional) Whether streaming was used + * - latencyMs: Query execution time in milliseconds + * - query: (optional) The query string that was executed + * - error: (optional) Error message if query failed + * - statusCode: (optional) HTTP status code if query failed + * - statusText: (optional) HTTP status text if query failed + * @param interactionName - (optional) Name of the interaction to report. + * Defaults to 'grafana_traces_traceql_response' + * + * @example + * ```typescript + * reportTempoQueryMetrics(options, { + * success: true, + * streaming: true, + * latencyMs: Math.round(performance.now() - startTime), + * query: 'my query' + * }); + * ``` + */ +function reportTempoQueryMetrics( + interactionName: string, + options: DataQueryRequest, + metrics: TempoQueryMetrics +) { + reportInteraction(interactionName, { + datasourceType: 'tempo', + app: options.app ?? '', + grafana_version: config.buildInfo.version, + ...metrics, + }); +} + export function buildExpr( metric: { expr: string; params: string[]; topk?: number }, extraParams: string,