From d0eeff2fa0a65a1bcb4735f57f3d37291c978df4 Mon Sep 17 00:00:00 2001 From: Sven Grossmann Date: Tue, 6 Dec 2022 21:54:20 +0100 Subject: [PATCH] Loki: Add tracking of executed queries (#59887) * add query tracking * add app * add comment * use `reportInteraction` not `console.log` * add test to `queryUtils` * organize imports * add datasource tests * change `metrics` to `metric` Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * change `parseToArray` to `parseToNodeNamesArray` Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --- .../datasource/loki/datasource.test.ts | 32 +++++++++++++++-- .../app/plugins/datasource/loki/datasource.ts | 29 ++++++++-------- .../datasource/loki/queryUtils.test.ts | 34 +++++++++++++++++++ .../app/plugins/datasource/loki/queryUtils.ts | 11 ++++++ .../app/plugins/datasource/loki/tracking.ts | 22 ++++++++++-- 5 files changed, 110 insertions(+), 18 deletions(-) diff --git a/public/app/plugins/datasource/loki/datasource.test.ts b/public/app/plugins/datasource/loki/datasource.test.ts index 0b9046ce348..4598d5ac948 100644 --- a/public/app/plugins/datasource/loki/datasource.test.ts +++ b/public/app/plugins/datasource/loki/datasource.test.ts @@ -15,17 +15,32 @@ import { LogRowModel, MutableDataFrame, } from '@grafana/data'; -import { BackendSrvRequest, FetchResponse, setBackendSrv, getBackendSrv, BackendSrv } from '@grafana/runtime'; +import { + BackendSrv, + BackendSrvRequest, + FetchResponse, + getBackendSrv, + reportInteraction, + setBackendSrv, +} from '@grafana/runtime'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { initialCustomVariableModelState } from '../../../features/variables/custom/reducer'; import { CustomVariableModel } from '../../../features/variables/types'; import { LokiDatasource } from './datasource'; -import { createMetadataRequest, createLokiDatasource } from './mocks'; +import { createLokiDatasource, createMetadataRequest } from './mocks'; +import { parseToNodeNamesArray } from './queryUtils'; import { LokiOptions, LokiQuery, LokiQueryType, LokiVariableQueryType } from './types'; import { LokiVariableSupport } from './variables'; +jest.mock('@grafana/runtime', () => { + return { + ...jest.requireActual('@grafana/runtime'), + reportInteraction: jest.fn(), + }; +}); + const templateSrvStub = { getAdhocFilters: jest.fn(() => [] as unknown[]), replace: jest.fn((a: string, ...rest: unknown[]) => a), @@ -112,6 +127,7 @@ describe('LokiDatasource', () => { afterEach(() => { setBackendSrv(origBackendSrv); + (reportInteraction as jest.Mock).mockClear(); }); describe('when doing logs queries with limits', () => { @@ -156,6 +172,18 @@ describe('LokiDatasource', () => { it('should use query max lines, if both exist, even if it is higher than ds max lines', async () => { await runTest(80, '40', 80); }); + + it('should report query interaction', async () => { + await runTest(80, '40', 80); + expect(reportInteraction).toHaveBeenCalledWith( + 'grafana_loki_query_executed', + expect.objectContaining({ + query_type: 'logs', + line_limit: 80, + parsed_query: parseToNodeNamesArray('{a="b"}').join(','), + }) + ); + }); }); describe('When using adhoc filters', () => { diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index c323694fa5d..debd615c4b3 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -1,8 +1,9 @@ import { cloneDeep, map as lodashMap } from 'lodash'; import { lastValueFrom, merge, Observable, of, throwError } from 'rxjs'; -import { catchError, map, switchMap } from 'rxjs/operators'; +import { catchError, map, switchMap, tap } from 'rxjs/operators'; import { + AbstractQuery, AnnotationEvent, AnnotationQueryRequest, CoreApp, @@ -19,21 +20,20 @@ import { dateMath, DateTime, FieldCache, - AbstractQuery, FieldType, + getDefaultTimeRange, Labels, LoadingState, LogLevel, LogRowModel, + QueryFixAction, + QueryHint, + rangeUtil, ScopedVars, TimeRange, - rangeUtil, toUtc, - QueryHint, - getDefaultTimeRange, - QueryFixAction, } from '@grafana/data'; -import { FetchError, config, DataSourceWithBackend } from '@grafana/runtime'; +import { config, DataSourceWithBackend, FetchError } from '@grafana/runtime'; import { queryLogsVolume } from 'app/core/logsModel'; import { convertToWebSocketUrl } from 'app/core/utils/explore'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; @@ -62,6 +62,7 @@ import { getQueryHints } from './queryHints'; import { getNormalizedLokiQuery, isLogsQuery, isValidQuery } from './queryUtils'; import { sortDataFrameByTime } from './sortDataFrame'; import { doLokiChannelStream } from './streaming'; +import { trackQuery } from './tracking'; import { LokiOptions, LokiQuery, @@ -191,13 +192,13 @@ export class LokiDatasource if (fixedRequest.liveStreaming) { return this.runLiveQueryThroughBackend(fixedRequest); } else { - return super - .query(fixedRequest) - .pipe( - map((response) => - transformBackendResult(response, fixedRequest.targets, this.instanceSettings.jsonData.derivedFields ?? []) - ) - ); + return super.query(fixedRequest).pipe( + // in case of an empty query, this is somehow run twice. `share()` is no workaround here as the observable is generated from `of()`. + map((response) => + transformBackendResult(response, fixedRequest.targets, this.instanceSettings.jsonData.derivedFields ?? []) + ), + tap((response) => trackQuery(response, fixedRequest.targets, fixedRequest.app)) + ); } } diff --git a/public/app/plugins/datasource/loki/queryUtils.test.ts b/public/app/plugins/datasource/loki/queryUtils.test.ts index 8650682cabd..449d30d5ae2 100644 --- a/public/app/plugins/datasource/loki/queryUtils.test.ts +++ b/public/app/plugins/datasource/loki/queryUtils.test.ts @@ -5,6 +5,7 @@ import { isQueryWithLabelFormat, isQueryWithParser, isValidQuery, + parseToNodeNamesArray, } from './queryUtils'; import { LokiQuery, LokiQueryType } from './types'; @@ -166,6 +167,39 @@ describe('isValidQuery', () => { }); }); +describe('parseToArray', () => { + it('returns on empty query', () => { + expect(parseToNodeNamesArray('{}')).toEqual(['LogQL', 'Expr', 'LogExpr', 'Selector', '⚠']); + }); + it('returns on invalid query', () => { + expect(parseToNodeNamesArray('{job="grafana"')).toEqual([ + 'LogQL', + 'Expr', + 'LogExpr', + 'Selector', + 'Matchers', + 'Matcher', + 'Identifier', + 'Eq', + 'String', + '⚠', + ]); + }); + it('returns on valid query', () => { + expect(parseToNodeNamesArray('{job="grafana"}')).toEqual([ + 'LogQL', + 'Expr', + 'LogExpr', + 'Selector', + 'Matchers', + 'Matcher', + 'Identifier', + 'Eq', + 'String', + ]); + }); +}); + describe('isLogsQuery', () => { it('returns false if metrics query', () => { expect(isLogsQuery('rate({job="grafana"}[5m])')).toBe(false); diff --git a/public/app/plugins/datasource/loki/queryUtils.ts b/public/app/plugins/datasource/loki/queryUtils.ts index 93613f3f2df..6b5c69777f0 100644 --- a/public/app/plugins/datasource/loki/queryUtils.ts +++ b/public/app/plugins/datasource/loki/queryUtils.ts @@ -108,6 +108,17 @@ export function getNormalizedLokiQuery(query: LokiQuery): LokiQuery { return { ...rest, queryType: LokiQueryType.Range }; } +export function parseToNodeNamesArray(query: string): string[] { + const queryParts: string[] = []; + const tree = parser.parse(query); + tree.iterate({ + enter: ({ name }): false | void => { + queryParts.push(name); + }, + }); + return queryParts; +} + export function isValidQuery(query: string): boolean { let isValid = true; const tree = parser.parse(query); diff --git a/public/app/plugins/datasource/loki/tracking.ts b/public/app/plugins/datasource/loki/tracking.ts index 99f32fa4a27..18fdb94d1e4 100644 --- a/public/app/plugins/datasource/loki/tracking.ts +++ b/public/app/plugins/datasource/loki/tracking.ts @@ -1,11 +1,11 @@ -import { DashboardLoadedEvent } from '@grafana/data'; +import { DashboardLoadedEvent, DataQueryResponse } from '@grafana/data'; import { reportInteraction } from '@grafana/runtime'; import { variableRegex } from 'app/features/variables/utils'; import { QueryEditorMode } from '../prometheus/querybuilder/shared/types'; import pluginJson from './plugin.json'; -import { getNormalizedLokiQuery, isLogsQuery } from './queryUtils'; +import { getNormalizedLokiQuery, isLogsQuery, parseToNodeNamesArray } from './queryUtils'; import { LokiQuery, LokiQueryType } from './types'; type LokiOnDashboardLoadedTrackingEvent = { @@ -116,3 +116,21 @@ const isQueryWithChangedLegend = (query: LokiQuery): boolean => { } return query.legendFormat !== ''; }; + +export function trackQuery(response: DataQueryResponse, queries: LokiQuery[], app: string): void { + for (const query of queries) { + reportInteraction('grafana_loki_query_executed', { + app, + editor_mode: query.editorMode, + has_data: response.data.some((frame) => frame.length > 0), + has_error: response.error !== undefined, + legend: query.legendFormat, + line_limit: query.maxLines, + parsed_query: parseToNodeNamesArray(query.expr).join(','), + query_type: isLogsQuery(query.expr) ? 'logs' : 'metric', + query_vector_type: query.queryType, + resolution: query.resolution, + simultaneously_sent_query_count: queries.length, + }); + } +}