From 8f7b980c95ae047de6e4f030b8ac803c25070e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Mon, 20 Feb 2023 14:55:56 +0100 Subject: [PATCH] Loki: query splitting: handle stats (#63436) * loki: query splitting: handle stats * update tests --- public/app/plugins/datasource/loki/mocks.ts | 26 ++++++-- .../datasource/loki/queryUtils.test.ts | 65 +++++++++++++++++-- .../app/plugins/datasource/loki/queryUtils.ts | 41 +++++++----- 3 files changed, 103 insertions(+), 29 deletions(-) diff --git a/public/app/plugins/datasource/loki/mocks.ts b/public/app/plugins/datasource/loki/mocks.ts index e84f1f6d257..c9241709161 100644 --- a/public/app/plugins/datasource/loki/mocks.ts +++ b/public/app/plugins/datasource/loki/mocks.ts @@ -150,6 +150,12 @@ export function getMockFrames() { values: new ArrayVector(['id1', 'id2']), }, ], + meta: { + stats: [ + { displayName: 'Summary: total bytes processed', value: 11 }, + { displayName: 'Ingester: total reached', value: 1 }, + ], + }, length: 2, }; @@ -192,7 +198,10 @@ export function getMockFrames() { }, ], meta: { - stats: [{ displayName: 'Ingester: total reached', value: 1 }], + stats: [ + { displayName: 'Summary: total bytes processed', value: 22 }, + { displayName: 'Ingester: total reached', value: 2 }, + ], }, length: 2, }; @@ -214,7 +223,10 @@ export function getMockFrames() { }, ], meta: { - stats: [{ displayName: 'Ingester: total reached', value: 1 }], + stats: [ + { displayName: 'Ingester: total reached', value: 1 }, + { displayName: 'Summary: total bytes processed', value: 11 }, + ], }, length: 2, }; @@ -236,7 +248,10 @@ export function getMockFrames() { }, ], meta: { - stats: [{ displayName: 'Ingester: total reached', value: 2 }], + stats: [ + { displayName: 'Ingester: total reached', value: 2 }, + { displayName: 'Summary: total bytes processed', value: 22 }, + ], }, length: 2, }; @@ -259,7 +274,10 @@ export function getMockFrames() { }, ], meta: { - stats: [{ displayName: 'Ingester: total reached', value: 2 }], + stats: [ + { displayName: 'Ingester: total reached', value: 2 }, + { displayName: 'Summary: total bytes processed', value: 33 }, + ], }, length: 2, }; diff --git a/public/app/plugins/datasource/loki/queryUtils.test.ts b/public/app/plugins/datasource/loki/queryUtils.test.ts index 17faea4fa81..25cc9577c2e 100644 --- a/public/app/plugins/datasource/loki/queryUtils.test.ts +++ b/public/app/plugins/datasource/loki/queryUtils.test.ts @@ -1,4 +1,4 @@ -import { ArrayVector, DataQueryResponse } from '@grafana/data'; +import { ArrayVector, DataQueryResponse, QueryResultMetaStat } from '@grafana/data'; import { getMockFrames } from './mocks'; import { @@ -368,8 +368,8 @@ describe('combineResponses', () => { meta: { stats: [ { - displayName: 'Ingester: total reached', - value: 1, + displayName: 'Summary: total bytes processed', + value: 33, }, ], }, @@ -408,8 +408,8 @@ describe('combineResponses', () => { meta: { stats: [ { - displayName: 'Ingester: total reached', - value: 3, + displayName: 'Summary: total bytes processed', + value: 33, }, ], }, @@ -448,8 +448,8 @@ describe('combineResponses', () => { meta: { stats: [ { - displayName: 'Ingester: total reached', - value: 3, + displayName: 'Summary: total bytes processed', + value: 33, }, ], }, @@ -471,4 +471,55 @@ describe('combineResponses', () => { expect(combineResponses(null, responseA)).not.toBe(responseA); expect(combineResponses(null, responseB)).not.toBe(responseB); }); + + describe('combine stats', () => { + const { metricFrameA } = getMockFrames(); + const makeResponse = (stats?: QueryResultMetaStat[]): DataQueryResponse => ({ + data: [ + { + ...metricFrameA, + meta: { + ...metricFrameA.meta, + stats, + }, + }, + ], + }); + it('two values', () => { + const responseA = makeResponse([ + { displayName: 'Ingester: total reached', value: 1 }, + { displayName: 'Summary: total bytes processed', value: 11 }, + ]); + const responseB = makeResponse([ + { displayName: 'Ingester: total reached', value: 2 }, + { displayName: 'Summary: total bytes processed', value: 22 }, + ]); + + expect(combineResponses(responseA, responseB).data[0].meta.stats).toStrictEqual([ + { displayName: 'Summary: total bytes processed', value: 33 }, + ]); + }); + + it('one value', () => { + const responseA = makeResponse([ + { displayName: 'Ingester: total reached', value: 1 }, + { displayName: 'Summary: total bytes processed', value: 11 }, + ]); + const responseB = makeResponse(); + + expect(combineResponses(responseA, responseB).data[0].meta.stats).toStrictEqual([ + { displayName: 'Summary: total bytes processed', value: 11 }, + ]); + + expect(combineResponses(responseB, responseA).data[0].meta.stats).toStrictEqual([ + { displayName: 'Summary: total bytes processed', value: 11 }, + ]); + }); + + it('no value', () => { + const responseA = makeResponse(); + const responseB = makeResponse(); + expect(combineResponses(responseA, responseB).data[0].meta.stats).toHaveLength(0); + }); + }); }); diff --git a/public/app/plugins/datasource/loki/queryUtils.ts b/public/app/plugins/datasource/loki/queryUtils.ts index aeb152560bb..79d1088e053 100644 --- a/public/app/plugins/datasource/loki/queryUtils.ts +++ b/public/app/plugins/datasource/loki/queryUtils.ts @@ -345,28 +345,33 @@ function combineFrames(dest: DataFrame, source: DataFrame) { ); } dest.length += source.length; - combineMetadata(dest, source); + dest.meta = { + ...dest.meta, + stats: getCombinedMetadataStats(dest.meta?.stats ?? [], source.meta?.stats ?? []), + }; } -function combineMetadata(dest: DataFrame, source: DataFrame) { - if (!source.meta?.stats) { - return; +const TOTAL_BYTES_STAT = 'Summary: total bytes processed'; + +function getCombinedMetadataStats( + destStats: QueryResultMetaStat[], + sourceStats: QueryResultMetaStat[] +): QueryResultMetaStat[] { + // in the current approach, we only handle a single stat + const destStat = destStats.find((s) => s.displayName === TOTAL_BYTES_STAT); + const sourceStat = sourceStats.find((s) => s.displayName === TOTAL_BYTES_STAT); + + if (sourceStat != null && destStat != null) { + return [{ value: sourceStat.value + destStat.value, displayName: TOTAL_BYTES_STAT }]; } - if (!dest.meta?.stats) { - if (!dest.meta) { - dest.meta = {}; - } - Object.assign(dest.meta, { stats: source.meta.stats }); - return; + + // maybe one of them exist + const eitherStat = sourceStat ?? destStat; + if (eitherStat != null) { + return [eitherStat]; } - dest.meta.stats.forEach((destStat: QueryResultMetaStat, i: number) => { - const sourceStat = source.meta?.stats?.find( - (sourceStat: QueryResultMetaStat) => destStat.displayName === sourceStat.displayName - ); - if (sourceStat) { - destStat.value += sourceStat.value; - } - }); + + return []; } /**