From 207628206453cea212fec0bb399858de61629607 Mon Sep 17 00:00:00 2001 From: Sven Grossmann Date: Fri, 3 Mar 2023 16:02:14 +0100 Subject: [PATCH] Logs: Log samples not being ordered correctly (#64097) * fix log samples being unsorted and multiple results when chunking * use `SortDirection` enum * changed to `sortDataFrame` to support other datasources than loki * update tests * change capitalization --- public/app/core/logsModel.test.ts | 28 +++++++++++-------- public/app/core/logsModel.ts | 9 +++++- .../app/plugins/datasource/loki/datasource.ts | 4 +-- .../datasource/loki/sortDataFrame.test.ts | 6 ++-- .../plugins/datasource/loki/sortDataFrame.ts | 7 +++-- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/public/app/core/logsModel.test.ts b/public/app/core/logsModel.test.ts index fdd947ba63d..5df6f2e1dbb 100644 --- a/public/app/core/logsModel.test.ts +++ b/public/app/core/logsModel.test.ts @@ -15,6 +15,7 @@ import { LogsMetaKind, LogsVolumeType, MutableDataFrame, + sortDataFrame, toDataFrame, } from '@grafana/data'; @@ -1345,14 +1346,14 @@ describe('logs sample', () => { const resultAFrame1 = createFrame([{ app: 'app01' }], [100, 200, 300], ['line 1', 'line 2', 'line 3']); const resultAFrame2 = createFrame( [{ app: 'app01', level: 'error' }], - [100, 200, 300], + [400, 500, 600], ['line 4', 'line 5', 'line 6'] ); - const resultBFrame1 = createFrame([{ app: 'app02' }], [100, 200, 300], ['line A', 'line B', 'line C']); + const resultBFrame1 = createFrame([{ app: 'app02' }], [700, 800, 900], ['line A', 'line B', 'line C']); const resultBFrame2 = createFrame( [{ app: 'app02', level: 'error' }], - [100, 200, 300], + [1000, 1100, 1200], ['line D', 'line E', 'line F'] ); @@ -1362,7 +1363,7 @@ describe('logs sample', () => { data: [resultAFrame1, resultAFrame2], }, { - data: [resultBFrame1, resultBFrame2], + data: [resultBFrame1, resultBFrame2, resultAFrame1, resultAFrame2], }, ]); } @@ -1374,14 +1375,17 @@ describe('logs sample', () => { it('returns data', async () => { setup(setupMultipleResults); await expect(logsSampleProvider).toEmitValuesWith((received) => { - expect(received).toMatchObject([ - { state: LoadingState.Loading, error: undefined, data: [] }, - { - state: LoadingState.Done, - error: undefined, - data: [resultAFrame1, resultAFrame2, resultBFrame1, resultBFrame2], - }, - ]); + expect(received).toContainEqual({ state: LoadingState.Loading, error: undefined, data: [] }); + expect(received).toContainEqual( + expect.objectContaining({ + data: expect.arrayContaining([ + sortDataFrame(resultAFrame1, 0), + sortDataFrame(resultAFrame2, 0), + sortDataFrame(resultBFrame1, 0), + sortDataFrame(resultBFrame2, 0), + ]), + }) + ); }); }); diff --git a/public/app/core/logsModel.ts b/public/app/core/logsModel.ts index 6331c67277c..04177206e3f 100644 --- a/public/app/core/logsModel.ts +++ b/public/app/core/logsModel.ts @@ -18,6 +18,7 @@ import { FieldWithIndex, findCommonLabels, findUniqueLabels, + getTimeField, Labels, LoadingState, LogLevel, @@ -30,6 +31,7 @@ import { MutableDataFrame, rangeUtil, ScopedVars, + sortDataFrame, textUtil, TimeRange, toDataFrame, @@ -41,6 +43,7 @@ import { ansicolor, colors } from '@grafana/ui'; import { getThemeColor } from 'app/core/utils/colors'; import { getLogLevel, getLogLevelFromKey, sortInAscendingOrder } from '../features/logs/utils'; + export const LIMIT_LABEL = 'Line limit'; export const COMMON_LABELS = 'Common labels'; @@ -798,7 +801,11 @@ export function queryLogsSample { + const frame = toDataFrame(dataFrame); + const { timeIndex } = getTimeField(frame); + return sortDataFrame(frame, timeIndex); + }); } }, error: (error) => { diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 67972b9f04d..a0e9fc43883 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -78,7 +78,7 @@ import { isValidQuery, requestSupportsPartitioning, } from './queryUtils'; -import { sortDataFrameByTime } from './sortDataFrame'; +import { sortDataFrameByTime, SortDirection } from './sortDataFrame'; import { doLokiChannelStream } from './streaming'; import { trackQuery } from './tracking'; import { @@ -686,7 +686,7 @@ export class LokiDatasource const processResults = (result: DataQueryResponse): DataQueryResponse => { const frames: DataFrame[] = result.data; const processedFrames = frames - .map((frame) => sortDataFrameByTime(frame, 'DESCENDING')) + .map((frame) => sortDataFrameByTime(frame, SortDirection.Descending)) .map((frame) => processDataFrame(frame)); // rename fields if needed return { diff --git a/public/app/plugins/datasource/loki/sortDataFrame.test.ts b/public/app/plugins/datasource/loki/sortDataFrame.test.ts index 165fc7cddfe..987c78ce3f3 100644 --- a/public/app/plugins/datasource/loki/sortDataFrame.test.ts +++ b/public/app/plugins/datasource/loki/sortDataFrame.test.ts @@ -1,6 +1,6 @@ import { ArrayVector, DataFrame, FieldType } from '@grafana/data'; -import { sortDataFrameByTime } from './sortDataFrame'; +import { sortDataFrameByTime, SortDirection } from './sortDataFrame'; const inputFrame: DataFrame = { refId: 'A', @@ -29,7 +29,7 @@ const inputFrame: DataFrame = { describe('loki sortDataFrame', () => { it('sorts a dataframe ascending', () => { - const sortedFrame = sortDataFrameByTime(inputFrame, 'ASCENDING'); + const sortedFrame = sortDataFrameByTime(inputFrame, SortDirection.Ascending); expect(sortedFrame.length).toBe(5); const timeValues = sortedFrame.fields[0].values.toArray(); const lineValues = sortedFrame.fields[1].values.toArray(); @@ -40,7 +40,7 @@ describe('loki sortDataFrame', () => { expect(tsNsValues).toStrictEqual([`1001000000`, `1002000000`, `1003000000`, `1004000000`, `1005000000`]); }); it('sorts a dataframe descending', () => { - const sortedFrame = sortDataFrameByTime(inputFrame, 'DESCENDING'); + const sortedFrame = sortDataFrameByTime(inputFrame, SortDirection.Descending); expect(sortedFrame.length).toBe(5); const timeValues = sortedFrame.fields[0].values.toArray(); const lineValues = sortedFrame.fields[1].values.toArray(); diff --git a/public/app/plugins/datasource/loki/sortDataFrame.ts b/public/app/plugins/datasource/loki/sortDataFrame.ts index c524de61160..dba69738950 100644 --- a/public/app/plugins/datasource/loki/sortDataFrame.ts +++ b/public/app/plugins/datasource/loki/sortDataFrame.ts @@ -1,6 +1,9 @@ import { DataFrame, Field, SortedVector } from '@grafana/data'; -type SortDirection = 'ASCENDING' | 'DESCENDING'; +export enum SortDirection { + Ascending, + Descending, +} // creates the `index` for the sorting. // this is needed by the `SortedVector`. @@ -21,7 +24,7 @@ function makeIndex(field: Field, dir: SortDirection): number[] { index[i] = i; } - const isAsc = dir === 'ASCENDING'; + const isAsc = dir === SortDirection.Ascending; index.sort((a: number, b: number): number => { // we need to answer this question: