From 5ffb28989e17a2939539761764afa9003d482be2 Mon Sep 17 00:00:00 2001 From: Matias Chomicki Date: Mon, 13 Mar 2023 10:33:01 +0100 Subject: [PATCH] Range splitting: remove hardcoded chunk limit (#64625) * Range splitting: allow splits larger than 30 days while keeping the fail safe * Remove limit * Fix test * Remove unnecessary try/catch * Remove unused export --- .../app/plugins/datasource/loki/logsTimeSplit.test.ts | 6 ------ public/app/plugins/datasource/loki/logsTimeSplit.ts | 8 +------- .../plugins/datasource/loki/metricTimeSplit.test.ts | 11 ++--------- public/app/plugins/datasource/loki/metricTimeSplit.ts | 10 ++-------- public/app/plugins/datasource/loki/querySplitting.ts | 6 +----- 5 files changed, 6 insertions(+), 35 deletions(-) diff --git a/public/app/plugins/datasource/loki/logsTimeSplit.test.ts b/public/app/plugins/datasource/loki/logsTimeSplit.test.ts index ae9c048dbf5..73b20d822ce 100644 --- a/public/app/plugins/datasource/loki/logsTimeSplit.test.ts +++ b/public/app/plugins/datasource/loki/logsTimeSplit.test.ts @@ -26,10 +26,4 @@ describe('querySplit', () => { [Date.parse('2022-02-06T14:10:43.567'), Date.parse('2022-02-06T14:11:03.567')], ]); }); - - it('should return null if too many chunks would be generated', () => { - const start = Date.parse('2022-02-06T14:10:03'); - const end = Date.parse('2022-02-06T14:30:03'); - expect(getRangeChunks(start, end, 10000)).toBeNull(); - }); }); diff --git a/public/app/plugins/datasource/loki/logsTimeSplit.ts b/public/app/plugins/datasource/loki/logsTimeSplit.ts index d77eb7f57b7..6814f74c487 100644 --- a/public/app/plugins/datasource/loki/logsTimeSplit.ts +++ b/public/app/plugins/datasource/loki/logsTimeSplit.ts @@ -2,8 +2,6 @@ // like returned by `new Date().getTime()`. this is needed because the "math" // has to be done on integer numbers. -const MAX_CHUNK_COUNT = 50; - // the way loki handles logs-range-queries is that if you specify start & end, // one of those will be included, but the other will not. this allows us to // make it easy to split ranges. @@ -22,7 +20,7 @@ export function getRangeChunks( startTime: number, endTime: number, idealRangeDuration: number -): Array<[number, number]> | null { +): Array<[number, number]> { if (endTime - startTime <= idealRangeDuration) { return [[startTime, endTime]]; } @@ -36,10 +34,6 @@ export function getRangeChunks( // to cross over the startTime const chunkStartTime = Math.max(chunkEndTime - idealRangeDuration, startTime); result.push([chunkStartTime, chunkEndTime]); - - if (result.length > MAX_CHUNK_COUNT) { - return null; - } } // because we walked backwards, we need to reverse the array diff --git a/public/app/plugins/datasource/loki/metricTimeSplit.test.ts b/public/app/plugins/datasource/loki/metricTimeSplit.test.ts index 34a0b65ef53..116e9c4a479 100644 --- a/public/app/plugins/datasource/loki/metricTimeSplit.test.ts +++ b/public/app/plugins/datasource/loki/metricTimeSplit.test.ts @@ -13,17 +13,10 @@ describe('querySplit', () => { ]); }); - it('should return null if too many chunks would be generated', () => { - const start = Date.parse('2022-02-06T14:10:03'); - const end = Date.parse('2022-02-06T14:35:01'); - const step = 10 * 1000; - expect(getRangeChunks(start, end, step, 20000)).toBeNull(); - }); - - it('should return null if requested duration is smaller than step', () => { + it('should return the original interval if requested duration is smaller than step', () => { const start = Date.parse('2022-02-06T14:10:03'); const end = Date.parse('2022-02-06T14:10:33'); const step = 10 * 1000; - expect(getRangeChunks(start, end, step, 1000)).toBeNull(); + expect(getRangeChunks(start, end, step, 1000)).toEqual([[start, end]]); }); }); diff --git a/public/app/plugins/datasource/loki/metricTimeSplit.ts b/public/app/plugins/datasource/loki/metricTimeSplit.ts index b0efb46fa4b..791decaa374 100644 --- a/public/app/plugins/datasource/loki/metricTimeSplit.ts +++ b/public/app/plugins/datasource/loki/metricTimeSplit.ts @@ -19,17 +19,15 @@ function expandTimeRange(startTime: number, endTime: number, step: number): [num return [newStartTime, newEndTime]; } -const MAX_CHUNK_COUNT = 50; - export function getRangeChunks( startTime: number, endTime: number, step: number, idealRangeDuration: number -): Array<[number, number]> | null { +): Array<[number, number]> { if (idealRangeDuration < step) { // we cannot create chunks smaller than `step` - return null; + return [[startTime, endTime]]; } // we make the duration a multiple of `step`, lowering it if necessary @@ -45,10 +43,6 @@ export function getRangeChunks( // to cross over the startTime const chunkStartTime = Math.max(chunkEndTime - alignedDuration, alignedStartTime); result.push([chunkStartTime, chunkEndTime]); - - if (result.length > MAX_CHUNK_COUNT) { - return null; - } } // because we walked backwards, we need to reverse the array diff --git a/public/app/plugins/datasource/loki/querySplitting.ts b/public/app/plugins/datasource/loki/querySplitting.ts index b3205d846d4..73f1a5e3ac2 100644 --- a/public/app/plugins/datasource/loki/querySplitting.ts +++ b/public/app/plugins/datasource/loki/querySplitting.ts @@ -41,11 +41,6 @@ export function partitionTimeRange( ? getLogsRangeChunks(start, end, duration) : getMetricRangeChunks(start, end, step, duration); - // if the split was not possible, go with the original range - if (ranges == null) { - return [originalTimeRange]; - } - return ranges.map(([start, end]) => { const from = dateTime(start); const to = dateTime(end); @@ -197,5 +192,6 @@ export function runPartitionedQueries(datasource: LokiDatasource, request: DataQ partition: [request.range], }); } + return runGroupedQueries(datasource, requests); }