From 40df27a4daa11cf38640d0ae23e31aec3b54d28f Mon Sep 17 00:00:00 2001 From: Nathan Marrs Date: Mon, 30 Oct 2023 11:33:40 -0600 Subject: [PATCH] Calculations: Update First * and Last * reducers to exclude NaNs (#77323) --- .../calculation-types/index.md | 4 +- .../src/transformations/fieldReducer.test.ts | 56 ++++++++++++++++++- .../src/transformations/fieldReducer.ts | 12 ++-- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/docs/sources/panels-visualizations/calculation-types/index.md b/docs/sources/panels-visualizations/calculation-types/index.md index 7aaa8d7a648..1471e0f00ce 100644 --- a/docs/sources/panels-visualizations/calculation-types/index.md +++ b/docs/sources/panels-visualizations/calculation-types/index.md @@ -29,9 +29,9 @@ The following table contains a list of calculations you can perform in Grafana. | Difference percent | Percentage change between first and last value of a field | | Distinct count | Number of unique values in a field | | First | First value in a field | -| First\* (not null) | First, not null value in a field | +| First\* (not null) | First, not null value in a field (also excludes NaNs) | | Last | Last value in a field | -| Last\* (not null) | Last, not null value in a field | +| Last\* (not null) | Last, not null value in a field (also excludes NaNs) | | Max | Maximum value of a field | | Mean | Mean value of all values in a field | | Variance | Variance of all values in a field | diff --git a/packages/grafana-data/src/transformations/fieldReducer.test.ts b/packages/grafana-data/src/transformations/fieldReducer.test.ts index c1d89c487a0..ebe4fd2a592 100644 --- a/packages/grafana-data/src/transformations/fieldReducer.test.ts +++ b/packages/grafana-data/src/transformations/fieldReducer.test.ts @@ -54,7 +54,7 @@ describe('Stats Calculators', () => { it('should calculate basic stats', () => { const stats = reduceField({ field: basicTable.fields[0], - reducers: ['first', 'last', 'mean', 'count'], + reducers: [ReducerID.first, ReducerID.last, ReducerID.mean, ReducerID.count], }); expect(stats.first).toEqual(10); @@ -67,7 +67,7 @@ describe('Stats Calculators', () => { basicTable.fields[0].state = undefined; // clear the cache const stats = reduceField({ field: basicTable.fields[0], - reducers: ['first'], + reducers: [ReducerID.first], }); // Should do the simple version that just looks up value @@ -172,6 +172,58 @@ describe('Stats Calculators', () => { } }); + it('consistent results for first/last value with NaN', () => { + const info = [ + { + data: [NaN, 200, NaN], // first/last value is NaN + result: 200, + }, + { + data: [NaN, NaN, NaN], // All NaN + result: null, + }, + { + data: [undefined, undefined, undefined], // Empty row + result: null, + }, + ]; + + const stats = reduceField({ + field: createField('x', info[0].data), + reducers: [ReducerID.first, ReducerID.last, ReducerID.firstNotNull, ReducerID.lastNotNull, ReducerID.diffperc], + }); + + expect(stats[ReducerID.first]).toEqual(NaN); + expect(stats[ReducerID.last]).toEqual(NaN); + expect(stats[ReducerID.firstNotNull]).toEqual(200); + expect(stats[ReducerID.lastNotNull]).toEqual(200); + expect(stats[ReducerID.diffperc]).toEqual(0); + + const reducers = [ReducerID.lastNotNull, ReducerID.firstNotNull]; + for (const input of info) { + for (const reducer of reducers) { + const v1 = reduceField({ + field: createField('x', input.data), + reducers: [reducer, ReducerID.mean], // uses standard path + })[reducer]; + + const v2 = reduceField({ + field: createField('x', input.data), + reducers: [reducer], // uses optimized path + })[reducer]; + + if (v1 !== v2 || v1 !== input.result) { + const msg = + `Invalid ${reducer} result for: ` + + input.data.join(', ') + + ` Expected: ${input.result}` + // configured + ` Received: Multiple: ${v1}, Single: ${v2}`; + expect(msg).toEqual(null); + } + } + } + }); + it('count should ignoreNulls by default', () => { const someNulls = createField('x', [1, null, null, 1]); expect(reduce(someNulls, ReducerID.count)).toEqual(2); diff --git a/packages/grafana-data/src/transformations/fieldReducer.ts b/packages/grafana-data/src/transformations/fieldReducer.ts index 8e2eea0f791..df2f690fbf3 100644 --- a/packages/grafana-data/src/transformations/fieldReducer.ts +++ b/packages/grafana-data/src/transformations/fieldReducer.ts @@ -137,7 +137,7 @@ export const fieldReducers = new Registry(() => [ { id: ReducerID.lastNotNull, name: 'Last *', - description: 'Last non-null value', + description: 'Last non-null value (also excludes NaNs)', standard: true, aliasIds: ['current'], reduce: calculateLastNotNull, @@ -152,7 +152,7 @@ export const fieldReducers = new Registry(() => [ { id: ReducerID.firstNotNull, name: 'First *', - description: 'First non-null value', + description: 'First non-null value (also excludes NaNs)', standard: true, reduce: calculateFirstNotNull, }, @@ -320,8 +320,8 @@ export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: calcs.count++; - if (currentValue != null) { - // null || undefined + if (currentValue != null && !Number.isNaN(currentValue)) { + // null || undefined || NaN const isFirst = calcs.firstNotNull === null; if (isFirst) { calcs.firstNotNull = currentValue; @@ -418,7 +418,7 @@ function calculateFirstNotNull(field: Field, ignoreNulls: boolean, nullAsZero: b const data = field.values; for (let idx = 0; idx < data.length; idx++) { const v = data[idx]; - if (v != null && v !== undefined) { + if (v != null && !Number.isNaN(v)) { return { firstNotNull: v }; } } @@ -435,7 +435,7 @@ function calculateLastNotNull(field: Field, ignoreNulls: boolean, nullAsZero: bo let idx = data.length - 1; while (idx >= 0) { const v = data[idx--]; - if (v != null && v !== undefined) { + if (v != null && !Number.isNaN(v)) { return { lastNotNull: v }; } }