From adcc634db6d11872ccc2117af50ede95dec5a4f8 Mon Sep 17 00:00:00 2001 From: Leon Sorokin Date: Fri, 24 Oct 2025 11:19:49 -0500 Subject: [PATCH] Histogram: Properly handle sparse heatmap-cells frames (#112907) * Histogram: Properly handle sparse heatmap-cells frames * add test * fix crash with float bounds and rounding * tweak comment * add test for float bounds --- .../transformers/histogram.test.ts | 611 ++++++++++++++++++ .../transformations/transformers/histogram.ts | 41 +- 2 files changed, 649 insertions(+), 3 deletions(-) diff --git a/packages/grafana-data/src/transformations/transformers/histogram.test.ts b/packages/grafana-data/src/transformations/transformers/histogram.test.ts index f8aa3f9c991..d7be8281adf 100644 --- a/packages/grafana-data/src/transformations/transformers/histogram.test.ts +++ b/packages/grafana-data/src/transformations/transformers/histogram.test.ts @@ -1,5 +1,6 @@ import { toDataFrame } from '../../dataframe/processDataFrame'; import { Field, FieldType } from '../../types/dataFrame'; +import { DataFrameType } from '../../types/dataFrameTypes'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { @@ -8,6 +9,7 @@ import { histogramFieldsToFrame, HistogramFields, joinHistograms, + getHistogramFields, } from './histogram'; describe('histogram frames frames', () => { @@ -287,6 +289,615 @@ describe('histogram frames frames', () => { }); }); +describe('getHistogramFields', () => { + it('densifies sparse native histograms (heatmap-cells), integer bounds', () => { + expect( + getHistogramFields( + toDataFrame({ + meta: { + type: DataFrameType.HeatmapCells, + }, + fields: [ + { name: 'yMin', type: FieldType.number, values: [512, 2048] }, + { name: 'yMax', type: FieldType.number, values: [1024, 4096] }, + { name: 'count', type: FieldType.number, values: [10, 10] }, + ], + }) + ) + ).toMatchInlineSnapshot(` + { + "counts": [ + { + "config": {}, + "name": "count", + "type": "number", + "values": [ + 10, + 0, + 10, + ], + }, + ], + "xMax": { + "config": {}, + "name": "xMax", + "type": "number", + "values": [ + 1024, + 2048, + 4096, + ], + }, + "xMin": { + "config": {}, + "name": "xMin", + "type": "number", + "values": [ + 512, + 1024, + 2048, + ], + }, + } + `); + }); + + it('densifies sparse native histograms (heatmap-cells), float bounds', () => { + let yMinVals = [ + 0.00005132424409507534, 0.00005596948505887885, 0.00006103515625, 0.0000725834420778028, 0.00041059395276060273, + 0.0005324744788404579, 0.0008211879055212055, 0.005065779510355506, 0.005524271728019902, 0.006569503244169644, + 0.007164094087536493, 0.011048543456039804, 0.012048522073499537, 0.013139006488339287, 0.014328188175072986, + 0.015625, 0.022097086912079608, 0.028656376350145972, 0.0340783666457893, 0.044194173824159216, + 0.04819408829399815, 0.05255602595335715, 0.057312752700291944, 0.0625, 0.0681567332915786, 0.07432544468767006, + 0.0810524721656881, 0.08838834764831843, 0.0963881765879963, 0.1051120519067143, 0.11462550540058389, 0.125, + 0.2102241038134286, 0.22925101080116778, 0.25, 0.29730177875068026, 0.3242098886627524, 0.35355339059327373, 0.5, + 0.8408964152537144, 1, 1.0905077326652577, 1.189207115002721, 1.2968395546510096, 1.5422108254079407, + 1.6817928305074288, 1.8340080864093422, 5.187358218604039, 6.727171322029715, 7.336032345637369, + 13.45434264405943, 17.448123722644123, 0.00005132424409507534, 0.00005596948505887885, 0.00006103515625, + 0.0000725834420778028, 0.00041059395276060273, 0.0005324744788404579, 0.005524271728019902, 0.006569503244169644, + 0.007164094087536493, 0.011048543456039804, 0.012048522073499537, 0.013139006488339287, 0.014328188175072986, + 0.015625, 0.044194173824159216, 0.0625, 0.0681567332915786, 0.07432544468767006, 0.0810524721656881, + 0.08838834764831843, 0.0963881765879963, 0.1051120519067143, 0.11462550540058389, 0.125, 0.2102241038134286, + 0.22925101080116778, 0.25, 0.3242098886627524, 0.35355339059327373, 0.5, 0.8408964152537144, 1, + 1.0905077326652577, 1.189207115002721, 1.2968395546510096, 1.5422108254079407, 1.6817928305074288, + 1.8340080864093422, 2, 5.187358218604039, 6.727171322029715, 7.336032345637369, 13.45434264405943, + 17.448123722644123, 19.027313840043536, + ]; + + let yMaxVals = [ + 0.00005596948505887885, 0.00006103515625, 0.00006655930985505723, 0.00007915280484930479, 0.0004477558804710308, + 0.0005806675366224224, 0.0008955117609420616, 0.005524271728019902, 0.0060242610367497685, 0.007164094087536493, + 0.0078125, 0.012048522073499537, 0.013139006488339287, 0.014328188175072986, 0.015625, 0.01703918332289465, + 0.024097044146999074, 0.03125, 0.03716272234383503, 0.04819408829399815, 0.05255602595335715, + 0.057312752700291944, 0.0625, 0.0681567332915786, 0.07432544468767006, 0.0810524721656881, 0.08838834764831843, + 0.0963881765879963, 0.1051120519067143, 0.11462550540058389, 0.125, 0.1363134665831572, 0.22925101080116778, 0.25, + 0.2726269331663144, 0.3242098886627524, 0.35355339059327373, 0.3855527063519852, 0.5452538663326288, + 0.9170040432046711, 1.0905077326652577, 1.189207115002721, 1.2968395546510096, 1.414213562373095, + 1.6817928305074288, 1.8340080864093422, 2, 5.65685424949238, 7.336032345637369, 8, 14.672064691274738, + 19.027313840043536, 0.00005596948505887885, 0.00006103515625, 0.00006655930985505723, 0.00007915280484930479, + 0.0004477558804710308, 0.0005806675366224224, 0.0060242610367497685, 0.007164094087536493, 0.0078125, + 0.012048522073499537, 0.013139006488339287, 0.014328188175072986, 0.015625, 0.01703918332289465, + 0.04819408829399815, 0.0681567332915786, 0.07432544468767006, 0.0810524721656881, 0.08838834764831843, + 0.0963881765879963, 0.1051120519067143, 0.11462550540058389, 0.125, 0.1363134665831572, 0.22925101080116778, 0.25, + 0.2726269331663144, 0.35355339059327373, 0.3855527063519852, 0.5452538663326288, 0.9170040432046711, + 1.0905077326652577, 1.189207115002721, 1.2968395546510096, 1.414213562373095, 1.6817928305074288, + 1.8340080864093422, 2, 2.1810154653305154, 5.65685424949238, 7.336032345637369, 8, 14.672064691274738, + 19.027313840043536, 20.749432874416154, + ]; + + let countVals = [ + 0.02222222222222222, 0.08888888888888888, 0.06666666666666665, 0.02222222222222222, 0.02222222222222222, + 0.04444444444444444, 0.02222222222222222, 0.02222222222222222, 0.02222222222222222, 0.04444444444444444, + 0.04444444444444444, 0.02222222222222222, 0.04444444444444444, 0.06666666666666665, 0.04444444444444444, + 0.04444444444444444, 0.04444444444444444, 0.04444444444444444, 0.06666666666666665, 0.02222222222222222, + 0.02222222222222222, 0.02222222222222222, 0.02222222222222222, 0.1333333333333333, 0.17777777777777776, + 0.15555555555555553, 0.17777777777777776, 0.19999999999999998, 0.1111111111111111, 0.08888888888888888, + 0.04444444444444444, 0.06666666666666665, 0.02222222222222222, 0.04444444444444444, 0.04444444444444444, + 0.02222222222222222, 0.06666666666666665, 0.02222222222222222, 0.02222222222222222, 0.02222222222222222, + 0.04444444444444444, 0.06666666666666665, 0.04444444444444444, 0.06666666666666665, 0.04444444444444444, + 0.06666666666666665, 0.02222222222222222, 0.04444444444444444, 0.04444444444444444, 0.08888888888888888, + 0.1333333333333333, 0.08888888888888888, 0.02222222222222222, 0.11111111111111109, 0.04444444444444444, + 0.02222222222222222, 0.02222222222222222, 0.04444444444444444, 0.04444444444444444, 0.04444444444444444, + 0.02222222222222222, 0.04444444444444444, 0.04444444444444444, 0.02222222222222222, 0.02222222222222222, + 0.02222222222222222, 0.02222222222222222, 0.08888888888888888, 0.13333333333333333, 0.17777777777777776, + 0.17777777777777776, 0.17777777777777776, 0.17777777777777776, 0.06666666666666665, 0.04444444444444444, + 0.02222222222222222, 0.02222222222222222, 0.04444444444444444, 0.04444444444444444, 0.04444444444444444, + 0.02222222222222222, 0.02222222222222222, 0.02222222222222222, 0.06666666666666665, 0.06666666666666665, + 0.04444444444444444, 0.04444444444444444, 0.04444444444444444, 0.06666666666666665, 0.02222222222222222, + 0.02222222222222222, 0.04444444444444444, 0.06666666666666665, 0.06666666666666665, 0.1333333333333333, + 0.04444444444444444, 0.02222222222222222, + ]; + + expect( + getHistogramFields( + toDataFrame({ + meta: { + type: DataFrameType.HeatmapCells, + }, + fields: [ + { name: 'yMin', type: FieldType.number, values: yMinVals }, + { name: 'yMax', type: FieldType.number, values: yMaxVals }, + { name: 'count', type: FieldType.number, values: countVals }, + ], + }) + ) + ).toMatchInlineSnapshot(` + { + "counts": [ + { + "config": {}, + "name": "count", + "type": "number", + "values": [ + 0.04444444444444444, + 0.19999999999999996, + 0.11111111111111109, + 0, + 0.04444444444444444, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.04444444444444444, + 0, + 0, + 0.08888888888888888, + 0, + 0, + 0, + 0, + 0.02222222222222222, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.02222222222222222, + 0.06666666666666665, + 0, + 0.08888888888888888, + 0.06666666666666665, + 0, + 0, + 0, + 0, + 0.06666666666666665, + 0.08888888888888888, + 0.08888888888888888, + 0.06666666666666665, + 0.06666666666666665, + 0, + 0, + 0, + 0.04444444444444444, + 0, + 0, + 0.04444444444444444, + 0, + 0.06666666666666665, + 0, + 0, + 0.04444444444444444, + 0.02222222222222222, + 0.02222222222222222, + 0.02222222222222222, + 0.22222222222222218, + 0.3111111111111111, + 0.33333333333333326, + 0.3555555555555555, + 0.37777777777777777, + 0.28888888888888886, + 0.15555555555555553, + 0.08888888888888888, + 0.08888888888888888, + 0, + 0, + 0, + 0, + 0, + 0.04444444444444444, + 0.08888888888888888, + 0.08888888888888888, + 0, + 0.02222222222222222, + 0.11111111111111109, + 0.04444444444444444, + 0, + 0, + 0, + 0.04444444444444444, + 0, + 0, + 0, + 0, + 0, + 0.04444444444444444, + 0, + 0.11111111111111109, + 0.1333333333333333, + 0.08888888888888888, + 0.11111111111111109, + 0, + 0.08888888888888888, + 0.1333333333333333, + 0.04444444444444444, + 0.02222222222222222, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.08888888888888888, + 0, + 0, + 0.11111111111111109, + 0.15555555555555553, + 0, + 0, + 0, + 0, + 0, + 0, + 0.2666666666666666, + 0, + 0, + 0.1333333333333333, + 0.02222222222222222, + ], + }, + ], + "xMax": { + "config": {}, + "name": "xMax", + "type": "number", + "values": [ + 0.00005596948505887885, + 0.00006103515625, + 0.00006655930985505723, + 0.0000725834420778028, + 0.00007915280484930479, + 0.00008631674575031098, + 0.00009412907869921515, + 0.00010264848819015071, + 0.00011193897011775773, + 0.00012207031250000003, + 0.0001331186197101145, + 0.00014516688415560565, + 0.00015830560969860963, + 0.00017263349150062202, + 0.00018825815739843035, + 0.00020529697638030147, + 0.00022387794023551551, + 0.0002441406250000001, + 0.00026623723942022904, + 0.00029033376831121135, + 0.0003166112193972193, + 0.0003452669830012441, + 0.00037651631479686075, + 0.00041059395276060306, + 0.0004477558804710308, + 0.00048828124999999995, + 0.0005324744788404577, + 0.0005806675366224224, + 0.0006332224387944383, + 0.0006905339660024879, + 0.0007530326295937212, + 0.0008211879055212057, + 0.0008955117609420616, + 0.0009765624999999999, + 0.0010649489576809155, + 0.0011613350732448445, + 0.0012664448775888764, + 0.0013810679320049755, + 0.0015060652591874421, + 0.0016423758110424111, + 0.0017910235218841235, + 0.001953125, + 0.0021298979153618314, + 0.0023226701464896895, + 0.002532889755177753, + 0.0027621358640099515, + 0.0030121305183748847, + 0.0032847516220848227, + 0.0035820470437682474, + 0.003906250000000001, + 0.004259795830723664, + 0.004645340292979381, + 0.005065779510355508, + 0.005524271728019902, + 0.0060242610367497685, + 0.0065695032441696445, + 0.007164094087536493, + 0.0078125, + 0.008519591661447326, + 0.009290680585958758, + 0.010131559020711013, + 0.011048543456039806, + 0.012048522073499537, + 0.013139006488339287, + 0.014328188175072986, + 0.015625, + 0.01703918332289465, + 0.018581361171917516, + 0.020263118041422026, + 0.02209708691207961, + 0.024097044146999074, + 0.026278012976678578, + 0.028656376350145975, + 0.03125, + 0.0340783666457893, + 0.03716272234383503, + 0.04052623608284405, + 0.04419417382415922, + 0.04819408829399815, + 0.05255602595335715, + 0.057312752700291944, + 0.0625, + 0.0681567332915786, + 0.07432544468767006, + 0.0810524721656881, + 0.08838834764831843, + 0.0963881765879963, + 0.1051120519067143, + 0.11462550540058389, + 0.125, + 0.1363134665831572, + 0.14865088937534013, + 0.1621049443313762, + 0.1767766952966369, + 0.19277635317599262, + 0.21022410381342865, + 0.22925101080116778, + 0.25, + 0.2726269331663144, + 0.29730177875068026, + 0.3242098886627524, + 0.35355339059327373, + 0.3855527063519852, + 0.42044820762685725, + 0.4585020216023356, + 0.5, + 0.5452538663326288, + 0.5946035575013605, + 0.6484197773255048, + 0.7071067811865476, + 0.7711054127039705, + 0.8408964152537146, + 0.9170040432046711, + 0.9999999999999999, + 1.0905077326652577, + 1.189207115002721, + 1.2968395546510096, + 1.414213562373095, + 1.5422108254079407, + 1.6817928305074288, + 1.8340080864093422, + 2, + 2.1810154653305154, + 2.378414230005442, + 2.5936791093020193, + 2.8284271247461903, + 3.084421650815882, + 3.3635856610148585, + 3.6680161728186853, + 4.000000000000001, + 4.362030930661032, + 4.756828460010886, + 5.18735821860404, + 5.65685424949238, + 6.168843301631763, + 6.727171322029716, + 7.336032345637369, + 8, + 8.724061861322062, + 9.513656920021768, + 10.374716437208077, + 11.313708498984761, + 12.337686603263528, + 13.454342644059434, + 14.672064691274738, + 15.999999999999998, + 17.44812372264412, + 19.027313840043536, + 20.749432874416154, + ], + }, + "xMin": { + "config": {}, + "name": "xMin", + "type": "number", + "values": [ + 0.00005132424409507534, + 0.00005596948505887885, + 0.00006103515625, + 0.00006655930985505723, + 0.0000725834420778028, + 0.00007915280484930479, + 0.00008631674575031098, + 0.00009412907869921515, + 0.00010264848819015071, + 0.00011193897011775773, + 0.00012207031250000003, + 0.0001331186197101145, + 0.00014516688415560565, + 0.00015830560969860963, + 0.00017263349150062202, + 0.00018825815739843035, + 0.00020529697638030147, + 0.00022387794023551551, + 0.0002441406250000001, + 0.00026623723942022904, + 0.00029033376831121135, + 0.0003166112193972193, + 0.0003452669830012441, + 0.00037651631479686075, + 0.00041059395276060273, + 0.0004477558804710308, + 0.00048828124999999995, + 0.0005324744788404579, + 0.0005806675366224224, + 0.0006332224387944383, + 0.0006905339660024879, + 0.0007530326295937212, + 0.0008211879055212055, + 0.0008955117609420616, + 0.0009765624999999999, + 0.0010649489576809155, + 0.0011613350732448445, + 0.0012664448775888764, + 0.0013810679320049755, + 0.0015060652591874421, + 0.0016423758110424111, + 0.0017910235218841235, + 0.001953125, + 0.0021298979153618314, + 0.0023226701464896895, + 0.002532889755177753, + 0.0027621358640099515, + 0.0030121305183748847, + 0.0032847516220848227, + 0.0035820470437682474, + 0.003906250000000001, + 0.004259795830723664, + 0.004645340292979381, + 0.005065779510355506, + 0.005524271728019902, + 0.0060242610367497685, + 0.006569503244169644, + 0.007164094087536493, + 0.007812499999999999, + 0.008519591661447324, + 0.009290680585958756, + 0.010131559020711011, + 0.011048543456039804, + 0.012048522073499537, + 0.013139006488339287, + 0.014328188175072986, + 0.015625, + 0.01703918332289465, + 0.018581361171917516, + 0.020263118041422026, + 0.022097086912079608, + 0.024097044146999074, + 0.026278012976678578, + 0.028656376350145972, + 0.031249999999999997, + 0.0340783666457893, + 0.03716272234383503, + 0.04052623608284405, + 0.044194173824159216, + 0.04819408829399815, + 0.05255602595335715, + 0.057312752700291944, + 0.0625, + 0.0681567332915786, + 0.07432544468767006, + 0.0810524721656881, + 0.08838834764831843, + 0.0963881765879963, + 0.1051120519067143, + 0.11462550540058389, + 0.125, + 0.1363134665831572, + 0.14865088937534013, + 0.1621049443313762, + 0.1767766952966369, + 0.19277635317599262, + 0.2102241038134286, + 0.22925101080116778, + 0.25, + 0.2726269331663144, + 0.29730177875068026, + 0.3242098886627524, + 0.35355339059327373, + 0.3855527063519852, + 0.42044820762685725, + 0.4585020216023356, + 0.5, + 0.5452538663326288, + 0.5946035575013605, + 0.6484197773255048, + 0.7071067811865476, + 0.7711054127039705, + 0.8408964152537144, + 0.9170040432046711, + 1, + 1.0905077326652577, + 1.189207115002721, + 1.2968395546510096, + 1.4142135623730951, + 1.5422108254079407, + 1.6817928305074288, + 1.8340080864093422, + 2, + 2.1810154653305154, + 2.378414230005442, + 2.5936791093020193, + 2.8284271247461903, + 3.084421650815882, + 3.3635856610148585, + 3.6680161728186853, + 4.000000000000001, + 4.362030930661032, + 4.756828460010886, + 5.187358218604039, + 5.656854249492381, + 6.168843301631764, + 6.727171322029715, + 7.336032345637369, + 7.999999999999999, + 8.72406186132206, + 9.513656920021766, + 10.374716437208075, + 11.31370849898476, + 12.337686603263526, + 13.45434264405943, + 14.672064691274738, + 15.999999999999998, + 17.448123722644123, + 19.027313840043536, + ], + }, + } + `); + }); +}); + describe('joinHistograms', () => { type TestHistogram = { xMin: number[]; diff --git a/packages/grafana-data/src/transformations/transformers/histogram.ts b/packages/grafana-data/src/transformations/transformers/histogram.ts index 8fa69f64a40..723fb07b0d8 100644 --- a/packages/grafana-data/src/transformations/transformers/histogram.ts +++ b/packages/grafana-data/src/transformations/transformers/histogram.ts @@ -198,8 +198,43 @@ export function getHistogramFields(frame: DataFrame): HistogramFields | undefine let uniqueMaxs = [...new Set(yMaxField.values)].sort((a, b) => a - b); let uniqueMins = [...new Set(yMinField.values)].sort((a, b) => a - b); + + // useLogScale is assumed for native histograms, we get the multiplier from first bucket's min/max bounds + let bucketFactor = uniqueMaxs[0] / uniqueMins[0]; + + // NOTES: + // native histograms default bucket growth factor is 8th root of 2 (schema 3) -> (2 ** (2 ** -3)) = 1.0905077326652577 + // the by-division inference from data results in 1.090507732665258 + + // densify the buckets using 0 counts (consumers of this fn assume a dense structure) + let denseMins: number[] = []; + let denseMaxs: number[] = []; + + for (let i = 0; i < uniqueMaxs.length; i++) { + let curMax = uniqueMaxs[i]; + let curMin = uniqueMins[i]; + + denseMaxs.push(curMax); + denseMins.push(curMin); + + let nextMax = uniqueMaxs[i + 1]; + + if (nextMax != null) { + curMax = curMax * bucketFactor; + curMin = curMin * bucketFactor; + + while (curMax < nextMax * 0.999999) { + denseMaxs.push(curMax); + denseMins.push(curMin); + + curMax = curMax * bucketFactor; + curMin = curMin * bucketFactor; + } + } + } + let countsByMax = new Map(); - uniqueMaxs.forEach((max) => countsByMax.set(max, 0)); + denseMaxs.forEach((max) => countsByMax.set(max, 0)); for (let i = 0; i < yMaxField.values.length; i++) { let max = yMaxField.values[i]; @@ -210,12 +245,12 @@ export function getHistogramFields(frame: DataFrame): HistogramFields | undefine xMin: { ...yMinField, name: 'xMin', - values: uniqueMins, + values: denseMins, }, xMax: { ...yMaxField, name: 'xMax', - values: uniqueMaxs, + values: denseMaxs, }, counts: [ {