diff --git a/package.json b/package.json index 41389e135e7..d36b4bfc5f8 100644 --- a/package.json +++ b/package.json @@ -347,6 +347,7 @@ "date-fns": "4.1.0", "debounce-promise": "3.1.2", "diff": "^8.0.0", + "downsample": "1.4.0", "fast-deep-equal": "^3.1.3", "fast-json-patch": "3.1.1", "file-saver": "2.0.5", diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index cc2b76fae69..a3d5536c670 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -42,5 +42,6 @@ export enum DataTransformerID { formatTime = 'formatTime', formatString = 'formatString', regression = 'regression', + smoothing = 'smoothing', groupToNestedTable = 'groupToNestedTable', } diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index aebbab8c6f9..06aa45d2275 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -1255,4 +1255,8 @@ export interface FeatureToggles { * Enables support for variables whose values can have multiple properties */ multiPropsVariables?: boolean; + /** + * Enables the ASAP smoothing transformation for time series data + */ + smoothingTransformation?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 3748db8e6b4..2933551d1ad 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -2075,6 +2075,13 @@ var ( FrontendOnly: true, Owner: grafanaDashboardsSquad, }, + { + Name: "smoothingTransformation", + Description: "Enables the ASAP smoothing transformation for time series data", + Stage: FeatureStageExperimental, + FrontendOnly: true, + Owner: grafanaDataProSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 0c85021cff8..e2d15a8466b 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -281,3 +281,4 @@ rudderstackUpgrade,experimental,@grafana/grafana-frontend-platform,false,false,t kubernetesAlertingHistorian,experimental,@grafana/alerting-squad,false,true,false useMTPlugins,experimental,@grafana/plugins-platform-backend,false,false,true multiPropsVariables,experimental,@grafana/dashboards-squad,false,false,true +smoothingTransformation,experimental,@grafana/datapro,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 6d55a6ca617..66910dc9d1c 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -3293,6 +3293,19 @@ "codeowner": "@grafana/dashboards-squad" } }, + { + "metadata": { + "name": "smoothingTransformation", + "resourceVersion": "1767349656275", + "creationTimestamp": "2026-01-02T10:27:36Z" + }, + "spec": { + "description": "Enables the ASAP smoothing transformation for time series data", + "stage": "experimental", + "codeowner": "@grafana/datapro", + "frontend": true + } + }, { "metadata": { "name": "sqlExpressions", diff --git a/public/app/features/transformers/docs/content.ts b/public/app/features/transformers/docs/content.ts index b41b14b57c8..cde7f209b19 100644 --- a/public/app/features/transformers/docs/content.ts +++ b/public/app/features/transformers/docs/content.ts @@ -1612,6 +1612,53 @@ ${buildImageContent( `; }, }, + smoothing: { + name: 'Smoothing', + getHelperDocs: function (imageRenderType: ImageRenderType = ImageRenderType.ShortcodeFigure) { + return ` +Use this transformation to reduce noise in time series data through adaptive smoothing. This transformation creates smoother, cleaner visualizations while preserving all original time points and important trends and patterns in your data. + +The smoothing transformation uses the ASAP (Automatic Smoothing for Attention Prioritization) algorithm internally to generate a smoothed curve, which is then interpolated back onto all original time points. This ensures your visualization maintains continuous lines without gaps while reducing noise. + +#### Available options + +- **Resolution** - Controls smoothing intensity (1-1000). Lower values create more aggressive smoothing, while higher values preserve more detail. The output preserves all original time points. + +#### When to use smoothing + +This transformation is useful for: + +- Noisy time series data that obscures underlying trends +- Clearer trend analysis and pattern recognition + +#### Example + +Consider noisy sensor data with thousands of points: + +**Before smoothing:** + +| Time | Temperature | +| ------------------- | ----------- | +| 2020-07-07 10:00:00 | 23.1 | +| 2020-07-07 10:00:01 | 23.3 | +| 2020-07-07 10:00:02 | 22.9 | +| 2020-07-07 10:00:03 | 23.2 | +| ... (thousands more) | ... | + +**After smoothing (Resolution: 100):** + +| Time | Temperature (smoothed) | +| ------------------- | ---------------------- | +| 2020-07-07 10:00:00 | 23.1 | +| 2020-07-07 10:00:01 | 23.1 | +| 2020-07-07 10:00:02 | 23.0 | +| 2020-07-07 10:00:03 | 23.0 | +| ... (same count) | ... | + +The transformation preserves all original time points while reducing noise, resulting in smoother curves that maintain continuous lines without gaps. + `; + }, + }, }; function buildImageContent(source: string, imageRenderType: ImageRenderType, imageAltText: string) { diff --git a/public/app/features/transformers/images/dark/smoothing.svg b/public/app/features/transformers/images/dark/smoothing.svg new file mode 100644 index 00000000000..e95ddc2f840 --- /dev/null +++ b/public/app/features/transformers/images/dark/smoothing.svg @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/app/features/transformers/images/light/smoothing.svg b/public/app/features/transformers/images/light/smoothing.svg new file mode 100644 index 00000000000..49651ec4b1e --- /dev/null +++ b/public/app/features/transformers/images/light/smoothing.svg @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/app/features/transformers/smoothing/asap.test.ts b/public/app/features/transformers/smoothing/asap.test.ts new file mode 100644 index 00000000000..195da8a7565 --- /dev/null +++ b/public/app/features/transformers/smoothing/asap.test.ts @@ -0,0 +1,130 @@ +import { asapSmooth, DataPoint, ASAPOptions } from './asap'; + +describe('asapSmooth', () => { + describe('Basic functionality', () => { + it('should return smoothed data with valid DataPoint objects', () => { + const data: DataPoint[] = [ + { x: 0, y: 0 }, + { x: 1, y: 1 }, + { x: 2, y: 2 }, + { x: 3, y: 3 }, + { x: 4, y: 4 }, + ]; + + const options: ASAPOptions = { resolution: 3 }; + const result = asapSmooth(data, options); + + expect(result.length).toBeGreaterThan(0); + result.forEach((point) => { + expect(point).toHaveProperty('x'); + expect(point).toHaveProperty('y'); + expect(typeof point.x).toBe('number'); + expect(typeof point.y).toBe('number'); + }); + }); + + it('should maintain x-axis ordering', () => { + const data: DataPoint[] = Array.from({ length: 20 }, (_, i) => ({ + x: i, + y: Math.random() * 100, + })); + + const options: ASAPOptions = { resolution: 10 }; + const result = asapSmooth(data, options); + + // check that x values are in ascending order + for (let i = 1; i < result.length; i++) { + expect(result[i].x).toBeGreaterThanOrEqual(result[i - 1].x); + } + }); + }); + + describe('Edge cases', () => { + it('should handle empty array', () => { + const data: DataPoint[] = []; + const options: ASAPOptions = { resolution: 10 }; + + const result = asapSmooth(data, options); + + expect(result).toEqual([]); + }); + + it('should handle single data point', () => { + const data: DataPoint[] = [{ x: 1, y: 42 }]; + const options: ASAPOptions = { resolution: 10 }; + + const result = asapSmooth(data, options); + + expect(result.length).toBeGreaterThan(0); + expect(result[0].x).toBe(1); + expect(result[0].y).toBe(42); + }); + + it('should filter out NaN values', () => { + const data: DataPoint[] = [ + { x: 0, y: 0 }, + { x: 1, y: NaN }, + { x: 2, y: 2 }, + { x: 3, y: NaN }, + { x: 4, y: 4 }, + ]; + + const options: ASAPOptions = { resolution: 3 }; + const result = asapSmooth(data, options); + + expect(result.length).toBeGreaterThan(0); + result.forEach((point) => { + expect(isNaN(point.x)).toBe(false); + expect(isNaN(point.y)).toBe(false); + }); + }); + + it('should return empty array when all values are NaN', () => { + const data: DataPoint[] = [ + { x: 0, y: NaN }, + { x: 1, y: NaN }, + { x: 2, y: NaN }, + ]; + + const options: ASAPOptions = { resolution: 3 }; + const result = asapSmooth(data, options); + + expect(result).toEqual([]); + }); + + it('should sort unsorted data', () => { + const data: DataPoint[] = [ + { x: 3, y: 3 }, + { x: 1, y: 1 }, + { x: 4, y: 4 }, + { x: 0, y: 0 }, + { x: 2, y: 2 }, + ]; + + const options: ASAPOptions = { resolution: 3 }; + const result = asapSmooth(data, options); + + expect(result.length).toBeGreaterThan(0); + + // result should be sorted by x + for (let i = 1; i < result.length; i++) { + expect(result[i].x).toBeGreaterThanOrEqual(result[i - 1].x); + } + }); + + it('should handle negative values', () => { + const data: DataPoint[] = Array.from({ length: 10 }, (_, i) => ({ + x: i, + y: -i * 2, + })); + + const options: ASAPOptions = { resolution: 5 }; + const result = asapSmooth(data, options); + + expect(result.length).toBeGreaterThan(0); + result.forEach((point) => { + expect(isFinite(point.y)).toBe(true); + }); + }); + }); +}); diff --git a/public/app/features/transformers/smoothing/asap.ts b/public/app/features/transformers/smoothing/asap.ts new file mode 100644 index 00000000000..93e28c3dbec --- /dev/null +++ b/public/app/features/transformers/smoothing/asap.ts @@ -0,0 +1,40 @@ +import { ASAP } from 'downsample'; + +export interface DataPoint { + x: number; + y: number; +} + +export interface ASAPOptions { + resolution: number; +} + +export function asapSmooth(data: DataPoint[], options: ASAPOptions): DataPoint[] { + const { resolution } = options; + + if (!data || data.length === 0) { + return []; + } + + // Filter invalid points and convert to tuple format for ASAP library + const inputData: Array<[number, number]> = data + .filter((point) => point != null && !isNaN(point.x) && !isNaN(point.y)) + .map((point) => [point.x, point.y]); + + if (inputData.length === 0) { + return []; + } + + // this prevents O(m×n) degradation if inputData is unsorted data + inputData.sort((a, b) => a[0] - b[0]); + + // ASAP always returns objects with x and y properties + const smoothedData = ASAP(inputData, resolution); + + // Convert back to DataPoint format + const result: DataPoint[] = Array.from(smoothedData).filter( + (item): item is DataPoint => item !== null && typeof item === 'object' && 'x' in item && 'y' in item + ); + + return result; +} diff --git a/public/app/features/transformers/smoothing/smoothing.test.ts b/public/app/features/transformers/smoothing/smoothing.test.ts new file mode 100644 index 00000000000..859ea75c7be --- /dev/null +++ b/public/app/features/transformers/smoothing/smoothing.test.ts @@ -0,0 +1,744 @@ +import { + DataFrame, + DataTransformContext, + FieldType, + toDataFrame, + TransformationApplicabilityLevels, +} from '@grafana/data'; + +import { calculateMaxSourcePoints, getSmoothingTransformer, SmoothingTransformerOptions } from './smoothing'; + +describe('Smoothing transformer', () => { + const smoothingTransformer = getSmoothingTransformer(); + const ctx: DataTransformContext = { + interpolate: (v: string) => v, + }; + + describe('isApplicable', () => { + it('should return Applicable for time series frames', () => { + const frames = [ + toDataFrame({ + name: 'time series', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + expect(smoothingTransformer.isApplicable!(frames)).toBe(TransformationApplicabilityLevels.Applicable); + }); + + it('should return NotApplicable for frames without time field', () => { + const frames = [ + toDataFrame({ + name: 'no time field', + fields: [ + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + expect(smoothingTransformer.isApplicable!(frames)).toBe(TransformationApplicabilityLevels.NotApplicable); + }); + + it('should return Applicable if at least one frame is a time series', () => { + const frames = [ + toDataFrame({ + name: 'not time series', + fields: [ + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + { name: 'label', type: FieldType.string, values: ['X', 'Y', 'Z'] }, + ], + }), + toDataFrame({ + name: 'time series', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + expect(smoothingTransformer.isApplicable!(frames)).toBe(TransformationApplicabilityLevels.Applicable); + }); + + it('should return NotApplicable for empty data', () => { + const frames: DataFrame[] = []; + + expect(smoothingTransformer.isApplicable!(frames)).toBe(TransformationApplicabilityLevels.NotApplicable); + }); + }); + + describe('Basic functionality', () => { + it('should smooth time series data with default settings', () => { + const source = [ + toDataFrame({ + name: 'test data', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15, 25, 18] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // first frame should be the original, unchanged + expect(result[0].name).toBe('test data'); + expect(result[0].fields).toHaveLength(2); + expect(result[0].fields[0].name).toBe('time'); + expect(result[0].fields[1].name).toBe('value'); + expect(result[0].fields[1].values).toEqual([10, 20, 15, 25, 18]); + + // second frame should be the smoothed version + expect(result[1].name).toBe('Smoothed'); + expect(result[1].fields).toHaveLength(2); + expect(result[1].fields[0].name).toBe('time'); + expect(result[1].fields[1].name).toBe('value'); + + // should preserve original time points + expect(result[1].fields[0].values).toEqual([1000, 2000, 3000, 4000, 5000]); + // should have corresponding smoothed values + expect(result[1].fields[1].values.length).toBe(5); + }); + + it('should handle multiple numeric fields', () => { + const source = [ + toDataFrame({ + name: 'multi field data', + refId: 'B', + fields: [ + { name: 'timestamp', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'cpu', type: FieldType.number, values: [50, 75, 60, 80] }, + { name: 'memory', type: FieldType.number, values: [40, 55, 45, 65] }, + { name: 'label', type: FieldType.string, values: ['a', 'b', 'c', 'd'] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 3 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // first frame is original + expect(result[0].name).toBe('multi field data'); + expect(result[0].fields[1].name).toBe('cpu'); + expect(result[0].fields[2].name).toBe('memory'); + + // second frame is smoothed + expect(result[1].fields).toHaveLength(4); + expect(result[1].fields[0].name).toBe('timestamp'); + expect(result[1].fields[1].name).toBe('cpu'); + expect(result[1].fields[2].name).toBe('memory'); + expect(result[1].fields[3].name).toBe('label'); + + // all numeric fields should be smoothed and preserve original time points + expect(result[1].fields[0].values.length).toBe(4); + expect(result[1].fields[1].values.length).toBe(4); + expect(result[1].fields[2].values.length).toBe(4); + }); + + it('should preserve non-numeric and non-time fields', () => { + const source = [ + toDataFrame({ + name: 'mixed data', + refId: 'C', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + { name: 'active', type: FieldType.boolean, values: [true, false, true] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 2 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // smoothed frame should preserve non-numeric fields + expect(result[1].fields[2].name).toBe('category'); + expect(result[1].fields[2].type).toBe(FieldType.string); + expect(result[1].fields[3].name).toBe('active'); + expect(result[1].fields[3].type).toBe(FieldType.boolean); + }); + }); + + describe('Configuration options', () => { + it('should use default resolution when not specified', () => { + const source = [ + toDataFrame({ + name: 'default test', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: Array.from({ length: 200 }, (_, i) => i * 1000) }, + { name: 'value', type: FieldType.number, values: Array.from({ length: 200 }, () => Math.random() * 100) }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // smoothed frame should preserve all original time points + expect(result[1].fields[0].values.length).toBe(200); + expect(result[1].fields[1].values.length).toBe(200); + }); + + it('should respect custom resolution settings', () => { + const source = [ + toDataFrame({ + name: 'resolution test', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: Array.from({ length: 100 }, (_, i) => i * 1000) }, + { name: 'value', type: FieldType.number, values: Array.from({ length: 100 }, () => Math.random() * 100) }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 25 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // smoothed frame should preserve all original time points regardless of resolution + expect(result[1].fields[0].values.length).toBe(100); + expect(result[1].fields[1].values.length).toBe(100); + }); + + it('should clamp resolution to minimum value', () => { + const source = [ + toDataFrame({ + name: 'small resolution test', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15, 25, 18] }, + ], + }), + ]; + + // request resolution below minimum, it should be clamped to 1 + const config: SmoothingTransformerOptions = { resolution: 2 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // smoothed frame should preserve all original time points and clamp resolution to minimum + expect(result[1].fields[0].values.length).toBe(5); + expect(result[1].fields[1].values.length).toBe(5); + }); + }); + + describe('Edge cases', () => { + it('should handle empty data frames', () => { + const source: DataFrame[] = []; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + expect(result).toEqual([]); + }); + + it('should handle frames without time fields', () => { + const source = [ + toDataFrame({ + name: 'no time field', + refId: 'A', + fields: [ + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return original frame unchanged + expect(result).toHaveLength(1); + expect(result[0]).toEqual(source[0]); + }); + + it('should handle frames without numeric fields', () => { + const source = [ + toDataFrame({ + name: 'no numeric fields', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return original frame unchanged + expect(result).toHaveLength(1); + expect(result[0]).toEqual(source[0]); + }); + + it('should filter out NaN values when smoothing', () => { + const source = [ + toDataFrame({ + name: 'data with NaN', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'value', type: FieldType.number, values: [10, NaN, 15, 25, NaN] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 3 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // smoothed frame should preserve all time points + expect(result[1].fields[0].values.length).toBe(5); + expect(result[1].fields[1].values.length).toBe(5); + + // all values should be interpolated from smoothed curve (no nulls) + const values = result[1].fields[1].values; + values.forEach((value) => { + expect(value).not.toBeNull(); + expect(typeof value).toBe('number'); + expect(isNaN(value)).toBe(false); + }); + }); + + it('should handle data with all NaN values', () => { + const source = [ + toDataFrame({ + name: 'all NaN data', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [NaN, NaN, NaN] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // When all values are NaN, only original frame should be returned (no smoothed frame) + expect(result).toHaveLength(1); + expect(result[0].fields[1].name).toBe('value'); // No "(smoothed)" suffix + expect(result[0].fields[1].values).toEqual([NaN, NaN, NaN]); + expect(result[0].name).toBe('all NaN data'); // Original name preserved + }); + + it('should handle data with null values', () => { + const source = [ + toDataFrame({ + name: 'data with nulls', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'value', type: FieldType.number, values: [10, null, 15, 25] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 3 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // smoothed frame should preserve all time points + expect(result[1].fields[0].values.length).toBe(4); + expect(result[1].fields[1].values.length).toBe(4); + + // all values should be interpolated (no nulls in output) + const values = result[1].fields[1].values; + values.forEach((value) => { + expect(value).not.toBeNull(); + expect(typeof value).toBe('number'); + expect(isNaN(value)).toBe(false); + }); + }); + + it('should handle single data point', () => { + const source = [ + toDataFrame({ + name: 'single point', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000] }, + { name: 'value', type: FieldType.number, values: [42] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + expect(result[1].fields[0].values).toHaveLength(1); + expect(result[1].fields[1].values).toHaveLength(1); + expect(result[1].fields[1].values[0]).toBe(42); + }); + + it('should handle empty numeric field values', () => { + const source = [ + toDataFrame({ + name: 'empty values', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return original frame since no numeric data to smooth + expect(result[0]).toEqual(source[0]); + }); + }); + + describe('Data integrity', () => { + it('should maintain time ordering in smoothed data', () => { + const source = [ + toDataFrame({ + name: 'ordered data', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15, 25, 18] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 4 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // check smoothed frame's time values + const timeValues = result[1].fields[0].values as number[]; + + // check that time values are in ascending order + for (let i = 1; i < timeValues.length; i++) { + expect(timeValues[i]).toBeGreaterThanOrEqual(timeValues[i - 1]); + } + }); + + it('should preserve original frame metadata', () => { + const source = [ + toDataFrame({ + name: 'original name', + refId: 'TEST', + meta: { custom: { test: 'value' } }, + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + + // original frame unchanged + expect(result[0].refId).toBe('TEST'); + expect(result[0].meta).toEqual(source[0].meta); + expect(result[0].name).toBe('original name'); + + // smoothed frame preserves metadata + expect(result[1].refId).toBe('TEST'); + expect(result[1].meta).toEqual(source[0].meta); + expect(result[1].name).toBe('Smoothed'); + }); + + it('should handle frames with no name', () => { + const source = [ + toDataFrame({ + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + expect(result[1].name).toBe('Smoothed'); + }); + }); + + describe('Real-world scenarios', () => { + it('should handle sparse data with irregular intervals', () => { + // based on real user data with ~10 points over 30 minutes + const source = [ + toDataFrame({ + name: 'temperature', + refId: 'A', + fields: [ + { + name: 'time', + type: FieldType.time, + values: [ + 1733999700000, 1733999790000, 1734000000000, 1734000210000, 1734000420000, 1734000630000, 1734000840000, + 1734001050000, 1734001260000, 1734001470000, + ], + }, + { + name: 'value', + type: FieldType.number, + values: [31.1, 31.1, 30.2, 30.8, 29.8, 30.0, 29.3, 28.6, 29.6, 30.5], + }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 20 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return both original and smoothed frames + expect(result).toHaveLength(2); + expect(result[1].fields[0].values.length).toBe(10); + expect(result[1].fields[1].values.length).toBe(10); + + // all values should be non-null numbers + const values = result[1].fields[1].values; + values.forEach((value) => { + expect(value).not.toBeNull(); + expect(typeof value).toBe('number'); + expect(isNaN(value)).toBe(false); + }); + }); + }); + + describe('Multiple frames', () => { + it('should process multiple frames independently', () => { + const source = [ + toDataFrame({ + name: 'frame1', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + toDataFrame({ + name: 'frame2', + refId: 'B', + fields: [ + { name: 'timestamp', type: FieldType.time, values: [4000, 5000, 6000] }, + { name: 'metric', type: FieldType.number, values: [30, 40, 35] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = { resolution: 2 }; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return original frames + smoothed frames (2 original + 2 smoothed = 4 total) + expect(result).toHaveLength(4); + + // original frames first + expect(result[0].name).toBe('frame1'); + expect(result[0].refId).toBe('A'); + expect(result[1].name).toBe('frame2'); + expect(result[1].refId).toBe('B'); + + // smoothed frames after + expect(result[2].name).toBe('Smoothed'); + expect(result[2].refId).toBe('A'); + expect(result[3].name).toBe('Smoothed'); + expect(result[3].refId).toBe('B'); + }); + + it('should handle mixed frame types', () => { + const source = [ + toDataFrame({ + name: 'valid frame', + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + toDataFrame({ + name: 'invalid frame', + refId: 'B', + fields: [ + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + { name: 'label', type: FieldType.string, values: ['X', 'Y', 'Z'] }, + ], + }), + ]; + + const config: SmoothingTransformerOptions = {}; + + const result = smoothingTransformer.transformer(config, ctx)(source); + + // should return 2 original frames + 1 smoothed frame (only valid frame gets smoothed) + expect(result).toHaveLength(3); + + // original frames first + expect(result[0].name).toBe('valid frame'); + expect(result[1]).toEqual(source[1]); + + // smoothed frame after + expect(result[2].name).toBe('Smoothed'); + }); + }); + + describe('calculateMaxSourcePoints', () => { + it('should return 0 for empty frames', () => { + expect(calculateMaxSourcePoints([])).toBe(0); + }); + + it('should return 0 for frames without time fields', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + ]; + + expect(calculateMaxSourcePoints(frames)).toBe(0); + }); + + it('should return 0 for frames without numeric fields', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'category', type: FieldType.string, values: ['A', 'B', 'C'] }, + ], + }), + ]; + + expect(calculateMaxSourcePoints(frames)).toBe(0); + }); + + it('should count valid data points, filtering out null and NaN', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'value', type: FieldType.number, values: [10, null, 15, NaN, 18] }, + ], + }), + ]; + + // Only 3 valid points: 10, 15, 18 + expect(calculateMaxSourcePoints(frames)).toBe(3); + }); + + it('should return maximum across multiple numeric fields', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'cpu', type: FieldType.number, values: [10, null, 15] }, // 2 valid points + { name: 'memory', type: FieldType.number, values: [20, 25, 30, 35] }, // 4 valid points + ], + }), + ]; + + expect(calculateMaxSourcePoints(frames)).toBe(4); + }); + + it('should return maximum across multiple frames', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 15] }, + ], + }), + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000, 5000] }, + { name: 'metric', type: FieldType.number, values: [30, 40, 35, 45, 50] }, + ], + }), + ]; + + expect(calculateMaxSourcePoints(frames)).toBe(5); + }); + + it('should handle frames with all valid points', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'value', type: FieldType.number, values: [10, 20, 30, 40] }, + ], + }), + ]; + + expect(calculateMaxSourcePoints(frames)).toBe(4); + }); + + it('should handle frames with all null values', () => { + const frames = [ + toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'value', type: FieldType.number, values: [null, null, null] }, + ], + }), + ]; + + expect(calculateMaxSourcePoints(frames)).toBe(0); + }); + }); +}); diff --git a/public/app/features/transformers/smoothing/smoothing.ts b/public/app/features/transformers/smoothing/smoothing.ts new file mode 100644 index 00000000000..ad829f8cc91 --- /dev/null +++ b/public/app/features/transformers/smoothing/smoothing.ts @@ -0,0 +1,267 @@ +import { map } from 'rxjs'; + +import { + DataFrame, + DataTransformerID, + FieldType, + SynchronousDataTransformerInfo, + isTimeSeriesFrame, + TransformationApplicabilityLevels, +} from '@grafana/data'; +import { t } from '@grafana/i18n'; + +import { asapSmooth, DataPoint } from './asap'; + +export interface SmoothingTransformerOptions { + resolution?: number; +} + +export const DEFAULTS = { + resolution: 100, +}; + +export const RESOLUTION_LIMITS = { + min: 1, + max: 1000, +}; + +const MAX_RESOLUTION_MULTIPLIER = 2; + +// converts time and value arrays into valid DataPoints, filtering out null/NaN values +export const createDataPoints = (timeValues: number[], sourceField: Array): DataPoint[] => { + return timeValues + .map((time, index) => ({ + x: time, + y: sourceField[index], + })) + .filter((point): point is DataPoint => point.y != null && !isNaN(point.y)); +}; + +// calculates effective resolution capped at 2x source points +export const calculateEffectiveResolution = (resolution: number, sourcePointCount: number): number => { + return Math.min(resolution, sourcePointCount * MAX_RESOLUTION_MULTIPLIER); +}; + +// calculates the maximum number of source points across all numeric fields in all frames +export const calculateMaxSourcePoints = (frames: DataFrame[]): number => { + let maxSourcePoints = 0; + + for (const frame of frames) { + const timeField = frame.fields.find((f) => f.type === FieldType.time); + if (!timeField) { + continue; + } + + for (const field of frame.fields) { + if (field.type === FieldType.number) { + const sourcePoints = createDataPoints(timeField.values, field.values); + if (sourcePoints.length > maxSourcePoints) { + maxSourcePoints = sourcePoints.length; + } + } + } + } + + return maxSourcePoints; +}; + +// performs linear interpolation between two points +export const linearInterpolate = (leftPoint: DataPoint, rightPoint: DataPoint, targetTime: number): number => { + // exact match + if (leftPoint.x === targetTime) { + return leftPoint.y; + } + if (rightPoint.x === targetTime) { + return rightPoint.y; + } + + // same point (shouldn't happen but handle gracefully) + if (leftPoint.x === rightPoint.x) { + return leftPoint.y; + } + + // linear interpolation + const ratio = (targetTime - leftPoint.x) / (rightPoint.x - leftPoint.x); + return leftPoint.y + ratio * (rightPoint.y - leftPoint.y); +}; + +// finds the two points in smoothedData that bracket the targetTime +export const findBracketingPoints = ( + smoothedData: DataPoint[], + targetTime: number, + lastIndex: number +): { leftPoint: DataPoint; rightPoint: DataPoint; newIndex: number } => { + // find the two points to interpolate between, starting from last known position + // if target is before our current search position, reset to beginning + let searchStart = Math.min(lastIndex, smoothedData.length - 2); + if (targetTime < smoothedData[searchStart].x) { + searchStart = 0; + } + + let leftPoint = smoothedData[searchStart]; + let rightPoint = smoothedData[searchStart + 1]; + let newIndex = searchStart; + + for (let i = searchStart; i < smoothedData.length - 1; i++) { + if (smoothedData[i].x <= targetTime && smoothedData[i + 1].x >= targetTime) { + leftPoint = smoothedData[i]; + rightPoint = smoothedData[i + 1]; + newIndex = i; + break; + } + } + + return { leftPoint, rightPoint, newIndex }; +}; + +// interpolates smoothed data back to original time points +export const interpolateToTimePoints = (smoothedData: DataPoint[], timeValues: number[]): number[] => { + const firstPoint = smoothedData[0]; + const lastPoint = smoothedData[smoothedData.length - 1]; + + let lastIndex = 0; + return timeValues.map((targetTime) => { + // handle out of bounds, use edge values instead of null + if (targetTime <= firstPoint.x) { + return firstPoint.y; + } + if (targetTime >= lastPoint.x) { + return lastPoint.y; + } + + const { leftPoint, rightPoint, newIndex } = findBracketingPoints(smoothedData, targetTime, lastIndex); + lastIndex = newIndex; + + return linearInterpolate(leftPoint, rightPoint, targetTime); + }); +}; + +// smooths a time series by creating a smoothed curve and interpolating back to original time points +export const interpolateFromSmoothedCurve = ( + sourceField: Array, + timeValues: number[], + resolution: number +): Array | null => { + const sourcePoints = createDataPoints(timeValues, sourceField); + + // if no valid source points, return null to signal this field should not be smoothed + if (sourcePoints.length === 0) { + return null; + } + + // smooth the source field's data with effective resolution + const effectiveFieldResolution = calculateEffectiveResolution(resolution, sourcePoints.length); + const smoothedData = asapSmooth(sourcePoints, { resolution: effectiveFieldResolution }); + + if (smoothedData.length === 0) { + return timeValues.map(() => null); + } + + // handle single point case - return the same value for all time points + if (smoothedData.length === 1) { + const singleValue = smoothedData[0].y; + return timeValues.map(() => singleValue); + } + + // this prevents O(m×n) degradation if asapSmooth returns unsorted data + smoothedData.sort((a, b) => a.x - b.x); + + // interpolate smoothed data back to original time points + return interpolateToTimePoints(smoothedData, timeValues); +}; + +export const getSmoothingTransformer: () => SynchronousDataTransformerInfo = () => ({ + id: DataTransformerID.smoothing, + name: t('transformers.smoothing.name', 'Smoothing'), + description: t( + 'transformers.smoothing.description', + 'Reduce noise in time series data through adaptive downsampling.' + ), + isApplicable: (data) => { + for (const frame of data) { + if (isTimeSeriesFrame(frame)) { + return TransformationApplicabilityLevels.Applicable; + } + } + + return TransformationApplicabilityLevels.NotApplicable; + }, + isApplicableDescription: t( + 'transformers.smoothing.is-applicable-description', + 'The Smoothing transformation requires at least one time series frame to function. You currently have none.' + ), + operator: (options, ctx) => { + const transformer = getSmoothingTransformer().transformer(options, ctx); + return (source) => source.pipe(map(transformer)); + }, + transformer: (options, ctx) => { + return (frames: DataFrame[]) => { + // clamp resolution to valid range to handle edge cases from API/plugins + const rawResolution = options.resolution ?? DEFAULTS.resolution; + const resolution = Math.max(RESOLUTION_LIMITS.min, Math.min(RESOLUTION_LIMITS.max, rawResolution)); + + if (frames.length === 0) { + return frames; + } + + const smoothedFrames: DataFrame[] = []; + + for (const frame of frames) { + const timeField = frame.fields.find((f) => f.type === FieldType.time); + if (!timeField) { + continue; + } + + // check if there's at least one numeric field with valid data + const hasValidNumericField = frame.fields.some((f) => { + if (f.type !== FieldType.number || f.values.length === 0) { + return false; + } + return f.values.some((v) => v != null && !isNaN(v)); + }); + + if (!hasValidNumericField) { + continue; + } + + // create smoothed fields for all numeric fields + const smoothedFields = [timeField]; // keep original time field + let anyFieldSmoothed = false; + + for (const field of frame.fields) { + if (field.type === FieldType.number) { + const smoothedValues = interpolateFromSmoothedCurve(field.values, timeField.values, resolution); + + // if smoothing returned null (no valid data), skip this field + if (smoothedValues === null) { + continue; + } + + anyFieldSmoothed = true; + smoothedFields.push({ + ...field, + values: smoothedValues, + state: undefined, + }); + } else if (field.type !== FieldType.time) { + // include other non-numeric, non-time fields (like labels) + smoothedFields.push(field); + } + } + + // only create a smoothed frame if at least one field was smoothed + if (anyFieldSmoothed) { + const smoothedFrame: DataFrame = { + ...frame, + name: 'Smoothed', + fields: smoothedFields, + }; + smoothedFrames.push(smoothedFrame); + } + } + + // return original frames followed by smoothed frames + return [...frames, ...smoothedFrames]; + }; + }, +}); diff --git a/public/app/features/transformers/smoothing/smoothingEditor.tsx b/public/app/features/transformers/smoothing/smoothingEditor.tsx new file mode 100644 index 00000000000..2f9ad2586d2 --- /dev/null +++ b/public/app/features/transformers/smoothing/smoothingEditor.tsx @@ -0,0 +1,93 @@ +import { css } from '@emotion/css'; +import { useMemo } from 'react'; + +import { DataTransformerID, TransformerRegistryItem, TransformerUIProps, TransformerCategory } from '@grafana/data'; +import { t } from '@grafana/i18n'; +import { InlineField, InlineFieldRow, Tooltip, useTheme2 } from '@grafana/ui'; +import { NumberInput } from 'app/core/components/OptionsUI/NumberInput'; + +import { getTransformationContent } from '../docs/getTransformationContent'; +import darkImage from '../images/dark/smoothing.svg'; +import lightImage from '../images/light/smoothing.svg'; + +import { + DEFAULTS, + RESOLUTION_LIMITS, + SmoothingTransformerOptions, + getSmoothingTransformer, + calculateEffectiveResolution, + calculateMaxSourcePoints, +} from './smoothing'; + +export const SmoothingTransformerEditor = ({ + input, + options, + onChange, +}: TransformerUIProps) => { + const theme = useTheme2(); + const resolution = options.resolution ?? DEFAULTS.resolution; + + const maxSourcePoints = useMemo(() => calculateMaxSourcePoints(input), [input]); + const effectiveResolution = maxSourcePoints > 0 ? calculateEffectiveResolution(resolution, maxSourcePoints) : null; + const showEffectiveResolution = effectiveResolution !== null && effectiveResolution < resolution; + + return ( + + + onChange({ ...options, resolution: v })} + min={RESOLUTION_LIMITS.min} + max={RESOLUTION_LIMITS.max} + width={20} + suffix={ + showEffectiveResolution ? ( + + + {t('transformers.smoothing.effective-resolution', 'Effective: {{value}}', { + value: effectiveResolution, + })} + + + ) : undefined + } + /> + + + ); +}; + +export const getSmoothingTransformerRegistryItem: () => TransformerRegistryItem = () => { + const smoothingTransformer = getSmoothingTransformer(); + return { + id: DataTransformerID.smoothing, + editor: SmoothingTransformerEditor, + transformation: smoothingTransformer, + name: smoothingTransformer.name, + description: smoothingTransformer.description, + categories: new Set([TransformerCategory.CalculateNewFields]), + imageDark: darkImage, + imageLight: lightImage, + help: getTransformationContent(DataTransformerID.smoothing).helperDocs, + tags: new Set(['ASAP', 'Autosmooth']), + }; +}; diff --git a/public/app/features/transformers/standardTransformers.ts b/public/app/features/transformers/standardTransformers.ts index 3dbe886bab3..5cebf190082 100644 --- a/public/app/features/transformers/standardTransformers.ts +++ b/public/app/features/transformers/standardTransformers.ts @@ -1,4 +1,5 @@ import { TransformerRegistryItem } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { getFilterByValueTransformRegistryItem } from './FilterByValueTransformer/FilterByValueTransformerEditor'; import { getHeatmapTransformRegistryItem } from './calculateHeatmap/HeatmapTransformerEditor'; @@ -31,6 +32,7 @@ import { getPartitionByValuesTransformRegistryItem } from './partitionByValues/P import { getPrepareTimeseriesTransformerRegistryItem } from './prepareTimeSeries/PrepareTimeSeriesEditor'; import { getRegressionTransformerRegistryItem } from './regression/regressionEditor'; import { getRowsToFieldsTransformRegistryItem } from './rowsToFields/RowsToFieldsTransformerEditor'; +import { getSmoothingTransformerRegistryItem } from './smoothing/smoothingEditor'; import { getSpatialTransformRegistryItem } from './spatial/SpatialTransformerEditor'; import { getTimeSeriesTableTransformRegistryItem } from './timeSeriesTable/TimeSeriesTableTransformEditor'; @@ -66,6 +68,7 @@ export const getStandardTransformers = (): TransformerRegistryItem[] => { getPartitionByValuesTransformRegistryItem(), getFormatStringTransformerRegistryItem(), getGroupToNestedTableTransformRegistryItem(), + ...(config.featureToggles.smoothingTransformation ? [getSmoothingTransformerRegistryItem()] : []), getFormatTimeTransformerRegistryItem(), getTimeSeriesTableTransformRegistryItem(), getTransposeTransformerRegistryItem(), diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index d4274aa98cd..99ed9b512a6 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -14399,6 +14399,17 @@ "series-to-rows": "Series to rows" } }, + "smoothing": { + "description": "Reduce noise in time series data through adaptive downsampling.", + "effective-resolution": "Effective: {{value}}", + "effective-resolution-tooltip": "Resolution is limited to 2× the number of data points ({{points}}).", + "is-applicable-description": "The Smoothing transformation requires at least one time series frame to function. You currently have none.", + "name": "Smoothing", + "resolution": { + "label": "Resolution", + "tooltip": "Controls smoothing intensity. Lower values create more aggressive smoothing. Both original and smoothed data are displayed." + } + }, "sort-by-transformer-editor": { "description": { "sort-fields": "Sort fields in a frame." diff --git a/yarn.lock b/yarn.lock index f9e4168eed8..afa76953435 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16497,6 +16497,13 @@ __metadata: languageName: node linkType: hard +"downsample@npm:1.4.0": + version: 1.4.0 + resolution: "downsample@npm:1.4.0" + checksum: 10/ad0ab937e368546b577b564b13d7f39cd85a92bf29d56562aaa6ed10bac19e91ee75ab58f38050a9e8bf601c1abcfda942541880a84c89ba78d1775a229636d1 + languageName: node + linkType: hard + "downshift@npm:^9.0.6": version: 9.0.10 resolution: "downshift@npm:9.0.10" @@ -19629,6 +19636,7 @@ __metadata: date-fns: "npm:4.1.0" debounce-promise: "npm:3.1.2" diff: "npm:^8.0.0" + downsample: "npm:1.4.0" enquirer: "npm:^2.4.1" esbuild: "npm:0.25.8" esbuild-loader: "npm:4.3.0"