From 479209f4832ce306d76bfb828ad5bd300c14ae8f Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Mon, 12 Mar 2018 17:13:05 +0300 Subject: [PATCH] prometheus: datasource refactor --- .../datasource/prometheus/datasource.ts | 210 ++---------------- .../prometheus/result_transformer.ts | 199 +++++++++++++++++ .../prometheus/specs/datasource_specs.ts | 50 +---- .../specs/result_transformer.jest.ts | 78 +++++++ 4 files changed, 293 insertions(+), 244 deletions(-) create mode 100644 public/app/plugins/datasource/prometheus/result_transformer.ts create mode 100644 public/app/plugins/datasource/prometheus/specs/result_transformer.jest.ts diff --git a/public/app/plugins/datasource/prometheus/datasource.ts b/public/app/plugins/datasource/prometheus/datasource.ts index 067c85cc3a4..9ad720f8917 100644 --- a/public/app/plugins/datasource/prometheus/datasource.ts +++ b/public/app/plugins/datasource/prometheus/datasource.ts @@ -3,7 +3,7 @@ import _ from 'lodash'; import kbn from 'app/core/utils/kbn'; import * as dateMath from 'app/core/utils/datemath'; import PrometheusMetricFindQuery from './metric_find_query'; -import TableModel from 'app/core/table_model'; +import { ResultTransformer } from './result_transformer'; function prometheusSpecialRegexEscape(value) { return value.replace(/[\\^$*+?.()|[\]{}]/g, '\\\\$&'); @@ -20,6 +20,7 @@ export class PrometheusDatasource { withCredentials: any; metricsNameCache: any; interval: string; + resultTransformer: ResultTransformer; /** @ngInject */ constructor(instanceSettings, private $q, private backendSrv, private templateSrv, private timeSrv) { @@ -32,6 +33,7 @@ export class PrometheusDatasource { this.basicAuth = instanceSettings.basicAuth; this.withCredentials = instanceSettings.withCredentials; this.interval = instanceSettings.jsonData.timeInterval || '15s'; + this.resultTransformer = new ResultTransformer(templateSrv); } _request(method, url, requestId?) { @@ -73,7 +75,6 @@ export class PrometheusDatasource { } query(options) { - var self = this; var start = this.getPrometheusTime(options.range.from, false); var end = this.getPrometheusTime(options.range.to, true); var range = Math.ceil(end - start); @@ -113,29 +114,17 @@ export class PrometheusDatasource { throw response.error; } - let prometheusResult = response.data.data.result; + let transformerOptions = { + format: activeTargets[index].format, + step: queries[index].step, + legendFormat: activeTargets[index].legendFormat, + start: start, + end: end, + responseListLength: responseList.length, + responseIndex: index, + }; - if (activeTargets[index].format === 'table') { - result.push(self.transformMetricDataToTable(prometheusResult, responseList.length, index)); - } else if (activeTargets[index].format === 'heatmap') { - let seriesList = []; - prometheusResult.sort(sortSeriesByLabel); - for (let metricData of prometheusResult) { - seriesList.push( - self.transformMetricData(metricData, activeTargets[index], start, end, queries[index].step) - ); - } - seriesList = self.transformToHistogramOverTime(seriesList); - result.push(...seriesList); - } else { - for (let metricData of prometheusResult) { - if (response.data.data.resultType === 'matrix') { - result.push(self.transformMetricData(metricData, activeTargets[index], start, end, queries[index].step)); - } else if (response.data.data.resultType === 'vector') { - result.push(self.transformInstantMetricData(metricData, activeTargets[index])); - } - } - } + this.resultTransformer.transform(result, response, transformerOptions); }); return { data: result }; @@ -276,9 +265,9 @@ export class PrometheusDatasource { var event = { annotation: annotation, time: Math.floor(parseFloat(value[0])) * 1000, - title: self.renderTemplate(titleFormat, series.metric), + title: self.resultTransformer.renderTemplate(titleFormat, series.metric), tags: tags, - text: self.renderTemplate(textFormat, series.metric), + text: self.resultTransformer.renderTemplate(textFormat, series.metric), }; eventList.push(event); @@ -296,145 +285,6 @@ export class PrometheusDatasource { }); } - transformMetricData(md, options, start, end, step) { - var dps = [], - metricLabel = null; - - metricLabel = this.createMetricLabel(md.metric, options); - - var stepMs = step * 1000; - var baseTimestamp = start * 1000; - for (let value of md.values) { - var dp_value = parseFloat(value[1]); - if (_.isNaN(dp_value)) { - dp_value = null; - } - - var timestamp = parseFloat(value[0]) * 1000; - for (let t = baseTimestamp; t < timestamp; t += stepMs) { - dps.push([null, t]); - } - baseTimestamp = timestamp + stepMs; - dps.push([dp_value, timestamp]); - } - - var endTimestamp = end * 1000; - for (let t = baseTimestamp; t <= endTimestamp; t += stepMs) { - dps.push([null, t]); - } - - return { target: metricLabel, datapoints: dps }; - } - - transformMetricDataToTable(md, resultCount: number, resultIndex: number) { - var table = new TableModel(); - var i, j; - var metricLabels = {}; - - if (md.length === 0) { - return table; - } - - // Collect all labels across all metrics - _.each(md, function(series) { - for (var label in series.metric) { - if (!metricLabels.hasOwnProperty(label)) { - metricLabels[label] = 1; - } - } - }); - - // Sort metric labels, create columns for them and record their index - var sortedLabels = _.keys(metricLabels).sort(); - table.columns.push({ text: 'Time', type: 'time' }); - _.each(sortedLabels, function(label, labelIndex) { - metricLabels[label] = labelIndex + 1; - table.columns.push({ text: label }); - }); - let valueText = resultCount > 1 ? `Value #${String.fromCharCode(65 + resultIndex)}` : 'Value'; - table.columns.push({ text: valueText }); - - // Populate rows, set value to empty string when label not present. - _.each(md, function(series) { - if (series.value) { - series.values = [series.value]; - } - if (series.values) { - for (i = 0; i < series.values.length; i++) { - var values = series.values[i]; - var reordered: any = [values[0] * 1000]; - if (series.metric) { - for (j = 0; j < sortedLabels.length; j++) { - var label = sortedLabels[j]; - if (series.metric.hasOwnProperty(label)) { - reordered.push(series.metric[label]); - } else { - reordered.push(''); - } - } - } - reordered.push(parseFloat(values[1])); - table.rows.push(reordered); - } - } - }); - - return table; - } - - transformInstantMetricData(md, options) { - var dps = [], - metricLabel = null; - metricLabel = this.createMetricLabel(md.metric, options); - dps.push([parseFloat(md.value[1]), md.value[0] * 1000]); - return { target: metricLabel, datapoints: dps }; - } - - transformToHistogramOverTime(seriesList, options?) { - /* t1 = timestamp1, t2 = timestamp2 etc. - t1 t2 t3 t1 t2 t3 - le10 10 10 0 => 10 10 0 - le20 20 10 30 => 10 0 30 - le30 30 10 35 => 10 0 5 - */ - for (let i = seriesList.length - 1; i > 0; i--) { - let topSeries = seriesList[i].datapoints; - let bottomSeries = seriesList[i - 1].datapoints; - for (let j = 0; j < topSeries.length; j++) { - topSeries[j][0] -= bottomSeries[j][0]; - } - } - - return seriesList; - } - - createMetricLabel(labelData, options) { - if (_.isUndefined(options) || _.isEmpty(options.legendFormat)) { - return this.getOriginalMetricName(labelData); - } - - return this.renderTemplate(this.templateSrv.replace(options.legendFormat), labelData) || '{}'; - } - - renderTemplate(aliasPattern, aliasData) { - var aliasRegex = /\{\{\s*(.+?)\s*\}\}/g; - return aliasPattern.replace(aliasRegex, function(match, g1) { - if (aliasData[g1]) { - return aliasData[g1]; - } - return g1; - }); - } - - getOriginalMetricName(labelData) { - var metricName = labelData.__name__ || ''; - delete labelData.__name__; - var labelPart = _.map(_.toPairs(labelData), function(label) { - return label[0] + '="' + label[1] + '"'; - }).join(','); - return metricName + '{' + labelPart + '}'; - } - getPrometheusTime(date, roundUp) { if (_.isString(date)) { date = dateMath.parse(date, roundUp); @@ -442,33 +292,3 @@ export class PrometheusDatasource { return Math.ceil(date.valueOf() / 1000); } } - -function sortSeriesByLabel(s1, s2) { - let le1, le2; - - try { - // fail if not integer. might happen with bad queries - le1 = parseHistogramLabel(s1.metric.le); - le2 = parseHistogramLabel(s2.metric.le); - } catch (err) { - console.log(err); - return 0; - } - - if (le1 > le2) { - return 1; - } - - if (le1 < le2) { - return -1; - } - - return 0; -} - -function parseHistogramLabel(le: string): number { - if (le === '+Inf') { - return +Infinity; - } - return Number(le); -} diff --git a/public/app/plugins/datasource/prometheus/result_transformer.ts b/public/app/plugins/datasource/prometheus/result_transformer.ts new file mode 100644 index 00000000000..6d97b783983 --- /dev/null +++ b/public/app/plugins/datasource/prometheus/result_transformer.ts @@ -0,0 +1,199 @@ +import _ from 'lodash'; +import TableModel from 'app/core/table_model'; + +export class ResultTransformer { + constructor(private templateSrv) {} + + transform(result: any, response: any, options: any) { + let prometheusResult = response.data.data.result; + + if (options.format === 'table') { + result.push(this.transformMetricDataToTable(prometheusResult, options.responseListLength, options.responseIndex)); + } else if (options.format === 'heatmap') { + let seriesList = []; + prometheusResult.sort(sortSeriesByLabel); + for (let metricData of prometheusResult) { + seriesList.push(this.transformMetricData(metricData, options, options.start, options.end)); + } + seriesList = this.transformToHistogramOverTime(seriesList); + result.push(...seriesList); + } else { + for (let metricData of prometheusResult) { + if (response.data.data.resultType === 'matrix') { + result.push(this.transformMetricData(metricData, options, options.start, options.end)); + } else if (response.data.data.resultType === 'vector') { + result.push(this.transformInstantMetricData(metricData, options)); + } + } + } + } + + transformMetricData(md, options, start, end) { + let dps = [], + metricLabel = null; + + metricLabel = this.createMetricLabel(md.metric, options); + + const stepMs = parseInt(options.step) * 1000; + let baseTimestamp = start * 1000; + for (let value of md.values) { + let dp_value = parseFloat(value[1]); + if (_.isNaN(dp_value)) { + dp_value = null; + } + + const timestamp = parseFloat(value[0]) * 1000; + for (let t = baseTimestamp; t < timestamp; t += stepMs) { + dps.push([null, t]); + } + baseTimestamp = timestamp + stepMs; + dps.push([dp_value, timestamp]); + } + + const endTimestamp = end * 1000; + for (let t = baseTimestamp; t <= endTimestamp; t += stepMs) { + dps.push([null, t]); + } + + return { target: metricLabel, datapoints: dps }; + } + + transformMetricDataToTable(md, resultCount: number, resultIndex: number) { + var table = new TableModel(); + var i, j; + var metricLabels = {}; + + if (md.length === 0) { + return table; + } + + // Collect all labels across all metrics + _.each(md, function(series) { + for (var label in series.metric) { + if (!metricLabels.hasOwnProperty(label)) { + metricLabels[label] = 1; + } + } + }); + + // Sort metric labels, create columns for them and record their index + var sortedLabels = _.keys(metricLabels).sort(); + table.columns.push({ text: 'Time', type: 'time' }); + _.each(sortedLabels, function(label, labelIndex) { + metricLabels[label] = labelIndex + 1; + table.columns.push({ text: label }); + }); + let valueText = resultCount > 1 ? `Value #${String.fromCharCode(65 + resultIndex)}` : 'Value'; + table.columns.push({ text: valueText }); + + // Populate rows, set value to empty string when label not present. + _.each(md, function(series) { + if (series.value) { + series.values = [series.value]; + } + if (series.values) { + for (i = 0; i < series.values.length; i++) { + var values = series.values[i]; + var reordered: any = [values[0] * 1000]; + if (series.metric) { + for (j = 0; j < sortedLabels.length; j++) { + var label = sortedLabels[j]; + if (series.metric.hasOwnProperty(label)) { + reordered.push(series.metric[label]); + } else { + reordered.push(''); + } + } + } + reordered.push(parseFloat(values[1])); + table.rows.push(reordered); + } + } + }); + + return table; + } + + transformInstantMetricData(md, options) { + var dps = [], + metricLabel = null; + metricLabel = this.createMetricLabel(md.metric, options); + dps.push([parseFloat(md.value[1]), md.value[0] * 1000]); + return { target: metricLabel, datapoints: dps }; + } + + createMetricLabel(labelData, options) { + if (_.isUndefined(options) || _.isEmpty(options.legendFormat)) { + return this.getOriginalMetricName(labelData); + } + + return this.renderTemplate(this.templateSrv.replace(options.legendFormat), labelData) || '{}'; + } + + renderTemplate(aliasPattern, aliasData) { + var aliasRegex = /\{\{\s*(.+?)\s*\}\}/g; + return aliasPattern.replace(aliasRegex, function(match, g1) { + if (aliasData[g1]) { + return aliasData[g1]; + } + return g1; + }); + } + + getOriginalMetricName(labelData) { + var metricName = labelData.__name__ || ''; + delete labelData.__name__; + var labelPart = _.map(_.toPairs(labelData), function(label) { + return label[0] + '="' + label[1] + '"'; + }).join(','); + return metricName + '{' + labelPart + '}'; + } + + transformToHistogramOverTime(seriesList) { + /* t1 = timestamp1, t2 = timestamp2 etc. + t1 t2 t3 t1 t2 t3 + le10 10 10 0 => 10 10 0 + le20 20 10 30 => 10 0 30 + le30 30 10 35 => 10 0 5 + */ + for (let i = seriesList.length - 1; i > 0; i--) { + let topSeries = seriesList[i].datapoints; + let bottomSeries = seriesList[i - 1].datapoints; + for (let j = 0; j < topSeries.length; j++) { + topSeries[j][0] -= bottomSeries[j][0]; + } + } + + return seriesList; + } +} + +function sortSeriesByLabel(s1, s2): number { + let le1, le2; + + try { + // fail if not integer. might happen with bad queries + le1 = parseHistogramLabel(s1.metric.le); + le2 = parseHistogramLabel(s2.metric.le); + } catch (err) { + console.log(err); + return 0; + } + + if (le1 > le2) { + return 1; + } + + if (le1 < le2) { + return -1; + } + + return 0; +} + +function parseHistogramLabel(le: string): number { + if (le === '+Inf') { + return +Infinity; + } + return Number(le); +} diff --git a/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts b/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts index 043bfcf25e0..55dd7ef7d42 100644 --- a/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/prometheus/specs/datasource_specs.ts @@ -223,43 +223,6 @@ describe('PrometheusDatasource', function() { expect(results[0].time).to.be(1443454528 * 1000); }); }); - describe('When resultFormat is table', function() { - var response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ - { - metric: { __name__: 'test', job: 'testjob' }, - values: [[1443454528, '3846']], - }, - { - metric: { - __name__: 'test', - instance: 'localhost:8080', - job: 'otherjob', - }, - values: [[1443454529, '3847']], - }, - ], - }, - }; - it('should return table model', function() { - var table = ctx.ds.transformMetricDataToTable(response.data.result); - expect(table.type).to.be('table'); - expect(table.rows).to.eql([ - [1443454528000, 'test', '', 'testjob', 3846], - [1443454529000, 'test', 'localhost:8080', 'otherjob', 3847], - ]); - expect(table.columns).to.eql([ - { text: 'Time', type: 'time' }, - { text: '__name__' }, - { text: 'instance' }, - { text: 'job' }, - { text: 'Value' }, - ]); - }); - }); describe('When resultFormat is table and instant = true', function() { var results; @@ -293,19 +256,8 @@ describe('PrometheusDatasource', function() { it('should return result', () => { expect(results).not.to.be(null); }); - - it('should return table model', function() { - var table = ctx.ds.transformMetricDataToTable(response.data.result); - expect(table.type).to.be('table'); - expect(table.rows).to.eql([[1443454528000, 'test', 'testjob', 3846]]); - expect(table.columns).to.eql([ - { text: 'Time', type: 'time' }, - { text: '__name__' }, - { text: 'job' }, - { text: 'Value' }, - ]); - }); }); + describe('The "step" query parameter', function() { var response = { status: 'success', diff --git a/public/app/plugins/datasource/prometheus/specs/result_transformer.jest.ts b/public/app/plugins/datasource/prometheus/specs/result_transformer.jest.ts new file mode 100644 index 00000000000..35e9780badd --- /dev/null +++ b/public/app/plugins/datasource/prometheus/specs/result_transformer.jest.ts @@ -0,0 +1,78 @@ +import { ResultTransformer } from '../result_transformer'; + +describe('Prometheus Result Transformer', () => { + let ctx: any = {}; + + beforeEach(() => { + ctx.templateSrv = { + replace: str => str, + }; + ctx.resultTransformer = new ResultTransformer(ctx.templateSrv); + }); + + describe('When resultFormat is table', () => { + var response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + values: [[1443454528, '3846']], + }, + { + metric: { + __name__: 'test', + instance: 'localhost:8080', + job: 'otherjob', + }, + values: [[1443454529, '3847']], + }, + ], + }, + }; + + it('should return table model', () => { + var table = ctx.resultTransformer.transformMetricDataToTable(response.data.result); + expect(table.type).toBe('table'); + expect(table.rows).toEqual([ + [1443454528000, 'test', '', 'testjob', 3846], + [1443454529000, 'test', 'localhost:8080', 'otherjob', 3847], + ]); + expect(table.columns).toEqual([ + { text: 'Time', type: 'time' }, + { text: '__name__' }, + { text: 'instance' }, + { text: 'job' }, + { text: 'Value' }, + ]); + }); + }); + + describe('When resultFormat is table and instant = true', () => { + var response = { + status: 'success', + data: { + resultType: 'vector', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + value: [1443454528, '3846'], + }, + ], + }, + }; + + it('should return table model', () => { + var table = ctx.resultTransformer.transformMetricDataToTable(response.data.result); + expect(table.type).toBe('table'); + expect(table.rows).toEqual([[1443454528000, 'test', 'testjob', 3846]]); + expect(table.columns).toEqual([ + { text: 'Time', type: 'time' }, + { text: '__name__' }, + { text: 'job' }, + { text: 'Value' }, + ]); + }); + }); +});