From be26c017f64c9cfffbc35a75c607e61031879541 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 6 Dec 2016 16:12:30 +0900 Subject: [PATCH 1/4] (cloudwatch) percentile support (#6634) * support extended statistics * handle different response format for extended statistics --- pkg/api/cloudwatch/cloudwatch.go | 50 +++++++++------ .../datasource/cloudwatch/datasource.js | 30 ++++++++- .../cloudwatch/query_parameter_ctrl.js | 13 ++-- .../cloudwatch/specs/datasource_specs.ts | 61 +++++++++++++++++++ 4 files changed, 125 insertions(+), 29 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 991d336411c..06c4b171d22 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -181,25 +181,33 @@ func handleGetMetricStatistics(req *cwRequest, c *middleware.Context) { reqParam := &struct { Parameters struct { - Namespace string `json:"namespace"` - MetricName string `json:"metricName"` - Dimensions []*cloudwatch.Dimension `json:"dimensions"` - Statistics []*string `json:"statistics"` - StartTime int64 `json:"startTime"` - EndTime int64 `json:"endTime"` - Period int64 `json:"period"` + Namespace string `json:"namespace"` + MetricName string `json:"metricName"` + Dimensions []*cloudwatch.Dimension `json:"dimensions"` + Statistics []*string `json:"statistics"` + ExtendedStatistics []*string `json:"extendedStatistics"` + StartTime int64 `json:"startTime"` + EndTime int64 `json:"endTime"` + Period int64 `json:"period"` } `json:"parameters"` }{} json.Unmarshal(req.Body, reqParam) params := &cloudwatch.GetMetricStatisticsInput{ - Namespace: aws.String(reqParam.Parameters.Namespace), - MetricName: aws.String(reqParam.Parameters.MetricName), - Dimensions: reqParam.Parameters.Dimensions, - Statistics: reqParam.Parameters.Statistics, - StartTime: aws.Time(time.Unix(reqParam.Parameters.StartTime, 0)), - EndTime: aws.Time(time.Unix(reqParam.Parameters.EndTime, 0)), - Period: aws.Int64(reqParam.Parameters.Period), + Namespace: aws.String(reqParam.Parameters.Namespace), + MetricName: aws.String(reqParam.Parameters.MetricName), + Dimensions: reqParam.Parameters.Dimensions, + Statistics: reqParam.Parameters.Statistics, + ExtendedStatistics: reqParam.Parameters.ExtendedStatistics, + StartTime: aws.Time(time.Unix(reqParam.Parameters.StartTime, 0)), + EndTime: aws.Time(time.Unix(reqParam.Parameters.EndTime, 0)), + Period: aws.Int64(reqParam.Parameters.Period), + } + if len(reqParam.Parameters.Statistics) != 0 { + params.Statistics = reqParam.Parameters.Statistics + } + if len(reqParam.Parameters.ExtendedStatistics) != 0 { + params.ExtendedStatistics = reqParam.Parameters.ExtendedStatistics } resp, err := svc.GetMetricStatistics(params) @@ -292,11 +300,12 @@ func handleDescribeAlarmsForMetric(req *cwRequest, c *middleware.Context) { reqParam := &struct { Parameters struct { - Namespace string `json:"namespace"` - MetricName string `json:"metricName"` - Dimensions []*cloudwatch.Dimension `json:"dimensions"` - Statistic string `json:"statistic"` - Period int64 `json:"period"` + Namespace string `json:"namespace"` + MetricName string `json:"metricName"` + Dimensions []*cloudwatch.Dimension `json:"dimensions"` + Statistic string `json:"statistic"` + ExtendedStatistic string `json:"extendedStatistic"` + Period int64 `json:"period"` } `json:"parameters"` }{} json.Unmarshal(req.Body, reqParam) @@ -312,6 +321,9 @@ func handleDescribeAlarmsForMetric(req *cwRequest, c *middleware.Context) { if reqParam.Parameters.Statistic != "" { params.Statistic = aws.String(reqParam.Parameters.Statistic) } + if reqParam.Parameters.ExtendedStatistic != "" { + params.ExtendedStatistic = aws.String(reqParam.Parameters.ExtendedStatistic) + } resp, err := svc.DescribeAlarmsForMetric(params) if err != nil { diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 07648779648..acef572367d 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -16,6 +16,13 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { this.supportMetrics = true; this.proxyUrl = instanceSettings.url; this.defaultRegion = instanceSettings.jsonData.defaultRegion; + this.standardStatistics = [ + 'Average', + 'Maximum', + 'Minimum', + 'Sum', + 'SampleCount' + ]; var self = this; this.query = function(options) { @@ -98,6 +105,8 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { }; this.performTimeSeriesQuery = function(query, start, end) { + var statistics = _.filter(query.statistics, function(s) { return _.includes(self.standardStatistics, s); }); + var extendedStatistics = _.reject(query.statistics, function(s) { return _.includes(self.standardStatistics, s); }); return this.awsRequest({ region: query.region, action: 'GetMetricStatistics', @@ -105,7 +114,8 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { namespace: query.namespace, metricName: query.metricName, dimensions: query.dimensions, - statistics: query.statistics, + statistics: statistics, + extendedStatistics: extendedStatistics, startTime: start, endTime: end, period: query.period @@ -268,10 +278,19 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { }; this.performDescribeAlarmsForMetric = function(region, namespace, metricName, dimensions, statistic, period) { + var s = _.includes(self.standardStatistics, statistic) ? statistic : ''; + var es = _.includes(self.standardStatistics, statistic) ? '' : statistic; return this.awsRequest({ region: region, action: 'DescribeAlarmsForMetric', - parameters: { namespace: namespace, metricName: metricName, dimensions: dimensions, statistic: statistic, period: period } + parameters: { + namespace: namespace, + metricName: metricName, + dimensions: dimensions, + statistic: s, + extendedStatistic: es, + period: period + } }); }; @@ -338,6 +357,7 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { var periodMs = options.period * 1000; return _.map(options.statistics, function(stat) { + var extended = !_.includes(self.standardStatistics, stat); var dps = []; var lastTimestamp = null; _.chain(md.Datapoints) @@ -350,7 +370,11 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { dps.push([null, lastTimestamp + periodMs]); } lastTimestamp = timestamp; - dps.push([dp[stat], timestamp]); + if (!extended) { + dps.push([dp[stat], timestamp]); + } else { + dps.push([dp.ExtendedStatistics[stat], timestamp]); + } }) .value(); diff --git a/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.js b/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.js index c239291ed8e..f7db203826b 100644 --- a/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.js +++ b/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.js @@ -61,14 +61,13 @@ function (angular, _) { }; $scope.getStatSegments = function() { - return $q.when([ + return $q.when(_.flatten([ angular.copy($scope.removeStatSegment), - uiSegmentSrv.getSegmentForValue('Average'), - uiSegmentSrv.getSegmentForValue('Maximum'), - uiSegmentSrv.getSegmentForValue('Minimum'), - uiSegmentSrv.getSegmentForValue('Sum'), - uiSegmentSrv.getSegmentForValue('SampleCount'), - ]); + _.map($scope.datasource.standardStatistics, function(s) { + return uiSegmentSrv.getSegmentForValue(s); + }), + uiSegmentSrv.getSegmentForValue('pNN.NN'), + ])); }; $scope.statSegmentChanged = function(segment, index) { diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 0e7ae5081e6..a834fdb2973 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -161,6 +161,67 @@ describe('CloudWatchDatasource', function() { }); }); + describe('When performing CloudWatch query for extended statistics', function() { + var requestParams; + + var query = { + range: { from: 'now-1h', to: 'now' }, + targets: [ + { + region: 'us-east-1', + namespace: 'AWS/ApplicationELB', + metricName: 'TargetResponseTime', + dimensions: { + LoadBalancer: 'lb', + TargetGroup: 'tg' + }, + statistics: ['p90.00'], + period: 300 + } + ] + }; + + var response = { + Datapoints: [ + { + ExtendedStatistics: { + 'p90.00': 1 + }, + Timestamp: 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' + }, + { + ExtendedStatistics: { + 'p90.00': 2 + }, + Timestamp: 'Wed Dec 31 1969 16:05:00 GMT-0800 (PST)' + }, + { + ExtendedStatistics: { + 'p90.00': 5 + }, + Timestamp: 'Wed Dec 31 1969 16:15:00 GMT-0800 (PST)' + } + ], + Label: 'TargetResponseTime' + }; + + beforeEach(function() { + ctx.backendSrv.datasourceRequest = function(params) { + requestParams = params; + return ctx.$q.when({data: response}); + }; + }); + + it('should return series list', function(done) { + ctx.ds.query(query).then(function(result) { + expect(result.data[0].target).to.be('TargetResponseTime_p90.00'); + expect(result.data[0].datapoints[0][0]).to.be(response.Datapoints[0].ExtendedStatistics['p90.00']); + done(); + }); + ctx.$rootScope.$apply(); + }); + }); + function describeMetricFindQuery(query, func) { describe('metricFindQuery ' + query, () => { let scenario: any = {}; From 5f6a3ecd56ac8fffa1d43e0b6bd9e3a439bdc82e Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 6 Dec 2016 16:16:41 +0900 Subject: [PATCH 2/4] (cloudwatch) don't expand variable when panel or row repeat is used (#6618) * (cloudwatch) don't expand variable when panel or row repeat is used * fix test --- public/app/features/templating/templateSrv.js | 12 ++++++++++-- .../app/plugins/datasource/cloudwatch/datasource.js | 6 +++--- .../datasource/cloudwatch/specs/datasource_specs.ts | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/public/app/features/templating/templateSrv.js b/public/app/features/templating/templateSrv.js index f2aa317351c..a41f5a9d0f3 100644 --- a/public/app/features/templating/templateSrv.js +++ b/public/app/features/templating/templateSrv.js @@ -112,10 +112,18 @@ function (angular, _, kbn) { this._grafanaVariables[name] = value; }; - this.variableExists = function(expression) { + this.getVariableName = function(expression) { this._regex.lastIndex = 0; var match = this._regex.exec(expression); - return match && (self._index[match[1] || match[2]] !== void 0); + if (!match) { + return null; + } + return match[1] || match[2]; + }; + + this.variableExists = function(expression) { + var name = this.getVariableName(expression); + return name && (self._index[name] !== void 0); }; this.highlightVariablesAsHtml = function(str) { diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index acef572367d..dc57c41b94d 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -31,7 +31,7 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { var queries = []; options = angular.copy(options); - options.targets = this.expandTemplateVariable(options.targets, templateSrv); + options.targets = this.expandTemplateVariable(options.targets, options.scopedVars, templateSrv); _.each(options.targets, function(target) { if (target.hide || !target.namespace || !target.metricName || _.isEmpty(target.statistics)) { return; @@ -412,12 +412,12 @@ function (angular, _, moment, dateMath, kbn, CloudWatchAnnotationQuery) { return str.indexOf('$' + variableName) !== -1; }; - this.expandTemplateVariable = function(targets, templateSrv) { + this.expandTemplateVariable = function(targets, scopedVars, templateSrv) { var self = this; return _.chain(targets) .map(function(target) { var dimensionKey = _.findKey(target.dimensions, function(v) { - return templateSrv.variableExists(v); + return templateSrv.variableExists(v) && !_.has(scopedVars, templateSrv.getVariableName(v)); }); if (dimensionKey) { diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index a834fdb2973..93dfa24036d 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -139,6 +139,7 @@ describe('CloudWatchDatasource', function() { ] } ], + getVariableName: function (e) { return 'instance_id'; }, variableExists: function (e) { return true; }, containsVariable: function (str, variableName) { return str.indexOf('$' + variableName) !== -1; } }; @@ -156,7 +157,7 @@ describe('CloudWatchDatasource', function() { } ]; - var result = ctx.ds.expandTemplateVariable(targets, templateSrv); + var result = ctx.ds.expandTemplateVariable(targets, {}, templateSrv); expect(result[0].dimensions.InstanceId).to.be('i-34567890'); }); }); From e51525dbdea955c18477bca38cb84ad3d9e61851 Mon Sep 17 00:00:00 2001 From: Ben RUBSON Date: Tue, 6 Dec 2016 08:36:23 +0100 Subject: [PATCH 3/4] Only take bar series to calculate barWidth (#6756) * Only take bar series to calculate barWidth solves #6751 * Typo * Typo * Typo * Typo * Refactor * Simplify --- public/app/plugins/panel/graph/graph.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/public/app/plugins/panel/graph/graph.ts b/public/app/plugins/panel/graph/graph.ts index 73fe37d5cad..d1997acc508 100755 --- a/public/app/plugins/panel/graph/graph.ts +++ b/public/app/plugins/panel/graph/graph.ts @@ -185,6 +185,7 @@ module.directive('grafanaGraph', function($rootScope, timeSrv) { // Series could have different timeSteps, // let's find the smallest one so that bars are correctly rendered. + // In addition, only take series which are rendered as bars for this. function getMinTimeStepOfSeries(data) { var min = Number.MAX_VALUE; @@ -192,6 +193,15 @@ module.directive('grafanaGraph', function($rootScope, timeSrv) { if (!data[i].stats.timeStep) { continue; } + if (panel.bars) { + if (data[i].bars && data[i].bars.show === false) { + continue; + } + } else { + if (typeof data[i].bars === 'undefined' || typeof data[i].bars.show === 'undefined' || !data[i].bars.show) { + continue; + } + } if (data[i].stats.timeStep < min) { min = data[i].stats.timeStep; From e73b82d54f86746a93060dc995626fff9b5eecf1 Mon Sep 17 00:00:00 2001 From: jifwin Date: Tue, 6 Dec 2016 08:37:36 +0100 Subject: [PATCH 4/4] Disable web-security for phantomjs rendering (#6469) --- pkg/components/renderer/renderer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/components/renderer/renderer.go b/pkg/components/renderer/renderer.go index 69a0b197ec8..6db6a6c3d35 100644 --- a/pkg/components/renderer/renderer.go +++ b/pkg/components/renderer/renderer.go @@ -52,6 +52,7 @@ func RenderToPng(params *RenderOpts) (string, error) { cmdArgs := []string{ "--ignore-ssl-errors=true", + "--web-security=false", scriptPath, "url=" + url, "width=" + params.Width,