From ef4bec1c6d870a38dd2f47b4543be8b6fef7715b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 10 Nov 2015 19:25:59 +0900 Subject: [PATCH 01/14] fix CloudWatch dimension value suggestion --- .../datasource/cloudwatch/datasource.js | 23 ++++++++++++------- .../datasource/cloudwatch/query_ctrl.js | 19 +++++++++++++-- .../cloudwatch/specs/datasource_specs.ts | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index eef2d77f5e2..d4c622eac55 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -124,13 +124,7 @@ function (angular, _) { }; return this.awsRequest(request).then(function(result) { - return _.chain(result.Metrics).map(function(metric) { - return _.pluck(metric.Dimensions, 'Value'); - }).flatten().uniq().sortBy(function(name) { - return name; - }).map(function(value) { - return {value: value, text: value}; - }).value(); + return _.pluck(result.Metrics, 'Dimensions'); }); }; @@ -191,7 +185,20 @@ function (angular, _) { }); } - return this.getDimensionValues(region, namespace, metricName, dimensions); + return this.getDimensionValues(region, namespace, metricName, dimensions).then(function(result) { + return _.map(result, function(dimensions) { + var values = _.chain(dimensions) + .sortBy(function(dimension) { + return dimension.Name; + }) + .map(function(dimension) { + return dimension.Name + '=' + dimension.Value; + }) + .value().join(','); + + return { text: values }; + }); + }); } var ebsVolumeIdsQuery = query.match(/^ebs_volume_ids\(([^,]+?),\s?([^,]+?)\)/); diff --git a/public/app/plugins/datasource/cloudwatch/query_ctrl.js b/public/app/plugins/datasource/cloudwatch/query_ctrl.js index 3869a5ec715..7bcc73b4323 100644 --- a/public/app/plugins/datasource/cloudwatch/query_ctrl.js +++ b/public/app/plugins/datasource/cloudwatch/query_ctrl.js @@ -76,7 +76,7 @@ function (angular, _) { } }; - $scope.getDimSegments = function(segment) { + $scope.getDimSegments = function(segment, $index) { if (segment.type === 'operator') { return $q.when([]); } var target = $scope.target; @@ -88,7 +88,22 @@ function (angular, _) { query = $scope.datasource.getDimensionValues(target.region, target.namespace, target.metricName, {}); } - return query.then($scope.transformToSegments(true)).then(function(results) { + return query.then(function(results) { + if (segment.type === 'value') { + results = _.chain(results) + .flatten(true) + .filter(function(dimension) { + return dimension.Name === templateSrv.replace($scope.dimSegments[$index-2].value); + }) + .pluck('Value') + .uniq() + .map(function(value) { + return {value: value, text: value}; + }) + .value(); + } + return $scope.transformToSegments(true)(results); + }).then(function(results) { if (segment.type === 'key') { results.splice(0, 0, angular.copy($scope.removeDimSegment)); } diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 4714a642d30..a6d4330a37e 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -165,7 +165,7 @@ describe('CloudWatchDatasource', function() { }); it('should call __ListMetrics and return result', () => { - expect(scenario.result[0].text).to.be('i-12345678'); + expect(scenario.result[0].text).to.be('InstanceId=i-12345678'); expect(scenario.request.data.action).to.be('ListMetrics'); }); }); From add5bb47d5a84a620abfe599e04c749b1901c1de Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sat, 14 Nov 2015 19:54:24 +0900 Subject: [PATCH 02/14] add dimensions() to CloudWatch templating query --- .../datasource/cloudwatch/datasource.js | 64 ++++++++++++++----- .../datasource/cloudwatch/query_ctrl.js | 20 +----- .../cloudwatch/specs/datasource_specs.ts | 26 +++++++- 3 files changed, 77 insertions(+), 33 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index d4c622eac55..4512aed972b 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -112,14 +112,14 @@ function (angular, _) { }); }; - CloudWatchDatasource.prototype.getDimensionValues = function(region, namespace, metricName, dimensions) { + CloudWatchDatasource.prototype.getDimensions = function(region, namespace, metricName, filterDimensions) { var request = { region: templateSrv.replace(region), action: 'ListMetrics', parameters: { namespace: templateSrv.replace(namespace), metricName: templateSrv.replace(metricName), - dimensions: convertDimensionFormat(dimensions, {}), + dimensions: convertDimensionFormat(filterDimensions, {}), } }; @@ -128,6 +128,17 @@ function (angular, _) { }); }; + CloudWatchDatasource.prototype.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { + return this.getDimensions(region, namespace, metricName, filterDimensions).then(function(dimensions) { + return _.chain(dimensions) + .flatten() + .filter(function(dimension) { + return dimension.Name === dimensionKey; + }) + .pluck('Value').uniq().sortBy().value(); + }); + }; + CloudWatchDatasource.prototype.performEC2DescribeInstances = function(region, filters, instanceIds) { return this.awsRequest({ region: region, @@ -140,6 +151,8 @@ function (angular, _) { var region; var namespace; var metricName; + var dimensionPart; + var dimensions; var transformSuggestData = function(suggestData) { return _.map(suggestData, function(v) { @@ -147,6 +160,21 @@ function (angular, _) { }); }; + var parseDimensions = function(dimensionPart) { + if (_.isEmpty(dimensionPart)) { + return {}; + } + var dimensions = {}; + _.each(dimensionPart.split(','), function(v) { + var t = v.split('='); + if (t.length !== 2) { + throw new Error('Invalid query format'); + } + dimensions[t[0]] = t[1]; + }); + return dimensions; + }; + var regionQuery = query.match(/^regions\(\)/); if (regionQuery) { return this.getRegions(); @@ -167,25 +195,31 @@ function (angular, _) { return this.getDimensionKeys(dimensionKeysQuery[1]); } - var dimensionValuesQuery = query.match(/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?)(,\s?([^)]*))?\)/); + var dimensionValuesQuery = query.match(/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?),\s?([^,]+?)(,\s?([^)]*))?\)/); if (dimensionValuesQuery) { region = templateSrv.replace(dimensionValuesQuery[1]); namespace = templateSrv.replace(dimensionValuesQuery[2]); metricName = templateSrv.replace(dimensionValuesQuery[3]); - var dimensionPart = templateSrv.replace(dimensionValuesQuery[5]); + var dimensionKey = templateSrv.replace(dimensionValuesQuery[4]); + dimensionPart = templateSrv.replace(dimensionValuesQuery[6]); - var dimensions = {}; - if (!_.isEmpty(dimensionPart)) { - _.each(dimensionPart.split(','), function(v) { - var t = v.split('='); - if (t.length !== 2) { - throw new Error('Invalid query format'); - } - dimensions[t[0]] = t[1]; + dimensions = parseDimensions(dimensionPart); + return this.getDimensionValues(region, namespace, metricName, dimensionKey, dimensions).then(function(result) { + return _.map(result, function(dimension_value) { + return { text: dimension_value }; }); - } + }); + } - return this.getDimensionValues(region, namespace, metricName, dimensions).then(function(result) { + var dimensionsQuery = query.match(/^dimensions\(([^,]+?),\s?([^,]+?),\s?([^,]+?)(,\s?([^)]*))?\)/); + if (dimensionsQuery) { + region = templateSrv.replace(dimensionsQuery[1]); + namespace = templateSrv.replace(dimensionsQuery[2]); + metricName = templateSrv.replace(dimensionsQuery[3]); + dimensionPart = templateSrv.replace(dimensionsQuery[5]); + + dimensions = parseDimensions(dimensionPart); + return this.getDimensions(region, namespace, metricName, dimensions).then(function(result) { return _.map(result, function(dimensions) { var values = _.chain(dimensions) .sortBy(function(dimension) { @@ -228,7 +262,7 @@ function (angular, _) { var metricName = 'EstimatedCharges'; var dimensions = {}; - return this.getDimensionValues(region, namespace, metricName, dimensions).then(function () { + return this.getDimensions(region, namespace, metricName, dimensions).then(function () { return { status: 'success', message: 'Data source is working', title: 'Success' }; }); }; diff --git a/public/app/plugins/datasource/cloudwatch/query_ctrl.js b/public/app/plugins/datasource/cloudwatch/query_ctrl.js index 7bcc73b4323..d0f6fe5b52a 100644 --- a/public/app/plugins/datasource/cloudwatch/query_ctrl.js +++ b/public/app/plugins/datasource/cloudwatch/query_ctrl.js @@ -85,25 +85,11 @@ function (angular, _) { if (segment.type === 'key' || segment.type === 'plus-button') { query = $scope.datasource.getDimensionKeys($scope.target.namespace); } else if (segment.type === 'value') { - query = $scope.datasource.getDimensionValues(target.region, target.namespace, target.metricName, {}); + var dimensionKey = $scope.dimSegments[$index-2].value; + query = $scope.datasource.getDimensionValues(target.region, target.namespace, target.metricName, dimensionKey, {}); } - return query.then(function(results) { - if (segment.type === 'value') { - results = _.chain(results) - .flatten(true) - .filter(function(dimension) { - return dimension.Name === templateSrv.replace($scope.dimSegments[$index-2].value); - }) - .pluck('Value') - .uniq() - .map(function(value) { - return {value: value, text: value}; - }) - .value(); - } - return $scope.transformToSegments(true)(results); - }).then(function(results) { + return query.then($scope.transformToSegments(true)).then(function(results) { if (segment.type === 'key') { results.splice(0, 0, angular.copy($scope.removeDimSegment)); } diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index a6d4330a37e..6c55eab55ca 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -146,7 +146,7 @@ describe('CloudWatchDatasource', function() { }); }); - describeMetricFindQuery('dimension_values(us-east-1,AWS/EC2,CPUUtilization)', scenario => { + describeMetricFindQuery('dimensions(us-east-1,AWS/EC2,CPUUtilization)', scenario => { scenario.setup(() => { scenario.requestResponse = { Metrics: [ @@ -170,4 +170,28 @@ describe('CloudWatchDatasource', function() { }); }); + describeMetricFindQuery('dimension_values(us-east-1,AWS/EC2,CPUUtilization,InstanceId)', scenario => { + scenario.setup(() => { + scenario.requestResponse = { + Metrics: [ + { + Namespace: 'AWS/EC2', + MetricName: 'CPUUtilization', + Dimensions: [ + { + Name: 'InstanceId', + Value: 'i-12345678' + } + ] + } + ] + }; + }); + + it('should call __ListMetrics and return result', () => { + expect(scenario.result[0].text).to.be('i-12345678'); + expect(scenario.request.data.action).to.be('ListMetrics'); + }); + }); + }); From 1bbd0567975c266f7884ee9b45acf3a4090e1511 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Mon, 16 Nov 2015 16:47:28 +0900 Subject: [PATCH 03/14] fix templating --- .../app/plugins/datasource/cloudwatch/datasource.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 4512aed972b..9b4bed35e90 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -135,7 +135,12 @@ function (angular, _) { .filter(function(dimension) { return dimension.Name === dimensionKey; }) - .pluck('Value').uniq().sortBy().value(); + .pluck('Value') + .uniq() + .sortBy() + .map(function(value) { + return {value: value, text: value}; + }).value(); }); }; @@ -204,11 +209,7 @@ function (angular, _) { dimensionPart = templateSrv.replace(dimensionValuesQuery[6]); dimensions = parseDimensions(dimensionPart); - return this.getDimensionValues(region, namespace, metricName, dimensionKey, dimensions).then(function(result) { - return _.map(result, function(dimension_value) { - return { text: dimension_value }; - }); - }); + return this.getDimensionValues(region, namespace, metricName, dimensionKey, dimensions); } var dimensionsQuery = query.match(/^dimensions\(([^,]+?),\s?([^,]+?),\s?([^,]+?)(,\s?([^)]*))?\)/); From ae7e7e96565055dadfca9fbbcaf42428d37c23ff Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 17 Nov 2015 22:49:46 +0900 Subject: [PATCH 04/14] remove getDimensions() --- .../datasource/cloudwatch/datasource.js | 37 ++----------------- .../cloudwatch/specs/datasource_specs.ts | 24 ------------ 2 files changed, 4 insertions(+), 57 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 9b4bed35e90..7297346b33c 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -112,7 +112,7 @@ function (angular, _) { }); }; - CloudWatchDatasource.prototype.getDimensions = function(region, namespace, metricName, filterDimensions) { + CloudWatchDatasource.prototype.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { var request = { region: templateSrv.replace(region), action: 'ListMetrics', @@ -124,13 +124,8 @@ function (angular, _) { }; return this.awsRequest(request).then(function(result) { - return _.pluck(result.Metrics, 'Dimensions'); - }); - }; - - CloudWatchDatasource.prototype.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { - return this.getDimensions(region, namespace, metricName, filterDimensions).then(function(dimensions) { - return _.chain(dimensions) + return _.chain(result.Metrics) + .pluck('Dimensions') .flatten() .filter(function(dimension) { return dimension.Name === dimensionKey; @@ -212,30 +207,6 @@ function (angular, _) { return this.getDimensionValues(region, namespace, metricName, dimensionKey, dimensions); } - var dimensionsQuery = query.match(/^dimensions\(([^,]+?),\s?([^,]+?),\s?([^,]+?)(,\s?([^)]*))?\)/); - if (dimensionsQuery) { - region = templateSrv.replace(dimensionsQuery[1]); - namespace = templateSrv.replace(dimensionsQuery[2]); - metricName = templateSrv.replace(dimensionsQuery[3]); - dimensionPart = templateSrv.replace(dimensionsQuery[5]); - - dimensions = parseDimensions(dimensionPart); - return this.getDimensions(region, namespace, metricName, dimensions).then(function(result) { - return _.map(result, function(dimensions) { - var values = _.chain(dimensions) - .sortBy(function(dimension) { - return dimension.Name; - }) - .map(function(dimension) { - return dimension.Name + '=' + dimension.Value; - }) - .value().join(','); - - return { text: values }; - }); - }); - } - var ebsVolumeIdsQuery = query.match(/^ebs_volume_ids\(([^,]+?),\s?([^,]+?)\)/); if (ebsVolumeIdsQuery) { region = templateSrv.replace(ebsVolumeIdsQuery[1]); @@ -263,7 +234,7 @@ function (angular, _) { var metricName = 'EstimatedCharges'; var dimensions = {}; - return this.getDimensions(region, namespace, metricName, dimensions).then(function () { + return this.getDimensionValues(region, namespace, metricName, 'ServiceName', dimensions).then(function () { return { status: 'success', message: 'Data source is working', title: 'Success' }; }); }; diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 6c55eab55ca..b97dff5ce3b 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -146,30 +146,6 @@ describe('CloudWatchDatasource', function() { }); }); - describeMetricFindQuery('dimensions(us-east-1,AWS/EC2,CPUUtilization)', scenario => { - scenario.setup(() => { - scenario.requestResponse = { - Metrics: [ - { - Namespace: 'AWS/EC2', - MetricName: 'CPUUtilization', - Dimensions: [ - { - Name: 'InstanceId', - Value: 'i-12345678' - } - ] - } - ] - }; - }); - - it('should call __ListMetrics and return result', () => { - expect(scenario.result[0].text).to.be('InstanceId=i-12345678'); - expect(scenario.request.data.action).to.be('ListMetrics'); - }); - }); - describeMetricFindQuery('dimension_values(us-east-1,AWS/EC2,CPUUtilization,InstanceId)', scenario => { scenario.setup(() => { scenario.requestResponse = { From ada9bfcae8140f53bd12139a58fc90a636029212 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Sun, 22 Nov 2015 21:39:56 +0900 Subject: [PATCH 05/14] keep track of elastic search version and generate query according to version --- public/app/features/org/datasourceEditCtrl.js | 6 ++++++ .../plugins/datasource/elasticsearch/datasource.js | 4 +++- .../datasource/elasticsearch/partials/config.html | 13 ++++++++++++- .../datasource/elasticsearch/query_builder.js | 11 +++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/public/app/features/org/datasourceEditCtrl.js b/public/app/features/org/datasourceEditCtrl.js index e2d5cc3d75d..4d17707792f 100644 --- a/public/app/features/org/datasourceEditCtrl.js +++ b/public/app/features/org/datasourceEditCtrl.js @@ -24,6 +24,12 @@ function (angular, _, config) { {name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY'}, ]; + $scope.elasticsearchVersions = [ + {name: '0.x', value: 0}, + {name: '1.x', value: 1}, + {name: '2.x', value: 2}, + ]; + $scope.init = function() { $scope.isNew = true; $scope.datasources = []; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 9c749b9459e..6f68951e06c 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -23,9 +23,11 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes this.name = datasource.name; this.index = datasource.index; this.timeField = datasource.jsonData.timeField; + this.elasticsearchVersion = datasource.jsonData.elasticsearchVersion; this.indexPattern = new IndexPattern(datasource.index, datasource.jsonData.interval); this.queryBuilder = new ElasticQueryBuilder({ - timeField: this.timeField + timeField: this.timeField, + elasticsearchVersion: this.elasticsearchVersion }); } diff --git a/public/app/plugins/datasource/elasticsearch/partials/config.html b/public/app/plugins/datasource/elasticsearch/partials/config.html index d1cb05801d6..ef596a4deef 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/config.html +++ b/public/app/plugins/datasource/elasticsearch/partials/config.html @@ -20,7 +20,7 @@
-
+
  • Time field name @@ -31,3 +31,14 @@
+
+
    +
  • + Elasticsearch version +
    • +
  • + +
    • +
+
+
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 129c52d3f02..f67cc716a42 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -6,11 +6,18 @@ function (angular) { function ElasticQueryBuilder(options) { this.timeField = options.timeField; + this.elasticsearchVersion = options.elasticsearchVersion; } ElasticQueryBuilder.prototype.getRangeFilter = function() { var filter = {}; filter[this.timeField] = {"gte": "$timeFrom", "lte": "$timeTo"}; + + // elastic search versions above 2.0 require the time format to be specified + if (this.elasticsearchVersion >= 2) { + filter[this.timeField]["format"] = "epoch_millis"; + } + return filter; }; @@ -129,6 +136,10 @@ function (angular) { "min_doc_count": 0, "extended_bounds": { "min": "$timeFrom", "max": "$timeTo" } }; + // elastic search versions above 2.0 require the time format to be specified + if (this.elasticsearchVersion >= 2) { + esAgg["date_histogram"]["format"] = "epoch_millis"; + } break; } case 'filters': { From a30ceefa6bacff9936972f41e77070d04005d2d5 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Wed, 25 Nov 2015 16:23:28 +0900 Subject: [PATCH 06/14] add tests for elastic search versioning in query builder and make es version 2 default --- public/app/features/org/datasourceEditCtrl.js | 2 +- .../datasource/elasticsearch/query_builder.js | 4 +-- .../specs/query_builder_specs.ts | 33 +++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/public/app/features/org/datasourceEditCtrl.js b/public/app/features/org/datasourceEditCtrl.js index 4d17707792f..dd526f69308 100644 --- a/public/app/features/org/datasourceEditCtrl.js +++ b/public/app/features/org/datasourceEditCtrl.js @@ -13,7 +13,7 @@ function (angular, _, config) { $scope.httpConfigPartialSrc = 'app/features/org/partials/datasourceHttpConfig.html'; - var defaults = {name: '', type: 'graphite', url: '', access: 'proxy' }; + var defaults = {name: '', type: 'graphite', url: '', access: 'proxy', jsonData: {'elasticsearchVersion': 2} }; $scope.indexPatternTypes = [ {name: 'No pattern', value: undefined}, diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index f67cc716a42..045df314b33 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -13,7 +13,7 @@ function (angular) { var filter = {}; filter[this.timeField] = {"gte": "$timeFrom", "lte": "$timeTo"}; - // elastic search versions above 2.0 require the time format to be specified + // elastic search versions 2.x require the time format to be specified if (this.elasticsearchVersion >= 2) { filter[this.timeField]["format"] = "epoch_millis"; } @@ -136,7 +136,7 @@ function (angular) { "min_doc_count": 0, "extended_bounds": { "min": "$timeFrom", "max": "$timeTo" } }; - // elastic search versions above 2.0 require the time format to be specified + // elastic search versions 2.x require the time format to be specified if (this.elasticsearchVersion >= 2) { esAgg["date_histogram"]["format"] = "epoch_millis"; } diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index bcae7d6e852..25aab0a084c 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -44,6 +44,39 @@ describe('ElasticQueryBuilder', function() { expect(query.aggs["2"].aggs["3"].date_histogram.field).to.be("@timestamp"); }); + it('with es1.x and es2.x date histogram queries check time format', function() { + var builder_2x = new ElasticQueryBuilder({ + timeField: '@timestamp', + elasticsearchVersion: 2 + }); + + var query_params = { + metrics: [], + bucketAggs: [ + {type: 'date_histogram', field: '@timestamp', id: '1'} + ], + }; + + // format should not be specified in 1.x queries + expect("format" in builder.build(query_params)["aggs"]["1"]["date_histogram"]).to.be(false); + + // 2.x query should specify format to be "epoch_millis" + expect(builder_2x.build(query_params)["aggs"]["1"]["date_histogram"]["format"]).to.be("epoch_millis"); + }); + + it('with es1.x and es2.x range filter check time format', function() { + var builder_2x = new ElasticQueryBuilder({ + timeField: '@timestamp', + elasticsearchVersion: 2 + }); + + // format should not be specified in 1.x queries + expect("format" in builder.getRangeFilter()["@timestamp"]).to.be(false); + + // 2.x query should specify format to be "epoch_millis" + expect(builder_2x.getRangeFilter()["@timestamp"]["format"]).to.be("epoch_millis"); + }); + it('with select field', function() { var query = builder.build({ metrics: [{type: 'avg', field: '@value', id: '1'}], From efbbb313702dc0f208b2fc8545c5a205983fbd33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 3 Dec 2015 12:03:06 +0100 Subject: [PATCH 07/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99854966938..3886c32a971 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Replace X.Y.Z by actual version number. cd $GOPATH/src/github.com/grafana/grafana go run build.go setup (only needed once to install godep) godep restore (will pull down all golang lib dependencies in your current GOPATH) -godep go run build.go build +go run build.go build ``` ### Building frontend assets From d6935847b4ebad698684b9428f0982baf8604bf6 Mon Sep 17 00:00:00 2001 From: Alexey Larkov Date: Thu, 3 Dec 2015 17:36:29 +0500 Subject: [PATCH 08/14] Web. Fix double slash --- public/app/plugins/datasource/elasticsearch/datasource.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 9c749b9459e..bd1773028fb 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -94,7 +94,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes var payload = angular.toJson(header) + '\n' + angular.toJson(data) + '\n'; - return this._post('/_msearch', payload).then(function(res) { + return this._post('_msearch', payload).then(function(res) { var list = []; var hits = res.responses[0].hits.hits; @@ -188,7 +188,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes payload = payload.replace(/\$timeTo/g, options.range.to.valueOf()); payload = templateSrv.replace(payload, options.scopedVars); - return this._post('/_msearch', payload).then(function(res) { + return this._post('_msearch', payload).then(function(res) { return new ElasticResponse(sentTargets, res).getTimeSeries(); }); }; From cf1f43dc9de5c66572d16cc89ca7b6e13511ad0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 3 Dec 2015 15:09:39 +0100 Subject: [PATCH 09/14] feat(influxdb): support for table queries, closes #3409, #3219 --- .../app/{panels/table => core}/table_model.ts | 23 +++------- public/app/panels/table/controller.ts | 20 ++++++++- .../app/panels/table/specs/renderer_specs.ts | 2 +- .../panels/table/specs/transformers_specs.ts | 13 +++--- public/app/panels/table/transformers.ts | 40 +++++++++++++++++- .../plugins/datasource/influxdb/datasource.js | 24 ++++++++--- .../datasource/influxdb/influx_query.ts | 2 + .../datasource/influxdb/influx_series.js | 42 ++++++++++++++++++- .../influxdb/partials/query.editor.html | 6 +++ .../plugins/datasource/influxdb/query_ctrl.js | 5 +++ .../influxdb/specs/influx_series_specs.ts | 23 ++++++++++ .../specs => test/core}/table_model_specs.ts | 2 +- 12 files changed, 166 insertions(+), 36 deletions(-) rename public/app/{panels/table => core}/table_model.ts (56%) rename public/{app/panels/table/specs => test/core}/table_model_specs.ts (95%) diff --git a/public/app/panels/table/table_model.ts b/public/app/core/table_model.ts similarity index 56% rename from public/app/panels/table/table_model.ts rename to public/app/core/table_model.ts index 1fa4007e6e3..7eb8d5ad92a 100644 --- a/public/app/panels/table/table_model.ts +++ b/public/app/core/table_model.ts @@ -1,12 +1,13 @@ -import {transformers} from './transformers'; -export class TableModel { +class TableModel { columns: any[]; rows: any[]; + type: string; constructor() { this.columns = []; this.rows = []; + this.type = 'table'; } sort(options) { @@ -33,20 +34,6 @@ export class TableModel { this.columns[options.col].desc = true; } } - - static transform(data, panel) { - var model = new TableModel(); - - if (!data || data.length === 0) { - return model; - } - - var transformer = transformers[panel.transform]; - if (!transformer) { - throw {message: 'Transformer ' + panel.transformer + ' not found'}; - } - - transformer.transform(data, panel, model); - return model; - } } + +export = TableModel; diff --git a/public/app/panels/table/controller.ts b/public/app/panels/table/controller.ts index 09e77108631..270e2f65a3d 100644 --- a/public/app/panels/table/controller.ts +++ b/public/app/panels/table/controller.ts @@ -5,7 +5,7 @@ import _ = require('lodash'); import moment = require('moment'); import PanelMeta = require('app/features/panel/panel_meta'); -import {TableModel} from './table_model'; +import {transformDataToTable} from './transformers'; export class TablePanelCtrl { @@ -104,7 +104,23 @@ export class TablePanelCtrl { }; $scope.render = function() { - $scope.table = TableModel.transform($scope.dataRaw, $scope.panel); + // automatically correct transform mode + // based on data + if ($scope.dataRaw && $scope.dataRaw.length) { + if ($scope.dataRaw[0].type === 'table') { + $scope.panel.transform = 'table'; + } else { + if ($scope.dataRaw[0].type === 'docs') { + $scope.panel.transform = 'json'; + } else { + if ($scope.panel.transform === 'table' || $scope.panel.transform === 'json') { + $scope.panel.transform = 'timeseries_to_rows'; + } + } + } + } + + $scope.table = transformDataToTable($scope.dataRaw, $scope.panel); $scope.table.sort($scope.panel.sort); panelHelper.broadcastRender($scope, $scope.table, $scope.dataRaw); }; diff --git a/public/app/panels/table/specs/renderer_specs.ts b/public/app/panels/table/specs/renderer_specs.ts index f8fdebb9ab0..f8af1baba17 100644 --- a/public/app/panels/table/specs/renderer_specs.ts +++ b/public/app/panels/table/specs/renderer_specs.ts @@ -1,6 +1,6 @@ import {describe, beforeEach, it, sinon, expect} from 'test/lib/common'; -import {TableModel} from '../table_model'; +import TableModel = require('app/core/table_model'); import {TableRenderer} from '../renderer'; describe('when rendering table', () => { diff --git a/public/app/panels/table/specs/transformers_specs.ts b/public/app/panels/table/specs/transformers_specs.ts index bb42b997d33..e3cdf44c8b2 100644 --- a/public/app/panels/table/specs/transformers_specs.ts +++ b/public/app/panels/table/specs/transformers_specs.ts @@ -1,7 +1,6 @@ import {describe, beforeEach, it, sinon, expect} from 'test/lib/common'; -import {TableModel} from '../table_model'; -import {transformers} from '../transformers'; +import {transformers, transformDataToTable} from '../transformers'; describe('when transforming time series table', () => { var table; @@ -26,7 +25,7 @@ describe('when transforming time series table', () => { }; beforeEach(() => { - table = TableModel.transform(timeSeries, panel); + table = transformDataToTable(timeSeries, panel); }); it('should return 3 rows', () => { @@ -51,7 +50,7 @@ describe('when transforming time series table', () => { }; beforeEach(() => { - table = TableModel.transform(timeSeries, panel); + table = transformDataToTable(timeSeries, panel); }); it ('should return 3 columns', () => { @@ -80,7 +79,7 @@ describe('when transforming time series table', () => { }; beforeEach(() => { - table = TableModel.transform(timeSeries, panel); + table = transformDataToTable(timeSeries, panel); }); it('should return 2 rows', () => { @@ -133,7 +132,7 @@ describe('when transforming time series table', () => { describe('transform', function() { beforeEach(() => { - table = TableModel.transform(rawData, panel); + table = transformDataToTable(rawData, panel); }); it ('should return 2 columns', () => { @@ -164,7 +163,7 @@ describe('when transforming time series table', () => { ]; beforeEach(() => { - table = TableModel.transform(rawData, panel); + table = transformDataToTable(rawData, panel); }); it ('should return 4 columns', () => { diff --git a/public/app/panels/table/transformers.ts b/public/app/panels/table/transformers.ts index a4d0d4395c5..843eb83c034 100644 --- a/public/app/panels/table/transformers.ts +++ b/public/app/panels/table/transformers.ts @@ -4,6 +4,7 @@ import moment = require('moment'); import _ = require('lodash'); import flatten = require('app/core/utils/flatten'); import TimeSeries = require('app/core/time_series'); +import TableModel = require('app/core/table_model'); var transformers = {}; @@ -136,6 +137,27 @@ transformers['annotations'] = { } }; +transformers['table'] = { + description: 'Table', + getColumns: function(data) { + if (!data || data.length === 0) { + return []; + } + }, + transform: function(data, panel, model) { + if (!data || data.length === 0) { + return; + } + + if (data[0].type !== 'table') { + throw {message: 'Query result is not in table format, try using another transform.'}; + } + + model.columns = data[0].columns; + model.rows = data[0].rows; + } +}; + transformers['json'] = { description: 'JSON Data', getColumns: function(data) { @@ -197,4 +219,20 @@ transformers['json'] = { } }; -export {transformers} +function transformDataToTable(data, panel) { + var model = new TableModel(); + + if (!data || data.length === 0) { + return model; + } + + var transformer = transformers[panel.transform]; + if (!transformer) { + throw {message: 'Transformer ' + panel.transformer + ' not found'}; + } + + transformer.transform(data, panel, model); + return model; +} + +export {transformers, transformDataToTable} diff --git a/public/app/plugins/datasource/influxdb/datasource.js b/public/app/plugins/datasource/influxdb/datasource.js index 9adabe9a83f..a23937cd74c 100644 --- a/public/app/plugins/datasource/influxdb/datasource.js +++ b/public/app/plugins/datasource/influxdb/datasource.js @@ -53,6 +53,7 @@ function (angular, _, dateMath, InfluxSeries, InfluxQuery) { // replace templated variables allQueries = templateSrv.replace(allQueries, options.scopedVars); + return this._seriesQuery(allQueries).then(function(data) { if (!data || !data.results) { return []; @@ -63,13 +64,26 @@ function (angular, _, dateMath, InfluxSeries, InfluxQuery) { var result = data.results[i]; if (!result || !result.series) { continue; } - var alias = (queryTargets[i] || {}).alias; + var target = queryTargets[i]; + var alias = target.alias; if (alias) { - alias = templateSrv.replace(alias, options.scopedVars); + alias = templateSrv.replace(target.alias, options.scopedVars); } - var targetSeries = new InfluxSeries({ series: data.results[i].series, alias: alias }).getTimeSeries(); - for (y = 0; y < targetSeries.length; y++) { - seriesList.push(targetSeries[y]); + + var influxSeries = new InfluxSeries({ series: data.results[i].series, alias: alias }); + + switch(target.resultFormat) { + case 'table': { + seriesList.push(influxSeries.getTable()); + break; + } + default: { + var timeSeries = influxSeries.getTimeSeries(); + for (y = 0; y < timeSeries.length; y++) { + seriesList.push(timeSeries[y]); + } + break; + } } } diff --git a/public/app/plugins/datasource/influxdb/influx_query.ts b/public/app/plugins/datasource/influxdb/influx_query.ts index 34b86e16930..f50627c7f89 100644 --- a/public/app/plugins/datasource/influxdb/influx_query.ts +++ b/public/app/plugins/datasource/influxdb/influx_query.ts @@ -12,6 +12,8 @@ class InfluxQuery { constructor(target) { this.target = target; + target.dsType = 'influxdb'; + target.resultFormat = target.resultFormat || 'time_series'; target.tags = target.tags || []; target.groupBy = target.groupBy || [ {type: 'time', params: ['$interval']}, diff --git a/public/app/plugins/datasource/influxdb/influx_series.js b/public/app/plugins/datasource/influxdb/influx_series.js index fff3536b5e8..63495ffb9b0 100644 --- a/public/app/plugins/datasource/influxdb/influx_series.js +++ b/public/app/plugins/datasource/influxdb/influx_series.js @@ -1,7 +1,8 @@ define([ 'lodash', + 'app/core/table_model', ], -function (_) { +function (_, TableModel) { 'use strict'; function InfluxSeries(options) { @@ -108,5 +109,44 @@ function (_) { return list; }; + p.getTable = function() { + var table = new TableModel(); + var self = this; + var i, j; + + if (self.series.length === 0) { + return table; + } + + _.each(self.series, function(series, seriesIndex) { + + if (seriesIndex === 0) { + table.columns.push({text: 'Time', type: 'time'}); + _.each(_.keys(series.tags), function(key) { + table.columns.push({text: key}); + }); + for (j = 1; j < series.columns.length; j++) { + table.columns.push({text: series.columns[j]}); + } + } + + if (series.values) { + for (i = 0; i < series.values.length; i++) { + var values = series.values[i]; + if (series.tags) { + for (var key in series.tags) { + if (series.tags.hasOwnProperty(key)) { + values.splice(1, 0, series.tags[key]); + } + } + } + table.rows.push(values); + } + } + }); + + return table; + }; + return InfluxSeries; }); diff --git a/public/app/plugins/datasource/influxdb/partials/query.editor.html b/public/app/plugins/datasource/influxdb/partials/query.editor.html index 0aeb8a44224..2b9f7e1a620 100644 --- a/public/app/plugins/datasource/influxdb/partials/query.editor.html +++ b/public/app/plugins/datasource/influxdb/partials/query.editor.html @@ -103,6 +103,12 @@
    • +
  • + Format as +
    • +
  • + +
    • diff --git a/public/app/plugins/datasource/influxdb/query_ctrl.js b/public/app/plugins/datasource/influxdb/query_ctrl.js index 38f87ecd84e..52b7e7f1b7a 100644 --- a/public/app/plugins/datasource/influxdb/query_ctrl.js +++ b/public/app/plugins/datasource/influxdb/query_ctrl.js @@ -20,6 +20,11 @@ function (angular, _, InfluxQueryBuilder, InfluxQuery, queryPart) { $scope.queryModel = new InfluxQuery($scope.target); $scope.queryBuilder = new InfluxQueryBuilder($scope.target); $scope.groupBySegment = uiSegmentSrv.newPlusButton(); + $scope.resultFormats = [ + {text: 'Time series', value: 'time_series'}, + {text: 'Table', value: 'table'}, + {text: 'JSON field', value: 'json_field'}, + ]; if (!$scope.target.measurement) { $scope.measurementSegment = uiSegmentSrv.newSelectMeasurement(); diff --git a/public/app/plugins/datasource/influxdb/specs/influx_series_specs.ts b/public/app/plugins/datasource/influxdb/specs/influx_series_specs.ts index c8c127ed759..8352e41d99a 100644 --- a/public/app/plugins/datasource/influxdb/specs/influx_series_specs.ts +++ b/public/app/plugins/datasource/influxdb/specs/influx_series_specs.ts @@ -186,5 +186,28 @@ describe('when generating timeseries from influxdb response', function() { }); }); + describe('given table response', function() { + var options = { + alias: '', + series: [ + { + name: 'app.prod.server1.count', + tags: {}, + columns: ['time', 'datacenter', 'value'], + values: [[1431946625000, 'America', 10], [1431946626000, 'EU', 12]] + } + ] + }; + + it('should return table', function() { + var series = new InfluxSeries(options); + var table = series.getTable(); + + expect(table.type).to.be('table'); + expect(table.columns.length).to.be(3); + expect(table.rows[0]).to.eql([1431946625000, 'America', 10]);; + }); + }); + }); diff --git a/public/app/panels/table/specs/table_model_specs.ts b/public/test/core/table_model_specs.ts similarity index 95% rename from public/app/panels/table/specs/table_model_specs.ts rename to public/test/core/table_model_specs.ts index ad515835730..8cdeb04f8d0 100644 --- a/public/app/panels/table/specs/table_model_specs.ts +++ b/public/test/core/table_model_specs.ts @@ -1,6 +1,6 @@ import {describe, beforeEach, it, sinon, expect} from 'test/lib/common'; -import {TableModel} from '../table_model'; +import TableModel = require('app/core/table_model'); describe('when sorting table desc', () => { var table; From d7c7f27207fa6a4682383672d664f4e28361a0f7 Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Thu, 3 Dec 2015 15:29:28 +0100 Subject: [PATCH 10/14] add npm-debug.log to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0ac42cbcb4b..3aa23b45149 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules +npm-debug.log coverage/ .aws-config.json awsconfig From 419251ed3514ca4b322ec1bcbed3416711873522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 3 Dec 2015 16:32:35 +0100 Subject: [PATCH 11/14] fix(elasticsearch): fixed issue with default state of elasticsearch query, result in error before query controller could set defaults, moved defaults to query builder, also removed raw query mode as it is pretty broken, fixes #3396 --- CHANGELOG.md | 4 +++- .../plugins/datasource/elasticsearch/datasource.js | 4 ++++ .../elasticsearch/partials/query.editor.html | 1 - .../datasource/elasticsearch/query_builder.js | 11 ++++++----- .../plugins/datasource/elasticsearch/query_ctrl.js | 12 +----------- .../elasticsearch/specs/query_builder_specs.ts | 8 -------- 6 files changed, 14 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b315d6323..4d6fa247b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ * **Elasticsearch**: Support for dynamic daily indices for annotations, closes [#3061](https://github.com/grafana/grafana/issues/3061) * **Graph Panel**: Option to hide series with all zeroes from legend and tooltip, closes [#1381](https://github.com/grafana/grafana/issues/1381), [#3336](https://github.com/grafana/grafana/issues/3336) - ### Bug Fixes * **cloudwatch**: fix for handling of period for long time ranges, fixes [#3086](https://github.com/grafana/grafana/issues/3086) * **dashboard**: fix for collapse row by clicking on row title, fixes [#3065](https://github.com/grafana/grafana/issues/3065) @@ -16,6 +15,9 @@ * **graph**: layout fix for color picker when right side legend was enabled, fixes [#3093](https://github.com/grafana/grafana/issues/3093) * **elasticsearch**: disabling elastic query (via eye) caused error, fixes [#3300](https://github.com/grafana/grafana/issues/3300) +### Breaking changes +* **elasticsearch**: Manual json edited queries are not supported any more (They very barely worked in 2.5) + # 2.5 (2015-10-28) **New Feature: Mix data sources** diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 9c749b9459e..9307031689c 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -183,6 +183,10 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes sentTargets.push(target); } + if (sentTargets.length === 0) { + return $q.when([]); + } + payload = payload.replace(/\$interval/g, options.interval); payload = payload.replace(/\$timeFrom/g, options.range.from.valueOf()); payload = payload.replace(/\$timeTo/g, options.range.to.valueOf()); diff --git a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html index dee2401e6f3..d027e1a5c14 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html +++ b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html @@ -14,7 +14,6 @@