From 073106437562332d0958e0aef286e30ea1275737 Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Tue, 8 Dec 2015 12:04:42 +0100 Subject: [PATCH 01/11] adds support for moving avg support in es queries --- .../datasource/elasticsearch/query_builder.js | 25 ++++++++++++++++--- .../specs/query_builder_specs.ts | 24 ++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index d736966285c..e388ad63fdb 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -175,8 +175,28 @@ function () { } var aggField = {}; - aggField[metric.type] = metricAgg; - nestedAggs.aggs[metric.id] = aggField; + if (metric.type === 'moving_avg') { + var pipeAgg = ''; + for(var aggname in nestedAggs.aggs) { + var agg = nestedAggs.aggs[aggname]; + for(prop in agg) { + if (agg.hasOwnProperty(prop) && agg[prop] !== null) { + if (metric.field === prop) { + pipeAgg = aggname; + break; + } + } + } + } + + if (pipeAgg !== '') { + aggField[metric.type] = { buckets_path: pipeAgg }; + nestedAggs.aggs[metric.id] = aggField; + } + } else { + aggField[metric.type] = metricAgg; + nestedAggs.aggs[metric.id] = aggField; + } } return query; @@ -217,5 +237,4 @@ function () { }; return ElasticQueryBuilder; - }); diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index 3cfdf61620e..931d9a3613e 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -155,4 +155,28 @@ describe('ElasticQueryBuilder', function() { expect(query.size).to.be(500); }); + it('with moving average', function() { + var query = builder.build({ + metrics: [ + { + id: '1', + type: 'sum', + field: '@value', + }, + { + id: '2', + type: 'moving_avg', + field: 'sum' + } + ], + bucketAggs: [ + {type: 'date_histogram', field: '@timestamp', id: '3'} + ], + }); + + var firstLevel = query.aggs["3"]; + expect(firstLevel.aggs["2"]).not.to.be(undefined); + expect(firstLevel.aggs["2"].moving_avg).not.to.be(undefined); + expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("1"); + }); }); From 8ad10149abddf279a07e0c337d251d6bfabe477a Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Tue, 8 Dec 2015 12:06:36 +0100 Subject: [PATCH 02/11] adds basic support for moving avg in es queries --- .../datasource/elasticsearch/metric_agg.js | 17 +++++++++++++++++ .../elasticsearch/partials/metricAgg.html | 5 ++++- .../datasource/elasticsearch/query_def.js | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index c4e04dad325..2073fbc3a55 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -78,6 +78,23 @@ function (angular, _, queryDef) { return $scope.getFields({$fieldType: 'number'}); }; + $scope.getMetrics = function() { + console.log($scope.target.metrics); + + var mets = _.filter($scope.target.metrics, function(x) { + return x.type !== 'moving_avg'; + }); + + mets = _.map(mets, function(m) { + return { text: m.type, type: m.id }; + }); + + console.log(mets); + + return $q.when(mets) + .then(uiSegmentSrv.transformToSegments(false)); + }; + $scope.addMetricAgg = function() { var addIndex = metricAggs.length; diff --git a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html index 65030af5655..09f48e487eb 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html @@ -7,7 +7,10 @@
  • - + +
    • +
  • +
  • {{settingsLinkText}} diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index baab2378e9f..1f7d352a2cc 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -13,6 +13,7 @@ function (_) { {text: "Min", value: 'min', requiresField: true}, {text: "Extended Stats", value: 'extended_stats', requiresField: true}, {text: "Percentiles", value: 'percentiles', requiresField: true}, + {text: "Moving Avg", value: 'moving_avg', requiresField: false, requiresBucketsPath: true}, {text: "Unique Count", value: "cardinality", requiresField: true}, {text: "Raw Document", value: "raw_document", requiresField: false} ], From dd7a13930f517b548e70eee8cea9148d59d8b7c5 Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Tue, 8 Dec 2015 12:07:56 +0100 Subject: [PATCH 03/11] adds null check for response parser --- .../elasticsearch/elastic_response.js | 8 +++- .../datasource/elasticsearch/metric_agg.js | 18 +++------ .../elasticsearch/partials/metricAgg.html | 2 +- .../datasource/elasticsearch/query_builder.js | 40 ++++++++----------- .../datasource/elasticsearch/query_def.js | 2 +- .../specs/query_builder_specs.ts | 11 ++--- 6 files changed, 34 insertions(+), 47 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index 73050ce7f2b..a7c8e6d14bf 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -76,8 +76,12 @@ function (_, queryDef) { newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; for (i = 0; i < esAgg.buckets.length; i++) { bucket = esAgg.buckets[i]; - value = bucket[metric.id].value; - newSeries.datapoints.push([value, bucket.key]); + + value = bucket[metric.id]; + if (value !== undefined) { + newSeries.datapoints.push([value.value, bucket.key]); + } + } seriesList.push(newSeries); break; diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index 2073fbc3a55..ab1afa6a017 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -35,6 +35,10 @@ function (angular, _, queryDef) { } switch($scope.agg.type) { + case 'moving_avg': { + $scope.agg.aggregation = $scope.agg.aggregation || 'sum'; + break; + } case 'percentiles': { $scope.agg.settings.percents = $scope.agg.settings.percents || [25,50,75,95,99]; $scope.settingsLinkText = 'values: ' + $scope.agg.settings.percents.join(','); @@ -79,19 +83,9 @@ function (angular, _, queryDef) { }; $scope.getMetrics = function() { - console.log($scope.target.metrics); + var aggs = [{ text: 'Sum', type: 'sum'}, { text: 'Average', type: 'avg'}]; - var mets = _.filter($scope.target.metrics, function(x) { - return x.type !== 'moving_avg'; - }); - - mets = _.map(mets, function(m) { - return { text: m.type, type: m.id }; - }); - - console.log(mets); - - return $q.when(mets) + return $q.when(aggs) .then(uiSegmentSrv.transformToSegments(false)); }; diff --git a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html index 09f48e487eb..fd487d52d3b 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html @@ -10,7 +10,7 @@
  • - +
  • {{settingsLinkText}} diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index e388ad63fdb..5d1492640dd 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -167,36 +167,28 @@ function () { continue; } - var metricAgg = {field: metric.field}; + var aggField = {}; + var metricAgg = null; + if (metric.type === 'moving_avg') { + var subBucket = metric.id + "_mavg"; + + var af = {}; + af[metric.aggregation] = {field: metric.field}; + nestedAggs.aggs[subBucket] = af; + + metricAgg = { buckets_path: subBucket }; + } else { + metricAgg = {field: metric.field}; + } + for (var prop in metric.settings) { if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) { metricAgg[prop] = metric.settings[prop]; } } - var aggField = {}; - if (metric.type === 'moving_avg') { - var pipeAgg = ''; - for(var aggname in nestedAggs.aggs) { - var agg = nestedAggs.aggs[aggname]; - for(prop in agg) { - if (agg.hasOwnProperty(prop) && agg[prop] !== null) { - if (metric.field === prop) { - pipeAgg = aggname; - break; - } - } - } - } - - if (pipeAgg !== '') { - aggField[metric.type] = { buckets_path: pipeAgg }; - nestedAggs.aggs[metric.id] = aggField; - } - } else { - aggField[metric.type] = metricAgg; - nestedAggs.aggs[metric.id] = aggField; - } + aggField[metric.type] = metricAgg; + nestedAggs.aggs[metric.id] = aggField; } return query; diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index 1f7d352a2cc..ec5141c1e8d 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -13,7 +13,7 @@ function (_) { {text: "Min", value: 'min', requiresField: true}, {text: "Extended Stats", value: 'extended_stats', requiresField: true}, {text: "Percentiles", value: 'percentiles', requiresField: true}, - {text: "Moving Avg", value: 'moving_avg', requiresField: false, requiresBucketsPath: true}, + {text: "Moving Avg", value: 'moving_avg', requiresField: true, requiresBucketsPath: true}, {text: "Unique Count", value: "cardinality", requiresField: true}, {text: "Raw Document", value: "raw_document", requiresField: false} ], diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index 931d9a3613e..0bda6ab73b1 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -158,15 +158,11 @@ describe('ElasticQueryBuilder', function() { it('with moving average', function() { var query = builder.build({ metrics: [ - { - id: '1', - type: 'sum', - field: '@value', - }, { id: '2', type: 'moving_avg', - field: 'sum' + field: '@value', + aggregation: 'sum' } ], bucketAggs: [ @@ -175,8 +171,9 @@ describe('ElasticQueryBuilder', function() { }); var firstLevel = query.aggs["3"]; + console.log(JSON.stringify(query)); expect(firstLevel.aggs["2"]).not.to.be(undefined); expect(firstLevel.aggs["2"].moving_avg).not.to.be(undefined); - expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("1"); + expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("99"); }); }); From b4763290de1f8ab88165fe13cb68550797d3d09b Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 08:31:01 +0100 Subject: [PATCH 04/11] move target extraction logic to query def --- .../datasource/elasticsearch/query_def.js | 14 +++++- .../elasticsearch/specs/query_def_specs.ts | 48 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index ec5141c1e8d..94401de778b 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -13,7 +13,7 @@ function (_) { {text: "Min", value: 'min', requiresField: true}, {text: "Extended Stats", value: 'extended_stats', requiresField: true}, {text: "Percentiles", value: 'percentiles', requiresField: true}, - {text: "Moving Avg", value: 'moving_avg', requiresField: true, requiresBucketsPath: true}, + {text: "Moving Avg", value: 'moving_avg', requiresField: false, requiresBucketsPath: true}, {text: "Unique Count", value: "cardinality", requiresField: true}, {text: "Raw Document", value: "raw_document", requiresField: false} ], @@ -67,6 +67,18 @@ function (_) { {text: '1d', value: '1d'}, ], + getMovingAverageSourceOptions: function(targets) { + var self = this; + var result = []; + _.each(targets.metrics, function(metric) { + if (metric.type !== 'moving_avg') { + result.push({text: self.describeMetric(metric), value: metric.id }); + } + }); + + return result; + }, + getOrderByOptions: function(target) { var self = this; var metricRefs = []; diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts new file mode 100644 index 00000000000..18d22f794d5 --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts @@ -0,0 +1,48 @@ +/// +/// + +import {describe, beforeEach, it, sinon, expect, angularMocks} from 'test/lib/common'; + +declare var helpers: any; +declare var QueryDef: any; + +describe('ElasticQueryDef', function() { + + describe('with zero targets', function() { + var response = QueryDef.getMovingAverageSourceOptions([]); + + it('should return zero', function() { + expect(response.length).to.be(0); + }); + }); + + describe('with count and sum targets', function() { + var targets = { + metrics: [ + { type: 'count', field: '@value' }, + { type: 'sum', field: '@value' } + ] + }; + + var response = QueryDef.getMovingAverageSourceOptions(targets); + + it('should return zero', function() { + expect(response.length).to.be(2); + }); + }); + + describe('with count and moving average targets', function() { + var targets = { + metrics: [ + { type: 'count', field: '@value' }, + { type: 'moving_avg', field: '@value' } + ] + }; + + var response = QueryDef.getMovingAverageSourceOptions(targets); + + it('should return zero', function() { + expect(response.length).to.be(1); + }); + }); +}); From ad79df9b74baafb7782d92ffcc261fae2004e4e4 Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 09:04:48 +0100 Subject: [PATCH 05/11] changes implementation direction moving average will now be based on another metric instead of having moving average on itself --- .../datasource/elasticsearch/metric_agg.js | 21 ++++++++++++------- .../elasticsearch/partials/metricAgg.html | 14 ++++++++++--- .../datasource/elasticsearch/query_builder.js | 9 ++------ .../datasource/elasticsearch/query_def.js | 2 +- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index ab1afa6a017..6e149e91384 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -13,6 +13,7 @@ function (angular, _, queryDef) { $scope.metricAggTypes = queryDef.metricAggTypes; $scope.extendedStats = queryDef.extendedStats; + $scope.mavgSourceOptions = []; $scope.init = function() { $scope.agg = metricAggs[$scope.index]; @@ -22,6 +23,7 @@ function (angular, _, queryDef) { $rootScope.onAppEvent('elastic-query-updated', function() { $scope.index = _.indexOf(metricAggs, $scope.agg); $scope.validateModel(); + $scope.updateMovingAverageOptions(); }, $scope); $scope.validateModel = function() { @@ -36,7 +38,8 @@ function (angular, _, queryDef) { switch($scope.agg.type) { case 'moving_avg': { - $scope.agg.aggregation = $scope.agg.aggregation || 'sum'; + $scope.agg.mavgSource = $scope.agg.mavgSource || ''; + $scope.settingsLinkText = 'Moving average options'; break; } case 'percentiles': { @@ -69,6 +72,11 @@ function (angular, _, queryDef) { $scope.toggleOptions = function() { $scope.showOptions = !$scope.showOptions; + $scope.updateMovingAverageOptions(); + }; + + $scope.onChangeInternal = function() { + $scope.onChange(); }; $scope.onTypeChange = function() { @@ -82,13 +90,6 @@ function (angular, _, queryDef) { return $scope.getFields({$fieldType: 'number'}); }; - $scope.getMetrics = function() { - var aggs = [{ text: 'Sum', type: 'sum'}, { text: 'Average', type: 'avg'}]; - - return $q.when(aggs) - .then(uiSegmentSrv.transformToSegments(false)); - }; - $scope.addMetricAgg = function() { var addIndex = metricAggs.length; @@ -100,6 +101,10 @@ function (angular, _, queryDef) { $scope.onChange(); }; + $scope.updateMovingAverageOptions = function() { + $scope.mvagSourceOptions = queryDef.getMovingAverageSourceOptions($scope.target); + }; + $scope.removeMetricAgg = function() { metricAggs.splice($scope.index, 1); $scope.onChange(); diff --git a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html index fd487d52d3b..2e34b28612d 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html @@ -9,9 +9,6 @@
    • -
  • - -
  • {{settingsLinkText}}
    • @@ -30,6 +27,17 @@
    +
    +
      +
    • + Based on +
      • +
    • + +
      • +
    +
    +
    • diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 5d1492640dd..5cd7d1dae50 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -169,14 +169,9 @@ function () { var aggField = {}; var metricAgg = null; + if (metric.type === 'moving_avg') { - var subBucket = metric.id + "_mavg"; - - var af = {}; - af[metric.aggregation] = {field: metric.field}; - nestedAggs.aggs[subBucket] = af; - - metricAgg = { buckets_path: subBucket }; + metricAgg = {buckets_path: "1"}; } else { metricAgg = {field: metric.field}; } diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index 94401de778b..865c74753b3 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -13,7 +13,7 @@ function (_) { {text: "Min", value: 'min', requiresField: true}, {text: "Extended Stats", value: 'extended_stats', requiresField: true}, {text: "Percentiles", value: 'percentiles', requiresField: true}, - {text: "Moving Avg", value: 'moving_avg', requiresField: false, requiresBucketsPath: true}, + {text: "Moving Avg", value: 'moving_avg', requiresField: false }, {text: "Unique Count", value: "cardinality", requiresField: true}, {text: "Raw Document", value: "raw_document", requiresField: false} ], From 78c6ce842e57c728550d7e96f528f4ee4723090c Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 09:10:50 +0100 Subject: [PATCH 06/11] revert elastic response parser --- .../plugins/datasource/elasticsearch/elastic_response.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index a7c8e6d14bf..73050ce7f2b 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -76,12 +76,8 @@ function (_, queryDef) { newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; for (i = 0; i < esAgg.buckets.length; i++) { bucket = esAgg.buckets[i]; - - value = bucket[metric.id]; - if (value !== undefined) { - newSeries.datapoints.push([value.value, bucket.key]); - } - + value = bucket[metric.id].value; + newSeries.datapoints.push([value, bucket.key]); } seriesList.push(newSeries); break; From f51d74fa6803df64b8419d36b63ced5b913855f5 Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 09:47:56 +0100 Subject: [PATCH 07/11] change the way options are added --- .../datasource/elasticsearch/elastic_response.js | 8 ++++++-- .../plugins/datasource/elasticsearch/metric_agg.js | 13 ++++++------- .../elasticsearch/partials/metricAgg.html | 2 +- .../plugins/datasource/elasticsearch/query_def.js | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index 73050ce7f2b..a7c8e6d14bf 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -76,8 +76,12 @@ function (_, queryDef) { newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; for (i = 0; i < esAgg.buckets.length; i++) { bucket = esAgg.buckets[i]; - value = bucket[metric.id].value; - newSeries.datapoints.push([value, bucket.key]); + + value = bucket[metric.id]; + if (value !== undefined) { + newSeries.datapoints.push([value.value, bucket.key]); + } + } seriesList.push(newSeries); break; diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index 6e149e91384..157ee95a6c4 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -13,7 +13,6 @@ function (angular, _, queryDef) { $scope.metricAggTypes = queryDef.metricAggTypes; $scope.extendedStats = queryDef.extendedStats; - $scope.mavgSourceOptions = []; $scope.init = function() { $scope.agg = metricAggs[$scope.index]; @@ -23,7 +22,6 @@ function (angular, _, queryDef) { $rootScope.onAppEvent('elastic-query-updated', function() { $scope.index = _.indexOf(metricAggs, $scope.agg); $scope.validateModel(); - $scope.updateMovingAverageOptions(); }, $scope); $scope.validateModel = function() { @@ -38,7 +36,7 @@ function (angular, _, queryDef) { switch($scope.agg.type) { case 'moving_avg': { - $scope.agg.mavgSource = $scope.agg.mavgSource || ''; + $scope.agg.mavgSource = $scope.agg.mavgSource || 'Basec on metric'; $scope.settingsLinkText = 'Moving average options'; break; } @@ -90,6 +88,11 @@ function (angular, _, queryDef) { return $scope.getFields({$fieldType: 'number'}); }; + $scope.mavgSourceOptions = function() { + return $q.when(queryDef.getMovingAverageSourceOptions($scope.target)) + .then(uiSegmentSrv.transformToSegments(false)); + }; + $scope.addMetricAgg = function() { var addIndex = metricAggs.length; @@ -101,10 +104,6 @@ function (angular, _, queryDef) { $scope.onChange(); }; - $scope.updateMovingAverageOptions = function() { - $scope.mvagSourceOptions = queryDef.getMovingAverageSourceOptions($scope.target); - }; - $scope.removeMetricAgg = function() { metricAggs.splice($scope.index, 1); $scope.onChange(); diff --git a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html index 2e34b28612d..e7f726c1616 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html @@ -33,7 +33,7 @@ Based on
    • - +
    diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index 865c74753b3..885704c4d25 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -13,7 +13,7 @@ function (_) { {text: "Min", value: 'min', requiresField: true}, {text: "Extended Stats", value: 'extended_stats', requiresField: true}, {text: "Percentiles", value: 'percentiles', requiresField: true}, - {text: "Moving Avg", value: 'moving_avg', requiresField: false }, + {text: "Moving Average", value: 'moving_avg', requiresField: false }, {text: "Unique Count", value: "cardinality", requiresField: true}, {text: "Raw Document", value: "raw_document", requiresField: false} ], From d3ff4bf75e5936027320acea56bc9bb82c7f8c37 Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 13:55:06 +0100 Subject: [PATCH 08/11] changes to using an array for mavg options --- .../datasource/elasticsearch/metric_agg.js | 19 +++++++++++-------- .../elasticsearch/partials/metricAgg.html | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index 157ee95a6c4..6db0ad7b48d 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -13,14 +13,21 @@ function (angular, _, queryDef) { $scope.metricAggTypes = queryDef.metricAggTypes; $scope.extendedStats = queryDef.extendedStats; + $scope.mavgOptions = []; $scope.init = function() { $scope.agg = metricAggs[$scope.index]; $scope.validateModel(); + $scope.updateMavgOptions(); + }; + + $scope.updateMavgOptions = function() { + $scope.mavgOptions = queryDef.getMovingAverageSourceOptions($scope.target); }; $rootScope.onAppEvent('elastic-query-updated', function() { $scope.index = _.indexOf(metricAggs, $scope.agg); + $scope.updateMavgOptions(); $scope.validateModel(); }, $scope); @@ -30,14 +37,15 @@ function (angular, _, queryDef) { $scope.settingsLinkText = ''; $scope.aggDef = _.findWhere($scope.metricAggTypes, {value: $scope.agg.type}); - if (!$scope.agg.field) { + if (!$scope.agg.field && $scope.agg.type !== 'moving_avg') { $scope.agg.field = 'select field'; } switch($scope.agg.type) { case 'moving_avg': { - $scope.agg.mavgSource = $scope.agg.mavgSource || 'Basec on metric'; + $scope.agg.mavgSource = $scope.agg.mavgSource || 'Metric to apply moving average'; $scope.settingsLinkText = 'Moving average options'; + $scope.agg.field = $scope.agg.mavgSource; break; } case 'percentiles': { @@ -70,7 +78,7 @@ function (angular, _, queryDef) { $scope.toggleOptions = function() { $scope.showOptions = !$scope.showOptions; - $scope.updateMovingAverageOptions(); + $scope.updateMavgOptions(); }; $scope.onChangeInternal = function() { @@ -88,11 +96,6 @@ function (angular, _, queryDef) { return $scope.getFields({$fieldType: 'number'}); }; - $scope.mavgSourceOptions = function() { - return $q.when(queryDef.getMovingAverageSourceOptions($scope.target)) - .then(uiSegmentSrv.transformToSegments(false)); - }; - $scope.addMetricAgg = function() { var addIndex = metricAggs.length; diff --git a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html index e7f726c1616..3ddec210ceb 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html @@ -33,7 +33,7 @@ Based on
  • - +
    • From 0b285845d1bda46a553881621b7fcaa7dab5a172 Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 14:21:48 +0100 Subject: [PATCH 09/11] adds spec for query builder --- .../datasource/elasticsearch/metric_agg.js | 2 +- .../datasource/elasticsearch/query_builder.js | 6 ++- .../specs/query_builder_specs.ts | 47 +++++++++++++++++-- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index 6db0ad7b48d..149623bdd9b 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -37,7 +37,7 @@ function (angular, _, queryDef) { $scope.settingsLinkText = ''; $scope.aggDef = _.findWhere($scope.metricAggTypes, {value: $scope.agg.type}); - if (!$scope.agg.field && $scope.agg.type !== 'moving_avg') { + if (!$scope.agg.field) { $scope.agg.field = 'select field'; } diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 5cd7d1dae50..967f8851495 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -171,7 +171,11 @@ function () { var metricAgg = null; if (metric.type === 'moving_avg') { - metricAgg = {buckets_path: "1"}; + if (metric.mavgSource && /^\d*$/.test(metric.mavgSource)) { + metricAgg = { buckets_path: metric.mavgSource }; + } else { + continue; + } } else { metricAgg = {field: metric.field}; } diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index 0bda6ab73b1..3c2c52916cb 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -158,11 +158,16 @@ describe('ElasticQueryBuilder', function() { it('with moving average', function() { var query = builder.build({ metrics: [ + { + id: '3', + type: 'sum', + field: '@value' + }, { id: '2', type: 'moving_avg', - field: '@value', - aggregation: 'sum' + field: '3', + mavgSource: '3' } ], bucketAggs: [ @@ -171,9 +176,43 @@ describe('ElasticQueryBuilder', function() { }); var firstLevel = query.aggs["3"]; - console.log(JSON.stringify(query)); + expect(firstLevel.aggs["2"]).not.to.be(undefined); expect(firstLevel.aggs["2"].moving_avg).not.to.be(undefined); - expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("99"); + expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("3"); + }); + + it('with broken moving average', function() { + var query = builder.build({ + metrics: [ + { + id: '3', + type: 'sum', + field: '@value' + }, + { + id: '2', + type: 'moving_avg', + field: '3', + mavgSource: '3' + }, + { + id: '4', + type: 'moving_avg', + field: '3', + mavgSource: 'Metric to apply moving average' + } + ], + bucketAggs: [ + { type: 'date_histogram', field: '@timestamp', id: '3' } + ], + }); + + var firstLevel = query.aggs["3"]; + + expect(firstLevel.aggs["2"]).not.to.be(undefined); + expect(firstLevel.aggs["2"].moving_avg).not.to.be(undefined); + expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("3"); + expect(firstLevel.aggs["4"]).to.be(undefined); }); }); From 0644bfe27cc62d9df7e903847ebc6d4970f5de4e Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 14:58:26 +0100 Subject: [PATCH 10/11] improves timeseries naming for moving average series --- .../app/plugins/datasource/elasticsearch/elastic_response.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index a7c8e6d14bf..1994102bc82 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -197,7 +197,10 @@ function (_, queryDef) { }); } - if (series.field) { + if (series.field && series.metric === 'moving_avg') { + var appliedAgg = _.findWhere(target.metrics, { id: series.field }); + metricName += ' ' + queryDef.describeMetric(appliedAgg); + } else if (series.field) { metricName += ' ' + series.field; } From 8e18f2c5d2fd093c71d6761059499133b6dc2dc5 Mon Sep 17 00:00:00 2001 From: carl bergquist Date: Wed, 9 Dec 2015 16:25:05 +0100 Subject: [PATCH 11/11] refactor es pipeline aggregation variables to match ES --- .../elasticsearch/elastic_response.js | 6 +- .../datasource/elasticsearch/metric_agg.js | 6 +- .../elasticsearch/partials/metricAgg.html | 2 +- .../datasource/elasticsearch/query_builder.js | 4 +- .../datasource/elasticsearch/query_def.js | 12 ++- .../specs/query_builder_specs.ts | 6 +- .../elasticsearch/specs/query_def_specs.ts | 74 ++++++++++++------- 7 files changed, 72 insertions(+), 38 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index 1994102bc82..fd53873aa8d 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -199,7 +199,11 @@ function (_, queryDef) { if (series.field && series.metric === 'moving_avg') { var appliedAgg = _.findWhere(target.metrics, { id: series.field }); - metricName += ' ' + queryDef.describeMetric(appliedAgg); + if (appliedAgg) { + metricName += ' ' + queryDef.describeMetric(appliedAgg); + } else { + metricName = 'Unset'; + } } else if (series.field) { metricName += ' ' + series.field; } diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index 149623bdd9b..1d6f82a292f 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -22,7 +22,7 @@ function (angular, _, queryDef) { }; $scope.updateMavgOptions = function() { - $scope.mavgOptions = queryDef.getMovingAverageSourceOptions($scope.target); + $scope.mavgOptions = queryDef.getMovingAverageOptions($scope.target); }; $rootScope.onAppEvent('elastic-query-updated', function() { @@ -43,9 +43,9 @@ function (angular, _, queryDef) { switch($scope.agg.type) { case 'moving_avg': { - $scope.agg.mavgSource = $scope.agg.mavgSource || 'Metric to apply moving average'; + $scope.agg.pipelineAgg = $scope.agg.pipelineAgg || 'Metric to apply moving average'; $scope.settingsLinkText = 'Moving average options'; - $scope.agg.field = $scope.agg.mavgSource; + $scope.agg.field = $scope.agg.pipelineAgg; break; } case 'percentiles': { diff --git a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html index 3ddec210ceb..7c21e64721e 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metricAgg.html @@ -33,7 +33,7 @@ Based on
  • - +
    • diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 967f8851495..64128943f40 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -171,8 +171,8 @@ function () { var metricAgg = null; if (metric.type === 'moving_avg') { - if (metric.mavgSource && /^\d*$/.test(metric.mavgSource)) { - metricAgg = { buckets_path: metric.mavgSource }; + if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { + metricAgg = { buckets_path: metric.pipelineAgg }; } else { continue; } diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index 885704c4d25..6fed5580576 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -67,7 +67,17 @@ function (_) { {text: '1d', value: '1d'}, ], - getMovingAverageSourceOptions: function(targets) { + pipelineAggs: ['moving_avg'], + + isPipelineAgg: function(metric) { + if (metric.type) { + return this.pipelineAggs.indexOf(metric.type) > -1; + } + + return false; + }, + + getMovingAverageOptions: function(targets) { var self = this; var result = []; _.each(targets.metrics, function(metric) { diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index 3c2c52916cb..91cdba7248e 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -167,7 +167,7 @@ describe('ElasticQueryBuilder', function() { id: '2', type: 'moving_avg', field: '3', - mavgSource: '3' + pipelineAgg: '3' } ], bucketAggs: [ @@ -194,13 +194,13 @@ describe('ElasticQueryBuilder', function() { id: '2', type: 'moving_avg', field: '3', - mavgSource: '3' + pipelineAgg: '3' }, { id: '4', type: 'moving_avg', field: '3', - mavgSource: 'Metric to apply moving average' + pipelineAgg: 'Metric to apply moving average' } ], bucketAggs: [ diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts index 18d22f794d5..cf375430ae1 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts @@ -8,41 +8,61 @@ declare var QueryDef: any; describe('ElasticQueryDef', function() { - describe('with zero targets', function() { - var response = QueryDef.getMovingAverageSourceOptions([]); + describe('getMovingAverageOptions', function() { + describe('with zero targets', function() { + var response = QueryDef.getMovingAverageOptions([]); - it('should return zero', function() { - expect(response.length).to.be(0); + it('should return zero', function() { + expect(response.length).to.be(0); + }); + }); + + describe('with count and sum targets', function() { + var targets = { + metrics: [ + { type: 'count', field: '@value' }, + { type: 'sum', field: '@value' } + ] + }; + + var response = QueryDef.getMovingAverageOptions(targets); + + it('should return zero', function() { + expect(response.length).to.be(2); + }); + }); + + describe('with count and moving average targets', function() { + var targets = { + metrics: [ + { type: 'count', field: '@value' }, + { type: 'moving_avg', field: '@value' } + ] + }; + + var response = QueryDef.getMovingAverageOptions(targets); + + it('should return zero', function() { + expect(response.length).to.be(1); + }); }); }); - describe('with count and sum targets', function() { - var targets = { - metrics: [ - { type: 'count', field: '@value' }, - { type: 'sum', field: '@value' } - ] - }; + describe('isPipelineMetric', function() { + describe('moving_avg', function() { + var result = QueryDef.isPipelineAgg({ type: 'moving_avg' }); - var response = QueryDef.getMovingAverageSourceOptions(targets); - - it('should return zero', function() { - expect(response.length).to.be(2); + it('is pipe line metric', function() { + expect(result).to.be(true); + }); }); - }); - describe('with count and moving average targets', function() { - var targets = { - metrics: [ - { type: 'count', field: '@value' }, - { type: 'moving_avg', field: '@value' } - ] - }; + describe('count', function() { + var result = QueryDef.isPipelineAgg({ type: 'count' }); - var response = QueryDef.getMovingAverageSourceOptions(targets); - - it('should return zero', function() { - expect(response.length).to.be(1); + it('is not pipe line metric', function() { + expect(result).to.be(false); + }); }); }); });