From b3bda02063ee96001089a4b66f1597751d545604 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torkel=20=C3=96degaard?= <torkel.odegaard@gmail.com>
Date: Thu, 3 Sep 2015 12:35:21 +0200
Subject: [PATCH] feat(elasticsearch): lots of work on elasticsearch metrics
 processing, handling grouped responses, etc, #1034

---
 .../datasource/elasticsearch/datasource.js    | 14 ++-
 .../elasticsearch/partials/query.editor.html  |  2 +-
 .../datasource/elasticsearch/queryBuilder.js  | 22 ++---
 public/test/specs/elasticsearch-specs.js      | 90 +++++++++++++++++--
 4 files changed, 105 insertions(+), 23 deletions(-)

diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js
index 0b8eb750306..b9a32df0066 100644
--- a/public/app/plugins/datasource/elasticsearch/datasource.js
+++ b/public/app/plugins/datasource/elasticsearch/datasource.js
@@ -172,6 +172,8 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       return date.getTime();
     };
 
+    // this is quite complex
+    // neeed to recurise down the nested buckets to build series
     ElasticDatasource.prototype._processBuckets = function(buckets, groupByFields, series, level, parentName, parentTime) {
       var points = [];
       var groupBy = groupByFields[level];
@@ -180,13 +182,14 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
         var bucket = buckets[i];
 
         if (groupBy) {
-          var seriesName = "";
+          var seriesName = level > 0 ? bucket.key : '';
           var time = parentTime || bucket.key;
           this._processBuckets(bucket[groupBy.field].buckets, groupByFields, series, level+1, seriesName, time)
         } else {
           var seriesName = parentName;
 
           if (level > 0) {
+            if (seriesName) { seriesName += " "; }
             seriesName += bucket.key;
           } else {
             parentTime = bucket.key;
@@ -202,12 +205,17 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       var series = [];
 
       for (var i = 0; i < results.responses.length; i++) {
-        var buckets = results.responses[i].aggregations.histogram.buckets;
+        var response = results.responses[i];
+        if (response.error) {
+          throw { message: response.error };
+        }
+
+        var buckets = response.aggregations.histogram.buckets;
         var target = targets[i];
         var points = [];
         var querySeries = {}
 
-        this._processBuckets(buckets, target.groupByFields, querySeries, 0, target.refId);
+        this._processBuckets(buckets, target.groupByFields, querySeries, 0);
 
         _.each(querySeries, function(value) {
           series.push(value);
diff --git a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html
index 190dcd88deb..29e66c477c5 100644
--- a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html
+++ b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html
@@ -75,7 +75,7 @@
 <div class="tight-form" ng-hide="target.rawQuery">
 	<ul class="tight-form-list">
 		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Group By
+			Group by
 		</li>
 		<li ng-repeat="segment in groupBySegments">
 			<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>
diff --git a/public/app/plugins/datasource/elasticsearch/queryBuilder.js b/public/app/plugins/datasource/elasticsearch/queryBuilder.js
index d4d89e1e213..9f6c76268bc 100644
--- a/public/app/plugins/datasource/elasticsearch/queryBuilder.js
+++ b/public/app/plugins/datasource/elasticsearch/queryBuilder.js
@@ -52,23 +52,23 @@ function () {
             "max": timeTo
           }
         }
-      }
+      },
     };
 
-    var nestedAggs = {};
+    var nestedAggs = query.aggs.histogram;
 
-    if (target.groupByFields && target.groupByFields.length > 0) {
-      var field = target.groupByFields[0].field;
-      nestedAggs[field] = {
-        terms:  {
-          field: field,
-        }
-      }
+    for (var i = 0; i < target.groupByFields.length; i++) {
+      var field = target.groupByFields[i].field;
+      var aggs = {terms: {field: field}};
+
+      nestedAggs.aggs = {};
+      nestedAggs.aggs[field] = aggs;
+      nestedAggs = aggs;
     }
 
-    query.aggs.histogram.aggs = nestedAggs;
+    console.log(angular.toJson(query, true));
 
-    query = JSON.stringify(query);
+    query = angular.toJson(query);
     return query;
   };
 
diff --git a/public/test/specs/elasticsearch-specs.js b/public/test/specs/elasticsearch-specs.js
index 1944c510f5a..0e4b7f04580 100644
--- a/public/test/specs/elasticsearch-specs.js
+++ b/public/test/specs/elasticsearch-specs.js
@@ -57,14 +57,7 @@ define([
         var result;
 
         beforeEach(function() {
-          result = ctx.ds._processTimeSeries([
-            {
-              refId: 'A',
-              groupByFields: [
-                {field: 'host' }
-              ]
-            }
-          ], {
+          result = ctx.ds._processTimeSeries([{refId: 'A', groupByFields: [{field: 'host' }]}], {
             responses: [{
               aggregations: {
                 histogram: {
@@ -103,6 +96,87 @@ define([
           expect(result.data[1].target).to.be('server2');
         });
       });
+
+      describe('two group by query', function() {
+        var result;
+
+        beforeEach(function() {
+          result = ctx.ds._processTimeSeries([{refId: 'A', groupByFields: [{field: 'host'}, {field: 'site'}]}], {
+            responses: [{
+              aggregations: {
+                histogram: {
+                  buckets: [
+                    {
+                      host: {
+                        buckets: [
+                           {
+                             site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                             },
+                             doc_count: 4, key: 'server1'
+                           },
+                           {
+                             site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                             },
+                             doc_count: 6, key: 'server2'
+                           },
+                        ]
+                      },
+                      doc_count: 10,
+                      key: 1000
+                    },
+                    {
+                      host: {
+                        buckets: [
+                          {
+                            site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                            },
+                            doc_count: 4,
+                            key: 'server1'
+                          },
+                          {
+                            site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                            },
+                            doc_count: 6,
+                            key: 'server2'
+                          },
+                        ]
+                      },
+                      doc_count: 15,
+                      key: 2000
+                    }
+                  ]
+                }
+              }
+            }]
+          })
+        });
+
+        it('should return 2 series', function() {
+          expect(result.data.length).to.be(4);
+          expect(result.data[0].datapoints.length).to.be(2);
+          expect(result.data[0].target).to.be('server1 backend');
+          expect(result.data[1].target).to.be('server1 frontend');
+          expect(result.data[2].target).to.be('server2 backend');
+          expect(result.data[3].target).to.be('server2 frontend');
+        });
+      });
+
     });
   });
 });