From 0d34740a621a5bcebfaa59c84196d415c2fafdcf Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Tue, 10 May 2016 19:38:22 +0200
Subject: [PATCH 1/7] Add support to ElasticSearch 5.0

---
 .../datasource/elasticsearch/config_ctrl.ts   |   1 +
 .../datasource/elasticsearch/datasource.js    |   7 +-
 .../datasource/elasticsearch/query_builder.js | 115 ++++++++++++------
 3 files changed, 86 insertions(+), 37 deletions(-)

diff --git a/public/app/plugins/datasource/elasticsearch/config_ctrl.ts b/public/app/plugins/datasource/elasticsearch/config_ctrl.ts
index 4c14e1133bf..c5b65dd3026 100644
--- a/public/app/plugins/datasource/elasticsearch/config_ctrl.ts
+++ b/public/app/plugins/datasource/elasticsearch/config_ctrl.ts
@@ -24,6 +24,7 @@ export class ElasticConfigCtrl {
   esVersions = [
     {name: '1.x', value: 1},
     {name: '2.x', value: 2},
+    {name: '5.x', value: 5},
   ];
 
   indexPatternTypeChanged() {
diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js
index c7c2351d0cc..4e1329dc2ab 100644
--- a/public/app/plugins/datasource/elasticsearch/datasource.js
+++ b/public/app/plugins/datasource/elasticsearch/datasource.js
@@ -182,7 +182,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
         luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
         esQuery = esQuery.replace("$lucene_query", luceneQuery);
 
-        var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch';
+        var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch';
         var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
         payload +=  header + '\n';
 
@@ -243,7 +243,8 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
     this.getTerms = function(queryDef) {
       var range = timeSrv.timeRange();
-      var header = this.getQueryHeader('count', range.from, range.to);
+      var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ;
+      var header = this.getQueryHeader(searchType, range.from, range.to);
       var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
 
       esQuery = esQuery.replace("$lucene_query", queryDef.query || '*');
@@ -251,7 +252,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
       esQuery = header + '\n' + esQuery + '\n';
 
-      return this._post('/_msearch?search_type=count', esQuery).then(function(res) {
+      return this._post('/_msearch?search_type=' + searchType, esQuery).then(function(res) {
         var buckets = res.responses[0].aggregations["1"].buckets;
         return _.map(buckets, function(bucket) {
           return {text: bucket.key, value: bucket.key};
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js
index 9c8217102aa..09b1d7ce8f1 100644
--- a/public/app/plugins/datasource/elasticsearch/query_builder.js
+++ b/public/app/plugins/datasource/elasticsearch/query_builder.js
@@ -75,14 +75,23 @@ function (queryDef) {
 
     for (var i = 0; i < aggDef.settings.filters.length; i++) {
       var query = aggDef.settings.filters[i].query;
-      filterObj[query] = {
-        query: {
+      if (this.esVersion >= 5) {
+        filterObj[query] = {
           query_string: {
             query: query,
             analyze_wildcard: true
           }
-        }
-      };
+        };
+      } else {
+        filterObj[query] = {
+          query: {
+            query_string: {
+              query: query,
+              analyze_wildcard: true
+            }
+          }
+        };
+      }
     }
 
     return filterObj;
@@ -106,24 +115,43 @@ function (queryDef) {
     target.timeField =  this.timeField;
 
     var i, nestedAggs, metric;
-    var query = {
-      "size": 0,
-      "query": {
-        "filtered": {
-          "query": {
-            "query_string": {
-              "analyze_wildcard": true,
-              "query": '$lucene_query',
-            }
-          },
-          "filter": {
-            "bool": {
-              "must": [{"range": this.getRangeFilter()}]
+    var query = {};
+    if (this.esVersion >= 5) {
+      query = {
+        "size": 0,
+        "query": {
+          "bool": {
+            "must": [
+              {"range": this.getRangeFilter()},
+              {"query_string": {
+                "analyze_wildcard": true,
+                "query": '$lucene_query'
+                }
+              }
+            ]
+          }
+        }
+      };
+    } else {
+      query = {
+        "size": 0,
+        "query": {
+          "filtered": {
+            "query": {
+              "query_string": {
+                "analyze_wildcard": true,
+                "query": '$lucene_query',
+              }
+            },
+            "filter": {
+              "bool": {
+                "must": [{"range": this.getRangeFilter()}]
+              }
             }
           }
         }
-      }
-    };
+      };
+    }
 
     // handle document query
     if (target.bucketAggs.length === 0) {
@@ -199,24 +227,43 @@ function (queryDef) {
   };
 
   ElasticQueryBuilder.prototype.getTermsQuery = function(queryDef) {
-    var query = {
-      "size": 0,
-      "query": {
-        "filtered": {
-          "query": {
-            "query_string": {
-              "analyze_wildcard": true,
-              "query": '$lucene_query',
-            }
-          },
-          "filter": {
-            "bool": {
-              "must": [{"range": this.getRangeFilter()}]
+    var query = {};
+    if (this.esVersion >= 5) {
+      query = {
+        "size": 0,
+        "query": {
+          "bool": {
+            "must": [
+              {"range": this.getRangeFilter()},
+              {"query_string": {
+                "analyze_wildcard": true,
+                "query": '$lucene_query'
+                }
+              }
+            ]
+          }
+        }
+      };
+    } else {
+      query = {
+        "size": 0,
+        "query": {
+          "filtered": {
+            "query": {
+              "query_string": {
+                "analyze_wildcard": true,
+                "query": '$lucene_query',
+              }
+            },
+            "filter": {
+              "bool": {
+                "must": [{"range": this.getRangeFilter()}]
+              }
             }
           }
         }
-      }
-    };
+      };
+    }
     query.aggs =  {
       "1": {
         "terms": {

From 2205ce53543de1be872cedf0fc21209ec715fbea Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Mon, 20 Jun 2016 00:40:16 +0200
Subject: [PATCH 2/7] Add some tests for ES 5.x

---
 .../elasticsearch/specs/datasource_specs.ts   | 73 ++++++++++++++++++-
 .../specs/query_builder_specs.ts              | 44 +++++++++++
 .../elasticsearch/specs/query_def_specs.ts    |  6 ++
 3 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
index 4d630ff9c40..4e89ab853fa 100644
--- a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
@@ -11,7 +11,7 @@ describe('ElasticDatasource', function() {
 
   beforeEach(angularMocks.module('grafana.core'));
   beforeEach(angularMocks.module('grafana.services'));
-  beforeEach(ctx.providePhase(['templateSrv', 'backendSrv']));
+  beforeEach(ctx.providePhase(['templateSrv', 'backendSrv','timeSrv']));
 
   beforeEach(angularMocks.inject(function($q, $rootScope, $httpBackend, $injector) {
     ctx.$q = $q;
@@ -112,4 +112,75 @@ describe('ElasticDatasource', function() {
     });
   });
 
+  describe('When issuing aggregation query on es5.x', function() {
+    var requestOptions, parts, header;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({data: {responses: []}});
+      };
+
+      ctx.ds.query({
+        range: { from: moment([2015, 4, 30, 10]), to: moment([2015, 5, 1, 10]) },
+        targets: [{
+            bucketAggs: [
+                {type: 'date_histogram', field: '@timestamp', id: '2'}
+            ],
+            metrics: [
+                {type: 'count'}], query: 'test' }
+            ]
+      });
+
+      ctx.$rootScope.$apply();
+      parts = requestOptions.data.split('\n');
+      header = angular.fromJson(parts[0]);
+    });
+
+    it('should not set search type to count', function() {
+      expect(header.search_type).to.not.eql('count');
+    });
+
+    it('should set size to 0', function() {
+      var body = angular.fromJson(parts[1]);
+      expect(body.size).to.be(0);
+    });
+
+  });
+
+  describe('When issuing metricFind query on es5.x', function() {
+    var requestOptions, parts, header;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({
+            data: {
+                responses: [{aggregations: {"1": [{buckets: {text: 'test', value: '1'}}]}}]
+            }
+        });
+      };
+
+      ctx.ds.metricFindQuery('{"find": "terms", "field": "test"}');
+      ctx.$rootScope.$apply();
+
+      parts = requestOptions.data.split('\n');
+      header = angular.fromJson(parts[0]);
+    });
+
+    it('should not set search type to count', function() {
+      expect(header.search_type).to.not.eql('count');
+    });
+
+    it('should set size to 0', function() {
+      var body = angular.fromJson(parts[1]);
+      expect(body.size).to.be(0);
+    });
+
+  });
+
 });
diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts
index bbd5711fd1f..cbb9488ab43 100644
--- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts
@@ -20,6 +20,22 @@ describe('ElasticQueryBuilder', function() {
     expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
   });
 
+  it('with defaults on es5.x', function() {
+    var builder_5x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 5
+    });
+
+    var query = builder_5x.build({
+      metrics: [{type: 'Count', id: '0'}],
+      timeField: '@timestamp',
+      bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
+    });
+
+    expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
+    expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
+  });
+
   it('with multiple bucket aggs', function() {
     var query = builder.build({
       metrics: [{type: 'count', id: '1'}],
@@ -143,6 +159,34 @@ describe('ElasticQueryBuilder', function() {
     expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
   });
 
+  it('with filters aggs on es5.x', function() {
+    var builder_5x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 5
+    });
+    var query = builder_5x.build({
+      metrics: [{type: 'count', id: '1'}],
+      timeField: '@timestamp',
+      bucketAggs: [
+        {
+          id: '2',
+          type: 'filters',
+          settings: {
+            filters: [
+              {query: '@metric:cpu' },
+              {query: '@metric:logins.count' },
+            ]
+          }
+        },
+        {type: 'date_histogram', field: '@timestamp', id: '4'}
+      ],
+    });
+
+    expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu");
+    expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count");
+    expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
+  });
+
   it('with raw_document metric', function() {
     var query = builder.build({
       metrics: [{type: 'raw_document', id: '1'}],
diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts
index 14a409a2eff..ed09f17a7c5 100644
--- a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts
@@ -95,5 +95,11 @@ describe('ElasticQueryDef', function() {
               expect(queryDef.getMetricAggTypes(2).length).to.be(11);
           });
       });
+
+      describe('using esversion 5', function() {
+          it('should get pipeline aggs', function() {
+              expect(queryDef.getMetricAggTypes(5).length).to.be(11);
+          });
+      });
   });
 });

From df9ace8ac22f4377b9c7f2e252efeef6cc93260f Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Thu, 21 Jul 2016 00:42:36 +0200
Subject: [PATCH 3/7] Change annotations query for ES 5.x

---
 .../datasource/elasticsearch/datasource.js        | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js
index c4ea52f64b5..8484e5aa258 100644
--- a/public/app/plugins/datasource/elasticsearch/datasource.js
+++ b/public/app/plugins/datasource/elasticsearch/datasource.js
@@ -79,11 +79,11 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       }
 
       var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene');
-      var filter = { "bool": { "must": [{ "range": range }] } };
-      var query = { "bool": { "should": [{ "query_string": { "query": queryInterpolated } }] } };
+      var query = { "bool": { "must": [{ "range": range }, { "query_string": { "query": queryInterpolated } }] }};
+
       var data = {
         "fields": [timeField, "_source"],
-        "query" : { "filtered": { "query" : query, "filter": filter } },
+        "query" : query,
         "size": 10000
       };
 
@@ -124,11 +124,12 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
         for (var i = 0; i < hits.length; i++) {
           var source = hits[i]._source;
-          var fields = hits[i].fields;
           var time = source[timeField];
-
-          if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
-            time = fields[timeField];
+          if (typeof hits[i].fields !== 'undefined') {
+            var fields = hits[i].fields;
+            if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
+              time = fields[timeField];
+            }
           }
 
           var event = {

From 3057c9747caf69bf0b1881890cbc1998c2c2928c Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Tue, 26 Jul 2016 22:05:49 +0200
Subject: [PATCH 4/7] Change size on terms aggregation for ES 5.x

---
 .../app/plugins/datasource/elasticsearch/query_builder.js  | 5 +++--
 .../datasource/elasticsearch/specs/datasource_specs.ts     | 7 +++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js
index 09b1d7ce8f1..02953a0d2eb 100644
--- a/public/app/plugins/datasource/elasticsearch/query_builder.js
+++ b/public/app/plugins/datasource/elasticsearch/query_builder.js
@@ -28,7 +28,8 @@ function (queryDef) {
       return queryNode;
     }
 
-    queryNode.terms.size = parseInt(aggDef.settings.size, 10);
+    queryNode.terms.size = parseInt(aggDef.settings.size, 10) === 0 ? 1000 : parseInt(aggDef.settings.size, 10);
+
     if (aggDef.settings.orderBy !== void 0) {
       queryNode.terms.order = {};
       queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order;
@@ -268,7 +269,7 @@ function (queryDef) {
       "1": {
         "terms": {
           "field": queryDef.field,
-          "size": 0,
+          "size": 1000,
           "order": {
             "_term": "asc"
           }
diff --git a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
index 4e89ab853fa..ed201583152 100644
--- a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
@@ -151,7 +151,7 @@ describe('ElasticDatasource', function() {
   });
 
   describe('When issuing metricFind query on es5.x', function() {
-    var requestOptions, parts, header;
+    var requestOptions, parts, header, body;
 
     beforeEach(function() {
       createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
@@ -170,6 +170,7 @@ describe('ElasticDatasource', function() {
 
       parts = requestOptions.data.split('\n');
       header = angular.fromJson(parts[0]);
+      body = angular.fromJson(parts[1]);
     });
 
     it('should not set search type to count', function() {
@@ -177,10 +178,12 @@ describe('ElasticDatasource', function() {
     });
 
     it('should set size to 0', function() {
-      var body = angular.fromJson(parts[1]);
       expect(body.size).to.be(0);
     });
 
+    it('should not set terms aggregation size to 0', function() {
+      expect(body['aggs']['1']['terms'].size).to.not.be(0);
+    });
   });
 
 });

From f844cd844badb41d3c1e271e4551ae902bee29dc Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Thu, 11 Aug 2016 22:24:58 +0200
Subject: [PATCH 5/7] Fix for annotations query on ES 5.x

---
 public/app/plugins/datasource/elasticsearch/datasource.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js
index 8484e5aa258..2f8249f00a7 100644
--- a/public/app/plugins/datasource/elasticsearch/datasource.js
+++ b/public/app/plugins/datasource/elasticsearch/datasource.js
@@ -82,11 +82,15 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       var query = { "bool": { "must": [{ "range": range }, { "query_string": { "query": queryInterpolated } }] }};
 
       var data = {
-        "fields": [timeField, "_source"],
         "query" : query,
         "size": 10000
       };
 
+      // fields field not supported on ES 5.x
+      if (this.esVersion < 5) {
+        data["fields"] = [timeField, "_source"];
+      }
+
       var header = {search_type: "query_then_fetch", "ignore_unavailable": true};
 
       // old elastic annotations had index specified on them

From 538e1fbc643c9a2725f08ad713a319c21d69fde7 Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Thu, 27 Oct 2016 18:59:24 +0200
Subject: [PATCH 6/7] Fix templating using query

---
 .../datasource/elasticsearch/query_builder.js | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js
index cb1081a3bc6..292020a032f 100644
--- a/public/app/plugins/datasource/elasticsearch/query_builder.js
+++ b/public/app/plugins/datasource/elasticsearch/query_builder.js
@@ -251,7 +251,7 @@ function (queryDef) {
   };
 
   ElasticQueryBuilder.prototype.getTermsQuery = function(queryDef) {
-    var query, queryPath;
+    var query;
 
     if (this.esVersion >= 5) {
       query = {
@@ -262,7 +262,15 @@ function (queryDef) {
           }
         }
       };
-      queryPath = query.query.bool.must;
+
+      if (queryDef.query) {
+        query.query.bool.must.push({
+          "query_string": {
+            "analyze_wildcard": true,
+            "query": queryDef.query,
+          }
+        });
+      }
 
     } else {
       query = {
@@ -277,16 +285,15 @@ function (queryDef) {
           }
         }
       };
-      queryPath = query.query.filtered.query;
-    }
 
-    if (queryDef.query) {
-      queryPath = {
-        "query_string": {
-          "analyze_wildcard": true,
-          "query": queryDef.query,
-        }
-      };
+      if (queryDef.query) {
+        query.query.filtered.query = {
+          "query_string": {
+            "analyze_wildcard": true,
+            "query": queryDef.query,
+          }
+        };
+      }
     }
 
     query.aggs =  {

From 717a96ab8ded92f921407d1a531b1388f3b0ea3e Mon Sep 17 00:00:00 2001
From: Leandro Piccilli <piccilli@gmail.com>
Date: Tue, 8 Nov 2016 22:18:59 +0100
Subject: [PATCH 7/7] Fix raw document query on ES 5.x

---
 .../app/plugins/datasource/elasticsearch/query_builder.js  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js
index 292020a032f..76608b7f90c 100644
--- a/public/app/plugins/datasource/elasticsearch/query_builder.js
+++ b/public/app/plugins/datasource/elasticsearch/query_builder.js
@@ -102,7 +102,12 @@ function (queryDef) {
     query.size = 500;
     query.sort = {};
     query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
-    query.fields = ["*", "_source"];
+
+    // fields field not supported on ES 5.x
+    if (this.esVersion < 5) {
+      query.fields = ["*", "_source"];
+    }
+
     query.script_fields = {},
     query.fielddata_fields = [this.timeField];
     return query;