feat(tablepanel/elasticsearch): extended elasticsearch data source and query editor to support document queries

public/app/panels/table/transformers.ts

@@ -14,8 +14,6 @@ transformers['timeseries_to_rows'] = {
       {text: 'Value'},
     ];
 
-    model.rows = [];
-
     for (var i = 0; i < data.length; i++) {
       var series = data[i];
       for (var y = 0; y < series.datapoints.length; y++) {
@@ -31,8 +29,7 @@ transformers['timeseries_to_rows'] = {
 transformers['timeseries_to_columns'] = {
   description: 'Time series to columns',
   transform: function(data, panel, model) {
-    model.columns = [{text: 'Time'}];
+    model.columns.push({text: 'Time'});
-    model.rows = [];
 
     // group by time
     var points = {};
@@ -75,8 +72,19 @@ transformers['annotations'] = {
 
 transformers['json'] = {
   description: 'JSON',
+  transform: function(data, panel, model) {
+    model.columns.push({text: 'JSON'});
+    debugger;
+
+    for (var i = 0; i < data.length; i++) {
+      var series = data[i];
+
+      for (var y = 0; y < series.datapoints.length; y++) {
+        var dp = series.datapoints[y];
+        model.rows.push([JSON.stringify(dp)]);
+      }
+    }
+  }
 };
 
 export {transformers}
-
-

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -153,8 +153,8 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       });
     };
 
-    ElasticDatasource.prototype.getQueryHeader = function(timeFrom, timeTo) {
+    ElasticDatasource.prototype.getQueryHeader = function(searchType, timeFrom, timeTo) {
-      var header = {search_type: "count", "ignore_unavailable": true};
+      var header = {search_type: searchType, "ignore_unavailable": true};
       header.index = this.indexPattern.getIndexList(timeFrom, timeTo);
       return angular.toJson(header);
     };
@@ -163,8 +163,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       var payload = "";
       var target;
       var sentTargets = [];
-
+      var headerAdded = false;
-      var header = this.getQueryHeader(options.range.from, options.range.to);
 
       for (var i = 0; i < options.targets.length; i++) {
         target = options.targets[i];
@@ -177,7 +176,14 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
         luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
         esQuery = esQuery.replace("$lucene_query", luceneQuery);
 
-        payload += header + '\n' + esQuery + '\n';
+        if (!headerAdded) {
+          var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch';
+          var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
+          payload +=  header + '\n';
+          headerAdded = true;
+        }
+
+        payload += esQuery + '\n';
         sentTargets.push(target);
       }
 
@@ -230,7 +236,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
     ElasticDatasource.prototype.getTerms = function(queryDef) {
       var range = timeSrv.timeRange();
-      var header = this.getQueryHeader(range.from, range.to);
+      var header = this.getQueryHeader('count', range.from, range.to);
       var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
 
       esQuery = esQuery.replace("$lucene_query", queryDef.query || '*');

public/app/plugins/datasource/elasticsearch/elastic_response.js

@@ -173,6 +173,33 @@ function (_, queryDef) {
     }
   };
 
+  ElasticResponse.prototype.processHits = function(hits, seriesList) {
+    var series = {target: 'docs', type: 'docs', datapoints: [], total: hits.total};
+    var propName, hit, doc, i;
+
+    for (i = 0; i < hits.hits.length; i++) {
+      hit = hits.hits[i];
+      doc = {
+        _id: hit._id,
+        _type: hit._type,
+        _index: hit._index
+      };
+
+      if (hit._source) {
+        for (propName in hit._source) {
+          doc[propName] = hit._source[propName];
+        }
+      }
+
+      for (propName in hit.fields) {
+        doc[propName] = hit.fields[propName];
+      }
+      series.datapoints.push(doc);
+    }
+
+    seriesList.push(series);
+  };
+
   ElasticResponse.prototype.getTimeSeries = function() {
     var seriesList = [];
 
@@ -182,15 +209,21 @@ function (_, queryDef) {
         throw { message: response.error };
       }
 
-      var aggregations = response.aggregations;
+      if (response.hits) {
-      var target = this.targets[i];
+        this.processHits(response.hits, seriesList);
-      var tmpSeriesList = [];
+      }
 
-      this.processBuckets(aggregations, target, tmpSeriesList, {});
+      if (response.aggregations) {
-      this.nameSeries(tmpSeriesList, target);
+        var aggregations = response.aggregations;
+        var target = this.targets[i];
+        var tmpSeriesList = [];
 
-      for (var y = 0; y < tmpSeriesList.length; y++) {
+        this.processBuckets(aggregations, target, tmpSeriesList, {});
-        seriesList.push(tmpSeriesList[y]);
+        this.nameSeries(tmpSeriesList, target);
+
+        for (var y = 0; y < tmpSeriesList.length; y++) {
+          seriesList.push(tmpSeriesList[y]);
+        }
       }
     }
 

public/app/plugins/datasource/elasticsearch/query_builder.js

@@ -71,6 +71,16 @@ function (angular) {
     return filterObj;
   };
 
+  ElasticQueryBuilder.prototype.documentQuery = function(query) {
+    query.size = 500;
+    query.sort = {};
+    query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
+    query.fields = ["*", "_source"];
+    query.script_fields = {},
+    query.fielddata_fields = [this.timeField];
+    return query;
+  };
+
   ElasticQueryBuilder.prototype.build = function(target) {
     if (target.rawQuery) {
       return angular.fromJson(target.rawQuery);
@@ -96,6 +106,15 @@ function (angular) {
       }
     };
 
+    // handle document query
+    if (target.bucketAggs.length === 0) {
+      metric = target.metrics[0];
+      if (metric && metric.type !== 'raw_document') {
+        throw {message: 'Invalid query'};
+      }
+      return this.documentQuery(query, target);
+    }
+
     nestedAggs = query;
 
     for (i = 0; i < target.bucketAggs.length; i++) {

public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts

@@ -108,7 +108,7 @@ describe('ElasticDatasource', function() {
 
     it('should set size', function() {
       var body = angular.fromJson(parts[1]);
-      expect(body.query.size).to.be(500);
+      expect(body.size).to.be(500);
     });
   });
 

public/app/plugins/datasource/elasticsearch/specs/elastic_response_specs.ts

@@ -411,4 +411,40 @@ describe('ElasticResponse', function() {
     });
   });
 
+  describe('Raw documents query', function() {
+    beforeEach(function() {
+      targets = [{ refId: 'A', metrics: [{type: 'raw_document', id: '1'}], bucketAggs: [] }];
+      response = {
+        responses: [{
+          hits: {
+            total: 100,
+            hits: [
+              {
+                _id: '1',
+                _type: 'type',
+                _index: 'index',
+                _source: {sourceProp: "asd"},
+                fields: {fieldProp: "field" },
+              },
+              {
+                _source: {sourceProp: "asd2"},
+                fields: {fieldProp: "field2" },
+              }
+            ]
+          }
+        }]
+      };
+
+      result = new ElasticResponse(targets, response).getTimeSeries();
+    });
+
+    it('should return docs', function() {
+      expect(result.data.length).to.be(1);
+      expect(result.data[0].type).to.be('docs');
+      expect(result.data[0].total).to.be(100);
+      expect(result.data[0].datapoints.length).to.be(2);
+      expect(result.data[0].datapoints[0].sourceProp).to.be("asd");
+      expect(result.data[0].datapoints[0].fieldProp).to.be("field");
+    });
+  });
 });

public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts

@@ -120,4 +120,14 @@ describe('ElasticQueryBuilder', function() {
     expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
   });
 
+  it('with raw_document metric', function() {
+    var query = builder.build({
+      metrics: [{type: 'raw_document', id: '1'}],
+      timeField: '@timestamp',
+      bucketAggs: [],
+    });
+
+    expect(query.size).to.be(500);
+  });
+
 });