diff --git a/public/app/plugins/datasource/elasticsearch/config_ctrl.ts b/public/app/plugins/datasource/elasticsearch/config_ctrl.ts index 4ed3fa8163a..427f49edd67 100644 --- a/public/app/plugins/datasource/elasticsearch/config_ctrl.ts +++ b/public/app/plugins/datasource/elasticsearch/config_ctrl.ts @@ -24,6 +24,7 @@ export class ElasticConfigCtrl { esVersions = [ {name: '1.x', value: 1}, {name: '2.x', value: 2}, + {name: '5.x', value: 5}, ]; indexPatternTypeChanged() { diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 9f98a794da4..183f22f4b3f 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -88,14 +88,18 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes } var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene'); - var filter = { "bool": { "must": [{ "range": range }] } }; - var query = { "bool": { "should": [{ "query_string": { "query": queryInterpolated } }] } }; + var query = { "bool": { "must": [{ "range": range }, { "query_string": { "query": queryInterpolated } }] }}; + var data = { - "fields": [timeField, "_source"], - "query" : { "filtered": { "query" : query, "filter": filter } }, + "query" : query, "size": 10000 }; + // fields field not supported on ES 5.x + if (this.esVersion < 5) { + data["fields"] = [timeField, "_source"]; + } + var header = {search_type: "query_then_fetch", "ignore_unavailable": true}; // old elastic annotations had index specified on them @@ -133,11 +137,12 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes for (var i = 0; i < hits.length; i++) { var source = hits[i]._source; - var fields = hits[i].fields; var time = source[timeField]; - - if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) { - time = fields[timeField]; + if (typeof hits[i].fields !== 'undefined') { + var fields = hits[i].fields; + if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) { + time = fields[timeField]; + } } var event = { @@ -194,7 +199,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2); esQuery = esQuery.replace("$lucene_query", luceneQuery); - var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch'; + var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch'; var header = this.getQueryHeader(searchType, options.range.from, options.range.to); payload += header + '\n'; @@ -277,14 +282,15 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes this.getTerms = function(queryDef) { var range = timeSrv.timeRange(); - var header = this.getQueryHeader('count', range.from, range.to); + var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ; + var header = this.getQueryHeader(searchType, range.from, range.to); var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef)); esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf()); esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf()); esQuery = header + '\n' + esQuery + '\n'; - return this._post('_msearch?search_type=count', esQuery).then(function(res) { + return this._post('_msearch?search_type=' + searchType, esQuery).then(function(res) { if (!res.responses[0].aggregations) { return []; } diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 4be5404a950..76608b7f90c 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -28,7 +28,8 @@ function (queryDef) { return queryNode; } - queryNode.terms.size = parseInt(aggDef.settings.size, 10); + queryNode.terms.size = parseInt(aggDef.settings.size, 10) === 0 ? 500 : parseInt(aggDef.settings.size, 10); + if (aggDef.settings.orderBy !== void 0) { queryNode.terms.order = {}; queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order; @@ -75,14 +76,23 @@ function (queryDef) { for (var i = 0; i < aggDef.settings.filters.length; i++) { var query = aggDef.settings.filters[i].query; - filterObj[query] = { - query: { + if (this.esVersion >= 5) { + filterObj[query] = { query_string: { query: query, analyze_wildcard: true } - } - }; + }; + } else { + filterObj[query] = { + query: { + query_string: { + query: query, + analyze_wildcard: true + } + } + }; + } } return filterObj; @@ -92,7 +102,12 @@ function (queryDef) { query.size = 500; query.sort = {}; query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'}; - query.fields = ["*", "_source"]; + + // fields field not supported on ES 5.x + if (this.esVersion < 5) { + query.fields = ["*", "_source"]; + } + query.script_fields = {}, query.fielddata_fields = [this.timeField]; return query; @@ -103,8 +118,13 @@ function (queryDef) { return; } - var i, filter, condition; - var must = query.query.filtered.filter.bool.must; + var i, filter, condition, must; + + if (this.esVersion >= 5) { + must = query.query.bool.must; + } else { + must = query.query.filtered.filter.bool.must; + } for (i = 0; i < adhocFilters.length; i++) { filter = adhocFilters[i]; @@ -122,24 +142,43 @@ function (queryDef) { target.timeField = this.timeField; var i, nestedAggs, metric; - var query = { - "size": 0, - "query": { - "filtered": { - "query": { - "query_string": { - "analyze_wildcard": true, - "query": '$lucene_query', - } - }, - "filter": { - "bool": { - "must": [{"range": this.getRangeFilter()}] + var query = {}; + if (this.esVersion >= 5) { + query = { + "size": 0, + "query": { + "bool": { + "must": [ + {"range": this.getRangeFilter()}, + {"query_string": { + "analyze_wildcard": true, + "query": '$lucene_query' + } + } + ] + } + } + }; + } else { + query = { + "size": 0, + "query": { + "filtered": { + "query": { + "query_string": { + "analyze_wildcard": true, + "query": '$lucene_query', + } + }, + "filter": { + "bool": { + "must": [{"range": this.getRangeFilter()}] + } } } } - } - }; + }; + } this.addAdhocFilters(query, adhocFilters); @@ -217,33 +256,56 @@ function (queryDef) { }; ElasticQueryBuilder.prototype.getTermsQuery = function(queryDef) { - var query = { - "size": 0, - "query": { - "filtered": { - "filter": { - "bool": { - "must": [{"range": this.getRangeFilter()}] + var query; + + if (this.esVersion >= 5) { + query = { + "size": 0, + "query": { + "bool": { + "must": [{"range": this.getRangeFilter()}] + } + } + }; + + if (queryDef.query) { + query.query.bool.must.push({ + "query_string": { + "analyze_wildcard": true, + "query": queryDef.query, + } + }); + } + + } else { + query = { + "size": 0, + "query": { + "filtered": { + "filter": { + "bool": { + "must": [{"range": this.getRangeFilter()}] + } } } } - } - }; - - if (queryDef.query) { - query.query.filtered.query = { - "query_string": { - "analyze_wildcard": true, - "query": queryDef.query, - } }; + + if (queryDef.query) { + query.query.filtered.query = { + "query_string": { + "analyze_wildcard": true, + "query": queryDef.query, + } + }; + } } query.aggs = { "1": { "terms": { "field": queryDef.field, - "size": 0, + "size": 500, "order": { "_term": "asc" } diff --git a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts index 43b295a3d64..6689b78c0b4 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts @@ -209,4 +209,79 @@ describe('ElasticDatasource', function() { }); }); + describe('When issuing aggregation query on es5.x', function() { + var requestOptions, parts, header; + + beforeEach(function() { + createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}}); + + ctx.backendSrv.datasourceRequest = function(options) { + requestOptions = options; + return ctx.$q.when({data: {responses: []}}); + }; + + ctx.ds.query({ + range: { from: moment([2015, 4, 30, 10]), to: moment([2015, 5, 1, 10]) }, + targets: [{ + bucketAggs: [ + {type: 'date_histogram', field: '@timestamp', id: '2'} + ], + metrics: [ + {type: 'count'}], query: 'test' } + ] + }); + + ctx.$rootScope.$apply(); + parts = requestOptions.data.split('\n'); + header = angular.fromJson(parts[0]); + }); + + it('should not set search type to count', function() { + expect(header.search_type).to.not.eql('count'); + }); + + it('should set size to 0', function() { + var body = angular.fromJson(parts[1]); + expect(body.size).to.be(0); + }); + + }); + + describe('When issuing metricFind query on es5.x', function() { + var requestOptions, parts, header, body; + + beforeEach(function() { + createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}}); + + ctx.backendSrv.datasourceRequest = function(options) { + requestOptions = options; + return ctx.$q.when({ + data: { + responses: [{aggregations: {"1": [{buckets: {text: 'test', value: '1'}}]}}] + } + }); + }; + + ctx.ds.metricFindQuery('{"find": "terms", "field": "test"}'); + ctx.$rootScope.$apply(); + + parts = requestOptions.data.split('\n'); + header = angular.fromJson(parts[0]); + body = angular.fromJson(parts[1]); + }); + + it('should not set search type to count', function() { + expect(header.search_type).to.not.eql('count'); + }); + + it('should set size to 0', function() { + expect(body.size).to.be(0); + }); + + it('should not set terms aggregation size to 0', function() { + expect(body['aggs']['1']['terms'].size).to.not.be(0); + }); + }); + }); + diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index d9174e73969..fab0319afda 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -20,6 +20,22 @@ describe('ElasticQueryBuilder', function() { expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom"); }); + it('with defaults on es5.x', function() { + var builder_5x = new ElasticQueryBuilder({ + timeField: '@timestamp', + esVersion: 5 + }); + + var query = builder_5x.build({ + metrics: [{type: 'Count', id: '0'}], + timeField: '@timestamp', + bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}], + }); + + expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom"); + expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom"); + }); + it('with multiple bucket aggs', function() { var query = builder.build({ metrics: [{type: 'count', id: '1'}], @@ -143,6 +159,34 @@ describe('ElasticQueryBuilder', function() { expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp"); }); + it('with filters aggs on es5.x', function() { + var builder_5x = new ElasticQueryBuilder({ + timeField: '@timestamp', + esVersion: 5 + }); + var query = builder_5x.build({ + metrics: [{type: 'count', id: '1'}], + timeField: '@timestamp', + bucketAggs: [ + { + id: '2', + type: 'filters', + settings: { + filters: [ + {query: '@metric:cpu' }, + {query: '@metric:logins.count' }, + ] + } + }, + {type: 'date_histogram', field: '@timestamp', id: '4'} + ], + }); + + expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu"); + expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count"); + expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp"); + }); + it('with raw_document metric', function() { var query = builder.build({ metrics: [{type: 'raw_document', id: '1'}], diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts index 14a409a2eff..ed09f17a7c5 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts @@ -95,5 +95,11 @@ describe('ElasticQueryDef', function() { expect(queryDef.getMetricAggTypes(2).length).to.be(11); }); }); + + describe('using esversion 5', function() { + it('should get pipeline aggs', function() { + expect(queryDef.getMetricAggTypes(5).length).to.be(11); + }); + }); }); });