diff --git a/docker/blocks/elastic1/elasticsearch.yml b/docker/blocks/elastic1/elasticsearch.yml new file mode 100644 index 00000000000..c57b2c12908 --- /dev/null +++ b/docker/blocks/elastic1/elasticsearch.yml @@ -0,0 +1,2 @@ +script.inline: on +script.indexed: on diff --git a/docker/blocks/elastic1/fig b/docker/blocks/elastic1/fig new file mode 100644 index 00000000000..c33e51f16a2 --- /dev/null +++ b/docker/blocks/elastic1/fig @@ -0,0 +1,8 @@ +elasticsearch1: + image: elasticsearch:1.7.6 + command: elasticsearch -Des.network.host=0.0.0.0 + ports: + - "11200:9200" + - "11300:9300" + volumes: + - ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml diff --git a/docker/blocks/elastic5/fig b/docker/blocks/elastic5/fig index c1c00bea168..6e5cd89ab3d 100644 --- a/docker/blocks/elastic5/fig +++ b/docker/blocks/elastic5/fig @@ -4,5 +4,5 @@ elasticsearch5: image: elasticsearch:5 command: elasticsearch ports: - - "9200:9200" - - "9300:9300" + - "10200:9200" + - "10300:9300" diff --git a/public/app/plugins/datasource/elasticsearch/config_ctrl.ts b/public/app/plugins/datasource/elasticsearch/config_ctrl.ts index 4ed3fa8163a..5c5704eb22c 100644 --- a/public/app/plugins/datasource/elasticsearch/config_ctrl.ts +++ b/public/app/plugins/datasource/elasticsearch/config_ctrl.ts @@ -22,8 +22,8 @@ export class ElasticConfigCtrl { ]; esVersions = [ - {name: '1.x', value: 1}, {name: '2.x', value: 2}, + {name: '5.x', value: 5}, ]; indexPatternTypeChanged() { diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 9f98a794da4..d17af0d27ae 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -81,21 +81,31 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes range[timeField]= { from: options.range.from.valueOf(), to: options.range.to.valueOf(), + format: "epoch_millis", }; - if (this.esVersion >= 2) { - range[timeField]["format"] = "epoch_millis"; - } - var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene'); - var filter = { "bool": { "must": [{ "range": range }] } }; - var query = { "bool": { "should": [{ "query_string": { "query": queryInterpolated } }] } }; + var query = { + "bool": { + "must": [ + { "range": range }, + { "query_string": { + "query": queryInterpolated } + } + ] + } + }; + var data = { - "fields": [timeField, "_source"], - "query" : { "filtered": { "query" : query, "filter": filter } }, + "query" : query, "size": 10000 }; + // fields field not supported on ES 5.x + if (this.esVersion < 5) { + data["fields"] = [timeField, "_source"]; + } + var header = {search_type: "query_then_fetch", "ignore_unavailable": true}; // old elastic annotations had index specified on them @@ -133,11 +143,12 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes for (var i = 0; i < hits.length; i++) { var source = hits[i]._source; - var fields = hits[i].fields; var time = source[timeField]; - - if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) { - time = fields[timeField]; + if (typeof hits[i].fields !== 'undefined') { + var fields = hits[i].fields; + if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) { + time = fields[timeField]; + } } var event = { @@ -194,7 +205,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2); esQuery = esQuery.replace("$lucene_query", luceneQuery); - var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch'; + var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch'; var header = this.getQueryHeader(searchType, options.range.from, options.range.to); payload += header + '\n'; @@ -277,14 +288,15 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes this.getTerms = function(queryDef) { var range = timeSrv.timeRange(); - var header = this.getQueryHeader('count', range.from, range.to); + var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ; + var header = this.getQueryHeader(searchType, range.from, range.to); var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef)); esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf()); esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf()); esQuery = header + '\n' + esQuery + '\n'; - return this._post('_msearch?search_type=count', esQuery).then(function(res) { + return this._post('_msearch?search_type=' + searchType, esQuery).then(function(res) { if (!res.responses[0].aggregations) { return []; } diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 073864d2097..8b00d092080 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -11,11 +11,11 @@ function (queryDef) { ElasticQueryBuilder.prototype.getRangeFilter = function() { var filter = {}; - filter[this.timeField] = {"gte": "$timeFrom", "lte": "$timeTo"}; - - if (this.esVersion >= 2) { - filter[this.timeField]["format"] = "epoch_millis"; - } + filter[this.timeField] = { + gte: "$timeFrom", + lte: "$timeTo", + format: "epoch_millis", + }; return filter; }; @@ -28,7 +28,8 @@ function (queryDef) { return queryNode; } - queryNode.terms.size = parseInt(aggDef.settings.size, 10); + queryNode.terms.size = parseInt(aggDef.settings.size, 10) === 0 ? 500 : parseInt(aggDef.settings.size, 10); + if (aggDef.settings.orderBy !== void 0) { queryNode.terms.order = {}; queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order; @@ -62,15 +63,12 @@ function (queryDef) { esAgg.field = this.timeField; esAgg.min_doc_count = settings.min_doc_count || 0; esAgg.extended_bounds = {min: "$timeFrom", max: "$timeTo"}; + esAgg.format = "epoch_millis"; if (esAgg.interval === 'auto') { esAgg.interval = "$interval"; } - if (this.esVersion >= 2) { - esAgg.format = "epoch_millis"; - } - if (settings.missing) { esAgg.missing = settings.missing; } @@ -80,15 +78,13 @@ function (queryDef) { ElasticQueryBuilder.prototype.getFiltersAgg = function(aggDef) { var filterObj = {}; - for (var i = 0; i < aggDef.settings.filters.length; i++) { var query = aggDef.settings.filters[i].query; + filterObj[query] = { - query: { - query_string: { - query: query, - analyze_wildcard: true - } + query_string: { + query: query, + analyze_wildcard: true } }; } @@ -100,7 +96,12 @@ function (queryDef) { query.size = 500; query.sort = {}; query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'}; - query.fields = ["*", "_source"]; + + // fields field not supported on ES 5.x + if (this.esVersion < 5) { + query.fields = ["*", "_source"]; + } + query.script_fields = {}, query.fielddata_fields = [this.timeField]; return query; @@ -112,13 +113,11 @@ function (queryDef) { } var i, filter, condition; - var must = query.query.filtered.filter.bool.must; - for (i = 0; i < adhocFilters.length; i++) { filter = adhocFilters[i]; condition = {}; condition[filter.key] = filter.value; - must.push({"term": condition}); + query.query.bool.must.push({"term": condition}); } }; @@ -133,18 +132,15 @@ function (queryDef) { var query = { "size": 0, "query": { - "filtered": { - "query": { - "query_string": { + "bool": { + "must": [ + {"range": this.getRangeFilter()}, + {"query_string": { "analyze_wildcard": true, - "query": '$lucene_query', + "query": '$lucene_query' + } } - }, - "filter": { - "bool": { - "must": [{"range": this.getRangeFilter()}] - } - } + ] } } }; @@ -228,30 +224,26 @@ function (queryDef) { var query = { "size": 0, "query": { - "filtered": { - "filter": { - "bool": { - "must": [{"range": this.getRangeFilter()}] - } - } + "bool": { + "must": [{"range": this.getRangeFilter()}] } } }; if (queryDef.query) { - query.query.filtered.query = { + query.query.bool.must.push({ "query_string": { "analyze_wildcard": true, "query": queryDef.query, } - }; + }); } query.aggs = { "1": { "terms": { "field": queryDef.field, - "size": 0, + "size": 500, "order": { "_term": "asc" } diff --git a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts index 43b295a3d64..0c08117f59c 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts @@ -28,7 +28,7 @@ describe('ElasticDatasource', function() { describe('When testing datasource with index pattern', function() { beforeEach(function() { - createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily'}}); + createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily', esVersion: '2'}}); }); it('should translate index pattern to current day', function() { @@ -50,7 +50,7 @@ describe('ElasticDatasource', function() { var requestOptions, parts, header; beforeEach(function() { - createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily'}}); + createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily', esVersion: '2'}}); ctx.backendSrv.datasourceRequest = function(options) { requestOptions = options; @@ -77,7 +77,7 @@ describe('ElasticDatasource', function() { it('should json escape lucene query', function() { var body = angular.fromJson(parts[1]); - expect(body.query.filtered.query.query_string.query).to.be('escape\\:test'); + expect(body.query.bool.must[1].query_string.query).to.be('escape\\:test'); }); }); @@ -85,7 +85,7 @@ describe('ElasticDatasource', function() { var requestOptions, parts, header; beforeEach(function() { - createDatasource({url: 'http://es.com', index: 'test'}); + createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '2'}}); ctx.backendSrv.datasourceRequest = function(options) { requestOptions = options; @@ -209,4 +209,79 @@ describe('ElasticDatasource', function() { }); }); + describe('When issuing aggregation query on es5.x', function() { + var requestOptions, parts, header; + + beforeEach(function() { + createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}}); + + ctx.backendSrv.datasourceRequest = function(options) { + requestOptions = options; + return ctx.$q.when({data: {responses: []}}); + }; + + ctx.ds.query({ + range: { from: moment([2015, 4, 30, 10]), to: moment([2015, 5, 1, 10]) }, + targets: [{ + bucketAggs: [ + {type: 'date_histogram', field: '@timestamp', id: '2'} + ], + metrics: [ + {type: 'count'}], query: 'test' } + ] + }); + + ctx.$rootScope.$apply(); + parts = requestOptions.data.split('\n'); + header = angular.fromJson(parts[0]); + }); + + it('should not set search type to count', function() { + expect(header.search_type).to.not.eql('count'); + }); + + it('should set size to 0', function() { + var body = angular.fromJson(parts[1]); + expect(body.size).to.be(0); + }); + + }); + + describe('When issuing metricFind query on es5.x', function() { + var requestOptions, parts, header, body; + + beforeEach(function() { + createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}}); + + ctx.backendSrv.datasourceRequest = function(options) { + requestOptions = options; + return ctx.$q.when({ + data: { + responses: [{aggregations: {"1": [{buckets: {text: 'test', value: '1'}}]}}] + } + }); + }; + + ctx.ds.metricFindQuery('{"find": "terms", "field": "test"}'); + ctx.$rootScope.$apply(); + + parts = requestOptions.data.split('\n'); + header = angular.fromJson(parts[0]); + body = angular.fromJson(parts[1]); + }); + + it('should not set search type to count', function() { + expect(header.search_type).to.not.eql('count'); + }); + + it('should set size to 0', function() { + expect(body.size).to.be(0); + }); + + it('should not set terms aggregation size to 0', function() { + expect(body['aggs']['1']['terms'].size).to.not.be(0); + }); + }); + }); + diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index d9174e73969..47d540500f4 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -16,7 +16,23 @@ describe('ElasticQueryBuilder', function() { bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}], }); - expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom"); + expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom"); + expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom"); + }); + + it('with defaults on es5.x', function() { + var builder_5x = new ElasticQueryBuilder({ + timeField: '@timestamp', + esVersion: 5 + }); + + var query = builder_5x.build({ + metrics: [{type: 'Count', id: '0'}], + timeField: '@timestamp', + bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}], + }); + + expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom"); expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom"); }); @@ -34,39 +50,6 @@ describe('ElasticQueryBuilder', function() { expect(query.aggs["2"].aggs["3"].date_histogram.field).to.be("@timestamp"); }); - it('with es1.x and es2.x date histogram queries check time format', function() { - var builder_2x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 2 - }); - - var query_params = { - metrics: [], - bucketAggs: [ - {type: 'date_histogram', field: '@timestamp', id: '1'} - ], - }; - - // format should not be specified in 1.x queries - expect("format" in builder.build(query_params)["aggs"]["1"]["date_histogram"]).to.be(false); - - // 2.x query should specify format to be "epoch_millis" - expect(builder_2x.build(query_params)["aggs"]["1"]["date_histogram"]["format"]).to.be("epoch_millis"); - }); - - it('with es1.x and es2.x range filter check time format', function() { - var builder_2x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 2 - }); - - // format should not be specified in 1.x queries - expect("format" in builder.getRangeFilter()["@timestamp"]).to.be(false); - - // 2.x query should specify format to be "epoch_millis" - expect(builder_2x.getRangeFilter()["@timestamp"]["format"]).to.be("epoch_millis"); - }); - it('with select field', function() { var query = builder.build({ metrics: [{type: 'avg', field: '@value', id: '1'}], @@ -138,8 +121,36 @@ describe('ElasticQueryBuilder', function() { ], }); - expect(query.aggs["2"].filters.filters["@metric:cpu"].query.query_string.query).to.be("@metric:cpu"); - expect(query.aggs["2"].filters.filters["@metric:logins.count"].query.query_string.query).to.be("@metric:logins.count"); + expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu"); + expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count"); + expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp"); + }); + + it('with filters aggs on es5.x', function() { + var builder_5x = new ElasticQueryBuilder({ + timeField: '@timestamp', + esVersion: 5 + }); + var query = builder_5x.build({ + metrics: [{type: 'count', id: '1'}], + timeField: '@timestamp', + bucketAggs: [ + { + id: '2', + type: 'filters', + settings: { + filters: [ + {query: '@metric:cpu' }, + {query: '@metric:logins.count' }, + ] + } + }, + {type: 'date_histogram', field: '@timestamp', id: '4'} + ], + }); + + expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu"); + expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count"); expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp"); }); @@ -247,7 +258,6 @@ describe('ElasticQueryBuilder', function() { {key: 'key1', operator: '=', value: 'value1'} ]); - expect(query.query.filtered.filter.bool.must[1].term["key1"]).to.be("value1"); + expect(query.query.bool.must[2].term["key1"]).to.be("value1"); }); - }); diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts index 14a409a2eff..ed09f17a7c5 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts @@ -95,5 +95,11 @@ describe('ElasticQueryDef', function() { expect(queryDef.getMetricAggTypes(2).length).to.be(11); }); }); + + describe('using esversion 5', function() { + it('should get pipeline aggs', function() { + expect(queryDef.getMetricAggTypes(5).length).to.be(11); + }); + }); }); });