From df1d56e7b19b09b647bd8ef003133b2e75a492a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 3 Sep 2015 11:14:25 +0200 Subject: [PATCH] feat(elasticsearch): lots of work on elasticsearch metrics processing, handling grouped responses, etc, #1034 --- public/app/panels/graph/module.js | 2 + .../datasource/elasticsearch/datasource.js | 121 +++++++++++++----- .../elasticsearch/partials/query.editor.html | 5 +- .../datasource/elasticsearch/queryBuilder.js | 15 ++- .../datasource/elasticsearch/queryCtrl.js | 56 +++++--- public/test/specs/elasticsearch-specs.js | 108 ++++++++++++++++ public/test/test-main.js | 1 + 7 files changed, 255 insertions(+), 53 deletions(-) create mode 100644 public/test/specs/elasticsearch-specs.js diff --git a/public/app/panels/graph/module.js b/public/app/panels/graph/module.js index ea0a1e958ad..54c6c554a7d 100644 --- a/public/app/panels/graph/module.js +++ b/public/app/panels/graph/module.js @@ -153,6 +153,8 @@ function (angular, app, $, _, kbn, moment, TimeSeries, PanelMeta) { return; } + console.log('graph data', results); + $scope.datapointsWarning = false; $scope.datapointsCount = 0; $scope.datapointsOutside = false; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 5b6408b493f..0b8eb750306 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -13,7 +13,7 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { var module = angular.module('grafana.services'); - module.factory('ElasticDatasource', function($q, backendSrv, templateSrv) { + module.factory('ElasticDatasource', function($q, backendSrv, templateSrv, timeSrv) { function ElasticDatasource(datasource) { this.type = 'elasticsearch'; @@ -160,7 +160,7 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { payload = payload.replace(/\$maxDataPoints/g, options.maxDataPoints); payload = templateSrv.replace(payload, options.scopedVars); - var processTimeSeries = _.partial(this._processTimeSeries, sentTargets); + var processTimeSeries = _.bind(this._processTimeSeries, this, sentTargets); return this._post('/_msearch?search_type=count', payload).then(processTimeSeries); }; @@ -172,54 +172,109 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { return date.getTime(); }; - ElasticDatasource._aggToSeries = function(agg) { - var datapoints = agg.date_histogram.buckets.map(function(entry) { - return [entry.stats.avg, entry.key]; - }); - return { target: agg.key, datapoints: datapoints }; - }; + ElasticDatasource.prototype._processBuckets = function(buckets, groupByFields, series, level, parentName, parentTime) { + var points = []; + var groupBy = groupByFields[level]; + for (var i = 0; i < buckets.length; i++) { + var bucket = buckets[i]; + + if (groupBy) { + var seriesName = ""; + var time = parentTime || bucket.key; + this._processBuckets(bucket[groupBy.field].buckets, groupByFields, series, level+1, seriesName, time) + } else { + var seriesName = parentName; + + if (level > 0) { + seriesName += bucket.key; + } else { + parentTime = bucket.key; + } + + var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []}; + serie.datapoints.push([bucket.doc_count, parentTime]); + } + } + }; ElasticDatasource.prototype._processTimeSeries = function(targets, results) { var series = []; - _.each(results.responses, function(response, index) { - var buckets = response.aggregations.date_histogram.buckets; - var target = targets[index]; + for (var i = 0; i < results.responses.length; i++) { + var buckets = results.responses[i].aggregations.histogram.buckets; + var target = targets[i]; var points = []; + var querySeries = {} - for (var i = 0; i < buckets.length; i++) { - var bucket = buckets[i]; - points[i] = [bucket.doc_count, bucket.key]; - } + this._processBuckets(buckets, target.groupByFields, querySeries, 0, target.refId); - series.push({target: 'name', datapoints: points}) - console.log('Nr DataPoints: ' + points.length); - }); - - console.log(series); + _.each(querySeries, function(value) { + series.push(value); + }); + }; return { data: series }; }; ElasticDatasource.prototype.metricFindQuery = function(query) { - var region; - var namespace; - var metricName; + var timeFrom = this.translateTime(timeSrv.time.from); + var timeTo = this.translateTime(timeSrv.time.to); - var transformSuggestData = function(suggestData) { - return _.map(suggestData, function(v) { - return { text: v }; - }); + var query = { + size: 10, + "query": { + "filtered": { + "filter": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": timeFrom, + "lte": timeTo + } + } + } + ], + } + } + } + } }; - var d = $q.defer(); + return this._post('/_search?', query).then(function(res) { + var fields = {}; - var regionQuery = query.match(/^region\(\)/); - if (regionQuery) { - d.resolve(transformSuggestData(this.performSuggestRegion())); - return d.promise; - } + for (var i = 0; i < res.hits.hits.length; i++) { + var hit = res.hits.hits[i]; + for (var field in hit) { + if (hit.hasOwnProperty(field) && field[0] !== '_') { + fields[field] = 1; + } + } + + if (hit._source) { + for (var field in hit._source) { + if (hit._source.hasOwnProperty(field)) { + fields[field] = 1; + } + } + } + } + + fields = _.map(_.keys(fields), function(field) { + return {text: field}; + }) + console.log('metricFindQuery:', fields); + return fields; + }); + // var d = $q.defer(); + // + // var fieldsQuery = query.match(/^fields\(\)/); + // if (fieldsQuery) { + // return d.promise; + // } }; return ElasticDatasource; diff --git a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html index a07c4d369a2..190dcd88deb 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html +++ b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html @@ -77,11 +77,10 @@
  • Group By
    • -
  • - +
  • +
    • -
    diff --git a/public/app/plugins/datasource/elasticsearch/queryBuilder.js b/public/app/plugins/datasource/elasticsearch/queryBuilder.js index 4bdeb9d1ff4..d4d89e1e213 100644 --- a/public/app/plugins/datasource/elasticsearch/queryBuilder.js +++ b/public/app/plugins/datasource/elasticsearch/queryBuilder.js @@ -42,7 +42,7 @@ function () { }; query.aggs = { - "date_histogram": { + "histogram": { "date_histogram": { "interval": target.interval || "$interval", "field": target.timeField, @@ -55,6 +55,19 @@ function () { } }; + var nestedAggs = {}; + + if (target.groupByFields && target.groupByFields.length > 0) { + var field = target.groupByFields[0].field; + nestedAggs[field] = { + terms: { + field: field, + } + } + } + + query.aggs.histogram.aggs = nestedAggs; + query = JSON.stringify(query); return query; }; diff --git a/public/app/plugins/datasource/elasticsearch/queryCtrl.js b/public/app/plugins/datasource/elasticsearch/queryCtrl.js index 39899355ce5..f2c3b1f0fbc 100644 --- a/public/app/plugins/datasource/elasticsearch/queryCtrl.js +++ b/public/app/plugins/datasource/elasticsearch/queryCtrl.js @@ -23,18 +23,25 @@ function (angular, _, ElasticQueryBuilder) { target.function = target.function || 'mean'; target.timeField = target.timeField || '@timestamp'; target.select = target.select || [{ agg: 'count' }]; + target.groupByFields = target.groupByFields || []; $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField); - $scope.groupByFieldSegment = uiSegmentSrv.getSegmentForValue(target.groupByField, 'add group by'); + + $scope.groupBySegments = _.map(target.groupByFields, function(group) { + return uiSegmentSrv.newSegment(group.field); + }); $scope.selectSegments = _.map(target.select, function(select) { return uiSegmentSrv.newSegment(select.agg); }); + + $scope.groupBySegments.push(uiSegmentSrv.newPlusButton()); + $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'}); + $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'}); }; $scope.getFields = function() { - return $scope.datasource.metricFindQuery('fields()') - .then($scope.transformToSegments(true)); + return $scope.datasource.metricFindQuery('fields()').then($scope.transformToSegments(true)); }; $scope.transformToSegments = function(addTemplateVars) { @@ -53,6 +60,36 @@ function (angular, _, ElasticQueryBuilder) { }; }; + $scope.getGroupByFields = function(segment) { + return $scope.datasource.metricFindQuery('fields()').then($scope.transformToSegments(false)) + .then(function(results) { + if (segment.type !== 'plus-button') { + results.splice(0, 0, angular.copy($scope.removeGroupBySegment)); + } + return results; + }) + .then(null, $scope.handleQueryError); + }; + + $scope.groupByChanged = function(segment, index) { + if (segment.value === $scope.removeGroupBySegment.value) { + $scope.target.groupByFields.splice(index, 1); + $scope.groupBySegments.splice(index, 1); + $scope.$parent.get_data(); + return; + } + + if (index === $scope.groupBySegments.length-1) { + $scope.groupBySegments.push(uiSegmentSrv.newPlusButton()); + } + + segment.type = 'group-by-key'; + segment.fake = false; + + $scope.target.groupByFields[index] = {field: segment.value}; + $scope.$parent.get_data(); + }; + $scope.valueFieldChanged = function() { $scope.target.valueField = $scope.valueFieldSegment.value; $scope.$parent.get_data(); @@ -88,19 +125,6 @@ function (angular, _, ElasticQueryBuilder) { return []; }; - $scope.transformToSegments = function(results) { - return _.map(results, function(segment) { - return new MetricSegment({ value: segment.text, expandable: segment.expandable }); - }); - }; - - $scope.addTemplateVariableSegments = function(segments) { - _.each(templateSrv.variables, function(variable) { - segments.unshift(new MetricSegment({ type: 'template', value: '$' + variable.name, expandable: true })); - }); - return segments; - }; - $scope.toggleQueryMode = function () { $scope.target.rawQuery = !$scope.target.rawQuery; }; diff --git a/public/test/specs/elasticsearch-specs.js b/public/test/specs/elasticsearch-specs.js new file mode 100644 index 00000000000..1944c510f5a --- /dev/null +++ b/public/test/specs/elasticsearch-specs.js @@ -0,0 +1,108 @@ +define([ + 'helpers', + 'plugins/datasource/elasticsearch/datasource', + 'aws-sdk', +], function(helpers) { + 'use strict'; + + describe('ElasticDatasource', function() { + var ctx = new helpers.ServiceTestContext(); + + beforeEach(module('grafana.services')); + beforeEach(ctx.providePhase(['templateSrv'])); + beforeEach(ctx.createService('ElasticDatasource')); + beforeEach(function() { + ctx.ds = new ctx.service({}); + }); + + describe('When processing es response', function() { + + describe('simple query', function() { + var result; + + beforeEach(function() { + result = ctx.ds._processTimeSeries([{ + refId: 'A', + groupByFields: [], + }], { + responses: [{ + aggregations: { + histogram: { + buckets: [ + { + doc_count: 10, + key: 1000 + }, + { + doc_count: 15, + key: 2000 + } + ] + } + } + }] + }) + }); + + it('should return 1 series', function() { + expect(result.data.length).to.be(1); + expect(result.data[0].datapoints.length).to.be(2); + expect(result.data[0].datapoints[0][0]).to.be(10); + expect(result.data[0].datapoints[0][1]).to.be(1000); + }); + + }); + + describe('single group by query', function() { + var result; + + beforeEach(function() { + result = ctx.ds._processTimeSeries([ + { + refId: 'A', + groupByFields: [ + {field: 'host' } + ] + } + ], { + responses: [{ + aggregations: { + histogram: { + buckets: [ + { + host: { + buckets: [ + {doc_count: 4, key: 'server1'}, + {doc_count: 6, key: 'server2'}, + ] + }, + doc_count: 10, + key: 1000 + }, + { + host: { + buckets: [ + {doc_count: 4, key: 'server1'}, + {doc_count: 6, key: 'server2'}, + ] + }, + doc_count: 15, + key: 2000 + } + ] + } + } + }] + }) + }); + + it('should return 2 series', function() { + expect(result.data.length).to.be(2); + expect(result.data[0].datapoints.length).to.be(2); + expect(result.data[0].target).to.be('server1'); + expect(result.data[1].target).to.be('server2'); + }); + }); + }); + }); +}); diff --git a/public/test/test-main.js b/public/test/test-main.js index 5d2316dfec1..336b5e306eb 100644 --- a/public/test/test-main.js +++ b/public/test/test-main.js @@ -152,6 +152,7 @@ require([ 'specs/valueSelectDropdown-specs', 'specs/opentsdbDatasource-specs', 'specs/cloudwatch-datasource-specs', + 'specs/elasticsearch-specs', ]; var pluginSpecs = (config.plugins.specs || []).map(function (spec) {