From 30b62e172df25bf27bc1d1e68c4ee21a64d3a1df Mon Sep 17 00:00:00 2001 From: Matt Page Date: Fri, 14 Mar 2014 12:14:16 -0700 Subject: [PATCH 01/17] Add an OpenTSDB datasource. This adds support for querying OpenTSDB for metric data. --- src/app/controllers/all.js | 3 +- src/app/controllers/influxTargetCtrl.js | 2 +- src/app/controllers/opentsdbTargetCtrl.js | 107 ++++++++++ src/app/partials/opentsdb/editor.html | 191 ++++++++++++++++++ src/app/services/datasourceSrv.js | 9 +- .../services/opentsdb/opentsdbDatasource.js | 155 ++++++++++++++ 6 files changed, 461 insertions(+), 6 deletions(-) create mode 100644 src/app/controllers/opentsdbTargetCtrl.js create mode 100644 src/app/partials/opentsdb/editor.html create mode 100644 src/app/services/opentsdb/opentsdbDatasource.js diff --git a/src/app/controllers/all.js b/src/app/controllers/all.js index 2626137657c..b3348bb0a58 100644 --- a/src/app/controllers/all.js +++ b/src/app/controllers/all.js @@ -10,4 +10,5 @@ define([ './graphiteImport', './influxTargetCtrl', './playlistCtrl', -], function () {}); \ No newline at end of file + './opentsdbTargetCtrl', +], function () {}); diff --git a/src/app/controllers/influxTargetCtrl.js b/src/app/controllers/influxTargetCtrl.js index 66439c5e757..467b135a961 100644 --- a/src/app/controllers/influxTargetCtrl.js +++ b/src/app/controllers/influxTargetCtrl.js @@ -64,4 +64,4 @@ function (angular) { }); -}); \ No newline at end of file +}); diff --git a/src/app/controllers/opentsdbTargetCtrl.js b/src/app/controllers/opentsdbTargetCtrl.js new file mode 100644 index 00000000000..ba66986532d --- /dev/null +++ b/src/app/controllers/opentsdbTargetCtrl.js @@ -0,0 +1,107 @@ +define([ + 'angular', + 'underscore', + 'kbn' +], +function (angular, _, kbn) { + 'use strict'; + + var module = angular.module('kibana.controllers'); + + module.controller('OpenTSDBTargetCtrl', function($scope) { + + $scope.init = function() { + $scope.target.errors = validateTarget($scope.target); + $scope.aggregators = ['avg', 'sum', 'min', 'max', 'dev', 'zimsum', 'mimmin', 'mimmax']; + }; + + $scope.targetBlur = function() { + $scope.target.errors = validateTarget($scope.target); + + if (!_.isEqual($scope.oldTarget, $scope.target) && _.isEmpty($scope.target.errors)) { + $scope.oldTarget = angular.copy($scope.target); + $scope.get_data(); + } + }; + + $scope.duplicate = function() { + var clone = angular.copy($scope.target); + $scope.panel.targets.push(clone); + }; + + $scope.suggestMetrics = function(query, callback) { + $scope.datasource + .performSuggestQuery(query, 'metrics') + .then(callback); + }; + + $scope.suggestTagKeys = function(query, callback) { + $scope.datasource + .performSuggestQuery(query, 'tagk') + .then(callback); + }; + + $scope.suggestTagValues = function(query, callback) { + $scope.datasource + .performSuggestQuery(query, 'tagv') + .then(callback); + }; + + $scope.addTag = function() { + if (!$scope.target.tags) { + $scope.target.tags = {}; + } + + $scope.target.errors = validateTarget($scope.target); + + if (!$scope.target.errors.tags) { + $scope.target.tags[$scope.target.currentTagKey] = $scope.target.currentTagValue; + $scope.target.currentTagKey = ''; + $scope.target.currentTagValue = ''; + $scope.targetBlur(); + } + }; + + $scope.removeTag = function(key) { + delete $scope.target.tags[key]; + $scope.targetBlur(); + }; + + function validateTarget(target) { + var errs = {}; + + if (!target.metric) { + errs.metric = "You must supply a metric name."; + } + + if (!target.aggregator) { + errs.aggregator = "You must choose an aggregation function."; + } + + if (target.shouldDownsample) { + try { + if (target.downsampleInterval) { + kbn.describe_interval(target.downsampleInterval); + } else { + errs.downsampleInterval = "You must supply a downsample interval (e.g. '1m' or '1h')."; + } + } catch(err) { + errs.downsampleInterval = err.message; + } + + if (!target.downsampleAggregator) { + errs.downsampleAggregator = "You must choose an aggregation function for downsampling."; + } + } + + if (target.tags && _.has(target.tags, target.currentTagKey)) { + errs.tags = "Duplicate tag key '" + target.currentTagKey + "'."; + } + + return errs; + } + + + }); + +}); diff --git a/src/app/partials/opentsdb/editor.html b/src/app/partials/opentsdb/editor.html new file mode 100644 index 00000000000..ebc0c6190b6 --- /dev/null +++ b/src/app/partials/opentsdb/editor.html @@ -0,0 +1,191 @@ +
+
+ +
+
+ + + + + + +
+
+
+
+
diff --git a/src/app/services/datasourceSrv.js b/src/app/services/datasourceSrv.js index d05bd11e278..7ad122539e9 100644 --- a/src/app/services/datasourceSrv.js +++ b/src/app/services/datasourceSrv.js @@ -4,19 +4,18 @@ define([ 'config', './graphite/graphiteDatasource', './influxdb/influxdbDatasource', + './opentsdb/opentsdbDatasource', ], function (angular, _, config) { 'use strict'; var module = angular.module('kibana.services'); - module.service('datasourceSrv', function($q, filterSrv, $http, GraphiteDatasource, InfluxDatasource) { + module.service('datasourceSrv', function($q, filterSrv, $http, GraphiteDatasource, InfluxDatasource, OpenTSDBDatasource) { this.init = function() { - var defaultDatasource = _.findWhere(_.values(config.datasources), { default: true } ); this.default = this.datasourceFactory(defaultDatasource); - }; this.datasourceFactory = function(ds) { @@ -25,6 +24,8 @@ function (angular, _, config) { return new GraphiteDatasource(ds); case 'influxdb': return new InfluxDatasource(ds); + case 'opentsdb': + return new OpenTSDBDatasource(ds); } }; @@ -50,4 +51,4 @@ function (angular, _, config) { this.init(); }); -}); \ No newline at end of file +}); diff --git a/src/app/services/opentsdb/opentsdbDatasource.js b/src/app/services/opentsdb/opentsdbDatasource.js new file mode 100644 index 00000000000..8766c224982 --- /dev/null +++ b/src/app/services/opentsdb/opentsdbDatasource.js @@ -0,0 +1,155 @@ +define([ + 'angular', + 'underscore', + 'kbn' +], +function (angular, _, kbn) { + 'use strict'; + + var module = angular.module('kibana.services'); + + module.factory('OpenTSDBDatasource', function($q, $http) { + + function OpenTSDBDatasource(datasource) { + this.type = 'opentsdb'; + this.editorSrc = 'app/partials/opentsdb/editor.html'; + this.url = datasource.url; + this.name = datasource.name; + } + + // Called once per panel (graph) + OpenTSDBDatasource.prototype.query = function(options) { + var start = convertToTSDBTime(options.range.from); + var end = convertToTSDBTime(options.range.to); + var queries = _.compact(_.map(options.targets, convertTargetToQuery)); + + // No valid targets, return the empty result to save a round trip. + if (_.isEmpty(queries)) { + var d = $q.defer(); + d.resolve({ data: [] }); + return d.promise; + } + + var groupByTags = {}; + _.each(queries, function(query) { + _.each(query.tags, function(val, key) { + if (val === "*") { + groupByTags[key] = true; + } + }); + }); + + return this.performTimeSeriesQuery(queries, start, end) + .then(function(response) { + var result = _.map(response.data, function(metricData) { + return transformMetricData(metricData, groupByTags); + }); + return { data: result }; + }); + }; + + OpenTSDBDatasource.prototype.performTimeSeriesQuery = function(queries, start, end) { + var reqBody = { + start: start, + queries: queries + }; + + // Relative queries (e.g. last hour) don't include an end time + if (end) { + reqBody.end = end; + } + + var options = { + method: 'POST', + url: this.url + '/api/query', + data: reqBody + }; + + return $http(options); + }; + + OpenTSDBDatasource.prototype.performSuggestQuery = function(query, type) { + var options = { + method: 'GET', + url: this.url + '/api/suggest', + params: { + type: type, + q: query + } + }; + return $http(options).then(function(result) { + return result.data; + }); + }; + + function transformMetricData(md, groupByTags) { + var dps = []; + + // TSDB returns datapoints has a hash of ts => value. + // Can't use _.pairs(invert()) because it stringifies keys/values + _.each(md.dps, function (v, k) { + dps.push([v, k]); + }); + + var target = md.metric; + if (!_.isEmpty(md.tags)) { + var tagData = []; + + _.each(_.pairs(md.tags), function(tag) { + if (_.has(groupByTags, tag[0])) { + tagData.push(tag[0] + "=" + tag[1]); + } + }); + + if (!_.isEmpty(tagData)) { + target = target + "{" + tagData.join(", ") + "}"; + } + } + + return { target: target, datapoints: dps }; + } + + function convertTargetToQuery(target) { + if (!target.metric) { + return null; + } + + var query = { + metric: target.metric, + aggregator: "avg" + }; + + if (target.aggregator) { + query.aggregator = target.aggregator; + } + + if (target.shouldComputeRate) { + query.rate = true; + query.rateOptions = { + counter: !!target.isCounter + }; + } + + if (target.shouldDownsample) { + query.downsample = target.downsampleInterval + "-" + target.downsampleAggregator; + } + + query.tags = angular.copy(target.tags); + + return query; + } + + function convertToTSDBTime(date) { + if (date === 'now') { + return null; + } + + date = kbn.parseDate(date); + + return date.getTime(); + } + + return OpenTSDBDatasource; + }); + +}); From 3da1c27692ef418018fe1810c49d44bcbf576d2b Mon Sep 17 00:00:00 2001 From: Clicky Date: Thu, 22 May 2014 15:20:10 -0700 Subject: [PATCH 02/17] Proof of concept of multiple groups using "host" Initial test to make sure this plan works using a predefined alternate group by. Now to actually figure this out based on the query and add some UI for it. --- .../services/influxdb/influxdbDatasource.js | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/app/services/influxdb/influxdbDatasource.js b/src/app/services/influxdb/influxdbDatasource.js index 0a47e2e2619..df17c08a7e7 100644 --- a/src/app/services/influxdb/influxdbDatasource.js +++ b/src/app/services/influxdb/influxdbDatasource.js @@ -162,20 +162,29 @@ function (angular, _, kbn) { _.each(data, function(series) { var timeCol = series.columns.indexOf('time'); + var groupByColumn = series.columns.indexOf('host'); _.each(series.columns, function(column, index) { - if (column === "time" || column === "sequence_number") { + if (column === "time" || column === "sequence_number" || column === "host") { return; } var target = data.alias || series.name + "." + column; - var datapoints = []; - - for(var i = 0; i < series.points.length; i++) { - datapoints[i] = [series.points[i][index], series.points[i][timeCol]]; - } - - output.push({ target:target, datapoints:datapoints }); + var datapoints = _.groupBy(series.points, function (point) { + if (groupByColumn == -1 ) return null; + else return point[groupByColumn]; + }); + datapoints = _.map(_.pairs(datapoints), function(values) { + return [values[0], _.map(values[1], function (point) { return [point[index], point[timeCol]]; }) ]; + }); + + _.each(datapoints, function(values) { + if (values[0] == null) { + output.push({ target: target, datapoints: values[1]}); + } else { + output.push({ target: values[0] + "-" + target, datapoints: values[1] }); + } + }); }); }); From f231af07f96248e68cced79d478a9241b3dec330 Mon Sep 17 00:00:00 2001 From: Clicky Date: Thu, 22 May 2014 16:08:25 -0700 Subject: [PATCH 03/17] Parse additional group by columns from a raw query This needs some further polishing but now parse out additional group by columns from a raw query, although it really only supports one additional column. --- .../services/influxdb/influxdbDatasource.js | 74 +++++++++++-------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/src/app/services/influxdb/influxdbDatasource.js b/src/app/services/influxdb/influxdbDatasource.js index df17c08a7e7..06fc933eb8b 100644 --- a/src/app/services/influxdb/influxdbDatasource.js +++ b/src/app/services/influxdb/influxdbDatasource.js @@ -17,7 +17,6 @@ function (angular, _, kbn) { this.username = datasource.username; this.password = datasource.password; this.name = datasource.name; - this.templateSettings = { interpolate : /\[\[([\s\S]+?)\]\]/g, }; @@ -33,7 +32,8 @@ function (angular, _, kbn) { } var timeFilter = getTimeFilter(options); - + var additionalGroups = []; + if (target.rawQuery) { query = target.query; query = query.replace(";", ""); @@ -42,6 +42,17 @@ function (angular, _, kbn) { var whereIndex = lowerCaseQueryElements.indexOf("where"); var groupByIndex = lowerCaseQueryElements.indexOf("group"); var orderIndex = lowerCaseQueryElements.indexOf("order"); + + var afterGroup = _.rest(lowerCaseQueryElements, groupByIndex); + for (var i = 0; i < afterGroup.length; i++) { + var el = afterGroup[i]; + if (el === "order") break; + if ( /,$/.test(el) && + _.size(afterGroup) > i && + ! /^time\(/.test(afterGroup[i + 1])) { + additionalGroups.push(queryElements[groupByIndex + i + 1]); + } + } if (whereIndex !== -1) { queryElements.splice(whereIndex+1, 0, timeFilter, "and"); @@ -88,7 +99,7 @@ function (angular, _, kbn) { target.query = query; } - return this.doInfluxRequest(query, target.alias).then(handleInfluxQueryResponse); + return this.doInfluxRequest(query, target.alias).then(handleInfluxQueryResponse(additionalGroups)); }, this); @@ -157,38 +168,43 @@ function (angular, _, kbn) { return deferred.promise; }; - function handleInfluxQueryResponse(data) { - var output = []; + function handleInfluxQueryResponse(additionalGroup) { + return function(data) { + var output = []; - _.each(data, function(series) { - var timeCol = series.columns.indexOf('time'); - var groupByColumn = series.columns.indexOf('host'); - - _.each(series.columns, function(column, index) { - if (column === "time" || column === "sequence_number" || column === "host") { - return; - } - - var target = data.alias || series.name + "." + column; - var datapoints = _.groupBy(series.points, function (point) { - if (groupByColumn == -1 ) return null; - else return point[groupByColumn]; + _.each(data, function(series) { + var timeCol = series.columns.indexOf('time'); + var groupCols = _.map(additionalGroup, function(col) { + return series.columns.indexOf(col); }); - datapoints = _.map(_.pairs(datapoints), function(values) { - return [values[0], _.map(values[1], function (point) { return [point[index], point[timeCol]]; }) ]; - }); - - _.each(datapoints, function(values) { - if (values[0] == null) { - output.push({ target: target, datapoints: values[1]}); - } else { - output.push({ target: values[0] + "-" + target, datapoints: values[1] }); + var groupByColumn = _.find(groupCols, function(col) { return col > -1; }); + + _.each(series.columns, function(column, index) { + if (column === "time" || column === "sequence_number" || _.contains(additionalGroup, column)) { + return; } + + var target = data.alias || series.name + "." + column; + var datapoints = _.groupBy(series.points, function (point) { + if (groupByColumn == undefined) return null; + else return point[groupByColumn]; + }); + datapoints = _.map(_.pairs(datapoints), function(values) { + return [values[0], _.map(values[1], function (point) { return [point[index], point[timeCol]]; }) ]; + }); + + _.each(datapoints, function(values) { + if (values[0] == null) { + output.push({ target: target, datapoints: values[1]}); + } else { + output.push({ target: values[0] + "-" + target, datapoints: values[1] }); + } + }); }); }); - }); - return output; + return output; + } } function getTimeFilter(options) { From 4a4708e3fdad48d0a6c42478ea18c6d26c191e03 Mon Sep 17 00:00:00 2001 From: Clicky Date: Thu, 22 May 2014 16:52:34 -0700 Subject: [PATCH 04/17] Fix single line naming --- src/app/services/influxdb/influxdbDatasource.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/services/influxdb/influxdbDatasource.js b/src/app/services/influxdb/influxdbDatasource.js index 06fc933eb8b..45ea716d68b 100644 --- a/src/app/services/influxdb/influxdbDatasource.js +++ b/src/app/services/influxdb/influxdbDatasource.js @@ -194,7 +194,8 @@ function (angular, _, kbn) { }); _.each(datapoints, function(values) { - if (values[0] == null) { + // this gets stringified on its way out of _.pair... sigh + if (values[0] == "null") { output.push({ target: target, datapoints: values[1]}); } else { output.push({ target: values[0] + "-" + target, datapoints: values[1] }); From 9b4c64298bd21143cb8f4ba59ea239444180ab8b Mon Sep 17 00:00:00 2001 From: Clicky Date: Thu, 22 May 2014 18:11:04 -0700 Subject: [PATCH 05/17] Hookup additional group by into UI Allow additional group by fields to be specified without writing a full on influx query by hand. --- .gitignore | 3 ++- src/app/partials/influxdb/editor.html | 20 ++++++++++++++++++- .../services/influxdb/influxdbDatasource.js | 15 +++++++++----- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 701b6bbbc11..383cfb4e5c9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ node_modules dist web.config config.js -*.sublime-workspace \ No newline at end of file +*.sublime-workspace +*.swp diff --git a/src/app/partials/influxdb/editor.html b/src/app/partials/influxdb/editor.html index 07f6ad52b7b..bbf4cbd5df4 100644 --- a/src/app/partials/influxdb/editor.html +++ b/src/app/partials/influxdb/editor.html @@ -76,7 +76,7 @@ +
  • + + + +
    • + +
  • + +
  • alias as diff --git a/src/app/services/influxdb/influxdbDatasource.js b/src/app/services/influxdb/influxdbDatasource.js index 45ea716d68b..4e72e93b586 100644 --- a/src/app/services/influxdb/influxdbDatasource.js +++ b/src/app/services/influxdb/influxdbDatasource.js @@ -73,14 +73,14 @@ function (angular, _, kbn) { query = queryElements.join(" "); } else { - var template = "select [[func]]([[column]]) as [[column]]_[[func]] from [[series]] " + + var template = "select [[group]][[group_add]] [[func]]([[column]]) as [[column]]_[[func]] from [[series]] " + "where [[timeFilter]] [[condition_add]] [[condition_key]] [[condition_op]] [[condition_value]] " + - "group by time([[interval]]) order asc"; + "group by time([[interval]])[[group_add]] [[group]] order asc"; if (target.column.indexOf('-') !== -1 || target.column.indexOf('.') !== -1) { - template = "select [[func]](\"[[column]]\") as \"[[column]]_[[func]]\" from [[series]] " + + template = "select [[group]][[group_add]] [[func]](\"[[column]]\") as \"[[column]]_[[func]]\" from [[series]] " + "where [[timeFilter]] [[condition_add]] [[condition_key]] [[condition_op]] [[condition_value]] " + - "group by time([[interval]]) order asc"; + "group by time([[interval]])[[group_add]] [[group]] order asc"; } var templateData = { @@ -92,9 +92,14 @@ function (angular, _, kbn) { condition_add: target.condiction_filter ? target.condition_add : '', condition_key: target.condiction_filter ? target.condition_key : '', condition_op: target.condiction_filter ? target.condition_op : '', - condition_value: target.condiction_filter ? target.condition_value: '' + condition_value: target.condiction_filter ? target.condition_value : '', + group_add: target.groupby_field_add && target.groupby_field ? ',' : '', + group: target.groupby_field_add ? target.groupby_field : '', }; + if (target.groupby_field_add) { + additionalGroups.push(target.groupby_field); + } query = _.template(template, templateData, this.templateSettings); target.query = query; } From 3b82ac00d895cb5717e384439115dbf0e50b8d20 Mon Sep 17 00:00:00 2001 From: Clicky Date: Thu, 22 May 2014 18:18:22 -0700 Subject: [PATCH 06/17] Fix typos in condition names --- src/app/partials/influxdb/editor.html | 4 ++-- src/app/services/influxdb/influxdbDatasource.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/app/partials/influxdb/editor.html b/src/app/partials/influxdb/editor.html index bbf4cbd5df4..cbd26ef9e01 100644 --- a/src/app/partials/influxdb/editor.html +++ b/src/app/partials/influxdb/editor.html @@ -94,12 +94,12 @@
    • -
  • +