From 85f241a4d89dc236dc54032de4a47c3cbf64859a Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Fri, 23 Aug 2013 13:20:31 -0700 Subject: [PATCH 1/4] Restored the "fill all the holes" strategy to the ZeroFill class, and introduced the fill_style option ("minimal" and "all" are possible values). Also allowed a set of required times to be specified when asking the times series for it's data. This way, we can ensure in the stacked bar chart that each data point in use has a value preventing the bars from stacking incorrectly. --- panels/histogram/module.js | 188 +++++++++++++++++++++++++++---------- 1 file changed, 137 insertions(+), 51 deletions(-) diff --git a/panels/histogram/module.js b/panels/histogram/module.js index 5593e6d9e90..ee184fdc3e5 100644 --- a/panels/histogram/module.js +++ b/panels/histogram/module.js @@ -197,12 +197,12 @@ angular.module('kibana.histogram', []) // we need to initialize the data variable on the first run, // and when we are working on the first segment of the data. if(_.isUndefined($scope.data[i]) || segment === 0) { - time_series = new timeSeries.ZeroFilled( - _interval, - // range may be false - _range && _range.from, - _range && _range.to - ); + time_series = new timeSeries.ZeroFilled({ + interval: _interval, + start_date: _range && _range.from, + end_date: _range && _range.to, + fillStyle: 'minimal' + }); hits = 0; } else { time_series = $scope.data[i].time_series; @@ -216,9 +216,8 @@ angular.module('kibana.histogram', []) $scope.hits += entry.count; // Entire dataset level hits counter }); $scope.data[i] = { - time_series: time_series, info: querySrv.list[id], - data: time_series.getFlotPairs(), + time_series: time_series, hits: hits }; @@ -310,7 +309,7 @@ angular.module('kibana.histogram', []) // Populate from the query service try { - _.each(scope.data,function(series) { + _.each(scope.data, function(series) { series.label = series.info.alias; series.color = series.info.color; }); @@ -383,6 +382,19 @@ angular.module('kibana.histogram', []) options.selection = { mode: "x", color: '#666' }; } + // when rendering stacked bars, we need to ensure each point that has data is zero-filled + // so that the stacking happens in the proper order + var required_times = []; + if (scope.panel.bars && stack) { + required_times = Array.prototype.concat.apply([], _.map(scope.data, function (series) { + return series.time_series.getOrderedTimes(); + })); + } + + for (var i = 0; i < scope.data.length; i++) { + scope.data[i].data = scope.data[i].time_series.getFlotPairs(required_times); + } + scope.plot = $.plot(elem, scope.data, options); } catch(e) { @@ -448,36 +460,53 @@ angular.module('kibana.histogram', []) }; }) .service('timeSeries', function () { + // map compatable parseInt + function base10Int(val) { + return parseInt(val, 10); + } + /** * Certain graphs require 0 entries to be specified for them to render * properly (like the line graph). So with this we will caluclate all of * the expected time measurements, and fill the missing ones in with 0 - * @param date start The start time for the result set - * @param date end The end time for the result set - * @param integer interval The length between measurements, in es interval - * notation (1m, 30s, 1h, 15d) + * @param object opts An object specifying some/all of the options + * + * OPTIONS: + * @opt string interval The interval notion describing the expected spacing between + * each data point. + * @opt date start_date (optional) The start point for the time series, setting this and the + * end_date will ensure that the series streches to resemble the entire + * expected result + * @opt date end_date (optional) The end point for the time series, see start_date + * @opt string fill_style Either "minimal", or "all" describing the strategy used to zero-fill + * the series. */ - var undef; - function base10Int(val) { - return parseInt(val, 10); - } - this.ZeroFilled = function (interval, start, end) { + this.ZeroFilled = function (opts) { + this.opts = _.defaults(opts, { + interval: '10m', + start_date: null, + end_date: null, + fill_style: 'minimal' + }); + // the expected differenece between readings. - this.interval_ms = base10Int(kbn.interval_to_seconds(interval)) * 1000; + this.interval_ms = base10Int(kbn.interval_to_seconds(opts.interval)) * 1000; + // will keep all values here, keyed by their time this._data = {}; - if (start) { - this.addValue(start, null); + if (opts.start_date) { + this.addValue(opts.start_date, null); } - if (end) { - this.addValue(end, null); + if (opts.end_date) { + this.addValue(opts.end_date, null); } }; + /** * Add a row - * @param int time The time for the value, in - * @param any value The value at this time + * @param {int} time The time for the value, in + * @param {any} value The value at this time */ this.ZeroFilled.prototype.addValue = function (time, value) { if (time instanceof Date) { @@ -486,44 +515,101 @@ angular.module('kibana.histogram', []) time = base10Int(time); } if (!isNaN(time)) { - this._data[time] = (value === undef ? 0 : value); + this._data[time] = (_.isUndefined(value) ? 0 : value); } + this._cached_times = null; }; + + /** + * Get an array of the times that have been explicitly set in the series + * @param {array} include (optional) list of timestamps to include in the response + * @return {array} An array of integer times. + */ + this.ZeroFilled.prototype.getOrderedTimes = function (include) { + var times = _.map(_.keys(this._data), base10Int).sort(); + if (_.isArray(include)) { + times = times.concat(include); + } + return times; + }; + /** * return the rows in the format: * [ [time, value], [time, value], ... ] - * @return array + * + * Heavy lifting is done by _get(Min|All)FlotPairs() + * @param {array} required_times An array of timestamps that must be in the resulting pairs + * @return {array} */ - this.ZeroFilled.prototype.getFlotPairs = function () { - // var startTime = performance.now(); - var times = _.map(_.keys(this._data), base10Int).sort(), - result = []; - _.each(times, function (time, i, times) { - var next, expected_next, prev, expected_prev; + this.ZeroFilled.prototype.getFlotPairs = function (required_times) { + var times = this.getOrderedTimes(required_times), + strategy, + pairs; - // check for previous measurement - if (i > 0) { - prev = times[i - 1]; - expected_prev = time - this.interval_ms; - if (prev < expected_prev) { - result.push([expected_prev, 0]); - } + if(this.opts.fill_style === 'all') { + strategy = this._getAllFlotPairs; + } else { + strategy = this._getMinFlotPairs; + } + + return _.reduce( + times, // what + strategy, // how + [], // where + this // context + ); + }; + + /** + * ** called as a reduce stragegy in getFlotPairs() ** + * Fill zero's on either side of the current time, unless there is already a measurement there or + * we are looking at an edge. + * @return {array} An array of points to plot with flot + */ + this.ZeroFilled.prototype._getMinFlotPairs = function (result, time, i, times) { + var next, expected_next, prev, expected_prev; + + // check for previous measurement + if (i > 0) { + prev = times[i - 1]; + expected_prev = time - this.interval_ms; + if (prev < expected_prev) { + result.push([expected_prev, 0]); } + } - // add the current time - result.push([ time, this._data[time] ]); + // add the current time + result.push([ time, this._data[time] || 0 ]); - // check for next measurement - if (times.length > i) { - next = times[i + 1]; - expected_next = time + this.interval_ms; - if (next > expected_next) { - result.push([expected_next, 0]); - } + // check for next measurement + if (times.length > i) { + next = times[i + 1]; + expected_next = time + this.interval_ms; + if (next > expected_next) { + result.push([expected_next, 0]); } + } - }, this); - // console.log(Math.round((performance.now() - startTime)*100)/100, 'ms to get', result.length, 'pairs'); return result; }; + + /** + * ** called as a reduce stragegy in getFlotPairs() ** + * Fill zero's to the right of each time, until the next measurement is reached or we are at the + * last measurement + * @return {array} An array of points to plot with flot + */ + this.ZeroFilled.prototype._getAllFlotPairs = function (result, time, i, times) { + var next, expected_next; + + result.push([ times[i], this._data[times[i]] || 0 ]); + next = times[i + 1]; + expected_next = times[i] + this.interval_ms; + for(; times.length > i && next > expected_next; expected_next+= this.interval_ms) { + result.push([expected_next, 0]); + } + + return result; + }; + }); \ No newline at end of file From 9b62617ede066d1b8c011248f8a1c72c76e0ed00 Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Fri, 23 Aug 2013 14:31:59 -0700 Subject: [PATCH 2/4] cleaned up the documentation a bit --- panels/histogram/module.js | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/panels/histogram/module.js b/panels/histogram/module.js index ee184fdc3e5..ed1eec0fcbc 100644 --- a/panels/histogram/module.js +++ b/panels/histogram/module.js @@ -108,11 +108,17 @@ angular.module('kibana.histogram', []) $scope.panel.interval = interval || '10m'; return $scope.panel.interval; }; + /** * Fetch the data for a chunk of a queries results. Multiple segments occur when several indicies * need to be consulted (like timestamped logstash indicies) - * @param number segment The segment count, (0 based) - * @param number query_id The id of the query, generated on the first run and passed back when + * + * The results of this function are stored on the scope's data property. This property will be an + * array of objects with the properties info, time_series, and hits. These objects are used in the + * render_panel function to create the historgram. + * + * @param {number} segment The segment count, (0 based) + * @param {number} query_id The id of the query, generated on the first run and passed back when * this call is made recursively for more segments */ $scope.get_data = function(segment, query_id) { @@ -469,17 +475,17 @@ angular.module('kibana.histogram', []) * Certain graphs require 0 entries to be specified for them to render * properly (like the line graph). So with this we will caluclate all of * the expected time measurements, and fill the missing ones in with 0 - * @param object opts An object specifying some/all of the options + * @param {object} opts An object specifying some/all of the options * * OPTIONS: - * @opt string interval The interval notion describing the expected spacing between - * each data point. - * @opt date start_date (optional) The start point for the time series, setting this and the - * end_date will ensure that the series streches to resemble the entire - * expected result - * @opt date end_date (optional) The end point for the time series, see start_date - * @opt string fill_style Either "minimal", or "all" describing the strategy used to zero-fill - * the series. + * @opt {string} interval The interval notion describing the expected spacing between + * each data point. + * @opt {date} start_date (optional) The start point for the time series, setting this and the + * end_date will ensure that the series streches to resemble the entire + * expected result + * @opt {date} end_date (optional) The end point for the time series, see start_date + * @opt {string} fill_style Either "minimal", or "all" describing the strategy used to zero-fill + * the series. */ this.ZeroFilled = function (opts) { this.opts = _.defaults(opts, { @@ -522,7 +528,7 @@ angular.module('kibana.histogram', []) /** * Get an array of the times that have been explicitly set in the series - * @param {array} include (optional) list of timestamps to include in the response + * @param {array} include (optional) list of timestamps to include in the response * @return {array} An array of integer times. */ this.ZeroFilled.prototype.getOrderedTimes = function (include) { @@ -538,7 +544,7 @@ angular.module('kibana.histogram', []) * [ [time, value], [time, value], ... ] * * Heavy lifting is done by _get(Min|All)FlotPairs() - * @param {array} required_times An array of timestamps that must be in the resulting pairs + * @param {array} required_times An array of timestamps that must be in the resulting pairs * @return {array} */ this.ZeroFilled.prototype.getFlotPairs = function (required_times) { From 252b2fd651e949e33d558c9bb83551e442c661dc Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Sat, 24 Aug 2013 14:24:54 -0700 Subject: [PATCH 3/4] fixed a property typo --- panels/histogram/module.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panels/histogram/module.js b/panels/histogram/module.js index ed1eec0fcbc..fe7de099773 100644 --- a/panels/histogram/module.js +++ b/panels/histogram/module.js @@ -207,7 +207,7 @@ angular.module('kibana.histogram', []) interval: _interval, start_date: _range && _range.from, end_date: _range && _range.to, - fillStyle: 'minimal' + fill_style: 'minimal' }); hits = 0; } else { From 556925f840f4680a3be61db51577ceada022d4f3 Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Mon, 26 Aug 2013 08:51:51 -0700 Subject: [PATCH 4/4] added check for 404 response from the index_save call, assumes that auto_create_index is disabled and attempts to create the index before saving a second time --- js/services.js | 4 +++ panels/dashcontrol/module.js | 59 +++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/js/services.js b/js/services.js index 101579b4de7..4ad364f744c 100644 --- a/js/services.js +++ b/js/services.js @@ -800,6 +800,10 @@ angular.module('kibana.services', []) ); }; + this.elasticsearch_create_index = function () { + return ejs.client.post('/'+config.kibana_index); + }; + this.elasticsearch_delete = function(id) { return ejs.Document(config.kibana_index,'dashboard',id).doDelete( // Success diff --git a/panels/dashcontrol/module.js b/panels/dashcontrol/module.js index 1119943b34b..015898e048a 100644 --- a/panels/dashcontrol/module.js +++ b/panels/dashcontrol/module.js @@ -17,7 +17,7 @@ * hide_control :: Upon save, hide this panel * elasticsearch_size :: show this many dashboards under the ES section in the load drop down * temp :: Allow saving of temp dashboards - * ttl :: Enable setting ttl. + * ttl :: Enable setting ttl. * temp_ttl :: How long should temp dashboards persist */ @@ -61,6 +61,10 @@ angular.module('kibana.dashcontrol', []) services: {} }; + function notice(type, title, message) { + alertSrv.set(title, message, type, 5000); + } + $scope.init = function() { $scope.gist_pattern = /(^\d{5,}$)|(^[a-z0-9]{10,}$)|(gist.github.com(\/*.*)\/[a-z0-9]{5,}\/*$)/; $scope.gist = {}; @@ -69,17 +73,17 @@ angular.module('kibana.dashcontrol', []) $scope.set_default = function() { if(dashboard.set_default()) { - alertSrv.set('Local Default Set',dashboard.current.title+' has been set as your local default','success',5000); + notice('success', 'Local Default Set', dashboard.current.title+' has been set as your local default'); } else { - alertSrv.set('Incompatible Browser','Sorry, your browser is too old for this feature','error',5000); + notice('error', 'Incompatible Browser', 'Sorry, your browser is too old for this feature'); } }; $scope.purge_default = function() { if(dashboard.purge_default()) { - alertSrv.set('Local Default Clear','Your local default dashboard has been cleared','success',5000); + notice('success', 'Local Default Clear', 'Your local default dashboard has been cleared'); } else { - alertSrv.set('Incompatible Browser','Sorry, your browser is too old for this feature','error',5000); + notice('error', 'Incompatible Browser', 'Sorry, your browser is too old for this feature'); } }; @@ -91,13 +95,30 @@ angular.module('kibana.dashcontrol', []) ).then( function(result) { if(!_.isUndefined(result._id)) { - alertSrv.set('Dashboard Saved','This dashboard has been saved to Elasticsearch as "' + - result._id + '"','success',5000); + notice( + 'success', + 'Dashboard Saved', + 'This dashboard has been saved to Elasticsearch as "'+result._id + '"' + ); if(type === 'temp') { - $scope.share = dashboard.share_link(dashboard.current.title,'temp',result._id); + $scope.share = dashboard.share_link(dashboard.current.title, 'temp', result._id); } } else { - alertSrv.set('Save failed','Dashboard could not be saved to Elasticsearch','error',5000); + if (result.status === 404) { + // auto create must be disabled and the index doesn't exist, create it! + return dashboard.elasticsearch_create_index().then(function () { + return $scope.elasticsearch_save(type, ttl); + }, function () { + notice( + 'error', + 'Save failed', + 'Dashboard could not be saved because the "'+config.kibana_index+'" '+ + 'index does not exist and could not be created.' + ); + }); + } else { + notice('error', 'Save failed', 'Dashboard could not be saved to Elasticsearch'); + } } }); }; @@ -107,15 +128,15 @@ angular.module('kibana.dashcontrol', []) function(result) { if(!_.isUndefined(result)) { if(result.found) { - alertSrv.set('Dashboard Deleted',id+' has been deleted','success',5000); + notice('success', 'Dashboard Deleted', id+' has been deleted'); // Find the deleted dashboard in the cached list and remove it var toDelete = _.where($scope.elasticsearch.dashboards,{_id:id})[0]; $scope.elasticsearch.dashboards = _.without($scope.elasticsearch.dashboards,toDelete); } else { - alertSrv.set('Dashboard Not Found','Could not find '+id+' in Elasticsearch','warning',5000); + notice('warning', 'Dashboard Not Found', 'Could not find '+id+' in Elasticsearch'); } } else { - alertSrv.set('Dashboard Not Deleted','An error occurred deleting the dashboard','error',5000); + notice('error', 'Dashboard Not Deleted', 'An error occurred deleting the dashboard'); } } ); @@ -137,10 +158,14 @@ angular.module('kibana.dashcontrol', []) function(link) { if(!_.isUndefined(link)) { $scope.gist.last = link; - alertSrv.set('Gist saved','You will be able to access your exported dashboard file at '+ - ''+link+' in a moment','success'); + notice( + 'success', + 'Gist saved', + 'You will be able to access your exported dashboard file at '+ + ''+link+' in a moment' + ); } else { - alertSrv.set('Save failed','Gist could not be saved','error',5000); + notice('error', 'Save failed', 'Gist could not be saved'); } }); }; @@ -151,7 +176,7 @@ angular.module('kibana.dashcontrol', []) if(files && files.length > 0) { $scope.gist.files = files; } else { - alertSrv.set('Gist Failed','Could not retrieve dashboard list from gist','error',5000); + notice('error', 'Gist Failed', 'Could not retrieve dashboard list from gist',5000); } }); }; @@ -183,7 +208,7 @@ angular.module('kibana.dashcontrol', []) // Something document.getElementById('dashupload').addEventListener('change', file_selected, false); } else { - alertSrv.set('Oops','Sorry, the HTML5 File APIs are not fully supported in this browser.','error'); + alertSrv.set('Oops', 'Sorry, the HTML5 File APIs are not fully supported in this browser.', 'error', 5000); } } };