From 33ac22bfdb53eeb7655966a1aed469a8a6f62a7b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 21 Jan 2018 22:08:18 +0100 Subject: [PATCH 0001/1100] start query builder ui --- .../postgres/partials/query.editor.html | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 163970a9ad5..635c3e6f222 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -1,10 +1,23 @@ - -
-
- - -
-
+ + +
+
+
+ + +
+
+
+ +
+
+
+ + + +
+
+
From 17be31e2167ef92af57884a2bb9975a33f7968fb Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 28 Jan 2018 22:16:51 +0100 Subject: [PATCH 0002/1100] call render in query --- .../app/plugins/datasource/postgres/datasource.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 8eee389d1a5..3a4bd27bb4c 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -1,5 +1,6 @@ import _ from 'lodash'; import ResponseParser from './response_parser'; +import PostgresQuery from 'app/plugins/datasource/postgres/postgres_query'; export class PostgresDatasource { id: any; @@ -33,16 +34,18 @@ export class PostgresDatasource { } query(options) { - var queries = _.filter(options.targets, item => { - return item.hide !== true; - }).map(item => { + var queries = _.filter(options.targets, target => { + return target.hide !== true; + }).map(target => { + var queryModel = new PostgresQuery(target, this.templateSrv, options.scopedVars); + return { - refId: item.refId, + refId: target.refId, intervalMs: options.intervalMs, maxDataPoints: options.maxDataPoints, datasourceId: this.id, - rawSql: this.templateSrv.replace(item.rawSql, options.scopedVars, this.interpolateVariable), - format: item.format, + rawSql: queryModel.render(this.interpolateVariable), + format: target.format, }; }); From a59e052a0f9ac9aab7467d49dd5586de0d08e124 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 14:08:10 +0100 Subject: [PATCH 0003/1100] more query builder components --- .../postgres/partials/query.editor.html | 94 +++++++++++++------ 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 635c3e6f222..400855cf82f 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -1,12 +1,12 @@
-
-
- - -
-
+
+
+ + +
+
@@ -15,42 +15,76 @@ +
+ +
+
+ +
+ +
+ + +
+ +
+ +
+ +
+
+
+
+ +
+
+ + + +
+
+
- -
- -
-
-
+ +
+ +
+
+
-
-
+
+
-
-
-
-
-
+ +
+
+
+ -
- 
{{ctrl.lastQueryMeta.sql}}
-
+
+ 
{{ctrl.lastQueryMeta.sql}}
+
-
- 
Time series:
+  

+    
Time series:
 - return column named time (UTC in seconds or timestamp)
 - return column(s) with numeric datatype as values
 - (Optional: return column named metric to represent the series name. If no column named metric is found the column name of the value column is used as series name)
@@ -78,13 +112,13 @@ Or build your own conditionals using these macros which just return the values:
 - $__timeTo() ->  to_timestamp(1492750877)
 - $__unixEpochFrom() ->  1492750877
 - $__unixEpochTo() ->  1492750877
-		

-	

+
+
- + -
- 
{{ctrl.lastQueryError}}
-
+
+ 
{{ctrl.lastQueryError}}
+
From 438b10bcd68abd115dbe86acad1a70a53a791c79 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 14:10:38 +0100 Subject: [PATCH 0004/1100] query builder changes --- .../plugins/datasource/postgres/query_ctrl.ts | 179 ++++++++++++++++-- 1 file changed, 168 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 7afd0cf7253..4e39105370b 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,12 +1,7 @@ import _ from 'lodash'; import { QueryCtrl } from 'app/plugins/sdk'; - -export interface PostgresQuery { - refId: string; - format: string; - alias: string; - rawSql: string; -} +import queryPart from './query_part'; +import PostgresQuery from './postgres_query'; export interface QueryMeta { sql: string; @@ -26,17 +21,21 @@ export class PostgresQueryCtrl extends QueryCtrl { showLastQuerySQL: boolean; formats: any[]; - target: PostgresQuery; + queryModel: PostgresQuery; lastQueryMeta: QueryMeta; lastQueryError: string; showHelp: boolean; + schemaSegment: any; + tableSegment: any; + timeColumnSegment: any; + selectMenu: any; /** @ngInject **/ - constructor($scope, $injector) { + constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { super($scope, $injector); + this.target = this.target; + this.queryModel = new PostgresQuery(this.target, templateSrv, this.panel.scopedVars); - this.target.format = this.target.format || 'time_series'; - this.target.alias = ''; this.formats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; if (!this.target.rawSql) { @@ -49,10 +48,104 @@ export class PostgresQueryCtrl extends QueryCtrl { } } + this.schemaSegment= uiSegmentSrv.newSegment(this.target.schema); + + if (!this.target.table) { + this.tableSegment = uiSegmentSrv.newSegment({value: 'select table',fake: true}); + } else { + this.tableSegment= uiSegmentSrv.newSegment(this.target.table); + } + + this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); + + this.buildSelectMenu(); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } + buildSelectMenu() { + var categories = queryPart.getCategories(); + this.selectMenu = _.reduce( + categories, + function(memo, cat, key) { + var menu = { + text: key, + submenu: cat.map(item => { + return { text: item.type, value: item.type }; + }), + }; + memo.push(menu); + return memo; + }, + [] + ); + } + + toggleEditorMode() { + try { +// this.target.query = this.queryModel.render(false); + } catch (err) { + console.log('query render error'); + } + this.target.rawQuery = !this.target.rawQuery; + } + + getSchemaSegments() { + var schemaQuery = "SELECT schema_name FROM information_schema.schemata WHERE"; + schemaQuery += " schema_name NOT LIKE 'pg_%' AND schema_name <> 'information_schema';"; + return this.datasource + .metricFindQuery(schemaQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + getTableSegments() { + var tableQuery = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + this.target.schema + "';"; + return this.datasource + .metricFindQuery(tableQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + getTimeColumnSegments() { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + columnQuery += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real');"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + getColumnSegments() { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + tableChanged() { + this.target.table = this.tableSegment.value; + this.panelCtrl.refresh(); + } + + schemaChanged() { + this.target.schema = this.schemaSegment.value; + this.panelCtrl.refresh(); + } + + timeColumnChanged() { + this.target.time = this.timeColumnSegment.value; + this.panelCtrl.refresh(); + } + onDataReceived(dataList) { this.lastQueryMeta = null; this.lastQueryError = null; @@ -72,4 +165,68 @@ export class PostgresQueryCtrl extends QueryCtrl { } } } + + transformToSegments(addTemplateVars) { + return results => { + var segments = _.map(results, segment => { + return this.uiSegmentSrv.newSegment({ + value: segment.text, + expandable: segment.expandable, + }); + }); + + if (addTemplateVars) { + for (let variable of this.templateSrv.variables) { + segments.unshift( + this.uiSegmentSrv.newSegment({ + type: 'template', + value: '/^$' + variable.name + '$/', + expandable: true, + }) + ); + } + } + + return segments; + }; + } + + addSelectPart(selectParts, cat, subitem) { + this.queryModel.addSelectPart(selectParts, subitem.value); + this.panelCtrl.refresh(); + } + + handleSelectPartEvent(selectParts, part, evt) { + switch (evt.name) { + case 'get-param-options': { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + case 'part-param-changed': { + this.panelCtrl.refresh(); + break; + } + case 'action': { + this.queryModel.removeSelectPart(selectParts, part); + this.panelCtrl.refresh(); + break; + } + case 'get-part-actions': { + return this.$q.when([{ text: 'Remove', value: 'remove-part' }]); + } + } + } + + handleQueryError(err) { + this.error = err.message || 'Failed to issue metric query'; + return []; + } + } From 443504517a1bcd58c533feef202530248a6d7050 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 22:13:55 +0100 Subject: [PATCH 0005/1100] add postgres_query.ts --- .../datasource/postgres/postgres_query.ts | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 public/app/plugins/datasource/postgres/postgres_query.ts diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts new file mode 100644 index 00000000000..3424ae24c78 --- /dev/null +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -0,0 +1,239 @@ +import _ from 'lodash'; +import queryPart from './query_part'; +import kbn from 'app/core/utils/kbn'; + +export default class PostgresQuery { + target: any; + selectModels: any[]; + queryBuilder: any; + groupByParts: any; + templateSrv: any; + scopedVars: any; + + /** @ngInject */ + constructor(target, templateSrv?, scopedVars?) { + this.target = target; + this.templateSrv = templateSrv; + this.scopedVars = scopedVars; + + target.schema = target.schema || 'public'; + target.format = target.format || 'time_series'; + target.timeColumn = target.timeColumn || 'time'; + target.alias = ''; + + target.orderByTime = target.orderByTime || 'ASC'; +// target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; + target.select = target.select || [[{ type: 'field', params: ['value'] }]]; + + this.updateProjection(); + } + + updateProjection() { + this.selectModels = _.map(this.target.select, function(parts: any) { + return _.map(parts, queryPart.create); + }); + this.groupByParts = _.map(this.target.groupBy, queryPart.create); + } + + updatePersistedParts() { + this.target.select = _.map(this.selectModels, function(selectParts) { + return _.map(selectParts, function(part: any) { + return { type: part.def.type, params: part.params }; + }); + }); + } + + hasGroupByTime() { + return _.find(this.target.groupBy, (g: any) => g.type === 'time'); + } + + hasFill() { + return _.find(this.target.groupBy, (g: any) => g.type === 'fill'); + } + + addGroupBy(value) { + var stringParts = value.match(/^(\w+)\((.*)\)$/); + var typePart = stringParts[1]; + var arg = stringParts[2]; + var partModel = queryPart.create({ type: typePart, params: [arg] }); + var partCount = this.target.groupBy.length; + + if (partCount === 0) { + this.target.groupBy.push(partModel.part); + } else if (typePart === 'time') { + this.target.groupBy.splice(0, 0, partModel.part); + } else if (typePart === 'tag') { + if (this.target.groupBy[partCount - 1].type === 'fill') { + this.target.groupBy.splice(partCount - 1, 0, partModel.part); + } else { + this.target.groupBy.push(partModel.part); + } + } else { + this.target.groupBy.push(partModel.part); + } + + this.updateProjection(); + } + + removeGroupByPart(part, index) { + var categories = queryPart.getCategories(); + + if (part.def.type === 'time') { + // remove fill + this.target.groupBy = _.filter(this.target.groupBy, (g: any) => g.type !== 'fill'); + // remove aggregations + this.target.select = _.map(this.target.select, (s: any) => { + return _.filter(s, (part: any) => { + var partModel = queryPart.create(part); + if (partModel.def.category === categories.Aggregations) { + return false; + } + if (partModel.def.category === categories.Selectors) { + return false; + } + return true; + }); + }); + } + + this.target.groupBy.splice(index, 1); + this.updateProjection(); + } + + removeSelect(index: number) { + this.target.select.splice(index, 1); + this.updateProjection(); + } + + removeSelectPart(selectParts, part) { + // if we remove the field remove the whole statement + if (part.def.type === 'field') { + if (this.selectModels.length > 1) { + var modelsIndex = _.indexOf(this.selectModels, selectParts); + this.selectModels.splice(modelsIndex, 1); + } + } else { + var partIndex = _.indexOf(selectParts, part); + selectParts.splice(partIndex, 1); + } + + this.updatePersistedParts(); + } + + addSelectPart(selectParts, type) { + var partModel = queryPart.create({ type: type }); + partModel.def.addStrategy(selectParts, partModel, this); + this.updatePersistedParts(); + } + + private renderTagCondition(tag, index, interpolate) { + var str = ''; + var operator = tag.operator; + var value = tag.value; + if (index > 0) { + str = (tag.condition || 'AND') + ' '; + } + + if (!operator) { + if (/^\/.*\/$/.test(value)) { + operator = '=~'; + } else { + operator = '='; + } + } + + // quote value unless regex + if (operator !== '=~' && operator !== '!~') { + if (interpolate) { + value = this.templateSrv.replace(value, this.scopedVars); + } + if (operator !== '>' && operator !== '<') { + value = "'" + value.replace(/\\/g, '\\\\') + "'"; + } + } else if (interpolate) { + value = this.templateSrv.replace(value, this.scopedVars, 'regex'); + } + + return str + '"' + tag.key + '" ' + operator + ' ' + value; + } + + interpolateQueryStr(value, variable, defaultFormatFn) { + // if no multi or include all do not regexEscape + if (!variable.multi && !variable.includeAll) { + return value; + } + + if (typeof value === 'string') { + return kbn.regexEscape(value); + } + + var escapedValues = _.map(value, kbn.regexEscape); + return '(' + escapedValues.join('|') + ')'; + } + + render(interpolate?) { + var target = this.target; + + if (target.rawQuery) { + if (interpolate) { + return this.templateSrv.replace(target.rawSql, this.scopedVars, this.interpolateQueryStr); + } else { + return target.rawSql; + } + } + + var query = 'SELECT '; + query += target.timeColumn + ' AS time,'; + + var i, y; + for (i = 0; i < this.selectModels.length; i++) { + let parts = this.selectModels[i]; + var selectText = ''; + for (y = 0; y < parts.length; y++) { + let part = parts[y]; + selectText = part.render(selectText); + } + + if (i > 0) { + query += ', '; + } + query += selectText; + } + + query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; + var conditions = _.map(target.tags, (tag, index) => { + return this.renderTagCondition(tag, index, interpolate); + }); + + if (conditions.length > 0) { + query += '(' + conditions.join(' ') + ') AND '; + } + + query += '$__timeFilter(time)'; + + var groupBySection = ''; + for (i = 0; i < this.groupByParts.length; i++) { + var part = this.groupByParts[i]; + if (i > 0) { + // for some reason fill has no seperator + groupBySection += part.def.type === 'fill' ? ' ' : ', '; + } + groupBySection += part.render(''); + } + + if (groupBySection.length) { + query += ' GROUP BY ' + groupBySection; + } + + query += ' ORDER BY time'; + + return query; + } + + renderAdhocFilters(filters) { + var conditions = _.map(filters, (tag, index) => { + return this.renderTagCondition(tag, index, false); + }); + return conditions.join(' '); + } +} From 571ecdc740b87aad130ce05bd827135fd2e570d4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 22:57:38 +0100 Subject: [PATCH 0006/1100] enhance render function --- .../datasource/postgres/postgres_query.ts | 8 +- .../plugins/datasource/postgres/query_part.ts | 380 ++++++++++++++++++ 2 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 public/app/plugins/datasource/postgres/query_part.ts diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 3424ae24c78..e6d84306a9b 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -28,6 +28,10 @@ export default class PostgresQuery { this.updateProjection(); } + quoteIdentifier(field) { + return '"' + field + '"'; + } + updateProjection() { this.selectModels = _.map(this.target.select, function(parts: any) { return _.map(parts, queryPart.create); @@ -183,7 +187,7 @@ export default class PostgresQuery { } var query = 'SELECT '; - query += target.timeColumn + ' AS time,'; + query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; var i, y; for (i = 0; i < this.selectModels.length; i++) { @@ -209,7 +213,7 @@ export default class PostgresQuery { query += '(' + conditions.join(' ') + ') AND '; } - query += '$__timeFilter(time)'; + query += '$__timeFilter(' + this.quoteIdentifier(target.timeColumn) + ')'; var groupBySection = ''; for (i = 0; i < this.groupByParts.length; i++) { diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts new file mode 100644 index 00000000000..dffea15dbd1 --- /dev/null +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -0,0 +1,380 @@ +import _ from 'lodash'; +import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/core/components/query_part/query_part'; + +var index = []; +var categories = { + Aggregations: [], + Selectors: [], + Transformations: [], + Predictors: [], + Math: [], + Aliasing: [], + Fields: [], +}; + +function createPart(part): any { + var def = index[part.type]; + if (!def) { + throw { message: 'Could not find query part ' + part.type }; + } + + return new QueryPart(part, def); +} + +function register(options: any) { + index[options.type] = new QueryPartDef(options); + options.category.push(index[options.type]); +} + +var groupByTimeFunctions = []; + +function aliasRenderer(part, innerExpr) { + return innerExpr + ' AS ' + '"' + part.params[0] + '"'; +} + +function fieldRenderer(part, innerExpr) { + return '"' + part.params[0] + '"'; +} + +function replaceAggregationAddStrategy(selectParts, partModel) { + // look for existing aggregation + for (var i = 0; i < selectParts.length; i++) { + var part = selectParts[i]; + if (part.def.category === categories.Aggregations) { + selectParts[i] = partModel; + return; + } + if (part.def.category === categories.Selectors) { + selectParts[i] = partModel; + return; + } + } + + selectParts.splice(1, 0, partModel); +} + +function addTransformationStrategy(selectParts, partModel) { + var i; + // look for index to add transformation + for (i = 0; i < selectParts.length; i++) { + var part = selectParts[i]; + if (part.def.category === categories.Math || part.def.category === categories.Aliasing) { + break; + } + } + + selectParts.splice(i, 0, partModel); +} + +function addMathStrategy(selectParts, partModel) { + var partCount = selectParts.length; + if (partCount > 0) { + // if last is math, replace it + if (selectParts[partCount - 1].def.type === 'math') { + selectParts[partCount - 1] = partModel; + return; + } + // if next to last is math, replace it + if (partCount > 1 && selectParts[partCount - 2].def.type === 'math') { + selectParts[partCount - 2] = partModel; + return; + } else if (selectParts[partCount - 1].def.type === 'alias') { + // if last is alias add it before + selectParts.splice(partCount - 1, 0, partModel); + return; + } + } + selectParts.push(partModel); +} + +function addAliasStrategy(selectParts, partModel) { + var partCount = selectParts.length; + if (partCount > 0) { + // if last is alias, replace it + if (selectParts[partCount - 1].def.type === 'alias') { + selectParts[partCount - 1] = partModel; + return; + } + } + selectParts.push(partModel); +} + +function addFieldStrategy(selectParts, partModel, query) { + // copy all parts + var parts = _.map(selectParts, function(part: any) { + return createPart({ type: part.def.type, params: _.clone(part.params) }); + }); + + query.selectModels.push(parts); +} + +register({ + type: 'field', + addStrategy: addFieldStrategy, + category: categories.Fields, + params: [{ type: 'field', dynamicLookup: true }], + defaultParams: ['value'], + renderer: fieldRenderer, +}); + +// Aggregations +register({ + type: 'avg', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'count', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'sum', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +// transformations + +register({ + type: 'derivative', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [ + { + name: 'duration', + type: 'interval', + options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['10s'], + renderer: functionRenderer, +}); + +register({ + type: 'spread', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'non_negative_derivative', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [ + { + name: 'duration', + type: 'interval', + options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['10s'], + renderer: functionRenderer, +}); + +register({ + type: 'difference', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'non_negative_difference', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'moving_average', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [{ name: 'window', type: 'int', options: [5, 10, 20, 30, 40] }], + defaultParams: [10], + renderer: functionRenderer, +}); + +register({ + type: 'cumulative_sum', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'stddev', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'time', + category: groupByTimeFunctions, + params: [ + { + name: 'interval', + type: 'time', + options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['$__interval'], + renderer: functionRenderer, +}); + +register({ + type: 'fill', + category: groupByTimeFunctions, + params: [ + { + name: 'fill', + type: 'string', + options: ['none', 'null', '0', 'previous', 'linear'], + }, + ], + defaultParams: ['null'], + renderer: functionRenderer, +}); + +register({ + type: 'elapsed', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [ + { + name: 'duration', + type: 'interval', + options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['10s'], + renderer: functionRenderer, +}); + +// predictions +register({ + type: 'holt_winters', + addStrategy: addTransformationStrategy, + category: categories.Predictors, + params: [ + { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, + { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, + ], + defaultParams: [10, 2], + renderer: functionRenderer, +}); + +register({ + type: 'holt_winters_with_fit', + addStrategy: addTransformationStrategy, + category: categories.Predictors, + params: [ + { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, + { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, + ], + defaultParams: [10, 2], + renderer: functionRenderer, +}); + +// Selectors +register({ + type: 'bottom', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [{ name: 'count', type: 'int' }], + defaultParams: [3], + renderer: functionRenderer, +}); + +register({ + type: 'max', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'min', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'percentile', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [{ name: 'nth', type: 'int' }], + defaultParams: [95], + renderer: functionRenderer, +}); + +register({ + type: 'top', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [{ name: 'count', type: 'int' }], + defaultParams: [3], + renderer: functionRenderer, +}); + +register({ + type: 'tag', + category: groupByTimeFunctions, + params: [{ name: 'tag', type: 'string', dynamicLookup: true }], + defaultParams: ['tag'], + renderer: fieldRenderer, +}); + +register({ + type: 'math', + addStrategy: addMathStrategy, + category: categories.Math, + params: [{ name: 'expr', type: 'string' }], + defaultParams: [' / 100'], + renderer: suffixRenderer, +}); + +register({ + type: 'alias', + addStrategy: addAliasStrategy, + category: categories.Aliasing, + params: [{ name: 'name', type: 'string', quote: 'double' }], + defaultParams: ['alias'], + renderMode: 'suffix', + renderer: aliasRenderer, +}); + +export default { + create: createPart, + getCategories: function() { + return categories; + }, +}; From 4dbd83fac18a3cc5e59208d396a32538f2ab1a5e Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 31 Jan 2018 15:32:32 +0100 Subject: [PATCH 0007/1100] add groupby to querybuilder remove unused aggregations --- .../postgres/partials/query.editor.html | 21 +++ .../datasource/postgres/postgres_query.ts | 5 +- .../plugins/datasource/postgres/query_ctrl.ts | 84 ++++++++++ .../plugins/datasource/postgres/query_part.ts | 150 +----------------- 4 files changed, 110 insertions(+), 150 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 400855cf82f..3e921e1c631 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -51,6 +51,27 @@ +
+
+ + + + +
+ +
+ +
+ +
+
+
+
+
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index e6d84306a9b..389ce85aae9 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -22,7 +22,7 @@ export default class PostgresQuery { target.alias = ''; target.orderByTime = target.orderByTime || 'ASC'; -// target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; + target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; target.select = target.select || [[{ type: 'field', params: ['value'] }]]; this.updateProjection(); @@ -92,9 +92,6 @@ export default class PostgresQuery { if (partModel.def.category === categories.Aggregations) { return false; } - if (partModel.def.category === categories.Selectors) { - return false; - } return true; }); }); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4e39105370b..020d8007789 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -29,6 +29,7 @@ export class PostgresQueryCtrl extends QueryCtrl { tableSegment: any; timeColumnSegment: any; selectMenu: any; + groupBySegment: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { @@ -59,6 +60,8 @@ export class PostgresQueryCtrl extends QueryCtrl { this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); this.buildSelectMenu(); + this.groupBySegment = this.uiSegmentSrv.newPlusButton(); + this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } @@ -224,6 +227,87 @@ export class PostgresQueryCtrl extends QueryCtrl { } } + handleGroupByPartEvent(part, index, evt) { + switch (evt.name) { + case 'get-param-options': { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + case 'part-param-changed': { + this.panelCtrl.refresh(); + break; + } + case 'action': { + this.queryModel.removeGroupByPart(part, index); + this.panelCtrl.refresh(); + break; + } + case 'get-part-actions': { + return this.$q.when([{ text: 'Remove', value: 'remove-part' }]); + } + } + } + + getGroupByOptions() { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + + + return this.datasource + .metricFindQuery(columnQuery) + .then(tags => { + var options = []; + if (!this.queryModel.hasFill()) { + options.push(this.uiSegmentSrv.newSegment({ value: 'fill(null)' })); + } + if (!this.target.limit) { + options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); + } + if (!this.target.slimit) { + options.push(this.uiSegmentSrv.newSegment({ value: 'SLIMIT' })); + } + if (this.target.orderByTime === 'ASC') { + options.push(this.uiSegmentSrv.newSegment({ value: 'ORDER BY time DESC' })); + } + if (!this.queryModel.hasGroupByTime()) { + options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); + } + for (let tag of tags) { + options.push(this.uiSegmentSrv.newSegment({ value: 'tag(' + tag.text + ')' })); + } + return options; + }) + .catch(this.handleQueryError.bind(this)); + } + + groupByAction() { + switch (this.groupBySegment.value) { + case 'LIMIT': { + this.target.limit = 10; + break; + } + case 'ORDER BY time DESC': { + this.target.orderByTime = 'DESC'; + break; + } + default: { + this.queryModel.addGroupBy(this.groupBySegment.value); + } + } + + var plusButton = this.uiSegmentSrv.newPlusButton(); + this.groupBySegment.value = plusButton.value; + this.groupBySegment.html = plusButton.html; + this.panelCtrl.refresh(); + } + handleQueryError(err) { this.error = err.message || 'Failed to issue metric query'; return []; diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index dffea15dbd1..5828515ec06 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -4,9 +4,6 @@ import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/c var index = []; var categories = { Aggregations: [], - Selectors: [], - Transformations: [], - Predictors: [], Math: [], Aliasing: [], Fields: [], @@ -44,10 +41,6 @@ function replaceAggregationAddStrategy(selectParts, partModel) { selectParts[i] = partModel; return; } - if (part.def.category === categories.Selectors) { - selectParts[i] = partModel; - return; - } } selectParts.splice(1, 0, partModel); @@ -147,34 +140,10 @@ register({ // transformations -register({ - type: 'derivative', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [ - { - name: 'duration', - type: 'interval', - options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - ], - defaultParams: ['10s'], - renderer: functionRenderer, -}); - -register({ - type: 'spread', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - register({ type: 'non_negative_derivative', addStrategy: addTransformationStrategy, - category: categories.Transformations, + category: categories.Aggregations, params: [ { name: 'duration', @@ -186,46 +155,10 @@ register({ renderer: functionRenderer, }); -register({ - type: 'difference', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'non_negative_difference', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'moving_average', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [{ name: 'window', type: 'int', options: [5, 10, 20, 30, 40] }], - defaultParams: [10], - renderer: functionRenderer, -}); - -register({ - type: 'cumulative_sum', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - register({ type: 'stddev', addStrategy: addTransformationStrategy, - category: categories.Transformations, + category: categories.Aggregations, params: [], defaultParams: [], renderer: functionRenderer, @@ -259,60 +192,11 @@ register({ renderer: functionRenderer, }); -register({ - type: 'elapsed', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [ - { - name: 'duration', - type: 'interval', - options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - ], - defaultParams: ['10s'], - renderer: functionRenderer, -}); - -// predictions -register({ - type: 'holt_winters', - addStrategy: addTransformationStrategy, - category: categories.Predictors, - params: [ - { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, - { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, - ], - defaultParams: [10, 2], - renderer: functionRenderer, -}); - -register({ - type: 'holt_winters_with_fit', - addStrategy: addTransformationStrategy, - category: categories.Predictors, - params: [ - { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, - { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, - ], - defaultParams: [10, 2], - renderer: functionRenderer, -}); - // Selectors -register({ - type: 'bottom', - addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, - params: [{ name: 'count', type: 'int' }], - defaultParams: [3], - renderer: functionRenderer, -}); - register({ type: 'max', addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, + category: categories.Aggregations, params: [], defaultParams: [], renderer: functionRenderer, @@ -321,38 +205,12 @@ register({ register({ type: 'min', addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, + category: categories.Aggregations, params: [], defaultParams: [], renderer: functionRenderer, }); -register({ - type: 'percentile', - addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, - params: [{ name: 'nth', type: 'int' }], - defaultParams: [95], - renderer: functionRenderer, -}); - -register({ - type: 'top', - addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, - params: [{ name: 'count', type: 'int' }], - defaultParams: [3], - renderer: functionRenderer, -}); - -register({ - type: 'tag', - category: groupByTimeFunctions, - params: [{ name: 'tag', type: 'string', dynamicLookup: true }], - defaultParams: ['tag'], - renderer: fieldRenderer, -}); - register({ type: 'math', addStrategy: addMathStrategy, From c65a964cdda6e4cabcc570f0199dcd5378331781 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Feb 2018 18:03:00 +0100 Subject: [PATCH 0008/1100] add metric column selector --- .../postgres/partials/query.editor.html | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 3e921e1c631..e266050dff1 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -17,6 +17,15 @@
+ +
+ +
+ +
+
+
+
@@ -46,9 +55,15 @@
- + +
+ +
+
+
+
From 382a5254772d29b67658dc1ed8e38a34b7df0a11 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Feb 2018 18:26:57 +0100 Subject: [PATCH 0009/1100] make metricColumn functional --- .../app/plugins/datasource/postgres/query_ctrl.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 020d8007789..0b2bce7fb05 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -27,7 +27,9 @@ export class PostgresQueryCtrl extends QueryCtrl { showHelp: boolean; schemaSegment: any; tableSegment: any; + whereSegment: any; timeColumnSegment: any; + metricColumnSegment: any; selectMenu: any; groupBySegment: any; @@ -58,6 +60,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); + this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); @@ -122,11 +125,11 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - getColumnSegments() { + getMetricColumnSegments() { var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; columnQuery += " table_schema = '" + this.target.schema + "'"; columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; + columnQuery += " AND data_type IN ('text','char','varchar');"; return this.datasource .metricFindQuery(columnQuery) @@ -145,7 +148,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } timeColumnChanged() { - this.target.time = this.timeColumnSegment.value; + this.target.timeColumn = this.timeColumnSegment.value; + this.panelCtrl.refresh(); + } + + metricColumnChanged() { + this.target.metricColumn = this.metricColumnSegment.value; this.panelCtrl.refresh(); } From 3bce45d8a66abf984644f0cad508e73e60b595bc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 8 Feb 2018 10:19:43 +0100 Subject: [PATCH 0010/1100] add query_builder --- .../datasource/postgres/query_builder.ts | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 public/app/plugins/datasource/postgres/query_builder.ts diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts new file mode 100644 index 00000000000..7a227f33b93 --- /dev/null +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -0,0 +1,50 @@ + +export class PostgresQueryBuilder { + constructor(private target, private queryModel) {} + + buildSchemaQuery() { + var query = "SELECT schema_name FROM information_schema.schemata WHERE"; + query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; + + return query; + } + + buildTableQuery() { + var query = "SELECT table_name FROM information_schema.tables WHERE "; + query += "table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + return query; + } + + buildColumnQuery(type?: string) { + var query = "SELECT column_name FROM information_schema.columns WHERE "; + query += "table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + query += " AND table_name = " + this.queryModel.quoteLiteral(this.target.table); + + switch (type) { + case "time": { + query += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real')"; + break; + } + case "metric": { + query += " AND data_type IN ('text','char','varchar')"; + break; + } + case "value": { + query += " AND data_type IN ('bigint','integer','double precision','real')"; + break; + } + } + + return query; + } + + buildValueQuery(column: string) { + var query = "SELECT DISTINCT " + this.queryModel.quoteIdentifier(column) + "::text"; + query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); + query += "." + this.queryModel.quoteIdentifier(this.target.table); + query += " ORDER BY " + this.queryModel.quoteIdentifier(column); + query += " LIMIT 100"; + return query; + } + +} From ef18eb7fcb0449c7bc9709b8837ff8fa202ccc93 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 8 Feb 2018 10:25:32 +0100 Subject: [PATCH 0011/1100] add where constraint handling --- .../postgres/partials/query.editor.html | 2 +- .../datasource/postgres/postgres_query.ts | 15 +- .../plugins/datasource/postgres/query_ctrl.ts | 192 ++++++++++++++---- 3 files changed, 163 insertions(+), 46 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index e266050dff1..e25a41b11c2 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -19,7 +19,7 @@
- +
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 389ce85aae9..9a5a14c2b6a 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -19,17 +19,22 @@ export default class PostgresQuery { target.schema = target.schema || 'public'; target.format = target.format || 'time_series'; target.timeColumn = target.timeColumn || 'time'; - target.alias = ''; + target.metricColumn = target.metricColumn || 'None'; target.orderByTime = target.orderByTime || 'ASC'; - target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; + target.groupBy = target.groupBy || []; + target.where = target.where || []; target.select = target.select || [[{ type: 'field', params: ['value'] }]]; this.updateProjection(); } - quoteIdentifier(field) { - return '"' + field + '"'; + quoteIdentifier(value) { + return '"' + value + '"'; + } + + quoteLiteral(value) { + return "'" + value + "'"; } updateProjection() { @@ -202,7 +207,7 @@ export default class PostgresQuery { } query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; - var conditions = _.map(target.tags, (tag, index) => { + var conditions = _.map(target.where, (tag, index) => { return this.renderTagCondition(tag, index, interpolate); }); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0b2bce7fb05..f6f6641eff0 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,4 +1,6 @@ +import angular from 'angular'; import _ from 'lodash'; +import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; import queryPart from './query_part'; import PostgresQuery from './postgres_query'; @@ -22,22 +24,25 @@ export class PostgresQueryCtrl extends QueryCtrl { showLastQuerySQL: boolean; formats: any[]; queryModel: PostgresQuery; + queryBuilder: PostgresQueryBuilder; lastQueryMeta: QueryMeta; lastQueryError: string; showHelp: boolean; schemaSegment: any; tableSegment: any; - whereSegment: any; + whereSegments: any; timeColumnSegment: any; metricColumnSegment: any; selectMenu: any; groupBySegment: any; + removeWhereFilterSegment: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { super($scope, $injector); this.target = this.target; this.queryModel = new PostgresQuery(this.target, templateSrv, this.panel.scopedVars); + this.queryBuilder = new PostgresQueryBuilder(this.target, this.queryModel); this.formats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; @@ -63,8 +68,32 @@ export class PostgresQueryCtrl extends QueryCtrl { this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); + this.whereSegments = []; + for (let tag of this.target.where) { + if (!tag.operator) { + if (/^\/.*\/$/.test(tag.value)) { + tag.operator = '=~'; + } else { + tag.operator = '='; + } + } + + if (tag.condition) { + this.whereSegments.push(uiSegmentSrv.newCondition(tag.condition)); + } + + this.whereSegments.push(uiSegmentSrv.newKey(tag.key)); + this.whereSegments.push(uiSegmentSrv.newOperator(tag.operator)); + this.whereSegments.push(uiSegmentSrv.newKeyValue(tag.value)); + } + + this.fixWhereSegments(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); + this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ + fake: true, + value: '-- remove tag filter --', + }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } @@ -97,42 +126,29 @@ export class PostgresQueryCtrl extends QueryCtrl { } getSchemaSegments() { - var schemaQuery = "SELECT schema_name FROM information_schema.schemata WHERE"; - schemaQuery += " schema_name NOT LIKE 'pg_%' AND schema_name <> 'information_schema';"; return this.datasource - .metricFindQuery(schemaQuery) + .metricFindQuery(this.queryBuilder.buildSchemaQuery()) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getTableSegments() { - var tableQuery = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + this.target.schema + "';"; return this.datasource - .metricFindQuery(tableQuery) + .metricFindQuery(this.queryBuilder.buildTableQuery()) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getTimeColumnSegments() { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real');"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery("time")) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getMetricColumnSegments() { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('text','char','varchar');"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery("metric")) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -210,13 +226,8 @@ export class PostgresQueryCtrl extends QueryCtrl { handleSelectPartEvent(selectParts, part, evt) { switch (evt.name) { case 'get-param-options': { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -238,12 +249,8 @@ export class PostgresQueryCtrl extends QueryCtrl { handleGroupByPartEvent(part, index, evt) { switch (evt.name) { case 'get-param-options': { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -262,14 +269,125 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - getGroupByOptions() { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; + fixWhereSegments() { + var count = this.whereSegments.length; + var lastSegment = this.whereSegments[Math.max(count - 1, 0)]; + if (!lastSegment || lastSegment.type !== 'plus-button') { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } + } + + getTagsOrValues(segment, index) { + if (segment.type === 'condition') { + return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); + } + if (segment.type === 'operator') { + var nextValue = this.whereSegments[index + 1].value; + if (/^\/.*\/$/.test(nextValue)) { + return this.$q.when(this.uiSegmentSrv.newOperators(['=~', '!~'])); + } else { + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); + } + } + + var query, addTemplateVars; + if (segment.type === 'key' || segment.type === 'plus-button') { + query = this.queryBuilder.buildColumnQuery(); + + addTemplateVars = false; + } else if (segment.type === 'value') { + query = this.queryBuilder.buildValueQuery(this.whereSegments[index -2].value); + addTemplateVars = true; + } return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(query) + .then(this.transformToSegments(addTemplateVars)) + .then(results => { + if (segment.type === 'key') { + results.splice(0, 0, angular.copy(this.removeWhereFilterSegment)); + } + return results; + }) + .catch(this.handleQueryError.bind(this)); + } + + getTagValueOperator(tagValue, tagOperator): string { + if (tagOperator !== '=~' && tagOperator !== '!~' && /^\/.*\/$/.test(tagValue)) { + return '=~'; + } else if ((tagOperator === '=~' || tagOperator === '!~') && /^(?!\/.*\/$)/.test(tagValue)) { + return '='; + } + return null; + } + + whereSegmentUpdated(segment, index) { + this.whereSegments[index] = segment; + + // handle remove where condition + if (segment.value === this.removeWhereFilterSegment.value) { + this.whereSegments.splice(index, 3); + if (this.whereSegments.length === 0) { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } else if (this.whereSegments.length > 2) { + this.whereSegments.splice(Math.max(index - 1, 0), 1); + if (this.whereSegments[this.whereSegments.length - 1].type !== 'plus-button') { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } + } + } else { + if (segment.type === 'plus-button') { + if (index > 2) { + this.whereSegments.splice(index, 0, this.uiSegmentSrv.newCondition('AND')); + } + this.whereSegments.push(this.uiSegmentSrv.newOperator('=')); + this.whereSegments.push(this.uiSegmentSrv.newFake('select value', 'value', 'query-segment-value')); + segment.type = 'key'; + segment.cssClass = 'query-segment-key'; + } + + if (index + 1 === this.whereSegments.length) { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } + } + + this.rebuildTargetWhereConditions(); + } + + rebuildTargetWhereConditions() { + var where = []; + var tagIndex = 0; + var tagOperator = ''; + + _.each(this.whereSegments, (segment2, index) => { + if (segment2.type === 'key') { + if (where.length === 0) { + where.push({}); + } + where[tagIndex].key = segment2.value; + } else if (segment2.type === 'value') { + tagOperator = this.getTagValueOperator(segment2.value, where[tagIndex].operator); + if (tagOperator) { + this.whereSegments[index - 1] = this.uiSegmentSrv.newOperator(tagOperator); + where[tagIndex].operator = tagOperator; + } + where[tagIndex].value = segment2.value; + } else if (segment2.type === 'condition') { + where.push({ condition: segment2.value }); + tagIndex += 1; + } else if (segment2.type === 'operator') { + where[tagIndex].operator = segment2.value; + } + }); + + this.target.where = where; + this.panelCtrl.refresh(); + } + + getGroupByOptions() { + return this.datasource + .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(tags => { var options = []; if (!this.queryModel.hasFill()) { @@ -278,12 +396,6 @@ export class PostgresQueryCtrl extends QueryCtrl { if (!this.target.limit) { options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); } - if (!this.target.slimit) { - options.push(this.uiSegmentSrv.newSegment({ value: 'SLIMIT' })); - } - if (this.target.orderByTime === 'ASC') { - options.push(this.uiSegmentSrv.newSegment({ value: 'ORDER BY time DESC' })); - } if (!this.queryModel.hasGroupByTime()) { options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); } From be2fa54459bd6aa9305ec6ee83be8599370f9ffe Mon Sep 17 00:00:00 2001 From: Martin Molnar Date: Tue, 20 Feb 2018 11:15:31 +0100 Subject: [PATCH 0012/1100] feat(ldap): Allow use of DN in user attribute filter (#3132) --- pkg/login/ldap.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/login/ldap.go b/pkg/login/ldap.go index be3babac02e..12e10557ffc 100644 --- a/pkg/login/ldap.go +++ b/pkg/login/ldap.go @@ -408,6 +408,10 @@ func (a *ldapAuther) searchForUser(username string) (*LdapUserInfo, error) { if a.server.GroupSearchFilterUserAttribute == "" { filter_replace = getLdapAttr(a.server.Attr.Username, searchResult) } + if a.server.GroupSearchFilterUserAttribute == "dn" { + filter_replace = searchResult.Entries[0].DN + } + filter := strings.Replace(a.server.GroupSearchFilter, "%s", ldap.EscapeFilter(filter_replace), -1) a.log.Info("Searching for user's groups", "filter", filter) @@ -430,7 +434,11 @@ func (a *ldapAuther) searchForUser(username string) (*LdapUserInfo, error) { if len(groupSearchResult.Entries) > 0 { for i := range groupSearchResult.Entries { - memberOf = append(memberOf, getLdapAttrN(a.server.Attr.MemberOf, groupSearchResult, i)) + if a.server.Attr.MemberOf == "dn" { + memberOf = append(memberOf, groupSearchResult.Entries[i].DN) + } else { + memberOf = append(memberOf, getLdapAttrN(a.server.Attr.MemberOf, groupSearchResult, i)) + } } break } From fd518846b1865385eb9775332ffd04fc5388dcc9 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Mar 2018 20:57:00 +0100 Subject: [PATCH 0013/1100] rename field to column --- .../datasource/postgres/postgres_query.ts | 22 ++++++++----- .../plugins/datasource/postgres/query_ctrl.ts | 7 ++--- .../plugins/datasource/postgres/query_part.ts | 31 +++++++------------ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9a5a14c2b6a..78e9c9c1a3f 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -24,7 +24,7 @@ export default class PostgresQuery { target.orderByTime = target.orderByTime || 'ASC'; target.groupBy = target.groupBy || []; target.where = target.where || []; - target.select = target.select || [[{ type: 'field', params: ['value'] }]]; + target.select = target.select || [[{ type: 'column', params: ['value'] }]]; this.updateProjection(); } @@ -88,8 +88,6 @@ export default class PostgresQuery { var categories = queryPart.getCategories(); if (part.def.type === 'time') { - // remove fill - this.target.groupBy = _.filter(this.target.groupBy, (g: any) => g.type !== 'fill'); // remove aggregations this.target.select = _.map(this.target.select, (s: any) => { return _.filter(s, (part: any) => { @@ -113,7 +111,7 @@ export default class PostgresQuery { removeSelectPart(selectParts, part) { // if we remove the field remove the whole statement - if (part.def.type === 'field') { + if (part.def.type === 'column') { if (this.selectModels.length > 1) { var modelsIndex = _.indexOf(this.selectModels, selectParts); this.selectModels.splice(modelsIndex, 1); @@ -189,7 +187,12 @@ export default class PostgresQuery { } var query = 'SELECT '; - query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; + + if (this.hasGroupByTime()) { + query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',1m),'; + } else { + query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; + } var i, y; for (i = 0; i < this.selectModels.length; i++) { @@ -221,10 +224,13 @@ export default class PostgresQuery { for (i = 0; i < this.groupByParts.length; i++) { var part = this.groupByParts[i]; if (i > 0) { - // for some reason fill has no seperator - groupBySection += part.def.type === 'fill' ? ' ' : ', '; + groupBySection += ', '; + } + if (part.def.type === 'time') { + groupBySection += 'time'; + } else { + groupBySection += part.render(''); } - groupBySection += part.render(''); } if (groupBySection.length) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index f6f6641eff0..0c30e2c51ff 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -92,7 +92,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ fake: true, - value: '-- remove tag filter --', + value: '-- remove filter --', }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); @@ -390,9 +390,6 @@ export class PostgresQueryCtrl extends QueryCtrl { .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(tags => { var options = []; - if (!this.queryModel.hasFill()) { - options.push(this.uiSegmentSrv.newSegment({ value: 'fill(null)' })); - } if (!this.target.limit) { options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); } @@ -400,7 +397,7 @@ export class PostgresQueryCtrl extends QueryCtrl { options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); } for (let tag of tags) { - options.push(this.uiSegmentSrv.newSegment({ value: 'tag(' + tag.text + ')' })); + options.push(this.uiSegmentSrv.newSegment({ value: tag.text })); } return options; }) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 5828515ec06..0086b45b848 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -6,7 +6,7 @@ var categories = { Aggregations: [], Math: [], Aliasing: [], - Fields: [], + Columns: [], }; function createPart(part): any { @@ -29,7 +29,7 @@ function aliasRenderer(part, innerExpr) { return innerExpr + ' AS ' + '"' + part.params[0] + '"'; } -function fieldRenderer(part, innerExpr) { +function columnRenderer(part, innerExpr) { return '"' + part.params[0] + '"'; } @@ -92,7 +92,7 @@ function addAliasStrategy(selectParts, partModel) { selectParts.push(partModel); } -function addFieldStrategy(selectParts, partModel, query) { +function addColumnStrategy(selectParts, partModel, query) { // copy all parts var parts = _.map(selectParts, function(part: any) { return createPart({ type: part.def.type, params: _.clone(part.params) }); @@ -102,12 +102,12 @@ function addFieldStrategy(selectParts, partModel, query) { } register({ - type: 'field', - addStrategy: addFieldStrategy, - category: categories.Fields, - params: [{ type: 'field', dynamicLookup: true }], + type: 'column', + addStrategy: addColumnStrategy, + category: categories.Columns, + params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], - renderer: fieldRenderer, + renderer: columnRenderer, }); // Aggregations @@ -170,25 +170,16 @@ register({ params: [ { name: 'interval', - type: 'time', + type: 'interval', options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], }, - ], - defaultParams: ['$__interval'], - renderer: functionRenderer, -}); - -register({ - type: 'fill', - category: groupByTimeFunctions, - params: [ { name: 'fill', type: 'string', - options: ['none', 'null', '0', 'previous', 'linear'], + options: ['none', 'null', '0'], }, ], - defaultParams: ['null'], + defaultParams: ['$__interval','none'], renderer: functionRenderer, }); From 7104e6f9f8dccd5bbba53519daaa40a7cce6a329 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Mar 2018 22:11:51 +0100 Subject: [PATCH 0014/1100] fix variable interpolation --- public/app/plugins/datasource/postgres/postgres_query.ts | 3 +++ public/app/plugins/datasource/postgres/query_ctrl.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 78e9c9c1a3f..2f8b3911ebc 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -239,6 +239,9 @@ export default class PostgresQuery { query += ' ORDER BY time'; + if (interpolate) { + query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); + } return query; } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0c30e2c51ff..bde4010e226 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -207,7 +207,7 @@ export class PostgresQueryCtrl extends QueryCtrl { segments.unshift( this.uiSegmentSrv.newSegment({ type: 'template', - value: '/^$' + variable.name + '$/', + value: '$' + variable.name, expandable: true, }) ); From e8c6341fed3811b3d6339dc5e2fc2a8815caa285 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 10:18:11 +0100 Subject: [PATCH 0015/1100] clean up aggregation functions --- .../plugins/datasource/postgres/query_part.ts | 87 ++++++------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 0086b45b848..600723be00d 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -46,19 +46,6 @@ function replaceAggregationAddStrategy(selectParts, partModel) { selectParts.splice(1, 0, partModel); } -function addTransformationStrategy(selectParts, partModel) { - var i; - // look for index to add transformation - for (i = 0; i < selectParts.length; i++) { - var part = selectParts[i]; - if (part.def.category === categories.Math || part.def.category === categories.Aliasing) { - break; - } - } - - selectParts.splice(i, 0, partModel); -} - function addMathStrategy(selectParts, partModel) { var partCount = selectParts.length; if (partCount > 0) { @@ -138,54 +125,8 @@ register({ renderer: functionRenderer, }); -// transformations - -register({ - type: 'non_negative_derivative', - addStrategy: addTransformationStrategy, - category: categories.Aggregations, - params: [ - { - name: 'duration', - type: 'interval', - options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - ], - defaultParams: ['10s'], - renderer: functionRenderer, -}); - register({ type: 'stddev', - addStrategy: addTransformationStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'time', - category: groupByTimeFunctions, - params: [ - { - name: 'interval', - type: 'interval', - options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - { - name: 'fill', - type: 'string', - options: ['none', 'null', '0'], - }, - ], - defaultParams: ['$__interval','none'], - renderer: functionRenderer, -}); - -// Selectors -register({ - type: 'max', addStrategy: replaceAggregationAddStrategy, category: categories.Aggregations, params: [], @@ -202,6 +143,15 @@ register({ renderer: functionRenderer, }); +register({ + type: 'max', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + register({ type: 'math', addStrategy: addMathStrategy, @@ -221,6 +171,25 @@ register({ renderer: aliasRenderer, }); +register({ + type: 'time', + category: groupByTimeFunctions, + params: [ + { + name: 'interval', + type: 'interval', + options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + { + name: 'fill', + type: 'string', + options: ['none', 'NULL', '0'], + }, + ], + defaultParams: ['$__interval','none'], + renderer: functionRenderer, +}); + export default { create: createPart, getCategories: function() { From 26e09b598c809e93a120dfbc30214af8ff9ac690 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 19:35:43 +0100 Subject: [PATCH 0016/1100] fix group by column --- .../datasource/postgres/postgres_query.ts | 24 ++++++++++++------- .../plugins/datasource/postgres/query_ctrl.ts | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 2f8b3911ebc..a236fa91b18 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -61,17 +61,17 @@ export default class PostgresQuery { } addGroupBy(value) { - var stringParts = value.match(/^(\w+)\((.*)\)$/); + var stringParts = value.match(/^(\w+)(\((.*)\))?$/); var typePart = stringParts[1]; - var arg = stringParts[2]; - var partModel = queryPart.create({ type: typePart, params: [arg] }); + var args = stringParts[3].split(","); + var partModel = queryPart.create({ type: typePart, params: args }); var partCount = this.target.groupBy.length; if (partCount === 0) { this.target.groupBy.push(partModel.part); } else if (typePart === 'time') { this.target.groupBy.splice(0, 0, partModel.part); - } else if (typePart === 'tag') { + } else if (typePart === 'column') { if (this.target.groupBy[partCount - 1].type === 'fill') { this.target.groupBy.splice(partCount - 1, 0, partModel.part); } else { @@ -188,8 +188,16 @@ export default class PostgresQuery { var query = 'SELECT '; - if (this.hasGroupByTime()) { - query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',1m),'; + var timeGroup = this.hasGroupByTime(); + + if (timeGroup) { + var args; + if (timeGroup.params.length > 1 && timeGroup.params[1] !== "none") { + args = timeGroup.params.join(","); + } else { + args = timeGroup.params[0]; + } + query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',' + args + '),'; } else { query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; } @@ -227,7 +235,7 @@ export default class PostgresQuery { groupBySection += ', '; } if (part.def.type === 'time') { - groupBySection += 'time'; + groupBySection += '1'; } else { groupBySection += part.render(''); } @@ -237,7 +245,7 @@ export default class PostgresQuery { query += ' GROUP BY ' + groupBySection; } - query += ' ORDER BY time'; + query += ' ORDER BY 1'; if (interpolate) { query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index bde4010e226..b0d5d7ab9ca 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -394,10 +394,10 @@ export class PostgresQueryCtrl extends QueryCtrl { options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); } if (!this.queryModel.hasGroupByTime()) { - options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'time', value: 'time(1m,none)' })); } for (let tag of tags) { - options.push(this.uiSegmentSrv.newSegment({ value: tag.text })); + options.push(this.uiSegmentSrv.newSegment({ type: 'column', value: 'column(' + tag.text + ')' })); } return options; }) From bf4a30d30f93aa7ac1cf01c319666242022272fe Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 19:46:11 +0100 Subject: [PATCH 0017/1100] set rawSQL when rendering query builder query --- public/app/plugins/datasource/postgres/postgres_query.ts | 1 + public/app/plugins/datasource/postgres/query_ctrl.ts | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a236fa91b18..a708f384972 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -247,6 +247,7 @@ export default class PostgresQuery { query += ' ORDER BY 1'; + this.target.rawSql = query; if (interpolate) { query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b0d5d7ab9ca..4e36e8699ae 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -117,11 +117,6 @@ export class PostgresQueryCtrl extends QueryCtrl { } toggleEditorMode() { - try { -// this.target.query = this.queryModel.render(false); - } catch (err) { - console.log('query render error'); - } this.target.rawQuery = !this.target.rawQuery; } From 83200c289a454e978ac4c0ade432ac7ccb31dbc4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 23:32:23 +0100 Subject: [PATCH 0018/1100] use metricColumn in query builder --- public/app/plugins/datasource/postgres/postgres_query.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a708f384972..4541ebae7de 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -217,6 +217,10 @@ export default class PostgresQuery { query += selectText; } + if (this.target.metricColumn !== 'None') { + query += "," + this.quoteIdentifier(this.target.metricColumn) + " AS metric"; + } + query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { return this.renderTagCondition(tag, index, interpolate); From 340f679d0f7c531eaa332c036a3141eb40119bff Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 9 Mar 2018 18:09:59 +0100 Subject: [PATCH 0019/1100] quote schema and table --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4541ebae7de..bd66ab5eca9 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -221,7 +221,7 @@ export default class PostgresQuery { query += "," + this.quoteIdentifier(this.target.metricColumn) + " AS metric"; } - query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; + query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { return this.renderTagCondition(tag, index, interpolate); }); From 1d8540ac69d37b84689eaf0016b8d155d8e899a3 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 9 Mar 2018 18:18:12 +0100 Subject: [PATCH 0020/1100] properly quote where constraint parts --- .../datasource/postgres/postgres_query.ts | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index bd66ab5eca9..9e682d2f186 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -130,7 +130,7 @@ export default class PostgresQuery { this.updatePersistedParts(); } - private renderTagCondition(tag, index, interpolate) { + private renderWhereConstraint(tag, index, interpolate) { var str = ''; var operator = tag.operator; var value = tag.value; @@ -138,27 +138,11 @@ export default class PostgresQuery { str = (tag.condition || 'AND') + ' '; } - if (!operator) { - if (/^\/.*\/$/.test(value)) { - operator = '=~'; - } else { - operator = '='; - } + if (interpolate) { + value = this.templateSrv.replace(value, this.scopedVars); } - // quote value unless regex - if (operator !== '=~' && operator !== '!~') { - if (interpolate) { - value = this.templateSrv.replace(value, this.scopedVars); - } - if (operator !== '>' && operator !== '<') { - value = "'" + value.replace(/\\/g, '\\\\') + "'"; - } - } else if (interpolate) { - value = this.templateSrv.replace(value, this.scopedVars, 'regex'); - } - - return str + '"' + tag.key + '" ' + operator + ' ' + value; + return str + this.quoteIdentifier(tag.key) + ' ' + operator + ' ' + this.quoteLiteral(value); } interpolateQueryStr(value, variable, defaultFormatFn) { @@ -223,7 +207,7 @@ export default class PostgresQuery { query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { - return this.renderTagCondition(tag, index, interpolate); + return this.renderWhereConstraint(tag, index, interpolate); }); if (conditions.length > 0) { @@ -260,7 +244,7 @@ export default class PostgresQuery { renderAdhocFilters(filters) { var conditions = _.map(filters, (tag, index) => { - return this.renderTagCondition(tag, index, false); + return this.renderWhereConstraint(tag, index, false); }); return conditions.join(' '); } From cb5278d413e85ec7fbd21490ca4edc467f691518 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 10 Mar 2018 20:18:03 +0100 Subject: [PATCH 0021/1100] handle variables in where constraints --- .../datasource/postgres/postgres_query.ts | 16 ++++++---------- .../plugins/datasource/postgres/query_ctrl.ts | 8 ++------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9e682d2f186..9c48a94862f 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -56,10 +56,6 @@ export default class PostgresQuery { return _.find(this.target.groupBy, (g: any) => g.type === 'time'); } - hasFill() { - return _.find(this.target.groupBy, (g: any) => g.type === 'fill'); - } - addGroupBy(value) { var stringParts = value.match(/^(\w+)(\((.*)\))?$/); var typePart = stringParts[1]; @@ -130,19 +126,19 @@ export default class PostgresQuery { this.updatePersistedParts(); } - private renderWhereConstraint(tag, index, interpolate) { + private renderWhereConstraint(constraint, index, interpolate) { var str = ''; - var operator = tag.operator; - var value = tag.value; + var operator = constraint.operator; + var value = constraint.value; if (index > 0) { - str = (tag.condition || 'AND') + ' '; + str = (constraint.condition || 'AND') + ' '; } if (interpolate) { value = this.templateSrv.replace(value, this.scopedVars); } - return str + this.quoteIdentifier(tag.key) + ' ' + operator + ' ' + this.quoteLiteral(value); + return str + this.quoteIdentifier(constraint.key) + ' ' + operator + ' ' + this.quoteLiteral(value); } interpolateQueryStr(value, variable, defaultFormatFn) { @@ -207,7 +203,7 @@ export default class PostgresQuery { query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { - return this.renderWhereConstraint(tag, index, interpolate); + return this.renderWhereConstraint(tag, index, false); }); if (conditions.length > 0) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4e36e8699ae..da8c28eab9c 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -353,7 +353,6 @@ export class PostgresQueryCtrl extends QueryCtrl { rebuildTargetWhereConditions() { var where = []; var tagIndex = 0; - var tagOperator = ''; _.each(this.whereSegments, (segment2, index) => { if (segment2.type === 'key') { @@ -362,11 +361,8 @@ export class PostgresQueryCtrl extends QueryCtrl { } where[tagIndex].key = segment2.value; } else if (segment2.type === 'value') { - tagOperator = this.getTagValueOperator(segment2.value, where[tagIndex].operator); - if (tagOperator) { - this.whereSegments[index - 1] = this.uiSegmentSrv.newOperator(tagOperator); - where[tagIndex].operator = tagOperator; - } + where[tagIndex].value = segment2.value; + } else if (segment2.type === 'template') { where[tagIndex].value = segment2.value; } else if (segment2.type === 'condition') { where.push({ condition: segment2.value }); From 0b358ff5f30930401a3cfa9ca35699d2903786dc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 10 Mar 2018 22:39:42 +0100 Subject: [PATCH 0022/1100] remove limit --- .../postgres/partials/query.editor.html | 2 +- .../plugins/datasource/postgres/query_ctrl.ts | 31 ++----------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index e25a41b11c2..0706cf3a6cc 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -19,7 +19,7 @@
- +
diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index da8c28eab9c..101d52ee096 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -273,17 +273,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - getTagsOrValues(segment, index) { + getWhereSegments(segment, index) { if (segment.type === 'condition') { return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); } if (segment.type === 'operator') { - var nextValue = this.whereSegments[index + 1].value; - if (/^\/.*\/$/.test(nextValue)) { - return this.$q.when(this.uiSegmentSrv.newOperators(['=~', '!~'])); - } else { - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); - } + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); } var query, addTemplateVars; @@ -308,15 +303,6 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - getTagValueOperator(tagValue, tagOperator): string { - if (tagOperator !== '=~' && tagOperator !== '!~' && /^\/.*\/$/.test(tagValue)) { - return '=~'; - } else if ((tagOperator === '=~' || tagOperator === '!~') && /^(?!\/.*\/$)/.test(tagValue)) { - return '='; - } - return null; - } - whereSegmentUpdated(segment, index) { this.whereSegments[index] = segment; @@ -381,14 +367,11 @@ export class PostgresQueryCtrl extends QueryCtrl { .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(tags => { var options = []; - if (!this.target.limit) { - options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); - } if (!this.queryModel.hasGroupByTime()) { options.push(this.uiSegmentSrv.newSegment({ type: 'time', value: 'time(1m,none)' })); } for (let tag of tags) { - options.push(this.uiSegmentSrv.newSegment({ type: 'column', value: 'column(' + tag.text + ')' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'column', value: tag.text })); } return options; }) @@ -397,14 +380,6 @@ export class PostgresQueryCtrl extends QueryCtrl { groupByAction() { switch (this.groupBySegment.value) { - case 'LIMIT': { - this.target.limit = 10; - break; - } - case 'ORDER BY time DESC': { - this.target.orderByTime = 'DESC'; - break; - } default: { this.queryModel.addGroupBy(this.groupBySegment.value); } From e780b1bce5140a6c74fcc4782090e15035c5268a Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 11 Mar 2018 12:06:54 +0100 Subject: [PATCH 0023/1100] cleanup where segment handling --- .../plugins/datasource/postgres/query_ctrl.ts | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 101d52ee096..afde342474b 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -68,26 +68,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); - this.whereSegments = []; - for (let tag of this.target.where) { - if (!tag.operator) { - if (/^\/.*\/$/.test(tag.value)) { - tag.operator = '=~'; - } else { - tag.operator = '='; - } - } - - if (tag.condition) { - this.whereSegments.push(uiSegmentSrv.newCondition(tag.condition)); - } - - this.whereSegments.push(uiSegmentSrv.newKey(tag.key)); - this.whereSegments.push(uiSegmentSrv.newOperator(tag.operator)); - this.whereSegments.push(uiSegmentSrv.newKeyValue(tag.value)); - } - - this.fixWhereSegments(); + this.buildWhereSegments(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ @@ -264,7 +245,18 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - fixWhereSegments() { + buildWhereSegments() { + this.whereSegments = []; + for (let constraint of this.target.where) { + + if (constraint.condition) { + this.whereSegments.push(this.uiSegmentSrv.newCondition(constraint.condition)); + } + this.whereSegments.push(this.uiSegmentSrv.newKey(constraint.key)); + this.whereSegments.push(this.uiSegmentSrv.newOperator(constraint.operator)); + this.whereSegments.push(this.uiSegmentSrv.newKeyValue(constraint.value)); + } + var count = this.whereSegments.length; var lastSegment = this.whereSegments[Math.max(count - 1, 0)]; From cdb4e2ba0b44741ed9efc09fbf060a7dd5d0c6ac Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 21:31:07 +0100 Subject: [PATCH 0024/1100] remove unused setting --- public/app/plugins/datasource/postgres/postgres_query.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9c48a94862f..f9b87e89541 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -21,7 +21,6 @@ export default class PostgresQuery { target.timeColumn = target.timeColumn || 'time'; target.metricColumn = target.metricColumn || 'None'; - target.orderByTime = target.orderByTime || 'ASC'; target.groupBy = target.groupBy || []; target.where = target.where || []; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; From af63a26be0a16acbada306fa053386f93e342af7 Mon Sep 17 00:00:00 2001 From: flopp999 <21694965+flopp999@users.noreply.github.com> Date: Tue, 13 Mar 2018 22:11:58 +0100 Subject: [PATCH 0025/1100] Added W/m2(energy) and l/h(flow) both as .fixedUnit --- public/app/core/utils/kbn.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/core/utils/kbn.ts b/public/app/core/utils/kbn.ts index 3b78ccfc001..3f2f0ad9419 100644 --- a/public/app/core/utils/kbn.ts +++ b/public/app/core/utils/kbn.ts @@ -496,6 +496,7 @@ kbn.valueFormats.watt = kbn.formatBuilders.decimalSIPrefix('W'); kbn.valueFormats.kwatt = kbn.formatBuilders.decimalSIPrefix('W', 1); kbn.valueFormats.mwatt = kbn.formatBuilders.decimalSIPrefix('W', -1); kbn.valueFormats.kwattm = kbn.formatBuilders.decimalSIPrefix('W/Min', 1); +kbn.valueFormats.Wm2 = kbn.formatBuilders.fixedUnit('W/m2'); kbn.valueFormats.voltamp = kbn.formatBuilders.decimalSIPrefix('VA'); kbn.valueFormats.kvoltamp = kbn.formatBuilders.decimalSIPrefix('VA', 1); kbn.valueFormats.voltampreact = kbn.formatBuilders.decimalSIPrefix('var'); @@ -576,6 +577,7 @@ kbn.valueFormats.flowgpm = kbn.formatBuilders.fixedUnit('gpm'); kbn.valueFormats.flowcms = kbn.formatBuilders.fixedUnit('cms'); kbn.valueFormats.flowcfs = kbn.formatBuilders.fixedUnit('cfs'); kbn.valueFormats.flowcfm = kbn.formatBuilders.fixedUnit('cfm'); +kbn.valueFormats.litreh = kbn.formatBuilders.fixedUnit('l/h'); // Angle kbn.valueFormats.degree = kbn.formatBuilders.fixedUnit('°'); @@ -1007,6 +1009,7 @@ kbn.getUnitFormats = function() { { text: 'Watt (W)', value: 'watt' }, { text: 'Kilowatt (kW)', value: 'kwatt' }, { text: 'Milliwatt (mW)', value: 'mwatt' }, + { text: 'Watt per square metre (W/m2)', value: 'Wm2' }, { text: 'Volt-ampere (VA)', value: 'voltamp' }, { text: 'Kilovolt-ampere (kVA)', value: 'kvoltamp' }, { text: 'Volt-ampere reactive (var)', value: 'voltampreact' }, @@ -1062,6 +1065,7 @@ kbn.getUnitFormats = function() { { text: 'Cubic meters/sec (cms)', value: 'flowcms' }, { text: 'Cubic feet/sec (cfs)', value: 'flowcfs' }, { text: 'Cubic feet/min (cfm)', value: 'flowcfm' }, + { text: 'Litre/hour', value: 'litreh' }, ], }, { From 08461408a279afa9af2bd8b746d474de373fd6fa Mon Sep 17 00:00:00 2001 From: flopp999 <21694965+flopp999@users.noreply.github.com> Date: Tue, 13 Mar 2018 22:17:56 +0100 Subject: [PATCH 0026/1100] Added Kilopascals(kPa) under pressure --- public/app/core/utils/kbn.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/app/core/utils/kbn.ts b/public/app/core/utils/kbn.ts index 3f2f0ad9419..87075e0de2e 100644 --- a/public/app/core/utils/kbn.ts +++ b/public/app/core/utils/kbn.ts @@ -526,6 +526,7 @@ kbn.valueFormats.pressurebar = kbn.formatBuilders.decimalSIPrefix('bar'); kbn.valueFormats.pressurembar = kbn.formatBuilders.decimalSIPrefix('bar', -1); kbn.valueFormats.pressurekbar = kbn.formatBuilders.decimalSIPrefix('bar', 1); kbn.valueFormats.pressurehpa = kbn.formatBuilders.fixedUnit('hPa'); +kbn.valueFormats.pressurekpa = kbn.formatBuilders.fixedUnit('kPa'); kbn.valueFormats.pressurehg = kbn.formatBuilders.fixedUnit('"Hg'); kbn.valueFormats.pressurepsi = kbn.formatBuilders.scaledUnits(1000, [' psi', ' ksi', ' Mpsi']); @@ -1045,6 +1046,7 @@ kbn.getUnitFormats = function() { { text: 'Bars', value: 'pressurebar' }, { text: 'Kilobars', value: 'pressurekbar' }, { text: 'Hectopascals', value: 'pressurehpa' }, + { text: 'Kilopascals', value: 'pressurekpa' }, { text: 'Inches of mercury', value: 'pressurehg' }, { text: 'PSI', value: 'pressurepsi' }, ], From b7c7030a4681e3c5c8d3e565588a02e9e8a2e9db Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:06:39 +0100 Subject: [PATCH 0027/1100] add regex operators --- .../datasource/postgres/query_builder.ts | 8 ++++++++ .../plugins/datasource/postgres/query_ctrl.ts | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 7a227f33b93..23691830b17 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -47,4 +47,12 @@ export class PostgresQueryBuilder { return query; } + buildDatatypeQuery(column: string) { + var query = "SELECT data_type FROM information_schema.columns WHERE "; + query += " table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + query += " AND table_name = " + this.queryModel.quoteLiteral(this.target.table); + query += " AND column_name = " + this.queryModel.quoteLiteral(column); + return query; + } + } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index afde342474b..b84b0ce0750 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -266,14 +266,28 @@ export class PostgresQueryCtrl extends QueryCtrl { } getWhereSegments(segment, index) { + var query, addTemplateVars; + if (segment.type === 'condition') { return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); } if (segment.type === 'operator') { - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); + var columnName = this.whereSegments[index - 1].value; + query = this.queryBuilder.buildDatatypeQuery(columnName); + return this.datasource.metricFindQuery(query) + .then(results => { + var datatype = results[0].text; + switch (datatype) { + case "text": + case "character varying": + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '~', '~*','!~','!~*','IN'])); + default: + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>='])); + } + }) + .catch(this.handleQueryError.bind(this)); } - var query, addTemplateVars; if (segment.type === 'key' || segment.type === 'plus-button') { query = this.queryBuilder.buildColumnQuery(); From 958646d976ad64a3b32707c2a7b686a906cd9fc1 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:15:25 +0100 Subject: [PATCH 0028/1100] dont quote where constraints --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index f9b87e89541..48f7efae760 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -137,7 +137,7 @@ export default class PostgresQuery { value = this.templateSrv.replace(value, this.scopedVars); } - return str + this.quoteIdentifier(constraint.key) + ' ' + operator + ' ' + this.quoteLiteral(value); + return str + constraint.key + ' ' + operator + ' ' + value; } interpolateQueryStr(value, variable, defaultFormatFn) { From 5e9a66de5f36c8e29ebeb5370f503529ac2dd7c7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:19:56 +0100 Subject: [PATCH 0029/1100] put values for IN in parens --- public/app/plugins/datasource/postgres/postgres_query.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 48f7efae760..e06cb3f2126 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -137,7 +137,11 @@ export default class PostgresQuery { value = this.templateSrv.replace(value, this.scopedVars); } - return str + constraint.key + ' ' + operator + ' ' + value; + if (operator === "IN") { + return str + constraint.key + ' ' + operator + ' (' + value + ')'; + } else { + return str + constraint.key + ' ' + operator + ' ' + value; + } } interpolateQueryStr(value, variable, defaultFormatFn) { From e6501f0f0ecec19ccafd9191bcfcdef4a1e19c6f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:24:26 +0100 Subject: [PATCH 0030/1100] revert special handling for IN --- public/app/plugins/datasource/postgres/postgres_query.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index e06cb3f2126..48f7efae760 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -137,11 +137,7 @@ export default class PostgresQuery { value = this.templateSrv.replace(value, this.scopedVars); } - if (operator === "IN") { - return str + constraint.key + ' ' + operator + ' (' + value + ')'; - } else { - return str + constraint.key + ' ' + operator + ' ' + value; - } + return str + constraint.key + ' ' + operator + ' ' + value; } interpolateQueryStr(value, variable, defaultFormatFn) { From 6793fa5e549dfe31c04dd3dacfa476a9091d3f3a Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:33:40 +0100 Subject: [PATCH 0031/1100] join multivalue variables with , --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 48f7efae760..3a11c344f3e 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -151,7 +151,7 @@ export default class PostgresQuery { } var escapedValues = _.map(value, kbn.regexEscape); - return '(' + escapedValues.join('|') + ')'; + return '(' + escapedValues.join(',') + ')'; } render(interpolate?) { From 64fa1ce8a0a3bfe6b7bb4e3c7cbc4227d0c42af4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 14 Mar 2018 18:09:47 +0100 Subject: [PATCH 0032/1100] properly handle IN queries --- .../app/plugins/datasource/postgres/postgres_query.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 3a11c344f3e..8f9f1261340 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -1,6 +1,5 @@ import _ from 'lodash'; import queryPart from './query_part'; -import kbn from 'app/core/utils/kbn'; export default class PostgresQuery { target: any; @@ -26,14 +25,16 @@ export default class PostgresQuery { target.select = target.select || [[{ type: 'column', params: ['value'] }]]; this.updateProjection(); + // give interpolateQueryStr access to this + this.interpolateQueryStr = this.interpolateQueryStr.bind(this); } quoteIdentifier(value) { - return '"' + value + '"'; + return '"' + value.replace('"','""') + '"'; } quoteLiteral(value) { - return "'" + value + "'"; + return "'" + value.replace("'","''") + "'"; } updateProjection() { @@ -147,10 +148,10 @@ export default class PostgresQuery { } if (typeof value === 'string') { - return kbn.regexEscape(value); + return this.quoteLiteral(value); } - var escapedValues = _.map(value, kbn.regexEscape); + var escapedValues = _.map(value, this.quoteLiteral); return '(' + escapedValues.join(',') + ')'; } From 0c3afd0e9c098c83f04f44f048f72468496aec5c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 14 Mar 2018 22:59:48 +0100 Subject: [PATCH 0033/1100] add buildAggregateQuery --- public/app/plugins/datasource/postgres/query_builder.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 23691830b17..275d15492fe 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -55,4 +55,12 @@ export class PostgresQueryBuilder { return query; } + buildAggregateQuery() { + var query = "SELECT DISTINCT proname FROM pg_aggregate "; + query += "INNER JOIN pg_proc ON pg_aggregate.aggfnoid = pg_proc.oid "; + query += "INNER JOIN pg_type ON pg_type.oid=pg_proc.prorettype "; + query += "WHERE pronargs=1 AND typname IN ('int8','float8') AND aggkind='n' ORDER BY 1"; + return query; + } + } From 46c229188e985f74ed35d7505c21d9ec723d7b99 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 14 Mar 2018 23:03:32 +0100 Subject: [PATCH 0034/1100] read aggregate functions from database --- .../datasource/postgres/postgres_query.ts | 3 ++- .../plugins/datasource/postgres/query_ctrl.ts | 11 +++++++++++ .../plugins/datasource/postgres/query_part.ts | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 8f9f1261340..4516bf4a4be 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -24,9 +24,10 @@ export default class PostgresQuery { target.where = target.where || []; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; - this.updateProjection(); // give interpolateQueryStr access to this this.interpolateQueryStr = this.interpolateQueryStr.bind(this); + + this.updateProjection(); } quoteIdentifier(value) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b84b0ce0750..4a2a0ce6c1c 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -77,9 +77,19 @@ export class PostgresQueryCtrl extends QueryCtrl { }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); + } buildSelectMenu() { + + if (!queryPart.hasAggregates()) { + this.datasource.metricFindQuery(this.queryBuilder.buildAggregateQuery()) + .then(results => { + queryPart.clearAggregates(); + _.map(results, segment => { queryPart.registerAggregate(segment.text); }); + }) + .catch(this.handleQueryError.bind(this)); + } var categories = queryPart.getCategories(); this.selectMenu = _.reduce( categories, @@ -279,6 +289,7 @@ export class PostgresQueryCtrl extends QueryCtrl { var datatype = results[0].text; switch (datatype) { case "text": + case "character": case "character varying": return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '~', '~*','!~','!~*','IN'])); default: diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 600723be00d..b63ebfae0c1 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -23,6 +23,17 @@ function register(options: any) { options.category.push(index[options.type]); } +function registerAggregate(name: string) { + register({ + type: name, + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, + }); +} + var groupByTimeFunctions = []; function aliasRenderer(part, innerExpr) { @@ -192,6 +203,12 @@ register({ export default { create: createPart, + registerAggregate: registerAggregate, + clearAggregates: function() { categories.Aggregations = []; }, + hasAggregates: function() { + // FIXME + return categories.Aggregations.length > 6; + }, getCategories: function() { return categories; }, From 8e7d23cdebc3df236d519777e3e4485d5ad32d12 Mon Sep 17 00:00:00 2001 From: wph95 Date: Fri, 23 Mar 2018 23:50:16 +0800 Subject: [PATCH 0035/1100] wip Signed-off-by: wph95 --- pkg/cmd/grafana-server/main.go | 1 + pkg/tsdb/elasticsearch/elasticsearch.go | 131 +++++++++++ pkg/tsdb/elasticsearch/model_parser.go | 97 +++++++++ pkg/tsdb/elasticsearch/models.go | 131 +++++++++++ pkg/tsdb/elasticsearch/query.go | 204 ++++++++++++++++++ pkg/tsdb/elasticsearch/response_parser.go | 111 ++++++++++ .../datasource/elasticsearch/plugin.json | 1 + 7 files changed, 676 insertions(+) create mode 100644 pkg/tsdb/elasticsearch/elasticsearch.go create mode 100644 pkg/tsdb/elasticsearch/model_parser.go create mode 100644 pkg/tsdb/elasticsearch/models.go create mode 100644 pkg/tsdb/elasticsearch/query.go create mode 100644 pkg/tsdb/elasticsearch/response_parser.go diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index ab0e12f2d9f..21090153bc0 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/grafana/grafana/pkg/services/alerting/conditions" _ "github.com/grafana/grafana/pkg/services/alerting/notifiers" _ "github.com/grafana/grafana/pkg/tsdb/cloudwatch" + _ "github.com/grafana/grafana/pkg/tsdb/elasticsearch" _ "github.com/grafana/grafana/pkg/tsdb/graphite" _ "github.com/grafana/grafana/pkg/tsdb/influxdb" _ "github.com/grafana/grafana/pkg/tsdb/mysql" diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go new file mode 100644 index 00000000000..d67b4ad902d --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -0,0 +1,131 @@ +package elasticsearch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/davecgh/go-spew/spew" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/net/context/ctxhttp" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +type ElasticsearchExecutor struct { + Transport *http.Transport +} + +var ( + glog log.Logger + intervalCalculator tsdb.IntervalCalculator +) + +func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + transport, err := dsInfo.GetHttpTransport() + if err != nil { + return nil, err + } + + return &ElasticsearchExecutor{ + Transport: transport, + }, nil +} + +func init() { + glog = log.New("tsdb.elasticsearch") + tsdb.RegisterTsdbQueryEndpoint("elasticsearch", NewElasticsearchExecutor) + intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Millisecond * 1}) +} + +func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + queryParser := ElasticSearchQueryParser{ + dsInfo, + tsdbQuery.TimeRange, + tsdbQuery.Queries, + glog, + } + + glog.Warn(spew.Sdump(dsInfo)) + glog.Warn(spew.Sdump(tsdbQuery)) + + payload, err := queryParser.Parse() + if err != nil { + return nil, err + } + + if setting.Env == setting.DEV { + glog.Debug("Elasticsearch playload", "raw playload", payload) + } + glog.Info("Elasticsearch playload", "raw playload", payload) + + req, err := e.createRequest(dsInfo, payload) + if err != nil { + return nil, err + } + + httpClient, err := dsInfo.GetHttpClient() + if err != nil { + return nil, err + } + + resp, err := ctxhttp.Do(ctx, httpClient, req) + if err != nil { + return nil, err + } + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("elasticsearch returned statuscode invalid status code: %v", resp.Status) + } + + var responses Responses + dec := json.NewDecoder(resp.Body) + defer resp.Body.Close() + dec.UseNumber() + err = dec.Decode(&responses) + if err != nil { + return nil, err + } + + glog.Warn(spew.Sdump(responses)) + for _, res := range responses.Responses { + if res.Err != nil { + return nil, errors.New(res.getErrMsg()) + } + + } + + return result, nil +} + +func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { + u, _ := url.Parse(dsInfo.Url) + u.Path = path.Join(u.Path, "_msearch") + req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(query)) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if dsInfo.BasicAuth { + req.SetBasicAuth(dsInfo.BasicAuthUser, dsInfo.BasicAuthPassword) + } + + if !dsInfo.BasicAuth && dsInfo.User != "" { + req.SetBasicAuth(dsInfo.User, dsInfo.Password) + } + + glog.Debug("Elasticsearch request", "url", req.URL.String()) + glog.Debug("Elasticsearch request", "body", query) + return req, nil +} diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go new file mode 100644 index 00000000000..136db6baed7 --- /dev/null +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -0,0 +1,97 @@ +package elasticsearch + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + "src/github.com/davecgh/go-spew/spew" + "strconv" + "strings" + "time" +) + +type ElasticSearchQueryParser struct { + DsInfo *models.DataSource + TimeRange *tsdb.TimeRange + Queries []*tsdb.Query + glog log.Logger +} + +func (qp *ElasticSearchQueryParser) Parse() (string, error) { + payload := bytes.Buffer{} + queryHeader := qp.getQueryHeader() + + for _, q := range qp.Queries { + timeField, err := q.Model.Get("timeField").String() + if err != nil { + return "", err + } + rawQuery := q.Model.Get("query").MustString("") + bucketAggs := q.Model.Get("bucketAggs").MustArray() + metrics := q.Model.Get("metrics").MustArray() + alias := q.Model.Get("alias").MustString("") + builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} + + query, err := builder.Build() + if err != nil { + return "", err + } + queryBytes, err := json.Marshal(query) + if err != nil { + return "", err + } + + payload.WriteString(queryHeader.String() + "\n") + payload.WriteString(string(queryBytes) + "\n") + } + + return qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) + +} + +func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { + var header QueryHeader + esVersion := qp.DsInfo.JsonData.Get("esVersion").MustInt() + + searchType := "query_then_fetch" + if esVersion < 5 { + searchType = "count" + } + header.SearchType = searchType + header.IgnoreUnavailable = true + header.Index = qp.getIndexList() + + if esVersion >= 56 { + header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + } + return &header +} +func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simplejson.Json) (string, error) { + parsedInterval, err := tsdb.GetIntervalFrom(qp.DsInfo, model, time.Millisecond) + if err != nil { + return "", nil + } + + interval := intervalCalculator.Calculate(qp.TimeRange, parsedInterval) + glog.Warn(spew.Sdump(interval)) + payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", qp.TimeRange.GetFromAsMsEpoch()), -1) + payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", qp.TimeRange.GetToAsMsEpoch()), -1) + payload = strings.Replace(payload, "$interval", interval.Text, -1) + payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + payload = strings.Replace(payload, "$__interval", interval.Text, -1) + + return payload, nil +} + +func (qp *ElasticSearchQueryParser) getIndexList() string { + _, err := qp.DsInfo.JsonData.Get("interval").String() + if err != nil { + return qp.DsInfo.Database + } + // todo: support interval + return qp.DsInfo.Database +} diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go new file mode 100644 index 00000000000..8662f6efbd3 --- /dev/null +++ b/pkg/tsdb/elasticsearch/models.go @@ -0,0 +1,131 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "bytes" + "fmt" + "encoding/json" +) + +type QueryHeader struct { + SearchType string `json:"search_type"` + IgnoreUnavailable bool `json:"ignore_unavailable"` + Index interface{} `json:"index"` + MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests"` +} + +func (q *QueryHeader) String() (string) { + r, _ := json.Marshal(q) + return string(r) +} + +type Query struct { + Query map[string]interface{} `json:"query"` + Aggs Aggs `json:"aggs"` + Size int `json:"size"` +} + +type Aggs map[string]interface{} + +type HistogramAgg struct { + Interval string `json:"interval,omitempty"` + Field string `json:"field"` + MinDocCount int `json:"min_doc_count"` + Missing string `json:"missing,omitempty"` +} + +type DateHistogramAgg struct { + HistogramAgg + ExtendedBounds ExtendedBounds `json:"extended_bounds"` + Format string `json:"format"` +} + +type FiltersAgg struct { + Filter map[string]interface{} `json:"filter"` +} + +type TermsAggSetting struct { + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + MinDocCount int `json:"min_doc_count"` + Missing string `json:"missing"` +} + +type TermsAgg struct { + Terms TermsAggSetting `json:"terms"` + Aggs Aggs `json:"aggs"` +} + +type ExtendedBounds struct { + Min string `json:"min"` + Max string `json:"max"` +} + +type RangeFilter struct { + Range map[string]RangeFilterSetting `json:"range"` +} +type RangeFilterSetting struct { + Gte string `json:"gte"` + Lte string `json:"lte"` + Format string `json:"format"` +} + +func newRangeFilter(field string, rangeFilterSetting RangeFilterSetting) *RangeFilter { + return &RangeFilter{ + map[string]RangeFilterSetting{field: rangeFilterSetting}} +} + +type QueryStringFilter struct { + QueryString QueryStringFilterSetting `json:"query_string"` +} +type QueryStringFilterSetting struct { + AnalyzeWildcard bool `json:"analyze_wildcard"` + Query string `json:"query"` +} + +func newQueryStringFilter(analyzeWildcard bool, query string) *QueryStringFilter { + return &QueryStringFilter{QueryStringFilterSetting{AnalyzeWildcard: analyzeWildcard, Query: query}} +} + +type BoolQuery struct { + Filter []interface{} `json:"filter"` +} + +type Metric map[string]interface{} + +type Responses struct { + Responses []Response `json:"responses"` +} + +type Response struct { + Status int `json:"status"` + Err map[string]interface{} `json:"error"` + Aggregations map[string]interface{} `json:"aggregations"` +} + +func (r *Response) getErrMsg() (string) { + var msg bytes.Buffer + errJson := simplejson.NewFromAny(r.Err) + errType, err := errJson.Get("type").String() + if err == nil { + msg.WriteString(fmt.Sprintf("type:%s", errType)) + } + + reason, err := errJson.Get("type").String() + if err == nil { + msg.WriteString(fmt.Sprintf("reason:%s", reason)) + } + return msg.String() +} + +type PercentilesResult struct { + Buckets struct { + map[string]struct { + Values map[string]string `json:"values"` + } + KeyAsString string `json:"key_as_string"` + Key int64 `json:"key"` + DocCount int `json:"doc_count"` + } `json:"buckets"` +} diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go new file mode 100644 index 00000000000..69dd5caa3b4 --- /dev/null +++ b/pkg/tsdb/elasticsearch/query.go @@ -0,0 +1,204 @@ +package elasticsearch + +import ( + "errors" + "github.com/grafana/grafana/pkg/components/simplejson" +) + +var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", + Lte: "$timeTo", + Format: "epoch_millis"} + +type QueryBuilder struct { + TimeField string + RawQuery string + BucketAggs []interface{} + Metrics []interface{} + Alias string +} + +func (b *QueryBuilder) Build() (Query, error) { + var err error + var res Query + res.Query = make(map[string]interface{}) + res.Size = 0 + + if err != nil { + return res, err + } + + boolQuery := BoolQuery{} + boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(b.TimeField, rangeFilterSetting)) + boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, b.RawQuery)) + res.Query["bool"] = boolQuery + + // handle document query + if len(b.BucketAggs) == 0 { + if len(b.Metrics) > 0 { + metric := simplejson.NewFromAny(b.Metrics[0]) + if metric.Get("type").MustString("") == "raw_document" { + return res, errors.New("alert not support Raw_Document") + } + } + } + aggs, err := b.parseAggs(b.BucketAggs, b.Metrics) + res.Aggs = aggs["aggs"].(Aggs) + + return res, err +} + +func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{}) (Aggs, error) { + query := make(Aggs) + nestedAggs := query + for _, aggRaw := range bucketAggs { + esAggs := make(Aggs) + aggJson := simplejson.NewFromAny(aggRaw) + aggType, err := aggJson.Get("type").String() + if err != nil { + return nil, err + } + id, err := aggJson.Get("id").String() + if err != nil { + return nil, err + } + + switch aggType { + case "date_histogram": + esAggs["date_histogram"] = b.getDateHistogramAgg(aggJson) + case "histogram": + esAggs["histogram"] = b.getHistogramAgg(aggJson) + case "filters": + esAggs["filters"] = b.getFilters(aggJson) + case "terms": + terms := b.getTerms(aggJson) + esAggs["terms"] = terms.Terms + esAggs["aggs"] = terms.Aggs + case "geohash_grid": + return nil, errors.New("alert not support Geo_Hash_Grid") + } + + if _, ok := nestedAggs["aggs"]; !ok { + nestedAggs["aggs"] = make(Aggs) + } + + if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { + aggs[id] = esAggs + } + nestedAggs = esAggs + + } + nestedAggs["aggs"] = make(Aggs) + + for _, metricRaw := range metrics { + metric := make(Metric) + metricJson := simplejson.NewFromAny(metricRaw) + + id, err := metricJson.Get("id").String() + if err != nil { + return nil, err + } + metricType, err := metricJson.Get("type").String() + if err != nil { + return nil, err + } + if metricType == "count" { + continue + } + + // todo support pipeline Agg + + settings := metricJson.Get("settings").MustMap() + settings["field"] = metricJson.Get("field").MustString() + metric[metricType] = settings + nestedAggs["aggs"].(Aggs)[id] = metric + } + return query, nil +} + +func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogramAgg { + agg := &DateHistogramAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + interval, err := settings.Get("interval").String() + if err == nil { + agg.Interval = interval + } + agg.Field = b.TimeField + agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} + agg.Format = "epoch_millis" + + if agg.Interval == "auto" { + agg.Interval = "$__interval" + } + + missing, err := settings.Get("missing").String() + if err == nil { + agg.Missing = missing + } + return *agg +} + +func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { + agg := &HistogramAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + interval, err := settings.Get("interval").String() + if err == nil { + agg.Interval = interval + } + field, err := model.Get("field").String() + if err == nil { + agg.Field = field + } + agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + missing, err := settings.Get("missing").String() + if err == nil { + agg.Missing = missing + } + return *agg +} + +func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { + agg := &FiltersAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + for filter := range settings.Get("filters").MustArray() { + filterJson := simplejson.NewFromAny(filter) + query := filterJson.Get("query").MustString("") + label := filterJson.Get("label").MustString("") + if label == "" { + label = query + } + agg.Filter[label] = newQueryStringFilter(true, query) + } + return *agg +} + +func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { + agg := &TermsAgg{} + settings := simplejson.NewFromAny(model.Get("settings").Interface()) + agg.Terms.Field = model.Get("field").MustString() + if settings == nil { + return *agg + } + agg.Terms.Size = settings.Get("size").MustInt(0) + if agg.Terms.Size == 0 { + agg.Terms.Size = 500 + } + orderBy := settings.Get("orderBy").MustString("") + if orderBy != "" { + agg.Terms.Order[orderBy] = settings.Get("order").MustString("") + // if orderBy is a int, means this fields is metric result value + // TODO set subAggs + } + + minDocCount, err := settings.Get("min_doc_count").Int() + if err == nil { + agg.Terms.MinDocCount = minDocCount + } + + missing, err := settings.Get("missing").String() + if err == nil { + agg.Terms.Missing = missing + } + + return *agg +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go new file mode 100644 index 00000000000..bc47a3f935e --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -0,0 +1,111 @@ +package elasticsearch + +import ( + "errors" + "fmt" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + "strconv" +) + +type ElasticsearchResponseParser struct { + Responses []Response + Targets []QueryBuilder +} + +func (rp *ElasticsearchResponseParser) getTimeSeries() []interface{} { + for i, res := range rp.Responses { + var series []interface{} + target := rp.Targets[i] + props := make(map[string]interface{}) + rp.processBuckets(res.Aggregations, target, &series, props, 0) + } +} + +func findAgg(target QueryBuilder, aggId string) (*simplejson.Json, error) { + for _, v := range target.BucketAggs { + aggDef := simplejson.NewFromAny(v) + if aggId == aggDef.Get("id").MustString() { + return aggDef, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggId) +} + +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target QueryBuilder, series *[]interface{}, props map[string]interface{}, depth int) error { + maxDepth := len(target.BucketAggs) - 1 + for aggId, v := range aggs { + aggDef, _ := findAgg(target, aggId) + esAgg := simplejson.NewFromAny(v) + if aggDef == nil { + continue + } + + if depth == maxDepth { + if aggDef.Get("type").MustString() == "date_histogram" { + rp.processMetrics(esAgg, target, series, props) + } + } + + } + +} + +func mapCopy(originalMap, newMap *map[string]string) { + for k, v := range originalMap { + newMap[k] = v + } + +} + +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target QueryBuilder, props map[string]string) ([]*tsdb.TimeSeries, error) { + var series []*tsdb.TimeSeries + for _, v := range target.Metrics { + metric := simplejson.NewFromAny(v) + if metric.Get("hide").MustBool(false) { + continue + } + metricId := fmt.Sprintf("%d", metric.Get("id").MustInt()) + metricField := metric.Get("field").MustString() + + switch metric.Get("type").MustString() { + case "count": + newSeries := tsdb.TimeSeries{} + for _, v := range esAgg.Get("buckets").MustMap() { + bucket := simplejson.NewFromAny(v) + value := bucket.Get("doc_count").MustFloat64() + key := bucket.Get("key").MustFloat64() + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + } + newSeries.Tags = props + newSeries.Tags["metric"] = "count" + series = append(series, &newSeries) + + case "percentiles": + buckets := esAgg.Get("buckets").MustArray() + if len(buckets) == 0 { + break + } + + firstBucket := simplejson.NewFromAny(buckets[0]) + percentiles := firstBucket.GetPath(metricId, "values").MustMap() + + for percentileName := range percentiles { + newSeries := tsdb.TimeSeries{} + newSeries.Tags = props + newSeries.Tags["metric"] = "p" + percentileName + newSeries.Tags["field"] = metricField + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + valueStr := bucket.GetPath(metricId, "values", percentileName).MustString() + value, _ := strconv.ParseFloat(valueStr, 64) + key := bucket.Get("key").MustFloat64() + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + } + series = append(series, &newSeries) + } + } + } + return series +} diff --git a/public/app/plugins/datasource/elasticsearch/plugin.json b/public/app/plugins/datasource/elasticsearch/plugin.json index 59d26b785ac..89cca1251d5 100644 --- a/public/app/plugins/datasource/elasticsearch/plugin.json +++ b/public/app/plugins/datasource/elasticsearch/plugin.json @@ -20,6 +20,7 @@ "version": "5.0.0" }, + "alerting": true, "annotations": true, "metrics": true, From bc5b59737c2f6f99b64b395de2e20b888d043c97 Mon Sep 17 00:00:00 2001 From: wph95 Date: Sat, 24 Mar 2018 13:06:21 +0800 Subject: [PATCH 0036/1100] finished CODING PHASE 1 Signed-off-by: wph95 --- pkg/tsdb/elasticsearch/elasticsearch.go | 13 ++++--------- pkg/tsdb/elasticsearch/model_parser.go | 16 ++++++++-------- pkg/tsdb/elasticsearch/models.go | 11 ----------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index d67b4ad902d..8fd82a179e8 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/davecgh/go-spew/spew" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -52,13 +51,9 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo dsInfo, tsdbQuery.TimeRange, tsdbQuery.Queries, - glog, } - glog.Warn(spew.Sdump(dsInfo)) - glog.Warn(spew.Sdump(tsdbQuery)) - - payload, err := queryParser.Parse() + payload, targets, err := queryParser.Parse() if err != nil { return nil, err } @@ -96,14 +91,14 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, err } - glog.Warn(spew.Sdump(responses)) for _, res := range responses.Responses { if res.Err != nil { return nil, errors.New(res.getErrMsg()) } - } - + responseParser := ElasticsearchResponseParser{responses.Responses, targets} + queryRes := responseParser.getTimeSeries() + result.Results["A"] = queryRes return result, nil } diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 136db6baed7..233a35efdc6 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "src/github.com/davecgh/go-spew/spew" @@ -18,38 +17,39 @@ type ElasticSearchQueryParser struct { DsInfo *models.DataSource TimeRange *tsdb.TimeRange Queries []*tsdb.Query - glog log.Logger } -func (qp *ElasticSearchQueryParser) Parse() (string, error) { +func (qp *ElasticSearchQueryParser) Parse() (string, []*QueryBuilder, error) { payload := bytes.Buffer{} queryHeader := qp.getQueryHeader() - + targets := make([]*QueryBuilder, 0) for _, q := range qp.Queries { timeField, err := q.Model.Get("timeField").String() if err != nil { - return "", err + return "", nil, err } rawQuery := q.Model.Get("query").MustString("") bucketAggs := q.Model.Get("bucketAggs").MustArray() metrics := q.Model.Get("metrics").MustArray() alias := q.Model.Get("alias").MustString("") builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} + targets = append(targets, &builder) query, err := builder.Build() if err != nil { - return "", err + return "", nil, err } queryBytes, err := json.Marshal(query) if err != nil { - return "", err + return "", nil, err } payload.WriteString(queryHeader.String() + "\n") payload.WriteString(string(queryBytes) + "\n") } + p, err := qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) - return qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) + return p, targets, err } diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 8662f6efbd3..d758e2159de 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -118,14 +118,3 @@ func (r *Response) getErrMsg() (string) { } return msg.String() } - -type PercentilesResult struct { - Buckets struct { - map[string]struct { - Values map[string]string `json:"values"` - } - KeyAsString string `json:"key_as_string"` - Key int64 `json:"key"` - DocCount int `json:"doc_count"` - } `json:"buckets"` -} From 1e275d0cd1ff976f44dfce6affe8661160cdd873 Mon Sep 17 00:00:00 2001 From: wph95 Date: Sun, 25 Mar 2018 02:18:28 +0800 Subject: [PATCH 0037/1100] set right series name Signed-off-by: wph95 --- pkg/tsdb/elasticsearch/query.go | 14 +- pkg/tsdb/elasticsearch/query_def.go | 26 +++ pkg/tsdb/elasticsearch/response_parser.go | 219 ++++++++++++++++++---- 3 files changed, 215 insertions(+), 44 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/query_def.go diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index 69dd5caa3b4..d6d70e79a2a 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -3,10 +3,11 @@ package elasticsearch import ( "errors" "github.com/grafana/grafana/pkg/components/simplejson" + "strconv" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", + Lte: "$timeTo", Format: "epoch_millis"} type QueryBuilder struct { @@ -173,18 +174,21 @@ func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { } func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { - agg := &TermsAgg{} + agg := &TermsAgg{Aggs: make(Aggs)} settings := simplejson.NewFromAny(model.Get("settings").Interface()) agg.Terms.Field = model.Get("field").MustString() if settings == nil { return *agg } - agg.Terms.Size = settings.Get("size").MustInt(0) - if agg.Terms.Size == 0 { - agg.Terms.Size = 500 + sizeStr := settings.Get("size").MustString("") + size, err := strconv.Atoi(sizeStr) + if err != nil { + size = 500 } + agg.Terms.Size = size orderBy := settings.Get("orderBy").MustString("") if orderBy != "" { + agg.Terms.Order = make(map[string]interface{}) agg.Terms.Order[orderBy] = settings.Get("order").MustString("") // if orderBy is a int, means this fields is metric result value // TODO set subAggs diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go new file mode 100644 index 00000000000..5dc02aa359e --- /dev/null +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -0,0 +1,26 @@ +package elasticsearch + +var metricAggType = map[string]string{ + "count": "Count", + "avg": "Average", + "sum": "Sum", + "max": "Max", + "min": "Min", + "extended_stats": "Extended Stats", + "percentiles": "Percentiles", + "cardinality": "Unique Count", + "moving_avg": "Moving Average", + "derivative": "Derivative", + "raw_document": "Raw Document", +} + +var extendedStats = map[string]string{ + "avg": "Avg", + "min": "Min", + "max": "Max", + "sum": "Sum", + "count": "Count", + "std_deviation": "Std Dev", + "std_deviation_bounds_upper": "Std Dev Upper", + "std_deviation_bounds_lower": "Std Dev Lower", +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index bc47a3f935e..a2a8565641f 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -7,33 +7,30 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" "strconv" + "regexp" + "strings" ) type ElasticsearchResponseParser struct { Responses []Response - Targets []QueryBuilder + Targets []*QueryBuilder } -func (rp *ElasticsearchResponseParser) getTimeSeries() []interface{} { +func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { + queryRes := tsdb.NewQueryResult() for i, res := range rp.Responses { - var series []interface{} target := rp.Targets[i] - props := make(map[string]interface{}) + props := make(map[string]string) + series := make([]*tsdb.TimeSeries, 0) rp.processBuckets(res.Aggregations, target, &series, props, 0) + rp.nameSeries(&series, target) + queryRes.Series = append(queryRes.Series, series...) } + return queryRes } -func findAgg(target QueryBuilder, aggId string) (*simplejson.Json, error) { - for _, v := range target.BucketAggs { - aggDef := simplejson.NewFromAny(v) - if aggId == aggDef.Get("id").MustString() { - return aggDef, nil - } - } - return nil, errors.New("can't found aggDef, aggID:" + aggId) -} - -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target QueryBuilder, series *[]interface{}, props map[string]interface{}, depth int) error { +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string, depth int) (error) { + var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { aggDef, _ := findAgg(target, aggId) @@ -44,43 +41,59 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ if depth == maxDepth { if aggDef.Get("type").MustString() == "date_histogram" { - rp.processMetrics(esAgg, target, series, props) + err = rp.processMetrics(esAgg, target, series, props) + if err != nil { + return err + } + } else { + return fmt.Errorf("not support type:%s", aggDef.Get("type").MustString()) + } + } else { + for i, b := range esAgg.Get("buckets").MustArray() { + field := aggDef.Get("field").MustString() + bucket := simplejson.NewFromAny(b) + newProps := props + if key, err := bucket.Get("key").String(); err == nil { + newProps[field] = key + } else { + props["filter"] = strconv.Itoa(i) + } + + if key, err := bucket.Get("key_as_string").String(); err == nil { + props[field] = key + } + rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } } } + return nil } -func mapCopy(originalMap, newMap *map[string]string) { - for k, v := range originalMap { - newMap[k] = v - } - -} - -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target QueryBuilder, props map[string]string) ([]*tsdb.TimeSeries, error) { - var series []*tsdb.TimeSeries +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string) (error) { for _, v := range target.Metrics { metric := simplejson.NewFromAny(v) if metric.Get("hide").MustBool(false) { continue } - metricId := fmt.Sprintf("%d", metric.Get("id").MustInt()) - metricField := metric.Get("field").MustString() - switch metric.Get("type").MustString() { + metricId := metric.Get("id").MustString() + metricField := metric.Get("field").MustString() + metricType := metric.Get("type").MustString() + + switch metricType { case "count": newSeries := tsdb.TimeSeries{} - for _, v := range esAgg.Get("buckets").MustMap() { + for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) - value := bucket.Get("doc_count").MustFloat64() - key := bucket.Get("key").MustFloat64() - newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + value := castToNullFloat(bucket.Get("doc_count")) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } newSeries.Tags = props newSeries.Tags["metric"] = "count" - series = append(series, &newSeries) + *series = append(*series, &newSeries) case "percentiles": buckets := esAgg.Get("buckets").MustArray() @@ -98,14 +111,142 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta newSeries.Tags["field"] = metricField for _, v := range buckets { bucket := simplejson.NewFromAny(v) - valueStr := bucket.GetPath(metricId, "values", percentileName).MustString() - value, _ := strconv.ParseFloat(valueStr, 64) - key := bucket.Get("key").MustFloat64() - newSeries.Points = append(newSeries.Points, tsdb.TimePoint{null.FloatFromPtr(&value), null.FloatFromPtr(&key)}) + value := castToNullFloat(bucket.GetPath(metricId, "values", percentileName)) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } - series = append(series, &newSeries) + *series = append(*series, &newSeries) + } + default: + newSeries := tsdb.TimeSeries{} + newSeries.Tags = props + newSeries.Tags["metric"] = metricType + newSeries.Tags["field"] = metricField + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + valueObj, err := bucket.Get(metricId).Map() + if err != nil { + break + } + var value null.Float + if _, ok := valueObj["normalized_value"]; ok { + value = castToNullFloat(bucket.GetPath(metricId, "normalized_value")) + } else { + value = castToNullFloat(bucket.GetPath(metricId, "value")) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } + } + return nil +} + +func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *QueryBuilder) { + set := make(map[string]string) + for _, v := range *seriesList { + if metricType, exists := v.Tags["metric"]; exists { + if _, ok := set[metricType]; !ok { + set[metricType] = "" } } } - return series + metricTypeCount := len(set) + for _, series := range *seriesList { + series.Name = rp.getSeriesName(series, target, metricTypeCount) + } + +} + +func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *QueryBuilder, metricTypeCount int) (string) { + metricName := rp.getMetricName(series.Tags["metric"]) + delete(series.Tags, "metric") + + field := "" + if v, ok := series.Tags["field"]; ok { + field = v + delete(series.Tags, "field") + } + + if target.Alias != "" { + var re = regexp.MustCompile(`{{([\s\S]+?)}}`) + for _, match := range re.FindAllString(target.Alias, -1) { + group := match[2:len(match)-2] + + if strings.HasPrefix(group, "term ") { + if term, ok := series.Tags["term "]; ok { + strings.Replace(target.Alias, match, term, 1) + } + } + if v, ok := series.Tags[group]; ok { + strings.Replace(target.Alias, match, v, 1) + } + + switch group { + case "metric": + strings.Replace(target.Alias, match, metricName, 1) + case "field": + strings.Replace(target.Alias, match, field, 1) + } + + } + } + // todo, if field and pipelineAgg + if field != "" { + metricName += " " + field + } + + if len(series.Tags) == 0 { + return metricName + } + + name := "" + for _, v := range series.Tags { + name += v + " " + } + + if metricTypeCount == 1 { + return strings.TrimSpace(name) + } + + return strings.TrimSpace(name) + " " + metricName + +} + +func (rp *ElasticsearchResponseParser) getMetricName(metric string) string { + if text, ok := metricAggType[metric]; ok { + return text + } + + if text, ok := extendedStats[metric]; ok { + return text + } + + return metric +} + +func castToNullFloat(j *simplejson.Json) null.Float { + f, err := j.Float64() + if err == nil { + return null.FloatFrom(f) + } + + s, err := j.String() + if err == nil { + v, _ := strconv.ParseFloat(s, 64) + return null.FloatFromPtr(&v) + } + + return null.NewFloat(0, false) +} + +func findAgg(target *QueryBuilder, aggId string) (*simplejson.Json, error) { + for _, v := range target.BucketAggs { + aggDef := simplejson.NewFromAny(v) + if aggId == aggDef.Get("id").MustString() { + return aggDef, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggId) } From d6cdc2497c929039f93830dd8b7a61661046ae57 Mon Sep 17 00:00:00 2001 From: wph95 Date: Mon, 26 Mar 2018 16:13:14 +0800 Subject: [PATCH 0038/1100] Handle Interval Date Format similar to the JS variant https://github.com/grafana/grafana/pull/10343/commits/7e14e272fa37df5b4d412c16845d1e525711f726 --- Gopkg.lock | 8 +- Gopkg.toml | 4 + pkg/tsdb/elasticsearch/model_parser.go | 46 +- pkg/tsdb/elasticsearch/model_parser_test.go | 49 + vendor/github.com/leibowitz/moment/diff.go | 75 ++ vendor/github.com/leibowitz/moment/moment.go | 1185 +++++++++++++++++ .../leibowitz/moment/moment_parser.go | 100 ++ .../github.com/leibowitz/moment/parse_day.go | 32 + .../leibowitz/moment/strftime_parser.go | 68 + 9 files changed, 1559 insertions(+), 8 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/model_parser_test.go create mode 100644 vendor/github.com/leibowitz/moment/diff.go create mode 100644 vendor/github.com/leibowitz/moment/moment.go create mode 100644 vendor/github.com/leibowitz/moment/moment_parser.go create mode 100644 vendor/github.com/leibowitz/moment/parse_day.go create mode 100644 vendor/github.com/leibowitz/moment/strftime_parser.go diff --git a/Gopkg.lock b/Gopkg.lock index ebadad8331b..78316b77664 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -295,6 +295,12 @@ packages = ["."] revision = "7cafcd837844e784b526369c9bce262804aebc60" +[[projects]] + branch = "master" + name = "github.com/leibowitz/moment" + packages = ["."] + revision = "8548108dcca204a1110b99e5fec966817499fe84" + [[projects]] branch = "master" name = "github.com/lib/pq" @@ -642,6 +648,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5e65aeace832f1b4be17e7ff5d5714513c40f31b94b885f64f98f2332968d7c6" + inputs-digest = "9895ff7b1516b9639d0fc280ca155c8958486656a2086fc45e91f727fccea0d2" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index df163e01ed3..1f8cbba6e11 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -201,3 +201,7 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" + +[[constraint]] + branch = "master" + name = "github.com/leibowitz/moment" diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 233a35efdc6..7da6765e06c 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" + "github.com/leibowitz/moment" "src/github.com/davecgh/go-spew/spew" "strconv" "strings" @@ -63,7 +64,7 @@ func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = qp.getIndexList() + header.Index = getIndexList(qp.DsInfo.Database, qp.DsInfo.JsonData.Get("interval").MustString(""), qp.TimeRange) if esVersion >= 56 { header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() @@ -87,11 +88,42 @@ func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simple return payload, nil } -func (qp *ElasticSearchQueryParser) getIndexList() string { - _, err := qp.DsInfo.JsonData.Get("interval").String() - if err != nil { - return qp.DsInfo.Database +func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { + if interval == "" { + return pattern } - // todo: support interval - return qp.DsInfo.Database + + var indexes []string + indexParts := strings.Split(strings.TrimLeft(pattern, "["), "]") + indexBase := indexParts[0] + if len(indexParts) <= 1 { + return pattern + } + + indexDateFormat := indexParts[1] + + start := moment.NewMoment(timeRange.MustGetFrom()) + end := moment.NewMoment(timeRange.MustGetTo()) + + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + for start.IsBefore(*end) { + switch interval { + case "Hourly": + start = start.AddHours(1) + + case "Daily": + start = start.AddDay() + + case "Weekly": + start = start.AddWeeks(1) + + case "Monthly": + start = start.AddMonths(1) + + case "Yearly": + start = start.AddYears(1) + } + indexes = append(indexes, fmt.Sprintf("%s%s", indexBase, start.Format(indexDateFormat))) + } + return strings.Join(indexes, ",") } diff --git a/pkg/tsdb/elasticsearch/model_parser_test.go b/pkg/tsdb/elasticsearch/model_parser_test.go new file mode 100644 index 00000000000..aa7336fb69b --- /dev/null +++ b/pkg/tsdb/elasticsearch/model_parser_test.go @@ -0,0 +1,49 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "strconv" + "strings" + "testing" +) + +func makeTime(hour int) string { + //unixtime 1500000000 == 2017-07-14T02:40:00+00:00 + return strconv.Itoa((1500000000 + hour*60*60) * 1000) +} + +func getIndexListByTime(pattern string, interval string, hour int) string { + timeRange := &tsdb.TimeRange{ + From: makeTime(0), + To: makeTime(hour), + } + return getIndexList(pattern, interval, timeRange) +} + +func TestElasticsearchGetIndexList(t *testing.T) { + Convey("Test Elasticsearch getIndex ", t, func() { + + Convey("Parse Interval Formats", func() { + So(getIndexListByTime("[logstash-]YYYY.MM.DD", "Daily", 48), + ShouldEqual, "logstash-2017.07.14,logstash-2017.07.15,logstash-2017.07.16") + + So(len(strings.Split(getIndexListByTime("[logstash-]YYYY.MM.DD.HH", "Hourly", 3), ",")), + ShouldEqual, 4) + + So(getIndexListByTime("[logstash-]YYYY.W", "Weekly", 100), + ShouldEqual, "logstash-2017.28,logstash-2017.29") + + So(getIndexListByTime("[logstash-]YYYY.MM", "Monthly", 700), + ShouldEqual, "logstash-2017.07,logstash-2017.08") + + So(getIndexListByTime("[logstash-]YYYY", "Yearly", 10000), + ShouldEqual, "logstash-2017,logstash-2018,logstash-2019") + }) + + Convey("No Interval", func() { + index := getIndexListByTime("logstash-test", "", 1) + So(index, ShouldEqual, "logstash-test") + }) + }) +} diff --git a/vendor/github.com/leibowitz/moment/diff.go b/vendor/github.com/leibowitz/moment/diff.go new file mode 100644 index 00000000000..0d6b3935adf --- /dev/null +++ b/vendor/github.com/leibowitz/moment/diff.go @@ -0,0 +1,75 @@ +package moment + +import ( + "fmt" + "math" + "time" +) + +// @todo In months/years requires the old and new to calculate correctly, right? +// @todo decide how to handle rounding (i.e. always floor?) +type Diff struct { + duration time.Duration +} + +func (d *Diff) InSeconds() int { + return int(d.duration.Seconds()) +} + +func (d *Diff) InMinutes() int { + return int(d.duration.Minutes()) +} + +func (d *Diff) InHours() int { + return int(d.duration.Hours()) +} + +func (d *Diff) InDays() int { + return int(math.Floor(float64(d.InSeconds()) / 86400)) +} + +// This depends on where the weeks fall? +func (d *Diff) InWeeks() int { + return int(math.Floor(float64(d.InDays() / 7))) +} + +func (d *Diff) InMonths() int { + return 0 +} + +func (d *Diff) InYears() int { + return 0 +} + +// http://momentjs.com/docs/#/durations/humanize/ +func (d *Diff) Humanize() string { + diffInSeconds := d.InSeconds() + + if diffInSeconds <= 45 { + return fmt.Sprintf("%d seconds ago", diffInSeconds) + } else if diffInSeconds <= 90 { + return "a minute ago" + } + + diffInMinutes := d.InMinutes() + + if diffInMinutes <= 45 { + return fmt.Sprintf("%d minutes ago", diffInMinutes) + } else if diffInMinutes <= 90 { + return "an hour ago" + } + + diffInHours := d.InHours() + + if diffInHours <= 22 { + return fmt.Sprintf("%d hours ago", diffInHours) + } else if diffInHours <= 36 { + return "a day ago" + } + + return "diff is in days" +} + +// In Months + +// In years diff --git a/vendor/github.com/leibowitz/moment/moment.go b/vendor/github.com/leibowitz/moment/moment.go new file mode 100644 index 00000000000..13c8ef7dbef --- /dev/null +++ b/vendor/github.com/leibowitz/moment/moment.go @@ -0,0 +1,1185 @@ +package moment + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// links +// http://en.wikipedia.org/wiki/ISO_week_date +// http://golang.org/src/pkg/time/format.go +// http://www.php.net/manual/en/class.datetime.php#datetime.constants.rfc822 +// http://php.net/manual/en/function.date.php +// http://www.php.net/manual/en/datetime.formats.relative.php + +// @todo are these constants needed if they are in the time package? +// There are a lot of extras here, and RFC822 doesn't match up. Why? +// Also, is timezone usage wrong? Double-check +const ( + ATOM = "2006-01-02T15:04:05Z07:00" + COOKIE = "Monday, 02-Jan-06 15:04:05 MST" + ISO8601 = "2006-01-02T15:04:05Z0700" + RFC822 = "Mon, 02 Jan 06 15:04:05 Z0700" + RFC850 = "Monday, 02-Jan-06 15:04:05 MST" + RFC1036 = "Mon, 02 Jan 06 15:04:05 Z0700" + RFC1123 = "Mon, 02 Jan 2006 15:04:05 Z0700" + RFC2822 = "Mon, 02 Jan 2006 15:04:05 Z0700" + RFC3339 = "2006-01-02T15:04:05Z07:00" + RSS = "Mon, 02 Jan 2006 15:04:05 Z0700" + W3C = "2006-01-02T15:04:05Z07:00" +) + +var ( + regex_days = "monday|mon|tuesday|tues|wednesday|wed|thursday|thurs|friday|fri|saturday|sat|sunday|sun" + regex_period = "second|minute|hour|day|week|month|year" + regex_numbers = "one|two|three|four|five|six|seven|eight|nine|ten" +) + +// regexp +var ( + compiled = regexp.MustCompile(`\s{2,}`) + relativeday = regexp.MustCompile(`(yesterday|today|tomorrow)`) + //relative1 = regexp.MustCompile(`(first|last) day of (this|next|last|previous) (week|month|year)`) + //relative2 = regexp.MustCompile(`(first|last) day of (` + "jan|january|feb|february|mar|march|apr|april|may|jun|june|jul|july|aug|august|sep|september|oct|october|nov|november|dec|december" + `)(?:\s(\d{4,4}))?`) + relative3 = regexp.MustCompile(`((?Pthis|next|last|previous) )?(` + regex_days + `)`) + //relativeval = regexp.MustCompile(`([0-9]+) (day|week|month|year)s? ago`) + ago = regexp.MustCompile(`([0-9]+) (` + regex_period + `)s? ago`) + ordinal = regexp.MustCompile("([0-9]+)(st|nd|rd|th)") + written = regexp.MustCompile(regex_numbers) + relativediff = regexp.MustCompile(`([\+\-])?([0-9]+),? ?(` + regex_period + `)s?`) + relativetime = regexp.MustCompile(`(?P\d\d?):(?P\d\d?)(:(?P\d\d?))?\s?(?Pam|pm)?\s?(?P[a-z]{3,3})?|(?Pnoon|midnight)`) + yearmonthday = regexp.MustCompile(`(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})`) + relativeperiod = regexp.MustCompile(`(?Pthis|next|last) (week|month|year)`) + numberRegex = regexp.MustCompile("([0-9]+)(?:)") +) + +// http://golang.org/src/pkg/time/format.go?s=12686:12728#L404 + +// Timezone implementation +// https://groups.google.com/forum/#!topic/golang-nuts/XEVN4QwTvHw +// http://en.wikipedia.org/wiki/Zone.tab + +// Support ISO8601 Duration Parsing? +// http://en.wikipedia.org/wiki/ISO_8601 + +// Differences +// Months are NOT zero-index, MOmentJS they are +// Weeks are 0 indexed +// -- Sunday being the last day of the week ISO-8601 - is that diff from Moment? +// From/FromNow Return a Diff object rather than strings + +// Support for locale and languages with English as default + +// Support for strftime +// https://github.com/benjaminoakes/moment-strftime +// Format: https://php.net/strftime + +type Moment struct { + time time.Time + + Parser +} + +type Parser interface { + Convert(string) string +} + +func New() *Moment { + m := &Moment{time.Now(), new(MomentParser)} + + return m +} + +func NewMoment(t time.Time) *Moment { + m := &Moment{t, new(MomentParser)} + + return m +} + +func (m *Moment) GetTime() time.Time { + return m.time +} + +func (m *Moment) Now() *Moment { + m.time = time.Now().In(m.GetTime().Location()) + + return m +} + +func (m *Moment) Moment(layout string, datetime string) *Moment { + return m.MomentGo(m.Convert(layout), datetime) +} + +func (m *Moment) MomentGo(layout string, datetime string) *Moment { + time, _ := time.Parse(layout, datetime) + + m.time = time + + return m +} + +// This method is nowhere near done - requires lots of work. +func (m *Moment) Strtotime(str string) *Moment { + str = strings.ToLower(strings.TrimSpace(str)) + str = compiled.ReplaceAllString(str, " ") + + // Replace written numbers (i.e. nine, ten) with actual numbers (9, 10) + str = written.ReplaceAllStringFunc(str, func(n string) string { + switch n { + case "one": + return "1" + case "two": + return "2" + case "three": + return "3" + case "four": + return "4" + case "five": + return "5" + case "six": + return "6" + case "seven": + return "7" + case "eight": + return "8" + case "nine": + return "9" + case "ten": + return "10" + } + + return "" + }) + + // Remove ordinal suffixes st, nd, rd, th + str = ordinal.ReplaceAllString(str, "$1") + + // Replace n second|minute|hour... ago to -n second|minute|hour... to consolidate parsing + str = ago.ReplaceAllString(str, "-$1 $2") + + // Look for relative +1day, +3 days 5 hours 15 minutes + if match := relativediff.FindAllStringSubmatch(str, -1); match != nil { + for i := range match { + switch match[i][1] { + case "-": + number, _ := strconv.Atoi(match[i][2]) + m.Subtract(match[i][3], number) + default: + number, _ := strconv.Atoi(match[i][2]) + m.Add(match[i][3], number) + } + + str = strings.Replace(str, match[i][0], "", 1) + } + } + + // Remove any words that aren't needed for consistency + str = strings.Replace(str, " at ", " ", -1) + str = strings.Replace(str, " on ", " ", -1) + + // Support for interchangeable previous/last + str = strings.Replace(str, "previous", "last", -1) + + var dateDefaults = map[string]int{ + "year": 0, + "month": 0, + "day": 0, + } + + dateMatches := dateDefaults + if match := yearmonthday.FindStringSubmatch(str); match != nil { + for i, name := range yearmonthday.SubexpNames() { + if i == 0 { + str = strings.Replace(str, match[i], "", 1) + continue + } + + if match[i] == "" { + continue + } + + if name == "year" || name == "month" || name == "day" { + dateMatches[name], _ = strconv.Atoi(match[i]) + } + + } + + defer m.strtotimeSetDate(dateMatches) + if str == "" { + // Nothing left to parse + return m + } + + str = strings.TrimSpace(str) + } + + // Try to parse out time from the string + var timeDefaults = map[string]int{ + "hour": 0, + "minutes": 0, + "seconds": 0, + } + + timeMatches := timeDefaults + var zone string + if match := relativetime.FindStringSubmatch(str); match != nil { + for i, name := range relativetime.SubexpNames() { + if i == 0 { + str = strings.Replace(str, match[i], "", 1) + continue + } + + if match[i] == "" { + continue + } + + // Midnight is all zero's so nothing to do + if name == "relativetime" && match[i] == "noon" { + timeDefaults["hour"] = 12 + } + + if name == "zone" { + zone = match[i] + } + + if name == "meridiem" && match[i] == "pm" && timeMatches["hour"] < 12 { + timeMatches["hour"] += 12 + } + + if name == "hour" || name == "minutes" || name == "seconds" { + timeMatches[name], _ = strconv.Atoi(match[i]) + } + } + + // Processing time is always last + defer m.strtotimeSetTime(timeMatches, zone) + + if str == "" { + // Nothing left to parse + return m + } + + str = strings.TrimSpace(str) + } + + // m.StartOf("month", "January").GoTo(time.Sunday) + + if match := relativeperiod.FindStringSubmatch(str); match != nil { + period := match[1] + unit := match[2] + + str = strings.Replace(str, match[0], "", 1) + + switch period { + case "next": + if unit == "year" { + m.AddYears(1) + } + if unit == "month" { + m.AddMonths(1) + } + if unit == "week" { + m.AddWeeks(1) + } + case "last": + if unit == "year" { + m.SubYears(1) + } + if unit == "month" { + m.SubMonths(1) + } + if unit == "week" { + m.SubWeeks(1) + } + } + + str = strings.TrimSpace(str) + + // first := regexp.MustCompile("(?Pfirst|last)?") + } + + /* + + relativeday: first day of + relativeperiod: this, last, next + relativeperiodunit week, month, year + day: monday, tues, wednesday + month: january, feb + + + YYYY-MM-DD (HH:MM:SS MST)? + MM-DD-YYYY (HH:MM:SS MST) + 10 September 2015 (HH:MM:SS MST)? + September, 10 2015 (HH:MM:SS MST)? + September 10 2015 (HH:MM:SS M + + this year 2014 + next year 2015 + last year 2013 + + this month April + next month May + last month Mar + + first day of April + last day of April + + + DONE 3PM + DONE 3:00 PM + DONE 3:00:05 MST + 3PM on January 5th + January 5th at 3:00PM + first saturday _of_ next month + first saturday _of_ next month _at_ 3:00PM + saturday of next week + saturday of last week + saturday next week + monday next week + saturday of this week + saturday at 3:00pm + saturday at 4:00PM + saturday at midn + first of january + last of january + january of next year + first day of january + last day of january + first day of February + + DONE midnight + DONE noon + DONE 3 days ago + DONE ten days + DONE 9 weeks ago // Convert to -9 weeks + DONE -9 weeks + + */ + + if match := relativeday.FindStringSubmatch(str); match != nil && len(match) > 1 { + day := match[1] + + str = strings.Replace(str, match[0], "", 1) + + switch day { + case "today": + m.Today() + case "yesterday": + m.Yesterday() + case "tomorrow": + m.Tomorrow() + } + } + + if match := relative3.FindStringSubmatch(str); match != nil { + var when string + for i, name := range relative3.SubexpNames() { + if name == "relperiod" { + when = match[i] + } + } + weekDay := match[len(match)-1] + + str = strings.Replace(str, match[0], "", 1) + + wDay, err := ParseWeekDay(weekDay) + if err == nil { + switch when { + case "last", "previous": + m.GoBackTo(wDay, true) + + case "next": + m.GoTo(wDay, true) + + case "", "this": + m.GoTo(wDay, false) + default: + m.GoTo(wDay, false) + } + } + } + + /* + + + yesterday 11:00 + today 11:00 + tomorrow 11:00 + midnight + noon + DONE +n (second|day|week|month|year)s? + DONE -n (second|day|week|month|year)s? + next (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 + last (monday|tuesday|wednesday|thursday|friday|saturday|sunday) 11:00 + next (month|year) + last (month|year) + first day of (january|february|march...|december) 2014 + last day of (january|february|march...|december) 2014 + first day of (this|next|last) (week|month|year) + last day of (this|next|last) (week|month|year) + first (monday|tuesday|wednesday) of July 2014 + last (monday|tuesday|wednesday) of July 2014 + n (day|week|month|year)s? ago + Monday|Tuesday|Wednesday|Thursday|Friday + Monday (last|this|next) week + + DONE +1 week 2 days 3 hours 4 minutes 5 seconds + */ + + return m +} + +// @todo deal with timezone +func (m *Moment) strtotimeSetTime(time map[string]int, zone string) { + m.SetHour(time["hour"]).SetMinute(time["minutes"]).SetSecond(time["seconds"]) +} + +func (m *Moment) strtotimeSetDate(date map[string]int) { + m.SetYear(date["year"]).SetMonth(time.Month(date["month"])).SetDay(date["day"]) +} + +func (m Moment) Clone() *Moment { + copy := New() + copy.time = m.GetTime() + + return copy +} + +/** + * Getters + * + */ +// https://groups.google.com/forum/#!topic/golang-nuts/pret7hjDc70 +func (m *Moment) Millisecond() { + +} + +func (m *Moment) Second() int { + return m.GetTime().Second() +} + +func (m *Moment) Minute() int { + return m.GetTime().Minute() +} + +func (m *Moment) Hour() int { + return m.GetTime().Hour() +} + +// Day of month +func (m *Moment) Date() int { + return m.DayOfMonth() +} + +// Carbon convenience method +func (m *Moment) DayOfMonth() int { + return m.GetTime().Day() +} + +// Day of week (int or string) +func (m *Moment) Day() time.Weekday { + return m.DayOfWeek() +} + +// Carbon convenience method +func (m *Moment) DayOfWeek() time.Weekday { + return m.GetTime().Weekday() +} + +func (m *Moment) DayOfWeekISO() int { + day := m.GetTime().Weekday() + + if day == time.Sunday { + return 7 + } + + return int(day) +} + +func (m *Moment) DayOfYear() int { + return m.GetTime().YearDay() +} + +// Day of Year with zero padding +func (m *Moment) dayOfYearZero() string { + day := m.GetTime().YearDay() + + if day < 10 { + return fmt.Sprintf("00%d", day) + } + + if day < 100 { + return fmt.Sprintf("0%d", day) + } + + return fmt.Sprintf("%d", day) +} + +// todo panic? +func (m *Moment) Weekday(index int) string { + if index > 6 { + panic("Weekday index must be between 0 and 6") + } + + return time.Weekday(index).String() +} + +func (m *Moment) Week() int { + return 0 +} + +// Is this the week number where as ISOWeekYear is the number of weeks in the year? +// @see http://stackoverflow.com/questions/18478741/get-weeks-in-year +func (m *Moment) ISOWeek() int { + _, week := m.GetTime().ISOWeek() + + return week +} + +// @todo Consider language support +func (m *Moment) Month() time.Month { + return m.GetTime().Month() +} + +func (m *Moment) Quarter() (quarter int) { + quarter = 4 + + switch m.Month() { + case time.January, time.February, time.March: + quarter = 1 + case time.April, time.May, time.June: + quarter = 2 + case time.July, time.August, time.September: + quarter = 3 + } + + return +} + +func (m *Moment) Year() int { + return m.GetTime().Year() +} + +// @see comments for ISOWeek +func (m *Moment) WeekYear() { + +} + +func (m *Moment) ISOWeekYear() { + +} + +/** + * Manipulate + * + */ +func (m *Moment) Add(key string, value int) *Moment { + switch key { + case "years", "year", "y": + m.AddYears(value) + case "months", "month", "M": + m.AddMonths(value) + case "weeks", "week", "w": + m.AddWeeks(value) + case "days", "day", "d": + m.AddDays(value) + case "hours", "hour", "h": + m.AddHours(value) + case "minutes", "minute", "m": + m.AddMinutes(value) + case "seconds", "second", "s": + m.AddSeconds(value) + case "milliseconds", "millisecond", "ms": + + } + + return m +} + +// Carbon +func (m *Moment) AddSeconds(seconds int) *Moment { + return m.addTime(time.Second * time.Duration(seconds)) +} + +// Carbon +func (m *Moment) AddMinutes(minutes int) *Moment { + return m.addTime(time.Minute * time.Duration(minutes)) +} + +// Carbon +func (m *Moment) AddHours(hours int) *Moment { + return m.addTime(time.Hour * time.Duration(hours)) +} + +// Carbon +func (m *Moment) AddDay() *Moment { + return m.AddDays(1) +} + +// Carbon +func (m *Moment) AddDays(days int) *Moment { + m.time = m.GetTime().AddDate(0, 0, days) + + return m +} + +// Carbon +func (m *Moment) AddWeeks(weeks int) *Moment { + return m.AddDays(weeks * 7) +} + +// Carbon +func (m *Moment) AddMonths(months int) *Moment { + m.time = m.GetTime().AddDate(0, months, 0) + + return m +} + +// Carbon +func (m *Moment) AddYears(years int) *Moment { + m.time = m.GetTime().AddDate(years, 0, 0) + + return m +} + +func (m *Moment) addTime(d time.Duration) *Moment { + m.time = m.GetTime().Add(d) + + return m +} + +func (m *Moment) Subtract(key string, value int) *Moment { + switch key { + case "years", "year", "y": + m.SubYears(value) + case "months", "month", "M": + m.SubMonths(value) + case "weeks", "week", "w": + m.SubWeeks(value) + case "days", "day", "d": + m.SubDays(value) + case "hours", "hour", "h": + m.SubHours(value) + case "minutes", "minute", "m": + m.SubMinutes(value) + case "seconds", "second", "s": + m.SubSeconds(value) + case "milliseconds", "millisecond", "ms": + + } + + return m +} + +// Carbon +func (m *Moment) SubSeconds(seconds int) *Moment { + return m.addTime(time.Second * time.Duration(seconds*-1)) +} + +// Carbon +func (m *Moment) SubMinutes(minutes int) *Moment { + return m.addTime(time.Minute * time.Duration(minutes*-1)) +} + +// Carbon +func (m *Moment) SubHours(hours int) *Moment { + return m.addTime(time.Hour * time.Duration(hours*-1)) +} + +// Carbon +func (m *Moment) SubDay() *Moment { + return m.SubDays(1) +} + +// Carbon +func (m *Moment) SubDays(days int) *Moment { + return m.AddDays(days * -1) +} + +func (m *Moment) SubWeeks(weeks int) *Moment { + return m.SubDays(weeks * 7) +} + +// Carbon +func (m *Moment) SubMonths(months int) *Moment { + return m.AddMonths(months * -1) +} + +// Carbon +func (m *Moment) SubYears(years int) *Moment { + return m.AddYears(years * -1) +} + +// Carbon +func (m *Moment) Today() *Moment { + return m.Now() +} + +// Carbon +func (m *Moment) Tomorrow() *Moment { + return m.Today().AddDay() +} + +// Carbon +func (m *Moment) Yesterday() *Moment { + return m.Today().SubDay() +} + +func (m *Moment) StartOf(key string) *Moment { + switch key { + case "year", "y": + m.StartOfYear() + case "month", "M": + m.StartOfMonth() + case "week", "w": + m.StartOfWeek() + case "day", "d": + m.StartOfDay() + case "hour", "h": + if m.Minute() > 0 { + m.SubMinutes(m.Minute()) + } + + if m.Second() > 0 { + m.SubSeconds(m.Second()) + } + case "minute", "m": + if m.Second() > 0 { + m.SubSeconds(m.Second()) + } + case "second", "s": + + } + + return m +} + +// Carbon +func (m *Moment) StartOfDay() *Moment { + if m.Hour() > 0 { + _, timeOffset := m.GetTime().Zone() + m.SubHours(m.Hour()) + + _, newTimeOffset := m.GetTime().Zone() + diffOffset := timeOffset - newTimeOffset + if diffOffset != 0 { + // we need to adjust for time zone difference + m.AddSeconds(diffOffset) + } + } + + return m.StartOf("hour") +} + +// @todo ISO8601 Starts on Monday +func (m *Moment) StartOfWeek() *Moment { + return m.GoBackTo(time.Monday, false).StartOfDay() +} + +// Carbon +func (m *Moment) StartOfMonth() *Moment { + return m.SetDay(1).StartOfDay() +} + +// Carbon +func (m *Moment) StartOfYear() *Moment { + return m.SetMonth(time.January).SetDay(1).StartOfDay() +} + +// Carbon +func (m *Moment) EndOf(key string) *Moment { + switch key { + case "year", "y": + m.EndOfYear() + case "month", "M": + m.EndOfMonth() + case "week", "w": + m.EndOfWeek() + case "day", "d": + m.EndOfDay() + case "hour", "h": + if m.Minute() < 59 { + m.AddMinutes(59 - m.Minute()) + } + case "minute", "m": + if m.Second() < 59 { + m.AddSeconds(59 - m.Second()) + } + case "second", "s": + + } + + return m +} + +// Carbon +func (m *Moment) EndOfDay() *Moment { + if m.Hour() < 23 { + _, timeOffset := m.GetTime().Zone() + m.AddHours(23 - m.Hour()) + + _, newTimeOffset := m.GetTime().Zone() + diffOffset := newTimeOffset - timeOffset + if diffOffset != 0 { + // we need to adjust for time zone difference + m.SubSeconds(diffOffset) + } + } + + return m.EndOf("hour") +} + +// @todo ISO8601 Ends on Sunday +func (m *Moment) EndOfWeek() *Moment { + return m.GoTo(time.Sunday, false).EndOfDay() +} + +// Carbon +func (m *Moment) EndOfMonth() *Moment { + return m.SetDay(m.DaysInMonth()).EndOfDay() +} + +// Carbon +func (m *Moment) EndOfYear() *Moment { + return m.GoToMonth(time.December, false).EndOfMonth() +} + +// Custom +func (m *Moment) GoTo(day time.Weekday, next bool) *Moment { + if m.Day() == day { + if !next { + return m + } else { + m.AddDay() + } + } + + var diff int + if diff = int(day) - int(m.Day()); diff > 0 { + return m.AddDays(diff) + } + + return m.AddDays(7 + diff) +} + +// Custom +func (m *Moment) GoBackTo(day time.Weekday, previous bool) *Moment { + if m.Day() == day { + if !previous { + return m + } else { + m.SubDay() + } + } + + var diff int + if diff = int(day) - int(m.Day()); diff > 0 { + return m.SubDays(7 - diff) + } + + return m.SubDays(diff * -1) +} + +// Custom +func (m *Moment) GoToMonth(month time.Month, next bool) *Moment { + if m.Month() == month { + if !next { + return m + } else { + m.AddMonths(1) + } + } + + var diff int + if diff = int(month - m.Month()); diff > 0 { + return m.AddMonths(diff) + } + + return m.AddMonths(12 + diff) +} + +// Custom +func (m *Moment) GoBackToMonth(month time.Month, previous bool) *Moment { + if m.Month() == month { + if !previous { + return m + } else { + m.SubMonths(1) + } + } + + var diff int + if diff = int(month) - int(m.Month()); diff > 0 { + return m.SubMonths(12 - diff) + } + + return m.SubMonths(diff * -1) +} + +func (m *Moment) SetSecond(seconds int) *Moment { + if seconds >= 0 && seconds <= 60 { + return m.AddSeconds(seconds - m.Second()) + } + + return m +} + +func (m *Moment) SetMinute(minute int) *Moment { + if minute >= 0 && minute <= 60 { + return m.AddMinutes(minute - m.Minute()) + } + + return m +} + +func (m *Moment) SetHour(hour int) *Moment { + if hour >= 0 && hour <= 23 { + return m.AddHours(hour - m.Hour()) + } + + return m +} + +// Custom +func (m *Moment) SetDay(day int) *Moment { + if m.DayOfMonth() == day { + return m + } + + return m.AddDays(day - m.DayOfMonth()) +} + +// Custom +func (m *Moment) SetMonth(month time.Month) *Moment { + if m.Month() > month { + return m.GoBackToMonth(month, false) + } + + return m.GoToMonth(month, false) +} + +// Custom +func (m *Moment) SetYear(year int) *Moment { + if m.Year() == year { + return m + } + + return m.AddYears(year - m.Year()) +} + +// UTC Mode. @see http://momentjs.com/docs/#/parsing/utc/ +func (m *Moment) UTC() *Moment { + return m +} + +// http://momentjs.com/docs/#/manipulating/timezone-offset/ +func (m *Moment) Zone() int { + _, offset := m.GetTime().Zone() + + return (offset / 60) * -1 +} + +/** + * Display + * + */ +func (m *Moment) Format(layout string) string { + format := m.Convert(layout) + hasCustom := false + + formatted := m.GetTime().Format(format) + + if strings.Contains(formatted, "", fmt.Sprintf("%d", m.Unix()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.ISOWeek()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeek()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfWeekISO()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.DayOfYear()), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Quarter()), -1) + formatted = strings.Replace(formatted, "", m.dayOfYearZero(), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", m.Hour()), -1) + } + + // This has to happen after time.Format + if hasCustom && strings.Contains(formatted, "") { + formatted = numberRegex.ReplaceAllStringFunc(formatted, func(n string) string { + ordinal, _ := strconv.Atoi(strings.Replace(n, "", "", 1)) + return m.ordinal(ordinal) + }) + } + + return formatted +} + +func (m *Moment) FormatGo(layout string) string { + return m.GetTime().Format(layout) +} + +// From Dmytro Shteflyuk @https://groups.google.com/forum/#!topic/golang-nuts/l8NhI74jl-4 +func (m *Moment) ordinal(x int) string { + suffix := "th" + switch x % 10 { + case 1: + if x%100 != 11 { + suffix = "st" + } + case 2: + if x%100 != 12 { + suffix = "nd" + } + case 3: + if x%100 != 13 { + suffix = "rd" + } + } + + return strconv.Itoa(x) + suffix +} + +func (m *Moment) FromNow() Diff { + now := new(Moment) + now.Now() + + return m.From(now) +} + +// Carbon +func (m *Moment) From(f *Moment) Diff { + return m.GetDiff(f) +} + +/** + * Difference + * + */ +func (m *Moment) Diff(t *Moment, unit string) int { + diff := m.GetDiff(t) + + switch unit { + case "years": + return diff.InYears() + case "months": + return diff.InMonths() + case "weeks": + return diff.InWeeks() + case "days": + return diff.InDays() + case "hours": + return diff.InHours() + case "minutes": + return diff.InMinutes() + case "seconds": + return diff.InSeconds() + } + + return 0 +} + +// Custom +func (m *Moment) GetDiff(t *Moment) Diff { + duration := m.GetTime().Sub(t.GetTime()) + + return Diff{duration} +} + +/** + * Display + * + */ +func (m *Moment) ValueOf() int64 { + return m.Unix() * 1000 +} + +func (m *Moment) Unix() int64 { + return m.GetTime().Unix() +} + +func (m *Moment) DaysInMonth() int { + days := 31 + switch m.Month() { + case time.April, time.June, time.September, time.November: + days = 30 + break + case time.February: + days = 28 + if m.IsLeapYear() { + days = 29 + } + break + } + + return days +} + +// or ToSlice? +func (m *Moment) ToArray() []int { + return []int{ + m.Year(), + int(m.Month()), + m.DayOfMonth(), + m.Hour(), + m.Minute(), + m.Second(), + } +} + +/** + * Query + * + */ +func (m *Moment) IsBefore(t Moment) bool { + return m.GetTime().Before(t.GetTime()) +} + +func (m *Moment) IsSame(t *Moment, layout string) bool { + return m.Format(layout) == t.Format(layout) +} + +func (m *Moment) IsAfter(t Moment) bool { + return m.GetTime().After(t.GetTime()) +} + +// Carbon +func (m *Moment) IsToday() bool { + today := m.Clone().Today() + + return m.Year() == today.Year() && m.Month() == today.Month() && m.Day() == today.Day() +} + +// Carbon +func (m *Moment) IsTomorrow() bool { + tomorrow := m.Clone().Tomorrow() + + return m.Year() == tomorrow.Year() && m.Month() == tomorrow.Month() && m.Day() == tomorrow.Day() +} + +// Carbon +func (m *Moment) IsYesterday() bool { + yesterday := m.Clone().Yesterday() + + return m.Year() == yesterday.Year() && m.Month() == yesterday.Month() && m.Day() == yesterday.Day() +} + +// Carbon +func (m *Moment) IsWeekday() bool { + return !m.IsWeekend() +} + +// Carbon +func (m *Moment) IsWeekend() bool { + return m.DayOfWeek() == time.Sunday || m.DayOfWeek() == time.Saturday +} + +func (m *Moment) IsLeapYear() bool { + year := m.Year() + return year%4 == 0 && (year%100 != 0 || year%400 == 0) +} + +// Custom +func (m *Moment) Range(start Moment, end Moment) bool { + return m.IsAfter(start) && m.IsBefore(end) +} diff --git a/vendor/github.com/leibowitz/moment/moment_parser.go b/vendor/github.com/leibowitz/moment/moment_parser.go new file mode 100644 index 00000000000..3361cfba113 --- /dev/null +++ b/vendor/github.com/leibowitz/moment/moment_parser.go @@ -0,0 +1,100 @@ +package moment + +import ( + "regexp" + "strings" +) + +type MomentParser struct{} + +var ( + date_pattern = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") +) + +/* + + S (makes any number before it ordinal) + + stdDayOfYear 1,2,365 + + stdDayOfYearZero 001, 002, 365 + + stdDayOfWeek w 0, 1, 2 numeric day of the week (0 = sunday) + + stdDayOfWeekISO N 1 = Monday + + stdWeekOfYear W Iso week number of year + + stdUnix U + + stdQuarter +*/ + +// Thanks to https://github.com/fightbulc/moment.php for replacement keys and regex +var moment_replacements = map[string]string{ + "M": "1", // stdNumMonth 1 2 ... 11 12 + "Mo": "1", // stdNumMonth 1st 2nd ... 11th 12th + "MM": "01", // stdZeroMonth 01 02 ... 11 12 + "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec + "MMMM": "January", // stdLongMonth January February ... November December + "D": "2", // stdDay 1 2 ... 30 30 + "Do": "2", // stdDay 1st 2nd ... 30th 31st @todo support st nd th etch + "DD": "02", // stdZeroDay 01 02 ... 30 31 + "DDD": "", // Day of the year 1 2 ... 364 365 + "DDDo": "", // Day of the year 1st 2nd ... 364th 365th + "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** + "d": "", // Numeric representation of day of the week 0 1 ... 5 6 + "do": "", // 0th 1st ... 5th 6th + "dd": "Mon", // ***Su Mo ... Fr Sa @todo + "ddd": "Mon", // Sun Mon ... Fri Sat + "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo + "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "w": "", // 1 2 ... 52 53 + "wo": "", // 1st 2nd ... 52nd 53rd + "ww": "", // ***01 02 ... 52 53 @todo + "W": "", // 1 2 ... 52 53 + "Wo": "", // 1st 2nd ... 52nd 53rd + "WW": "", // ***01 02 ... 52 53 @todo + "YY": "06", // stdYear 70 71 ... 29 30 + "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 + // "gg" : "o", // ISO-8601 year number 70 71 ... 29 30 @todo + // "gggg" : "o", // ***1970 1971 ... 2029 2030 @todo + // "GG" : "o", //70 71 ... 29 30 @todo + // "GGGG" : "o", // ***1970 1971 ... 2029 2030 @todo + "Q": "", + "A": "PM", // stdPM AM PM + "a": "pm", // stdpm am pm + "H": "", // stdHour 0 1 ... 22 23 + "HH": "15", // 00 01 ... 22 23 + "h": "3", // stdHour12 1 2 ... 11 12 + "hh": "03", // stdZeroHour12 01 02 ... 11 12 + "m": "4", // stdZeroMinute 0 1 ... 58 59 + "mm": "04", // stdZeroMinute 00 01 ... 58 59 + "s": "5", // stdSecond 0 1 ... 58 59 + "ss": "05", // stdZeroSecond ***00 01 ... 58 59 + // "S" : "", //0 1 ... 8 9 + // "SS" : "", //0 1 ... 98 99 + // "SSS" : "", //0 1 ... 998 999 + "z": "MST", //EST CST ... MST PST + "zz": "MST", //EST CST ... MST PST + "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 + "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "X": "", // Seconds since unix epoch 1360013296 + "LT": "3:04 PM", // 8:30 PM + "L": "01/02/2006", //09/04/1986 + "l": "1/2/2006", //9/4/1986 + "LL": "January 2 2006", //September 4th 1986 the php s flag isn't supported + "ll": "Jan 2 2006", //Sep 4 1986 + "LLL": "January 2 2006 3:04 PM", //September 4th 1986 8:30 PM @todo the php s flag isn't supported + "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM + "LLLL": "Monday, January 2 2006 3:04 PM", //Thursday, September 4th 1986 8:30 PM the php s flag isn't supported + "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM +} + +func (p *MomentParser) Convert(layout string) string { + var match [][]string + if match = date_pattern.FindAllStringSubmatch(layout, -1); match == nil { + return layout + } + + for i := range match { + if replace, ok := moment_replacements[match[i][0]]; ok { + layout = strings.Replace(layout, match[i][0], replace, 1) + } + } + + return layout +} diff --git a/vendor/github.com/leibowitz/moment/parse_day.go b/vendor/github.com/leibowitz/moment/parse_day.go new file mode 100644 index 00000000000..e8e890a462e --- /dev/null +++ b/vendor/github.com/leibowitz/moment/parse_day.go @@ -0,0 +1,32 @@ +package moment + +import ( + "fmt" + "strings" + "time" +) + +var ( + days = []time.Weekday{ + time.Sunday, + time.Monday, + time.Tuesday, + time.Wednesday, + time.Thursday, + time.Friday, + time.Saturday, + } +) + +func ParseWeekDay(day string) (time.Weekday, error) { + + day = strings.ToLower(day) + + for _, d := range days { + if day == strings.ToLower(d.String()) { + return d, nil + } + } + + return -1, fmt.Errorf("Unable to parse %s as week day", day) +} diff --git a/vendor/github.com/leibowitz/moment/strftime_parser.go b/vendor/github.com/leibowitz/moment/strftime_parser.go new file mode 100644 index 00000000000..3c024376535 --- /dev/null +++ b/vendor/github.com/leibowitz/moment/strftime_parser.go @@ -0,0 +1,68 @@ +package moment + +import ( + "regexp" + "strings" +) + +type StrftimeParser struct{} + +var ( + replacements_pattern = regexp.MustCompile("%[mbhBedjwuaAVgyGYpPkHlIMSZzsTrRTDFXx]") +) + +// Not implemented +// U +// C + +var strftime_replacements = map[string]string{ + "%m": "01", // stdZeroMonth 01 02 ... 11 12 + "%b": "Jan", // stdMonth Jan Feb ... Nov Dec + "%h": "Jan", + "%B": "January", // stdLongMonth January February ... November December + "%e": "2", // stdDay 1 2 ... 30 30 + "%d": "02", // stdZeroDay 01 02 ... 30 31 + "%j": "", // Day of the year ***001 002 ... 364 365 @todo**** + "%w": "", // Numeric representation of day of the week 0 1 ... 5 6 + "%u": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "%a": "Mon", // Sun Mon ... Fri Sat + "%A": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "%V": "", // ***01 02 ... 52 53 @todo begin with zeros + "%g": "06", // stdYear 70 71 ... 29 30 + "%y": "06", + "%G": "2006", // stdLongYear 1970 1971 ... 2029 2030 + "%Y": "2006", + "%p": "PM", // stdPM AM PM + "%P": "pm", // stdpm am pm + "%k": "15", // stdHour 0 1 ... 22 23 + "%H": "15", // 00 01 ... 22 23 + "%l": "3", // stdHour12 1 2 ... 11 12 + "%I": "03", // stdZeroHour12 01 02 ... 11 12 + "%M": "04", // stdZeroMinute 00 01 ... 58 59 + "%S": "05", // stdZeroSecond ***00 01 ... 58 59 + "%Z": "MST", //EST CST ... MST PST + "%z": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "%s": "", // Seconds since unix epoch 1360013296 + "%r": "03:04:05 PM", + "%R": "15:04", + "%T": "15:04:05", + "%D": "01/02/06", + "%F": "2006-01-02", + "%X": "15:04:05", + "%x": "01/02/06", +} + +func (p *StrftimeParser) Convert(layout string) string { + var match [][]string + if match = replacements_pattern.FindAllStringSubmatch(layout, -1); match == nil { + return layout + } + + for i := range match { + if replace, ok := strftime_replacements[match[i][0]]; ok { + layout = strings.Replace(layout, match[i][0], replace, 1) + } + } + + return layout +} From 12600a0e959866036058092d35f6b7414e98dd65 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 13:19:14 +0200 Subject: [PATCH 0039/1100] support non-nested menu entries --- public/app/plugins/datasource/postgres/query_ctrl.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4a2a0ce6c1c..8a5cb273ef7 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -205,7 +205,11 @@ export class PostgresQueryCtrl extends QueryCtrl { } addSelectPart(selectParts, cat, subitem) { - this.queryModel.addSelectPart(selectParts, subitem.value); + if ("submenu" in cat) { + this.queryModel.addSelectPart(selectParts, subitem.value); + } else { + this.queryModel.addSelectPart(selectParts, cat.value); + } this.panelCtrl.refresh(); } From 63a200686e065a79fdd7ade563fd942236c4feda Mon Sep 17 00:00:00 2001 From: wph95 Date: Mon, 26 Mar 2018 19:48:57 +0800 Subject: [PATCH 0040/1100] - pipeline aggs support - add some test --- pkg/tsdb/elasticsearch/elasticsearch.go | 42 ++- pkg/tsdb/elasticsearch/model_parser.go | 81 ++---- pkg/tsdb/elasticsearch/models.go | 21 +- pkg/tsdb/elasticsearch/query.go | 182 +++++++----- pkg/tsdb/elasticsearch/query_def.go | 18 ++ pkg/tsdb/elasticsearch/query_test.go | 331 ++++++++++++++++++++++ pkg/tsdb/elasticsearch/response_parser.go | 34 ++- 7 files changed, 557 insertions(+), 152 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/query_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 8fd82a179e8..0ce9eca0972 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -1,6 +1,7 @@ package elasticsearch import ( + "bytes" "context" "encoding/json" "errors" @@ -18,7 +19,8 @@ import ( ) type ElasticsearchExecutor struct { - Transport *http.Transport + QueryParser *ElasticSearchQueryParser + Transport *http.Transport } var ( @@ -47,17 +49,21 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo result := &tsdb.Response{} result.Results = make(map[string]*tsdb.QueryResult) - queryParser := ElasticSearchQueryParser{ - dsInfo, - tsdbQuery.TimeRange, - tsdbQuery.Queries, - } - - payload, targets, err := queryParser.Parse() + queries, err := e.getQuery(dsInfo, tsdbQuery) if err != nil { return nil, err } + buff := bytes.Buffer{} + for _, q := range queries { + s, err := q.Build(tsdbQuery, dsInfo) + if err != nil { + return nil, err + } + buff.WriteString(s) + } + payload := buff.String() + if setting.Env == setting.DEV { glog.Debug("Elasticsearch playload", "raw playload", payload) } @@ -96,12 +102,30 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, errors.New(res.getErrMsg()) } } - responseParser := ElasticsearchResponseParser{responses.Responses, targets} + responseParser := ElasticsearchResponseParser{responses.Responses, queries} queryRes := responseParser.getTimeSeries() result.Results["A"] = queryRes return result, nil } +func (e *ElasticsearchExecutor) getQuery(dsInfo *models.DataSource, context *tsdb.TsdbQuery) ([]*Query, error) { + queries := make([]*Query, 0) + if len(context.Queries) == 0 { + return nil, fmt.Errorf("query request contains no queries") + } + for _, v := range context.Queries { + + query, err := e.QueryParser.Parse(v.Model, dsInfo) + if err != nil { + return nil, err + } + queries = append(queries, query) + + } + return queries, nil + +} + func (e *ElasticsearchExecutor) createRequest(dsInfo *models.DataSource, query string) (*http.Request, error) { u, _ := url.Parse(dsInfo.Url) u.Path = path.Join(u.Path, "_msearch") diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 7da6765e06c..0d016dc58a5 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -1,62 +1,45 @@ package elasticsearch import ( - "bytes" - "encoding/json" "fmt" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "github.com/leibowitz/moment" - "src/github.com/davecgh/go-spew/spew" - "strconv" "strings" "time" ) type ElasticSearchQueryParser struct { - DsInfo *models.DataSource - TimeRange *tsdb.TimeRange - Queries []*tsdb.Query } -func (qp *ElasticSearchQueryParser) Parse() (string, []*QueryBuilder, error) { - payload := bytes.Buffer{} - queryHeader := qp.getQueryHeader() - targets := make([]*QueryBuilder, 0) - for _, q := range qp.Queries { - timeField, err := q.Model.Get("timeField").String() - if err != nil { - return "", nil, err - } - rawQuery := q.Model.Get("query").MustString("") - bucketAggs := q.Model.Get("bucketAggs").MustArray() - metrics := q.Model.Get("metrics").MustArray() - alias := q.Model.Get("alias").MustString("") - builder := QueryBuilder{timeField, rawQuery, bucketAggs, metrics, alias} - targets = append(targets, &builder) - - query, err := builder.Build() - if err != nil { - return "", nil, err - } - queryBytes, err := json.Marshal(query) - if err != nil { - return "", nil, err - } - - payload.WriteString(queryHeader.String() + "\n") - payload.WriteString(string(queryBytes) + "\n") +func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource) (*Query, error) { + //payload := bytes.Buffer{} + //queryHeader := qp.getQueryHeader() + timeField, err := model.Get("timeField").String() + if err != nil { + return nil, err + } + rawQuery := model.Get("query").MustString("") + bucketAggs := model.Get("bucketAggs").MustArray() + metrics := model.Get("metrics").MustArray() + alias := model.Get("alias").MustString("") + parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) + if err != nil { + return nil, err } - p, err := qp.payloadReplace(payload.String(), qp.DsInfo.JsonData) - - return p, targets, err + return &Query{timeField, + rawQuery, + bucketAggs, + metrics, + alias, + parsedInterval}, nil } -func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { +func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { var header QueryHeader - esVersion := qp.DsInfo.JsonData.Get("esVersion").MustInt() + esVersion := dsInfo.JsonData.Get("esVersion").MustInt() searchType := "query_then_fetch" if esVersion < 5 { @@ -64,29 +47,13 @@ func (qp *ElasticSearchQueryParser) getQueryHeader() *QueryHeader { } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = getIndexList(qp.DsInfo.Database, qp.DsInfo.JsonData.Get("interval").MustString(""), qp.TimeRange) + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(""), timeRange) if esVersion >= 56 { - header.MaxConcurrentShardRequests = qp.DsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() + header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() } return &header } -func (qp *ElasticSearchQueryParser) payloadReplace(payload string, model *simplejson.Json) (string, error) { - parsedInterval, err := tsdb.GetIntervalFrom(qp.DsInfo, model, time.Millisecond) - if err != nil { - return "", nil - } - - interval := intervalCalculator.Calculate(qp.TimeRange, parsedInterval) - glog.Warn(spew.Sdump(interval)) - payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", qp.TimeRange.GetFromAsMsEpoch()), -1) - payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", qp.TimeRange.GetToAsMsEpoch()), -1) - payload = strings.Replace(payload, "$interval", interval.Text, -1) - payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) - payload = strings.Replace(payload, "$__interval", interval.Text, -1) - - return payload, nil -} func getIndexList(pattern string, interval string, timeRange *tsdb.TimeRange) string { if interval == "" { diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index d758e2159de..822df2dd4d1 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -1,25 +1,25 @@ package elasticsearch import ( - "github.com/grafana/grafana/pkg/components/simplejson" "bytes" - "fmt" "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" ) type QueryHeader struct { SearchType string `json:"search_type"` IgnoreUnavailable bool `json:"ignore_unavailable"` Index interface{} `json:"index"` - MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests"` + MaxConcurrentShardRequests int `json:"max_concurrent_shard_requests,omitempty"` } -func (q *QueryHeader) String() (string) { +func (q *QueryHeader) String() string { r, _ := json.Marshal(q) return string(r) } -type Query struct { +type Request struct { Query map[string]interface{} `json:"query"` Aggs Aggs `json:"aggs"` Size int `json:"size"` @@ -45,11 +45,10 @@ type FiltersAgg struct { } type TermsAggSetting struct { - Field string `json:"field"` - Size int `json:"size"` - Order map[string]interface{} `json:"order"` - MinDocCount int `json:"min_doc_count"` - Missing string `json:"missing"` + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + Missing string `json:"missing,omitempty"` } type TermsAgg struct { @@ -104,7 +103,7 @@ type Response struct { Aggregations map[string]interface{} `json:"aggregations"` } -func (r *Response) getErrMsg() (string) { +func (r *Response) getErrMsg() string { var msg bytes.Buffer errJson := simplejson.NewFromAny(r.Err) errType, err := errJson.Get("type").String() diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index d6d70e79a2a..51f1ebb5d7a 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -1,81 +1,103 @@ package elasticsearch import ( + "bytes" + "encoding/json" "errors" + "fmt" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" "strconv" + "strings" + "time" ) var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", - Lte: "$timeTo", + Lte: "$timeTo", Format: "epoch_millis"} -type QueryBuilder struct { - TimeField string - RawQuery string - BucketAggs []interface{} - Metrics []interface{} - Alias string +type Query struct { + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []interface{} `json:"bucketAggs"` + Metrics []interface{} `json:"metrics"` + Alias string `json:"Alias"` + Interval time.Duration } -func (b *QueryBuilder) Build() (Query, error) { - var err error - var res Query - res.Query = make(map[string]interface{}) - res.Size = 0 +func (q *Query) Build(queryContext *tsdb.TsdbQuery, dsInfo *models.DataSource) (string, error) { + var req Request + payload := bytes.Buffer{} - if err != nil { - return res, err - } - - boolQuery := BoolQuery{} - boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(b.TimeField, rangeFilterSetting)) - boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, b.RawQuery)) - res.Query["bool"] = boolQuery + req.Size = 0 + q.renderReqQuery(&req) // handle document query - if len(b.BucketAggs) == 0 { - if len(b.Metrics) > 0 { - metric := simplejson.NewFromAny(b.Metrics[0]) + if q.isRawDocumentQuery() { + return "", errors.New("alert not support Raw_Document") + } + + err := q.parseAggs(&req) + if err != nil { + return "", err + } + + reqBytes, err := json.Marshal(req) + reqHeader := getRequestHeader(queryContext.TimeRange, dsInfo) + payload.WriteString(reqHeader.String() + "\n") + payload.WriteString(string(reqBytes) + "\n") + return q.renderTemplate(payload.String(), queryContext) +} + +func (q *Query) isRawDocumentQuery() bool { + if len(q.BucketAggs) == 0 { + if len(q.Metrics) > 0 { + metric := simplejson.NewFromAny(q.Metrics[0]) if metric.Get("type").MustString("") == "raw_document" { - return res, errors.New("alert not support Raw_Document") + return true } } } - aggs, err := b.parseAggs(b.BucketAggs, b.Metrics) - res.Aggs = aggs["aggs"].(Aggs) - - return res, err + return false } -func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{}) (Aggs, error) { - query := make(Aggs) - nestedAggs := query - for _, aggRaw := range bucketAggs { +func (q *Query) renderReqQuery(req *Request) { + req.Query = make(map[string]interface{}) + boolQuery := BoolQuery{} + boolQuery.Filter = append(boolQuery.Filter, newRangeFilter(q.TimeField, rangeFilterSetting)) + boolQuery.Filter = append(boolQuery.Filter, newQueryStringFilter(true, q.RawQuery)) + req.Query["bool"] = boolQuery +} + +func (q *Query) parseAggs(req *Request) error { + aggs := make(Aggs) + nestedAggs := aggs + for _, aggRaw := range q.BucketAggs { esAggs := make(Aggs) aggJson := simplejson.NewFromAny(aggRaw) aggType, err := aggJson.Get("type").String() if err != nil { - return nil, err + return err } id, err := aggJson.Get("id").String() if err != nil { - return nil, err + return err } switch aggType { case "date_histogram": - esAggs["date_histogram"] = b.getDateHistogramAgg(aggJson) + esAggs["date_histogram"] = q.getDateHistogramAgg(aggJson) case "histogram": - esAggs["histogram"] = b.getHistogramAgg(aggJson) + esAggs["histogram"] = q.getHistogramAgg(aggJson) case "filters": - esAggs["filters"] = b.getFilters(aggJson) + esAggs["filters"] = q.getFilters(aggJson) case "terms": - terms := b.getTerms(aggJson) + terms := q.getTerms(aggJson) esAggs["terms"] = terms.Terms esAggs["aggs"] = terms.Aggs case "geohash_grid": - return nil, errors.New("alert not support Geo_Hash_Grid") + return errors.New("alert not support Geo_Hash_Grid") } if _, ok := nestedAggs["aggs"]; !ok { @@ -90,40 +112,51 @@ func (b *QueryBuilder) parseAggs(bucketAggs []interface{}, metrics []interface{} } nestedAggs["aggs"] = make(Aggs) - for _, metricRaw := range metrics { + for _, metricRaw := range q.Metrics { metric := make(Metric) metricJson := simplejson.NewFromAny(metricRaw) id, err := metricJson.Get("id").String() if err != nil { - return nil, err + return err } metricType, err := metricJson.Get("type").String() if err != nil { - return nil, err + return err } if metricType == "count" { continue } - // todo support pipeline Agg + settings := metricJson.Get("settings").MustMap(map[string]interface{}{}) + + if isPipelineAgg(metricType) { + pipelineAgg := metricJson.Get("pipelineAgg").MustString("") + if _, err := strconv.Atoi(pipelineAgg); err == nil { + settings["buckets_path"] = pipelineAgg + } else { + continue + } + + } else { + settings["field"] = metricJson.Get("field").MustString() + } - settings := metricJson.Get("settings").MustMap() - settings["field"] = metricJson.Get("field").MustString() metric[metricType] = settings nestedAggs["aggs"].(Aggs)[id] = metric } - return query, nil + req.Aggs = aggs["aggs"].(Aggs) + return nil } -func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogramAgg { +func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { agg := &DateHistogramAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) interval, err := settings.Get("interval").String() if err == nil { agg.Interval = interval } - agg.Field = b.TimeField + agg.Field = q.TimeField agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} agg.Format = "epoch_millis" @@ -136,10 +169,10 @@ func (b *QueryBuilder) getDateHistogramAgg(model *simplejson.Json) DateHistogram if err == nil { agg.Missing = missing } - return *agg + return agg } -func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { +func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { agg := &HistogramAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) interval, err := settings.Get("interval").String() @@ -155,10 +188,10 @@ func (b *QueryBuilder) getHistogramAgg(model *simplejson.Json) HistogramAgg { if err == nil { agg.Missing = missing } - return *agg + return agg } -func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { +func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { agg := &FiltersAgg{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) for filter := range settings.Get("filters").MustArray() { @@ -170,15 +203,15 @@ func (b *QueryBuilder) getFilters(model *simplejson.Json) FiltersAgg { } agg.Filter[label] = newQueryStringFilter(true, query) } - return *agg + return agg } -func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { +func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { agg := &TermsAgg{Aggs: make(Aggs)} settings := simplejson.NewFromAny(model.Get("settings").Interface()) agg.Terms.Field = model.Get("field").MustString() if settings == nil { - return *agg + return agg } sizeStr := settings.Get("size").MustString("") size, err := strconv.Atoi(sizeStr) @@ -186,17 +219,25 @@ func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { size = 500 } agg.Terms.Size = size - orderBy := settings.Get("orderBy").MustString("") - if orderBy != "" { + orderBy, err := settings.Get("orderBy").String() + if err == nil { agg.Terms.Order = make(map[string]interface{}) agg.Terms.Order[orderBy] = settings.Get("order").MustString("") - // if orderBy is a int, means this fields is metric result value - // TODO set subAggs - } - - minDocCount, err := settings.Get("min_doc_count").Int() - if err == nil { - agg.Terms.MinDocCount = minDocCount + if _, err := strconv.Atoi(orderBy); err != nil { + for _, metricI := range q.Metrics { + metric := simplejson.NewFromAny(metricI) + metricId := metric.Get("id").MustString() + if metricId == orderBy { + subAggs := make(Aggs) + metricField := metric.Get("field").MustString() + metricType := metric.Get("type").MustString() + subAggs[metricType] = map[string]string{"field": metricField} + agg.Aggs = make(Aggs) + agg.Aggs[metricId] = subAggs + break + } + } + } } missing, err := settings.Get("missing").String() @@ -204,5 +245,16 @@ func (b *QueryBuilder) getTerms(model *simplejson.Json) TermsAgg { agg.Terms.Missing = missing } - return *agg + return agg +} + +func (q *Query) renderTemplate(payload string, queryContext *tsdb.TsdbQuery) (string, error) { + timeRange := queryContext.TimeRange + interval := intervalCalculator.Calculate(timeRange, q.Interval) + payload = strings.Replace(payload, "$timeFrom", fmt.Sprintf("%d", timeRange.GetFromAsMsEpoch()), -1) + payload = strings.Replace(payload, "$timeTo", fmt.Sprintf("%d", timeRange.GetToAsMsEpoch()), -1) + payload = strings.Replace(payload, "$interval", interval.Text, -1) + payload = strings.Replace(payload, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + payload = strings.Replace(payload, "$__interval", interval.Text, -1) + return payload, nil } diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go index 5dc02aa359e..6f78f02f346 100644 --- a/pkg/tsdb/elasticsearch/query_def.go +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -24,3 +24,21 @@ var extendedStats = map[string]string{ "std_deviation_bounds_upper": "Std Dev Upper", "std_deviation_bounds_lower": "Std Dev Lower", } + +var pipelineOptions = map[string]string{ + "moving_avg": "moving_avg", + "derivative": "derivative", +} + +func isPipelineAgg(metricType string) bool { + if _, ok := pipelineOptions[metricType]; ok { + return true + } + return false +} + +func describeMetric(metricType, field string) string { + text := metricAggType[metricType] + return text + " " + field + +} diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go new file mode 100644 index 00000000000..992469175b6 --- /dev/null +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -0,0 +1,331 @@ +package elasticsearch + +import ( + "encoding/json" + "fmt" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "reflect" + "strconv" + "strings" + "testing" +) + +func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJSON string) { + var queryExpectedJSONInterface, queryJSONInterface interface{} + parser := ElasticSearchQueryParser{} + model := &Query{} + + err := json.Unmarshal([]byte(requestJSON), model) + So(err, ShouldBeNil) + jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) + dsInfo := &models.DataSource{ + Database: "grafana-test", + JsonData: jsonDate, + } + + testTimeRange := tsdb.NewTimeRange("5m", "now") + + req, _ := simplejson.NewJson([]byte(requestJSON)) + query, err := parser.Parse(req, dsInfo) + s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) + + queryJSON := strings.Split(s, "\n")[1] + err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) + So(err, ShouldBeNil) + + expectedElasticSearchRequestJSON = strings.Replace( + expectedElasticSearchRequestJSON, + "", + strconv.FormatInt(testTimeRange.GetFromAsMsEpoch(), 10), + -1, + ) + + expectedElasticSearchRequestJSON = strings.Replace( + expectedElasticSearchRequestJSON, + "", + strconv.FormatInt(testTimeRange.GetToAsMsEpoch(), 10), + -1, + ) + + err = json.Unmarshal([]byte(expectedElasticSearchRequestJSON), &queryExpectedJSONInterface) + So(err, ShouldBeNil) + + result := reflect.DeepEqual(queryExpectedJSONInterface, queryJSONInterface) + if !result { + fmt.Printf("ERROR: %s \n != \n %s", expectedElasticSearchRequestJSON, queryJSON) + } + So(result, ShouldBeTrue) +} +func TestElasticSearchQueryBuilder(t *testing.T) { + Convey("Elasticsearch QueryBuilder query testing", t, func() { + Convey("Build test average metric with moving average", func() { + var testElasticsearchModelRequestJSON = ` + { + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "metrics": [ + { + "field": "value", + "id": "1", + "inlineScript": "_value * 2", + "meta": {}, + "settings": { + "script": { + "inline": "_value * 2" + } + }, + "type": "avg" + }, + { + "field": "1", + "id": "3", + "meta": {}, + "pipelineAgg": "1", + "settings": { + "minimize": false, + "model": "simple", + "window": 5 + }, + "type": "moving_avg" + } + ], + "query": "(test:query) AND (name:sample)", + "refId": "A", + "timeField": "timestamp" + } + ` + + var expectedElasticsearchQueryJSON = ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "(test:query) AND (name:sample)" + } + } + ] + } + }, + "aggs": { + "2": { + "date_histogram": { + "interval": "200ms", + "field": "timestamp", + "min_doc_count": 0, + "extended_bounds": { + "min": "", + "max": "" + }, + "format": "epoch_millis" + }, + "aggs": { + "1": { + "avg": { + "field": "value", + "script": { + "inline": "_value * 2" + } + } + }, + "3": { + "moving_avg": { + "buckets_path": "1", + "window": 5, + "model": "simple", + "minimize": false + } + } + } + } + } + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Wildcards and Quotes", func() { + testElasticsearchModelRequestJSON := ` + { + "alias": "New", + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "metrics": [ + { + "type": "sum", + "field": "value", + "id": "1" + } + ], + "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", + "refId": "A", + "timeField": "timestamp" + }` + + expectedElasticsearchQueryJSON := ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"" + } + } + ] + } + }, + "aggs": { + "2": { + "aggs": { + "1": { + "sum": { + "field": "value" + } + } + }, + "date_histogram": { + "extended_bounds": { + "max": "", + "min": "" + }, + "field": "timestamp", + "format": "epoch_millis", + "min_doc_count": 0 + } + } + } + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Term Aggregates", func() { + testElasticsearchModelRequestJSON := ` + { + "bucketAggs": [{ + "field": "name_raw", + "id": "4", + "settings": { + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "1m", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + }], + "dsType": "elasticsearch", + "filters": [{ + "boolOp": "AND", + "not": false, + "type": "rfc190Scope", + "value": "*.hmp.metricsd" + }, { + "boolOp": "AND", + "not": false, + "type": "name_raw", + "value": "builtin.general.*_instance_count" + }], + "metricObject": {}, + "metrics": [{ + "field": "value", + "id": "1", + "meta": {}, + "options": {}, + "settings": {}, + "type": "sum" + }], + "mode": 0, + "numToGraph": 10, + "prependHostName": false, + "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", + "refId": "A", + "regexAlias": false, + "selectedApplication": "", + "selectedHost": "", + "selectedLocation": "", + "timeField": "timestamp", + "useFullHostName": "", + "useQuery": false + }` + + expectedElasticsearchQueryJSON := ` + { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "timestamp": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)" + } + } + ] + } + }, + "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} + }` + + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + }) +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index a2a8565641f..01b8cb1d235 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -6,14 +6,14 @@ import ( "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" - "strconv" "regexp" + "strconv" "strings" ) type ElasticsearchResponseParser struct { Responses []Response - Targets []*QueryBuilder + Targets []*Query } func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { @@ -29,7 +29,7 @@ func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { return queryRes } -func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string, depth int) (error) { +func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { @@ -71,7 +71,7 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } -func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *QueryBuilder, series *[]*tsdb.TimeSeries, props map[string]string) (error) { +func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { for _, v := range target.Metrics { metric := simplejson.NewFromAny(v) if metric.Get("hide").MustBool(false) { @@ -143,7 +143,7 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta return nil } -func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *QueryBuilder) { +func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries, target *Query) { set := make(map[string]string) for _, v := range *seriesList { if metricType, exists := v.Tags["metric"]; exists { @@ -159,8 +159,9 @@ func (rp *ElasticsearchResponseParser) nameSeries(seriesList *[]*tsdb.TimeSeries } -func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *QueryBuilder, metricTypeCount int) (string) { - metricName := rp.getMetricName(series.Tags["metric"]) +func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { + metricType := series.Tags["metric"] + metricName := rp.getMetricName(metricType) delete(series.Tags, "metric") field := "" @@ -172,7 +173,7 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta if target.Alias != "" { var re = regexp.MustCompile(`{{([\s\S]+?)}}`) for _, match := range re.FindAllString(target.Alias, -1) { - group := match[2:len(match)-2] + group := match[2 : len(match)-2] if strings.HasPrefix(group, "term ") { if term, ok := series.Tags["term "]; ok { @@ -193,7 +194,20 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta } } // todo, if field and pipelineAgg - if field != "" { + if field != "" && isPipelineAgg(metricType) { + found := false + for _, targetMetricI := range target.Metrics { + targetMetric := simplejson.NewFromAny(targetMetricI) + if targetMetric.Get("id").MustString() == field { + metricName += " " + describeMetric(targetMetric.Get("type").MustString(), field) + found = true + } + } + if !found { + metricName = "Unset" + } + + } else if field != "" { metricName += " " + field } @@ -241,7 +255,7 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *QueryBuilder, aggId string) (*simplejson.Json, error) { +func findAgg(target *Query, aggId string) (*simplejson.Json, error) { for _, v := range target.BucketAggs { aggDef := simplejson.NewFromAny(v) if aggId == aggDef.Get("id").MustString() { From 6c2ef7dca6b34f189ef44c416e98c386117d6010 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 18:34:40 +0200 Subject: [PATCH 0041/1100] handle aggregate functions more generic --- .../plugins/datasource/postgres/query_ctrl.ts | 47 +++++------- .../plugins/datasource/postgres/query_part.ts | 75 ++----------------- 2 files changed, 27 insertions(+), 95 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 8a5cb273ef7..d0ec5dc59be 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -81,30 +81,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } buildSelectMenu() { - - if (!queryPart.hasAggregates()) { - this.datasource.metricFindQuery(this.queryBuilder.buildAggregateQuery()) - .then(results => { - queryPart.clearAggregates(); - _.map(results, segment => { queryPart.registerAggregate(segment.text); }); - }) - .catch(this.handleQueryError.bind(this)); - } - var categories = queryPart.getCategories(); - this.selectMenu = _.reduce( - categories, - function(memo, cat, key) { - var menu = { - text: key, - submenu: cat.map(item => { - return { text: item.type, value: item.type }; - }), - }; - memo.push(menu); - return memo; - }, - [] - ); + this.selectMenu = [ + {text: "aggregate", value: "aggregate"}, + {text: "math", value: "math"}, + {text: "alias", value: "alias"}, + {text: "column", value: "column"}, + ]; } toggleEditorMode() { @@ -216,10 +198,19 @@ export class PostgresQueryCtrl extends QueryCtrl { handleSelectPartEvent(selectParts, part, evt) { switch (evt.name) { case 'get-param-options': { - return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) - .then(this.transformToSegments(true)) - .catch(this.handleQueryError.bind(this)); + switch (part.def.type) { + case "aggregate": + return this.datasource + .metricFindQuery(this.queryBuilder.buildAggregateQuery()) + .then(this.transformToSegments(false)) + .catch(this.handleQueryError.bind(this)); + case "column": + return this.datasource + .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + } case 'part-param-changed': { this.panelCtrl.refresh(); diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index b63ebfae0c1..044f51a2457 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -23,23 +23,16 @@ function register(options: any) { options.category.push(index[options.type]); } -function registerAggregate(name: string) { - register({ - type: name, - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, - }); -} - var groupByTimeFunctions = []; function aliasRenderer(part, innerExpr) { return innerExpr + ' AS ' + '"' + part.params[0] + '"'; } +function aggregateRenderer(part, innerExpr) { + return part.params[0] + '(' + innerExpr + ')'; +} + function columnRenderer(part, innerExpr) { return '"' + part.params[0] + '"'; } @@ -108,59 +101,13 @@ register({ renderer: columnRenderer, }); -// Aggregations register({ - type: 'avg', + type: 'aggregate', addStrategy: replaceAggregationAddStrategy, category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'count', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'sum', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'stddev', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'min', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'max', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, + params: [{name: 'name', type: 'string', dynamicLookup: true}], + defaultParams: ['avg'], + renderer: aggregateRenderer, }); register({ @@ -203,12 +150,6 @@ register({ export default { create: createPart, - registerAggregate: registerAggregate, - clearAggregates: function() { categories.Aggregations = []; }, - hasAggregates: function() { - // FIXME - return categories.Aggregations.length > 6; - }, getCategories: function() { return categories; }, From d6ac7aee899db14d2306ab9828cb39ea9854d853 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 18:50:03 +0200 Subject: [PATCH 0042/1100] remove unused import --- public/app/plugins/datasource/postgres/query_ctrl.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index d0ec5dc59be..dd1da1c75cf 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -2,7 +2,6 @@ import angular from 'angular'; import _ from 'lodash'; import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; -import queryPart from './query_part'; import PostgresQuery from './postgres_query'; export interface QueryMeta { From 4042e4b225ad4b989f3f1ecabb52271547ff2af2 Mon Sep 17 00:00:00 2001 From: wph95 Date: Tue, 27 Mar 2018 02:12:43 +0800 Subject: [PATCH 0043/1100] fix a terms bug and add test --- pkg/tsdb/elasticsearch/models.go | 2 +- pkg/tsdb/elasticsearch/query.go | 6 +- pkg/tsdb/elasticsearch/query_test.go | 97 ++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 822df2dd4d1..6ab6fa9f43e 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -41,7 +41,7 @@ type DateHistogramAgg struct { } type FiltersAgg struct { - Filter map[string]interface{} `json:"filter"` + Filters map[string]interface{} `json:"filters"` } type TermsAggSetting struct { diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index 51f1ebb5d7a..c4e30cfcbf4 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -193,15 +193,17 @@ func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { agg := &FiltersAgg{} + agg.Filters = map[string]interface{}{} settings := simplejson.NewFromAny(model.Get("settings").Interface()) - for filter := range settings.Get("filters").MustArray() { + + for _, filter := range settings.Get("filters").MustArray() { filterJson := simplejson.NewFromAny(filter) query := filterJson.Get("query").MustString("") label := filterJson.Get("label").MustString("") if label == "" { label = query } - agg.Filter[label] = newQueryStringFilter(true, query) + agg.Filters[label] = newQueryStringFilter(true, query) } return agg } diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index 992469175b6..4f7b4d9147e 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -325,6 +325,103 @@ func TestElasticSearchQueryBuilder(t *testing.T) { "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} }` + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + }) + Convey("Test Filters Aggregates", func() { + testElasticsearchModelRequestJSON := ` + { + "bucketAggs": [ + { + "id": "3", + "settings": { + "filters": [{ + "label": "hello", + "query": "host:\"67.65.185.232\"" + }] + }, + "type": "filters" + }, + { + "field": "time", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "pipelineAgg": "select metric", + "field": "bytesSent", + "id": "1", + "meta": {}, + "settings": {}, + "type": "count" + } + ], + "query": "*", + "refId": "A", + "timeField": "time" + }` + + expectedElasticsearchQueryJSON := `{ + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "time": { + "gte": "", + "lte": "", + "format": "epoch_millis" + } + } + }, + { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + ] + } + }, + "aggs": { + "3": { + "filters": { + "filters": { + "hello": { + "query_string": { + "query": "host:\"67.65.185.232\"", + "analyze_wildcard": true + } + } + } + }, + "aggs": { + "2": { + "date_histogram": { + "interval": "200ms", + "field": "time", + "min_doc_count": 0, + "extended_bounds": { + "min": "", + "max": "" + }, + "format": "epoch_millis" + }, + "aggs": {} + } + } + } + } + } + ` + testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) }) }) From 8b3c3081689236be24a21645ca852a928d33d9c7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 20:15:16 +0200 Subject: [PATCH 0044/1100] remove categories from queryPart --- .../datasource/postgres/postgres_query.ts | 5 +---- .../plugins/datasource/postgres/query_part.ts | 19 +------------------ 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4516bf4a4be..a804cb70294 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -82,14 +82,11 @@ export default class PostgresQuery { } removeGroupByPart(part, index) { - var categories = queryPart.getCategories(); - if (part.def.type === 'time') { // remove aggregations this.target.select = _.map(this.target.select, (s: any) => { return _.filter(s, (part: any) => { - var partModel = queryPart.create(part); - if (partModel.def.category === categories.Aggregations) { + if (part.type === "aggregate") { return false; } return true; diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 044f51a2457..0d2b365fe66 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -2,12 +2,6 @@ import _ from 'lodash'; import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/core/components/query_part/query_part'; var index = []; -var categories = { - Aggregations: [], - Math: [], - Aliasing: [], - Columns: [], -}; function createPart(part): any { var def = index[part.type]; @@ -20,11 +14,8 @@ function createPart(part): any { function register(options: any) { index[options.type] = new QueryPartDef(options); - options.category.push(index[options.type]); } -var groupByTimeFunctions = []; - function aliasRenderer(part, innerExpr) { return innerExpr + ' AS ' + '"' + part.params[0] + '"'; } @@ -41,7 +32,7 @@ function replaceAggregationAddStrategy(selectParts, partModel) { // look for existing aggregation for (var i = 0; i < selectParts.length; i++) { var part = selectParts[i]; - if (part.def.category === categories.Aggregations) { + if (part.def.type === "aggregate") { selectParts[i] = partModel; return; } @@ -95,7 +86,6 @@ function addColumnStrategy(selectParts, partModel, query) { register({ type: 'column', addStrategy: addColumnStrategy, - category: categories.Columns, params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], renderer: columnRenderer, @@ -104,7 +94,6 @@ register({ register({ type: 'aggregate', addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, params: [{name: 'name', type: 'string', dynamicLookup: true}], defaultParams: ['avg'], renderer: aggregateRenderer, @@ -113,7 +102,6 @@ register({ register({ type: 'math', addStrategy: addMathStrategy, - category: categories.Math, params: [{ name: 'expr', type: 'string' }], defaultParams: [' / 100'], renderer: suffixRenderer, @@ -122,7 +110,6 @@ register({ register({ type: 'alias', addStrategy: addAliasStrategy, - category: categories.Aliasing, params: [{ name: 'name', type: 'string', quote: 'double' }], defaultParams: ['alias'], renderMode: 'suffix', @@ -131,7 +118,6 @@ register({ register({ type: 'time', - category: groupByTimeFunctions, params: [ { name: 'interval', @@ -150,7 +136,4 @@ register({ export default { create: createPart, - getCategories: function() { - return categories; - }, }; From 06f73321560defb2ac074e4d90af1c94f459943d Mon Sep 17 00:00:00 2001 From: wph95 Date: Wed, 28 Mar 2018 01:42:25 +0800 Subject: [PATCH 0045/1100] cleanup and add more test --- pkg/tsdb/elasticsearch/elasticsearch_test.go | 121 ++++++++++++ pkg/tsdb/elasticsearch/model_parser.go | 65 ++++++- pkg/tsdb/elasticsearch/models.go | 26 ++- pkg/tsdb/elasticsearch/query.go | 113 +++++------ pkg/tsdb/elasticsearch/query_def.go | 1 - pkg/tsdb/elasticsearch/query_test.go | 186 +------------------ pkg/tsdb/elasticsearch/response_parser.go | 50 +++-- 7 files changed, 274 insertions(+), 288 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/elasticsearch_test.go diff --git a/pkg/tsdb/elasticsearch/elasticsearch_test.go b/pkg/tsdb/elasticsearch/elasticsearch_test.go new file mode 100644 index 00000000000..ad905299166 --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch_test.go @@ -0,0 +1,121 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "time" +) + +var avgWithMovingAvg = Query{ + TimeField: "timestamp", + RawQuery: "(test:query) AND (name:sample)", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "avg", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "script": map[string]string{ + "inline": "_value * 2", + }, + }), + }, { + Field: "1", + ID: "3", + Type: "moving_avg", + PipelineAggregate: "1", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "minimize": false, + "model": "simple", + "window": 5, + }), + }}, +} + +var wildcardsAndQuotes = Query{ + TimeField: "timestamp", + RawQuery: "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "sum", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} +var termAggs = Query{ + TimeField: "timestamp", + RawQuery: "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + Field: "name_raw", + ID: "4", + Type: "terms", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "order": "desc", + "orderBy": "_term", + "size": "10", + }), + }, { + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "value", + ID: "1", + Type: "sum", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} + +var filtersAggs = Query{ + TimeField: "time", + RawQuery: "*", + Interval: time.Millisecond, + BucketAggs: []*BucketAgg{{ + ID: "3", + Type: "filters", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "filters": []interface{}{ + map[string]interface{}{"label": "hello", "query": "host:\"67.65.185.232\""}, + }, + }), + }, { + Field: "timestamp", + ID: "2", + Type: "date_histogram", + Settings: simplejson.NewFromAny(map[string]interface{}{ + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0, + }), + }}, + Metrics: []*Metric{{ + Field: "bytesSent", + ID: "1", + Type: "count", + PipelineAggregate: "select metric", + Settings: simplejson.NewFromAny(map[string]interface{}{}), + }}, +} diff --git a/pkg/tsdb/elasticsearch/model_parser.go b/pkg/tsdb/elasticsearch/model_parser.go index 0d016dc58a5..5d94aebef1a 100644 --- a/pkg/tsdb/elasticsearch/model_parser.go +++ b/pkg/tsdb/elasticsearch/model_parser.go @@ -20,9 +20,15 @@ func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models if err != nil { return nil, err } - rawQuery := model.Get("query").MustString("") - bucketAggs := model.Get("bucketAggs").MustArray() - metrics := model.Get("metrics").MustArray() + rawQuery := model.Get("query").MustString() + bucketAggs, err := qp.parseBucketAggs(model) + if err != nil { + return nil, err + } + metrics, err := qp.parseMetrics(model) + if err != nil { + return nil, err + } alias := model.Get("alias").MustString("") parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond) if err != nil { @@ -37,6 +43,57 @@ func (qp *ElasticSearchQueryParser) Parse(model *simplejson.Json, dsInfo *models parsedInterval}, nil } +func (qp *ElasticSearchQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { + var err error + var result []*BucketAgg + for _, t := range model.Get("bucketAggs").MustArray() { + aggJson := simplejson.NewFromAny(t) + agg := &BucketAgg{} + + agg.Type, err = aggJson.Get("type").String() + if err != nil { + return nil, err + } + + agg.ID, err = aggJson.Get("id").String() + if err != nil { + return nil, err + } + + agg.Field = aggJson.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJson.Get("settings").MustMap()) + + result = append(result, agg) + } + return result, nil +} + +func (qp *ElasticSearchQueryParser) parseMetrics(model *simplejson.Json) ([]*Metric, error) { + var err error + var result []*Metric + for _, t := range model.Get("metrics").MustArray() { + metricJson := simplejson.NewFromAny(t) + metric := &Metric{} + + metric.Field = metricJson.Get("field").MustString() + metric.Hide = metricJson.Get("hide").MustBool(false) + metric.ID, err = metricJson.Get("id").String() + if err != nil { + return nil, err + } + + metric.PipelineAggregate = metricJson.Get("pipelineAgg").MustString() + metric.Settings = simplejson.NewFromAny(metricJson.Get("settings").MustMap()) + + metric.Type, err = metricJson.Get("type").String() + if err != nil { + return nil, err + } + + result = append(result, metric) + } + return result, nil +} func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *QueryHeader { var header QueryHeader esVersion := dsInfo.JsonData.Get("esVersion").MustInt() @@ -47,7 +104,7 @@ func getRequestHeader(timeRange *tsdb.TimeRange, dsInfo *models.DataSource) *Que } header.SearchType = searchType header.IgnoreUnavailable = true - header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(""), timeRange) + header.Index = getIndexList(dsInfo.Database, dsInfo.JsonData.Get("interval").MustString(), timeRange) if esVersion >= 56 { header.MaxConcurrentShardRequests = dsInfo.JsonData.Get("maxConcurrentShardRequests").MustInt() diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 6ab6fa9f43e..9cf295cbd0e 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -7,6 +7,22 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" ) +type BucketAgg struct { + Field string `json:"field"` + ID string `json:"id"` + Settings *simplejson.Json `json:"settings"` + Type string `jsons:"type"` +} + +type Metric struct { + Field string `json:"field"` + Hide bool `json:"hide"` + ID string `json:"id"` + PipelineAggregate string `json:"pipelineAgg"` + Settings *simplejson.Json `json:"settings"` + Type string `json:"type"` +} + type QueryHeader struct { SearchType string `json:"search_type"` IgnoreUnavailable bool `json:"ignore_unavailable"` @@ -44,16 +60,16 @@ type FiltersAgg struct { Filters map[string]interface{} `json:"filters"` } -type TermsAggSetting struct { +type TermsAgg struct { Field string `json:"field"` Size int `json:"size"` Order map[string]interface{} `json:"order"` Missing string `json:"missing,omitempty"` } -type TermsAgg struct { - Terms TermsAggSetting `json:"terms"` - Aggs Aggs `json:"aggs"` +type TermsAggWrap struct { + Terms TermsAgg `json:"terms"` + Aggs Aggs `json:"aggs"` } type ExtendedBounds struct { @@ -91,8 +107,6 @@ type BoolQuery struct { Filter []interface{} `json:"filter"` } -type Metric map[string]interface{} - type Responses struct { Responses []Response `json:"responses"` } diff --git a/pkg/tsdb/elasticsearch/query.go b/pkg/tsdb/elasticsearch/query.go index c4e30cfcbf4..a63529df2df 100644 --- a/pkg/tsdb/elasticsearch/query.go +++ b/pkg/tsdb/elasticsearch/query.go @@ -18,11 +18,11 @@ var rangeFilterSetting = RangeFilterSetting{Gte: "$timeFrom", Format: "epoch_millis"} type Query struct { - TimeField string `json:"timeField"` - RawQuery string `json:"query"` - BucketAggs []interface{} `json:"bucketAggs"` - Metrics []interface{} `json:"metrics"` - Alias string `json:"Alias"` + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []*BucketAgg `json:"bucketAggs"` + Metrics []*Metric `json:"metrics"` + Alias string `json:"Alias"` Interval time.Duration } @@ -73,27 +73,17 @@ func (q *Query) renderReqQuery(req *Request) { func (q *Query) parseAggs(req *Request) error { aggs := make(Aggs) nestedAggs := aggs - for _, aggRaw := range q.BucketAggs { + for _, agg := range q.BucketAggs { esAggs := make(Aggs) - aggJson := simplejson.NewFromAny(aggRaw) - aggType, err := aggJson.Get("type").String() - if err != nil { - return err - } - id, err := aggJson.Get("id").String() - if err != nil { - return err - } - - switch aggType { + switch agg.Type { case "date_histogram": - esAggs["date_histogram"] = q.getDateHistogramAgg(aggJson) + esAggs["date_histogram"] = q.getDateHistogramAgg(agg) case "histogram": - esAggs["histogram"] = q.getHistogramAgg(aggJson) + esAggs["histogram"] = q.getHistogramAgg(agg) case "filters": - esAggs["filters"] = q.getFilters(aggJson) + esAggs["filters"] = q.getFilters(agg) case "terms": - terms := q.getTerms(aggJson) + terms := q.getTerms(agg) esAggs["terms"] = terms.Terms esAggs["aggs"] = terms.Aggs case "geohash_grid": @@ -105,59 +95,47 @@ func (q *Query) parseAggs(req *Request) error { } if aggs, ok := (nestedAggs["aggs"]).(Aggs); ok { - aggs[id] = esAggs + aggs[agg.ID] = esAggs } nestedAggs = esAggs } nestedAggs["aggs"] = make(Aggs) - for _, metricRaw := range q.Metrics { - metric := make(Metric) - metricJson := simplejson.NewFromAny(metricRaw) + for _, metric := range q.Metrics { + subAgg := make(Aggs) - id, err := metricJson.Get("id").String() - if err != nil { - return err - } - metricType, err := metricJson.Get("type").String() - if err != nil { - return err - } - if metricType == "count" { + if metric.Type == "count" { continue } + settings := metric.Settings.MustMap(make(map[string]interface{})) - settings := metricJson.Get("settings").MustMap(map[string]interface{}{}) - - if isPipelineAgg(metricType) { - pipelineAgg := metricJson.Get("pipelineAgg").MustString("") - if _, err := strconv.Atoi(pipelineAgg); err == nil { - settings["buckets_path"] = pipelineAgg + if isPipelineAgg(metric.Type) { + if _, err := strconv.Atoi(metric.PipelineAggregate); err == nil { + settings["buckets_path"] = metric.PipelineAggregate } else { continue } } else { - settings["field"] = metricJson.Get("field").MustString() + settings["field"] = metric.Field } - metric[metricType] = settings - nestedAggs["aggs"].(Aggs)[id] = metric + subAgg[metric.Type] = settings + nestedAggs["aggs"].(Aggs)[metric.ID] = subAgg } req.Aggs = aggs["aggs"].(Aggs) return nil } -func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { +func (q *Query) getDateHistogramAgg(target *BucketAgg) *DateHistogramAgg { agg := &DateHistogramAgg{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - interval, err := settings.Get("interval").String() + interval, err := target.Settings.Get("interval").String() if err == nil { agg.Interval = interval } agg.Field = q.TimeField - agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) + agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) agg.ExtendedBounds = ExtendedBounds{"$timeFrom", "$timeTo"} agg.Format = "epoch_millis" @@ -165,66 +143,63 @@ func (q *Query) getDateHistogramAgg(model *simplejson.Json) *DateHistogramAgg { agg.Interval = "$__interval" } - missing, err := settings.Get("missing").String() + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Missing = missing } return agg } -func (q *Query) getHistogramAgg(model *simplejson.Json) *HistogramAgg { +func (q *Query) getHistogramAgg(target *BucketAgg) *HistogramAgg { agg := &HistogramAgg{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - interval, err := settings.Get("interval").String() + interval, err := target.Settings.Get("interval").String() if err == nil { agg.Interval = interval } - field, err := model.Get("field").String() - if err == nil { - agg.Field = field + + if target.Field != "" { + agg.Field = target.Field } - agg.MinDocCount = settings.Get("min_doc_count").MustInt(0) - missing, err := settings.Get("missing").String() + agg.MinDocCount = target.Settings.Get("min_doc_count").MustInt(0) + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Missing = missing } return agg } -func (q *Query) getFilters(model *simplejson.Json) *FiltersAgg { +func (q *Query) getFilters(target *BucketAgg) *FiltersAgg { agg := &FiltersAgg{} agg.Filters = map[string]interface{}{} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - - for _, filter := range settings.Get("filters").MustArray() { + for _, filter := range target.Settings.Get("filters").MustArray() { filterJson := simplejson.NewFromAny(filter) query := filterJson.Get("query").MustString("") label := filterJson.Get("label").MustString("") if label == "" { label = query } + agg.Filters[label] = newQueryStringFilter(true, query) } return agg } -func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { - agg := &TermsAgg{Aggs: make(Aggs)} - settings := simplejson.NewFromAny(model.Get("settings").Interface()) - agg.Terms.Field = model.Get("field").MustString() - if settings == nil { +func (q *Query) getTerms(target *BucketAgg) *TermsAggWrap { + agg := &TermsAggWrap{Aggs: make(Aggs)} + agg.Terms.Field = target.Field + if len(target.Settings.MustMap()) == 0 { return agg } - sizeStr := settings.Get("size").MustString("") + sizeStr := target.Settings.Get("size").MustString("") size, err := strconv.Atoi(sizeStr) if err != nil { size = 500 } agg.Terms.Size = size - orderBy, err := settings.Get("orderBy").String() + orderBy, err := target.Settings.Get("orderBy").String() if err == nil { agg.Terms.Order = make(map[string]interface{}) - agg.Terms.Order[orderBy] = settings.Get("order").MustString("") + agg.Terms.Order[orderBy] = target.Settings.Get("order").MustString("") if _, err := strconv.Atoi(orderBy); err != nil { for _, metricI := range q.Metrics { metric := simplejson.NewFromAny(metricI) @@ -242,7 +217,7 @@ func (q *Query) getTerms(model *simplejson.Json) *TermsAgg { } } - missing, err := settings.Get("missing").String() + missing, err := target.Settings.Get("missing").String() if err == nil { agg.Terms.Missing = missing } diff --git a/pkg/tsdb/elasticsearch/query_def.go b/pkg/tsdb/elasticsearch/query_def.go index 6f78f02f346..128e752d97a 100644 --- a/pkg/tsdb/elasticsearch/query_def.go +++ b/pkg/tsdb/elasticsearch/query_def.go @@ -40,5 +40,4 @@ func isPipelineAgg(metricType string) bool { func describeMetric(metricType, field string) string { text := metricAggType[metricType] return text + " " + field - } diff --git a/pkg/tsdb/elasticsearch/query_test.go b/pkg/tsdb/elasticsearch/query_test.go index 4f7b4d9147e..aecca9f4734 100644 --- a/pkg/tsdb/elasticsearch/query_test.go +++ b/pkg/tsdb/elasticsearch/query_test.go @@ -13,13 +13,8 @@ import ( "testing" ) -func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJSON string) { +func testElasticSearchResponse(query Query, expectedElasticSearchRequestJSON string) { var queryExpectedJSONInterface, queryJSONInterface interface{} - parser := ElasticSearchQueryParser{} - model := &Query{} - - err := json.Unmarshal([]byte(requestJSON), model) - So(err, ShouldBeNil) jsonDate, _ := simplejson.NewJson([]byte(`{"esVersion":2}`)) dsInfo := &models.DataSource{ Database: "grafana-test", @@ -28,10 +23,8 @@ func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJ testTimeRange := tsdb.NewTimeRange("5m", "now") - req, _ := simplejson.NewJson([]byte(requestJSON)) - query, err := parser.Parse(req, dsInfo) s, err := query.Build(&tsdb.TsdbQuery{TimeRange: testTimeRange}, dsInfo) - + So(err, ShouldBeNil) queryJSON := strings.Split(s, "\n")[1] err = json.Unmarshal([]byte(queryJSON), &queryJSONInterface) So(err, ShouldBeNil) @@ -62,53 +55,6 @@ func testElasticSearchResponse(requestJSON string, expectedElasticSearchRequestJ func TestElasticSearchQueryBuilder(t *testing.T) { Convey("Elasticsearch QueryBuilder query testing", t, func() { Convey("Build test average metric with moving average", func() { - var testElasticsearchModelRequestJSON = ` - { - "bucketAggs": [ - { - "field": "timestamp", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "metrics": [ - { - "field": "value", - "id": "1", - "inlineScript": "_value * 2", - "meta": {}, - "settings": { - "script": { - "inline": "_value * 2" - } - }, - "type": "avg" - }, - { - "field": "1", - "id": "3", - "meta": {}, - "pipelineAgg": "1", - "settings": { - "minimize": false, - "model": "simple", - "window": 5 - }, - "type": "moving_avg" - } - ], - "query": "(test:query) AND (name:sample)", - "refId": "A", - "timeField": "timestamp" - } - ` - var expectedElasticsearchQueryJSON = ` { "size": 0, @@ -167,32 +113,9 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(avgWithMovingAvg, expectedElasticsearchQueryJSON) }) Convey("Test Wildcards and Quotes", func() { - testElasticsearchModelRequestJSON := ` - { - "alias": "New", - "bucketAggs": [ - { - "field": "timestamp", - "id": "2", - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "metrics": [ - { - "type": "sum", - "field": "value", - "id": "1" - } - ], - "query": "scope:$location.leagueconnect.api AND name:*CreateRegistration AND name:\"*.201-responses.rate\"", - "refId": "A", - "timeField": "timestamp" - }` - expectedElasticsearchQueryJSON := ` { "size": 0, @@ -239,65 +162,9 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(wildcardsAndQuotes, expectedElasticsearchQueryJSON) }) Convey("Test Term Aggregates", func() { - testElasticsearchModelRequestJSON := ` - { - "bucketAggs": [{ - "field": "name_raw", - "id": "4", - "settings": { - "order": "desc", - "orderBy": "_term", - "size": "10" - }, - "type": "terms" - }, { - "field": "timestamp", - "id": "2", - "settings": { - "interval": "1m", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - }], - "dsType": "elasticsearch", - "filters": [{ - "boolOp": "AND", - "not": false, - "type": "rfc190Scope", - "value": "*.hmp.metricsd" - }, { - "boolOp": "AND", - "not": false, - "type": "name_raw", - "value": "builtin.general.*_instance_count" - }], - "metricObject": {}, - "metrics": [{ - "field": "value", - "id": "1", - "meta": {}, - "options": {}, - "settings": {}, - "type": "sum" - }], - "mode": 0, - "numToGraph": 10, - "prependHostName": false, - "query": "(scope:*.hmp.metricsd) AND (name_raw:builtin.general.*_instance_count)", - "refId": "A", - "regexAlias": false, - "selectedApplication": "", - "selectedHost": "", - "selectedLocation": "", - "timeField": "timestamp", - "useFullHostName": "", - "useQuery": false - }` - expectedElasticsearchQueryJSON := ` { "size": 0, @@ -322,51 +189,12 @@ func TestElasticSearchQueryBuilder(t *testing.T) { ] } }, - "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"1m","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} + "aggs": {"4":{"aggs":{"2":{"aggs":{"1":{"sum":{"field":"value"}}},"date_histogram":{"extended_bounds":{"max":"","min":""},"field":"timestamp","format":"epoch_millis","interval":"200ms","min_doc_count":0}}},"terms":{"field":"name_raw","order":{"_term":"desc"},"size":10}}} }` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(termAggs, expectedElasticsearchQueryJSON) }) Convey("Test Filters Aggregates", func() { - testElasticsearchModelRequestJSON := ` - { - "bucketAggs": [ - { - "id": "3", - "settings": { - "filters": [{ - "label": "hello", - "query": "host:\"67.65.185.232\"" - }] - }, - "type": "filters" - }, - { - "field": "time", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "metrics": [ - { - "pipelineAgg": "select metric", - "field": "bytesSent", - "id": "1", - "meta": {}, - "settings": {}, - "type": "count" - } - ], - "query": "*", - "refId": "A", - "timeField": "time" - }` - expectedElasticsearchQueryJSON := `{ "size": 0, "query": { @@ -422,7 +250,7 @@ func TestElasticSearchQueryBuilder(t *testing.T) { } ` - testElasticSearchResponse(testElasticsearchModelRequestJSON, expectedElasticsearchQueryJSON) + testElasticSearchResponse(filtersAggs, expectedElasticsearchQueryJSON) }) }) } diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 01b8cb1d235..24d5ebebfc4 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -40,27 +40,26 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } if depth == maxDepth { - if aggDef.Get("type").MustString() == "date_histogram" { + if aggDef.Type == "date_histogram" { err = rp.processMetrics(esAgg, target, series, props) if err != nil { return err } } else { - return fmt.Errorf("not support type:%s", aggDef.Get("type").MustString()) + return fmt.Errorf("not support type:%s", aggDef.Type) } } else { for i, b := range esAgg.Get("buckets").MustArray() { - field := aggDef.Get("field").MustString() bucket := simplejson.NewFromAny(b) newProps := props if key, err := bucket.Get("key").String(); err == nil { - newProps[field] = key + newProps[aggDef.Field] = key } else { props["filter"] = strconv.Itoa(i) } if key, err := bucket.Get("key_as_string").String(); err == nil { - props[field] = key + props[aggDef.Field] = key } rp.processBuckets(bucket.MustMap(), target, series, newProps, depth+1) } @@ -72,17 +71,12 @@ func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{ } func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *[]*tsdb.TimeSeries, props map[string]string) error { - for _, v := range target.Metrics { - metric := simplejson.NewFromAny(v) - if metric.Get("hide").MustBool(false) { + for _, metric := range target.Metrics { + if metric.Hide { continue } - metricId := metric.Get("id").MustString() - metricField := metric.Get("field").MustString() - metricType := metric.Get("type").MustString() - - switch metricType { + switch metric.Type { case "count": newSeries := tsdb.TimeSeries{} for _, v := range esAgg.Get("buckets").MustArray() { @@ -102,16 +96,16 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } firstBucket := simplejson.NewFromAny(buckets[0]) - percentiles := firstBucket.GetPath(metricId, "values").MustMap() + percentiles := firstBucket.GetPath(metric.ID, "values").MustMap() for percentileName := range percentiles { newSeries := tsdb.TimeSeries{} newSeries.Tags = props newSeries.Tags["metric"] = "p" + percentileName - newSeries.Tags["field"] = metricField + newSeries.Tags["field"] = metric.Field for _, v := range buckets { bucket := simplejson.NewFromAny(v) - value := castToNullFloat(bucket.GetPath(metricId, "values", percentileName)) + value := castToNullFloat(bucket.GetPath(metric.ID, "values", percentileName)) key := castToNullFloat(bucket.Get("key")) newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } @@ -120,20 +114,20 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta default: newSeries := tsdb.TimeSeries{} newSeries.Tags = props - newSeries.Tags["metric"] = metricType - newSeries.Tags["field"] = metricField + newSeries.Tags["metric"] = metric.Type + newSeries.Tags["field"] = metric.Field for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) key := castToNullFloat(bucket.Get("key")) - valueObj, err := bucket.Get(metricId).Map() + valueObj, err := bucket.Get(metric.ID).Map() if err != nil { break } var value null.Float if _, ok := valueObj["normalized_value"]; ok { - value = castToNullFloat(bucket.GetPath(metricId, "normalized_value")) + value = castToNullFloat(bucket.GetPath(metric.ID, "normalized_value")) } else { - value = castToNullFloat(bucket.GetPath(metricId, "value")) + value = castToNullFloat(bucket.GetPath(metric.ID, "value")) } newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) } @@ -196,10 +190,9 @@ func (rp *ElasticsearchResponseParser) getSeriesName(series *tsdb.TimeSeries, ta // todo, if field and pipelineAgg if field != "" && isPipelineAgg(metricType) { found := false - for _, targetMetricI := range target.Metrics { - targetMetric := simplejson.NewFromAny(targetMetricI) - if targetMetric.Get("id").MustString() == field { - metricName += " " + describeMetric(targetMetric.Get("type").MustString(), field) + for _, metric := range target.Metrics { + if metric.ID == field { + metricName += " " + describeMetric(metric.Type, field) found = true } } @@ -255,11 +248,10 @@ func castToNullFloat(j *simplejson.Json) null.Float { return null.NewFloat(0, false) } -func findAgg(target *Query, aggId string) (*simplejson.Json, error) { +func findAgg(target *Query, aggId string) (*BucketAgg, error) { for _, v := range target.BucketAggs { - aggDef := simplejson.NewFromAny(v) - if aggId == aggDef.Get("id").MustString() { - return aggDef, nil + if aggId == v.ID { + return v, nil } } return nil, errors.New("can't found aggDef, aggID:" + aggId) From 4050fce2205f9fdb6c217eae49686730cabd92c7 Mon Sep 17 00:00:00 2001 From: wph95 Date: Wed, 28 Mar 2018 12:35:05 +0800 Subject: [PATCH 0046/1100] add response_parser test --- pkg/tsdb/elasticsearch/response_parser.go | 9 +- .../elasticsearch/response_parser_test.go | 109 ++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 pkg/tsdb/elasticsearch/response_parser_test.go diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 24d5ebebfc4..ec7d2f9eb08 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -30,6 +30,7 @@ func (rp *ElasticsearchResponseParser) getTimeSeries() *tsdb.QueryResult { } func (rp *ElasticsearchResponseParser) processBuckets(aggs map[string]interface{}, target *Query, series *[]*tsdb.TimeSeries, props map[string]string, depth int) error { + var err error maxDepth := len(target.BucketAggs) - 1 for aggId, v := range aggs { @@ -113,7 +114,11 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta } default: newSeries := tsdb.TimeSeries{} - newSeries.Tags = props + newSeries.Tags = map[string]string{} + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = metric.Type newSeries.Tags["field"] = metric.Field for _, v := range esAgg.Get("buckets").MustArray() { @@ -121,7 +126,7 @@ func (rp *ElasticsearchResponseParser) processMetrics(esAgg *simplejson.Json, ta key := castToNullFloat(bucket.Get("key")) valueObj, err := bucket.Get(metric.ID).Map() if err != nil { - break + continue } var value null.Float if _, ok := valueObj["normalized_value"]; ok { diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go new file mode 100644 index 00000000000..c5b877c1925 --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -0,0 +1,109 @@ +package elasticsearch + +import ( + "encoding/json" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func testElasticsearchResponse(body string, target Query) *tsdb.QueryResult { + var responses Responses + err := json.Unmarshal([]byte(body), &responses) + So(err, ShouldBeNil) + + responseParser := ElasticsearchResponseParser{responses.Responses, []*Query{&target}} + return responseParser.getTimeSeries() +} + +func TestElasticSearchResponseParser(t *testing.T) { + Convey("Elasticsearch Response query testing", t, func() { + Convey("Build test average metric with moving average", func() { + responses := `{ + "responses": [ + { + "took": 1, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 4500, + "max_score": 0, + "hits": [] + }, + "aggregations": { + "2": { + "buckets": [ + { + "1": { + "value": null + }, + "key_as_string": "1522205880000", + "key": 1522205880000, + "doc_count": 0 + }, + { + "1": { + "value": 10 + }, + "key_as_string": "1522205940000", + "key": 1522205940000, + "doc_count": 300 + }, + { + "1": { + "value": 10 + }, + "3": { + "value": 20 + }, + "key_as_string": "1522206000000", + "key": 1522206000000, + "doc_count": 300 + }, + { + "1": { + "value": 10 + }, + "3": { + "value": 20 + }, + "key_as_string": "1522206060000", + "key": 1522206060000, + "doc_count": 300 + } + ] + } + }, + "status": 200 + } + ] +} +` + res := testElasticsearchResponse(responses, avgWithMovingAvg) + So(len(res.Series), ShouldEqual, 2) + So(res.Series[0].Name, ShouldEqual, "Average value") + So(len(res.Series[0].Points), ShouldEqual, 4) + for i, p := range res.Series[0].Points { + if i == 0 { + So(p[0].Valid, ShouldBeFalse) + } else { + So(p[0].Float64, ShouldEqual, 10) + } + So(p[1].Float64, ShouldEqual, 1522205880000+60000*i) + } + + So(res.Series[1].Name, ShouldEqual, "Moving Average Average 1") + So(len(res.Series[1].Points), ShouldEqual, 2) + + for _, p := range res.Series[1].Points { + So(p[0].Float64, ShouldEqual, 20) + } + + }) + }) +} From 64c16eb912dd469e44744b2070e1fb076b2c3651 Mon Sep 17 00:00:00 2001 From: Marcel Anacker Date: Wed, 4 Apr 2018 15:56:27 +0200 Subject: [PATCH 0047/1100] Alerting: Fixing mobile notifications in Microsoft Teams --- pkg/services/alerting/notifiers/teams.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go index 9a9e93dbc47..43d628a4415 100644 --- a/pkg/services/alerting/notifiers/teams.go +++ b/pkg/services/alerting/notifiers/teams.go @@ -41,10 +41,8 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { type TeamsNotifier struct { NotifierBase - Url string - Recipient string - Mention string - log log.Logger + Url string + log log.Logger } func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error { @@ -75,17 +73,17 @@ func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error { }) } - message := this.Mention + message := "" if evalContext.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok. - message += " " + evalContext.Rule.Message - } else { - message += " " // summary must not be empty + message = evalContext.Rule.Message } body := map[string]interface{}{ - "@type": "MessageCard", - "@context": "http://schema.org/extensions", - "summary": message, + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + // summary MUST not be empty or the webhook request fails + // summary SHOULD contain some meaningful information, since it is used for mobile notifications + "summary": evalContext.GetNotificationTitle(), "title": evalContext.GetNotificationTitle(), "themeColor": evalContext.GetStateModel().Color, "sections": []map[string]interface{}{ From d83f886519c1e20d3eadfa03a575ed8d0022cdea Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Tue, 3 Apr 2018 16:36:43 +0200 Subject: [PATCH 0048/1100] migrated jquery.flot.events to ts --- .../plugins/panel/graph/jquery.flot.events.js | 604 ---------------- .../plugins/panel/graph/jquery.flot.events.ts | 663 ++++++++++++++++++ 2 files changed, 663 insertions(+), 604 deletions(-) delete mode 100644 public/app/plugins/panel/graph/jquery.flot.events.js create mode 100644 public/app/plugins/panel/graph/jquery.flot.events.ts diff --git a/public/app/plugins/panel/graph/jquery.flot.events.js b/public/app/plugins/panel/graph/jquery.flot.events.js deleted file mode 100644 index 3ea3ca8f330..00000000000 --- a/public/app/plugins/panel/graph/jquery.flot.events.js +++ /dev/null @@ -1,604 +0,0 @@ -define([ - 'jquery', - 'lodash', - 'angular', - 'tether-drop', -], -function ($, _, angular, Drop) { - 'use strict'; - - function createAnnotationToolip(element, event, plot) { - var injector = angular.element(document).injector(); - var content = document.createElement('div'); - content.innerHTML = ''; - - injector.invoke(["$compile", "$rootScope", function($compile, $rootScope) { - var eventManager = plot.getOptions().events.manager; - var tmpScope = $rootScope.$new(true); - tmpScope.event = event; - tmpScope.onEdit = function() { - eventManager.editEvent(event); - }; - - $compile(content)(tmpScope); - tmpScope.$digest(); - tmpScope.$destroy(); - - var drop = new Drop({ - target: element[0], - content: content, - position: "bottom center", - classes: 'drop-popover drop-popover--annotation', - openOn: 'hover', - hoverCloseDelay: 200, - tetherOptions: { - constraints: [{to: 'window', pin: true, attachment: "both"}] - } - }); - - drop.open(); - - drop.on('close', function() { - setTimeout(function() { - drop.destroy(); - }); - }); - }]); - } - - var markerElementToAttachTo = null; - - function createEditPopover(element, event, plot) { - var eventManager = plot.getOptions().events.manager; - if (eventManager.editorOpen) { - // update marker element to attach to (needed in case of legend on the right - // when there is a double render pass and the initial marker element is removed) - markerElementToAttachTo = element; - return; - } - - // mark as openend - eventManager.editorOpened(); - // set marker element to attache to - markerElementToAttachTo = element; - - // wait for element to be attached and positioned - setTimeout(function() { - - var injector = angular.element(document).injector(); - var content = document.createElement('div'); - content.innerHTML = ''; - - injector.invoke(["$compile", "$rootScope", function($compile, $rootScope) { - var scope = $rootScope.$new(true); - var drop; - - scope.event = event; - scope.panelCtrl = eventManager.panelCtrl; - scope.close = function() { - drop.close(); - }; - - $compile(content)(scope); - scope.$digest(); - - drop = new Drop({ - target: markerElementToAttachTo[0], - content: content, - position: "bottom center", - classes: 'drop-popover drop-popover--form', - openOn: 'click', - tetherOptions: { - constraints: [{to: 'window', pin: true, attachment: "both"}] - } - }); - - drop.open(); - eventManager.editorOpened(); - - drop.on('close', function() { - // need timeout here in order call drop.destroy - setTimeout(function() { - eventManager.editorClosed(); - scope.$destroy(); - drop.destroy(); - }); - }); - }]); - - }, 100); - } - - /* - * jquery.flot.events - * - * description: Flot plugin for adding events/markers to the plot - * version: 0.2.5 - * authors: - * Alexander Wunschik - * Joel Oughton - * Nicolas Joseph - * - * website: https://github.com/mojoaxel/flot-events - * - * released under MIT License and GPLv2+ - */ - - /** - * A class that allows for the drawing an remove of some object - */ - var DrawableEvent = function(object, drawFunc, clearFunc, moveFunc, left, top, width, height) { - var _object = object; - var _drawFunc = drawFunc; - var _clearFunc = clearFunc; - var _moveFunc = moveFunc; - var _position = { left: left, top: top }; - var _width = width; - var _height = height; - - this.width = function() { return _width; }; - this.height = function() { return _height; }; - this.position = function() { return _position; }; - this.draw = function() { _drawFunc(_object); }; - this.clear = function() { _clearFunc(_object); }; - this.getObject = function() { return _object; }; - this.moveTo = function(position) { - _position = position; - _moveFunc(_object, _position); - }; - }; - - /** - * Event class that stores options (eventType, min, max, title, description) and the object to draw. - */ - var VisualEvent = function(options, drawableEvent) { - var _parent; - var _options = options; - var _drawableEvent = drawableEvent; - var _hidden = false; - - this.visual = function() { return _drawableEvent; }; - this.getOptions = function() { return _options; }; - this.getParent = function() { return _parent; }; - this.isHidden = function() { return _hidden; }; - this.hide = function() { _hidden = true; }; - this.unhide = function() { _hidden = false; }; - }; - - /** - * A Class that handles the event-markers inside the given plot - */ - var EventMarkers = function(plot) { - var _events = []; - - this._types = []; - this._plot = plot; - this.eventsEnabled = false; - - this.getEvents = function() { - return _events; - }; - - this.setTypes = function(types) { - return this._types = types; - }; - - /** - * create internal objects for the given events - */ - this.setupEvents = function(events) { - var that = this; - var parts = _.partition(events, 'isRegion'); - var regions = parts[0]; - events = parts[1]; - - $.each(events, function(index, event) { - var ve = new VisualEvent(event, that._buildDiv(event)); - _events.push(ve); - }); - - $.each(regions, function (index, event) { - var vre = new VisualEvent(event, that._buildRegDiv(event)); - _events.push(vre); - }); - - _events.sort(function(a, b) { - var ao = a.getOptions(), bo = b.getOptions(); - if (ao.min > bo.min) { return 1; } - if (ao.min < bo.min) { return -1; } - return 0; - }); - }; - - /** - * draw the events to the plot - */ - this.drawEvents = function() { - var that = this; - // var o = this._plot.getPlotOffset(); - - $.each(_events, function(index, event) { - // check event is inside the graph range - if (that._insidePlot(event.getOptions().min) && !event.isHidden()) { - event.visual().draw(); - } else { - event.visual().getObject().hide(); - } - }); - }; - - /** - * update the position of the event-markers (e.g. after scrolling or zooming) - */ - this.updateEvents = function() { - var that = this; - var o = this._plot.getPlotOffset(), left, top; - var xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - - $.each(_events, function(index, event) { - top = o.top + that._plot.height() - event.visual().height(); - left = xaxis.p2c(event.getOptions().min) + o.left - event.visual().width() / 2; - event.visual().moveTo({ top: top, left: left }); - }); - }; - - /** - * remove all events from the plot - */ - this._clearEvents = function() { - $.each(_events, function(index, val) { - val.visual().clear(); - }); - _events = []; - }; - - /** - * create a DOM element for the given event - */ - this._buildDiv = function(event) { - var that = this; - - var container = this._plot.getPlaceholder(); - var o = this._plot.getPlotOffset(); - var axes = this._plot.getAxes(); - var xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - var yaxis, top, left, color, markerSize, markerShow, lineStyle, lineWidth; - var markerTooltip; - - // determine the y axis used - if (axes.yaxis && axes.yaxis.used) { yaxis = axes.yaxis; } - if (axes.yaxis2 && axes.yaxis2.used) { yaxis = axes.yaxis2; } - - // map the eventType to a types object - var eventTypeId = event.eventType; - - if (this._types === null || !this._types[eventTypeId] || !this._types[eventTypeId].color) { - color = '#666'; - } else { - color = this._types[eventTypeId].color; - } - - if (this._types === null || !this._types[eventTypeId] || !this._types[eventTypeId].markerSize) { - markerSize = 8; //default marker size - } else { - markerSize = this._types[eventTypeId].markerSize; - } - - if (this._types === null || !this._types[eventTypeId] || this._types[eventTypeId].markerShow === undefined) { - markerShow = true; - } else { - markerShow = this._types[eventTypeId].markerShow; - } - - if (this._types === null || !this._types[eventTypeId] || this._types[eventTypeId].markerTooltip === undefined) { - markerTooltip = true; - } else { - markerTooltip = this._types[eventTypeId].markerTooltip; - } - - if (this._types == null || !this._types[eventTypeId] || !this._types[eventTypeId].lineStyle) { - lineStyle = 'dashed'; //default line style - } else { - lineStyle = this._types[eventTypeId].lineStyle.toLowerCase(); - } - - if (this._types == null || !this._types[eventTypeId] || this._types[eventTypeId].lineWidth === undefined) { - lineWidth = 1; //default line width - } else { - lineWidth = this._types[eventTypeId].lineWidth; - } - - var topOffset = xaxis.options.eventSectionHeight || 0; - topOffset = topOffset / 3; - - top = o.top + this._plot.height() + topOffset; - left = xaxis.p2c(event.min) + o.left; - - var line = $('
').css({ - "position": "absolute", - "opacity": 0.8, - "left": left + 'px', - "top": 8, - "width": lineWidth + "px", - "height": this._plot.height() + topOffset * 0.8, - "border-left-width": lineWidth + "px", - "border-left-style": lineStyle, - "border-left-color": color, - "color": color - }) - .appendTo(container); - - if (markerShow) { - var marker = $('
').css({ - "position": "absolute", - "left": (-markerSize - Math.round(lineWidth / 2)) + "px", - "font-size": 0, - "line-height": 0, - "width": 0, - "height": 0, - "border-left": markerSize+"px solid transparent", - "border-right": markerSize+"px solid transparent" - }); - - marker.appendTo(line); - - if (this._types[eventTypeId] && this._types[eventTypeId].position && this._types[eventTypeId].position.toUpperCase() === 'BOTTOM') { - marker.css({ - "top": top-markerSize-8 +"px", - "border-top": "none", - "border-bottom": markerSize+"px solid " + color - }); - } else { - marker.css({ - "top": "0px", - "border-top": markerSize+"px solid " + color, - "border-bottom": "none" - }); - } - - marker.data({ - "event": event - }); - - var mouseenter = function() { - createAnnotationToolip(marker, $(this).data("event"), that._plot); - }; - - if (event.editModel) { - createEditPopover(marker, event.editModel, that._plot); - } - - var mouseleave = function() { - that._plot.clearSelection(); - }; - - if (markerTooltip) { - marker.css({ "cursor": "help" }); - marker.hover(mouseenter, mouseleave); - } - } - - var drawableEvent = new DrawableEvent( - line, - function drawFunc(obj) { obj.show(); }, - function(obj) { obj.remove(); }, - function(obj, position) { - obj.css({ - top: position.top, - left: position.left - }); - }, - left, - top, - line.width(), - line.height() - ); - - return drawableEvent; - }; - - /** - * create a DOM element for the given region - */ - this._buildRegDiv = function (event) { - var that = this; - - var container = this._plot.getPlaceholder(); - var o = this._plot.getPlotOffset(); - var axes = this._plot.getAxes(); - var xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - var yaxis, top, left, lineWidth, regionWidth, lineStyle, color, markerTooltip; - - // determine the y axis used - if (axes.yaxis && axes.yaxis.used) { yaxis = axes.yaxis; } - if (axes.yaxis2 && axes.yaxis2.used) { yaxis = axes.yaxis2; } - - // map the eventType to a types object - var eventTypeId = event.eventType; - - if (this._types === null || !this._types[eventTypeId] || !this._types[eventTypeId].color) { - color = '#666'; - } else { - color = this._types[eventTypeId].color; - } - - if (this._types === null || !this._types[eventTypeId] || this._types[eventTypeId].markerTooltip === undefined) { - markerTooltip = true; - } else { - markerTooltip = this._types[eventTypeId].markerTooltip; - } - - if (this._types == null || !this._types[eventTypeId] || this._types[eventTypeId].lineWidth === undefined) { - lineWidth = 1; //default line width - } else { - lineWidth = this._types[eventTypeId].lineWidth; - } - - if (this._types == null || !this._types[eventTypeId] || !this._types[eventTypeId].lineStyle) { - lineStyle = 'dashed'; //default line style - } else { - lineStyle = this._types[eventTypeId].lineStyle.toLowerCase(); - } - - var topOffset = 2; - top = o.top + this._plot.height() + topOffset; - - var timeFrom = Math.min(event.min, event.timeEnd); - var timeTo = Math.max(event.min, event.timeEnd); - left = xaxis.p2c(timeFrom) + o.left; - var right = xaxis.p2c(timeTo) + o.left; - regionWidth = right - left; - - _.each([left, right], function(position) { - var line = $('
').css({ - "position": "absolute", - "opacity": 0.8, - "left": position + 'px', - "top": 8, - "width": lineWidth + "px", - "height": that._plot.height() + topOffset, - "border-left-width": lineWidth + "px", - "border-left-style": lineStyle, - "border-left-color": color, - "color": color - }); - line.appendTo(container); - }); - - var region = $('
').css({ - "position": "absolute", - "opacity": 0.5, - "left": left + 'px', - "top": top, - "width": Math.round(regionWidth + lineWidth) + "px", - "height": "0.5rem", - "border-left-color": color, - "color": color, - "background-color": color - }); - region.appendTo(container); - - region.data({ - "event": event - }); - - var mouseenter = function () { - createAnnotationToolip(region, $(this).data("event"), that._plot); - }; - - if (event.editModel) { - createEditPopover(region, event.editModel, that._plot); - } - - var mouseleave = function () { - that._plot.clearSelection(); - }; - - if (markerTooltip) { - region.css({ "cursor": "help" }); - region.hover(mouseenter, mouseleave); - } - - var drawableEvent = new DrawableEvent( - region, - function drawFunc(obj) { obj.show(); }, - function (obj) { obj.remove(); }, - function (obj, position) { - obj.css({ - top: position.top, - left: position.left - }); - }, - left, - top, - region.width(), - region.height() - ); - - return drawableEvent; - }; - - /** - * check if the event is inside visible range - */ - this._insidePlot = function(x) { - var xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - var xc = xaxis.p2c(x); - return xc > 0 && xc < xaxis.p2c(xaxis.max); - }; - }; - - /** - * initialize the plugin for the given plot - */ - function init(plot) { - /*jshint validthis:true */ - var that = this; - var eventMarkers = new EventMarkers(plot); - - plot.getEvents = function() { - return eventMarkers._events; - }; - - plot.hideEvents = function() { - $.each(eventMarkers._events, function(index, event) { - event.visual().getObject().hide(); - }); - }; - - plot.showEvents = function() { - plot.hideEvents(); - $.each(eventMarkers._events, function(index, event) { - event.hide(); - }); - - that.eventMarkers.drawEvents(); - }; - - // change events on an existing plot - plot.setEvents = function(events) { - if (eventMarkers.eventsEnabled) { - eventMarkers.setupEvents(events); - } - }; - - plot.hooks.processOptions.push(function(plot, options) { - // enable the plugin - if (options.events.data != null) { - eventMarkers.eventsEnabled = true; - } - }); - - plot.hooks.draw.push(function(plot) { - var options = plot.getOptions(); - - if (eventMarkers.eventsEnabled) { - // check for first run - if (eventMarkers.getEvents().length < 1) { - eventMarkers.setTypes(options.events.types); - eventMarkers.setupEvents(options.events.data); - } else { - eventMarkers.updateEvents(); - } - } - - eventMarkers.drawEvents(); - }); - } - - var defaultOptions = { - events: { - data: null, - types: null, - xaxis: 1, - position: 'BOTTOM' - } - }; - - $.plot.plugins.push({ - init: init, - options: defaultOptions, - name: "events", - version: "0.2.5" - }); -}); diff --git a/public/app/plugins/panel/graph/jquery.flot.events.ts b/public/app/plugins/panel/graph/jquery.flot.events.ts new file mode 100644 index 00000000000..642883ff75c --- /dev/null +++ b/public/app/plugins/panel/graph/jquery.flot.events.ts @@ -0,0 +1,663 @@ +import $ from 'jquery'; +import _ from 'lodash'; +import angular from 'angular'; +import Drop from 'tether-drop'; + +function createAnnotationToolip(element, event, plot) { + let injector = angular.element(document).injector(); + let content = document.createElement('div'); + content.innerHTML = ''; + + injector.invoke([ + '$compile', + '$rootScope', + function($compile, $rootScope) { + let eventManager = plot.getOptions().events.manager; + let tmpScope = $rootScope.$new(true); + tmpScope.event = event; + tmpScope.onEdit = function() { + eventManager.editEvent(event); + }; + + $compile(content)(tmpScope); + tmpScope.$digest(); + tmpScope.$destroy(); + + let drop = new Drop({ + target: element[0], + content: content, + position: 'bottom center', + classes: 'drop-popover drop-popover--annotation', + openOn: 'hover', + hoverCloseDelay: 200, + tetherOptions: { + constraints: [{ to: 'window', pin: true, attachment: 'both' }], + }, + }); + + drop.open(); + + drop.on('close', function() { + setTimeout(function() { + drop.destroy(); + }); + }); + }, + ]); +} + +let markerElementToAttachTo = null; + +function createEditPopover(element, event, plot) { + let eventManager = plot.getOptions().events.manager; + if (eventManager.editorOpen) { + // update marker element to attach to (needed in case of legend on the right + // when there is a double render pass and the inital marker element is removed) + markerElementToAttachTo = element; + return; + } + + // mark as openend + eventManager.editorOpened(); + // set marker elment to attache to + markerElementToAttachTo = element; + + // wait for element to be attached and positioned + setTimeout(function() { + let injector = angular.element(document).injector(); + let content = document.createElement('div'); + content.innerHTML = ''; + + injector.invoke([ + '$compile', + '$rootScope', + function($compile, $rootScope) { + let scope = $rootScope.$new(true); + let drop; + + scope.event = event; + scope.panelCtrl = eventManager.panelCtrl; + scope.close = function() { + drop.close(); + }; + + $compile(content)(scope); + scope.$digest(); + + drop = new Drop({ + target: markerElementToAttachTo[0], + content: content, + position: 'bottom center', + classes: 'drop-popover drop-popover--form', + openOn: 'click', + tetherOptions: { + constraints: [{ to: 'window', pin: true, attachment: 'both' }], + }, + }); + + drop.open(); + eventManager.editorOpened(); + + drop.on('close', function() { + // need timeout here in order call drop.destroy + setTimeout(function() { + eventManager.editorClosed(); + scope.$destroy(); + drop.destroy(); + }); + }); + }, + ]); + }, 100); +} + +/* + * jquery.flot.events + * + * description: Flot plugin for adding events/markers to the plot + * version: 0.2.5 + * authors: + * Alexander Wunschik + * Joel Oughton + * Nicolas Joseph + * + * website: https://github.com/mojoaxel/flot-events + * + * released under MIT License and GPLv2+ + */ + +/** + * A class that allows for the drawing an remove of some object + */ +let DrawableEvent = function(object, drawFunc, clearFunc, moveFunc, left, top, width, height) { + let _object = object; + let _drawFunc = drawFunc; + let _clearFunc = clearFunc; + let _moveFunc = moveFunc; + let _position = { left: left, top: top }; + let _width = width; + let _height = height; + + this.width = function() { + return _width; + }; + this.height = function() { + return _height; + }; + this.position = function() { + return _position; + }; + this.draw = function() { + _drawFunc(_object); + }; + this.clear = function() { + _clearFunc(_object); + }; + this.getObject = function() { + return _object; + }; + this.moveTo = function(position) { + _position = position; + _moveFunc(_object, _position); + }; +}; + +/** + * Event class that stores options (eventType, min, max, title, description) and the object to draw. + */ +let VisualEvent = function(options, drawableEvent) { + let _parent; + let _options = options; + let _drawableEvent = drawableEvent; + let _hidden = false; + + this.visual = function() { + return _drawableEvent; + }; + this.getOptions = function() { + return _options; + }; + this.getParent = function() { + return _parent; + }; + this.isHidden = function() { + return _hidden; + }; + this.hide = function() { + _hidden = true; + }; + this.unhide = function() { + _hidden = false; + }; +}; + +/** + * A Class that handles the event-markers inside the given plot + */ +let EventMarkers = function(plot) { + let _events = []; + + this._types = []; + this._plot = plot; + this.eventsEnabled = false; + + this.getEvents = function() { + return _events; + }; + + this.setTypes = function(types) { + return (this._types = types); + }; + + /** + * create internal objects for the given events + */ + this.setupEvents = function(events) { + let that = this; + let parts = _.partition(events, 'isRegion'); + let regions = parts[0]; + events = parts[1]; + + $.each(events, function(index, event) { + let ve = new VisualEvent(event, that._buildDiv(event)); + _events.push(ve); + }); + + $.each(regions, function(index, event) { + let vre = new VisualEvent(event, that._buildRegDiv(event)); + _events.push(vre); + }); + + _events.sort(function(a, b) { + let ao = a.getOptions(), + bo = b.getOptions(); + if (ao.min > bo.min) { + return 1; + } + if (ao.min < bo.min) { + return -1; + } + return 0; + }); + }; + + /** + * draw the events to the plot + */ + this.drawEvents = function() { + let that = this; + // let o = this._plot.getPlotOffset(); + + $.each(_events, function(index, event) { + // check event is inside the graph range + if (that._insidePlot(event.getOptions().min) && !event.isHidden()) { + event.visual().draw(); + } else { + event + .visual() + .getObject() + .hide(); + } + }); + }; + + /** + * update the position of the event-markers (e.g. after scrolling or zooming) + */ + this.updateEvents = function() { + let that = this; + let o = this._plot.getPlotOffset(), + left, + top; + let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; + + $.each(_events, function(index, event) { + top = o.top + that._plot.height() - event.visual().height(); + left = xaxis.p2c(event.getOptions().min) + o.left - event.visual().width() / 2; + event.visual().moveTo({ top: top, left: left }); + }); + }; + + /** + * remove all events from the plot + */ + this._clearEvents = function() { + $.each(_events, function(index, val) { + val.visual().clear(); + }); + _events = []; + }; + + /** + * create a DOM element for the given event + */ + this._buildDiv = function(event) { + let that = this; + + let container = this._plot.getPlaceholder(); + let o = this._plot.getPlotOffset(); + let axes = this._plot.getAxes(); + let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; + let yaxis, top, left, color, markerSize, markerShow, lineStyle, lineWidth; + let markerTooltip; + + // determine the y axis used + if (axes.yaxis && axes.yaxis.used) { + yaxis = axes.yaxis; + } + if (axes.yaxis2 && axes.yaxis2.used) { + yaxis = axes.yaxis2; + } + + // map the eventType to a types object + let eventTypeId = event.eventType; + + if (this._types === null || !this._types[eventTypeId] || !this._types[eventTypeId].color) { + color = '#666'; + } else { + color = this._types[eventTypeId].color; + } + + if (this._types === null || !this._types[eventTypeId] || !this._types[eventTypeId].markerSize) { + markerSize = 8; //default marker size + } else { + markerSize = this._types[eventTypeId].markerSize; + } + + if (this._types === null || !this._types[eventTypeId] || this._types[eventTypeId].markerShow === undefined) { + markerShow = true; + } else { + markerShow = this._types[eventTypeId].markerShow; + } + + if (this._types === null || !this._types[eventTypeId] || this._types[eventTypeId].markerTooltip === undefined) { + markerTooltip = true; + } else { + markerTooltip = this._types[eventTypeId].markerTooltip; + } + + if (this._types == null || !this._types[eventTypeId] || !this._types[eventTypeId].lineStyle) { + lineStyle = 'dashed'; //default line style + } else { + lineStyle = this._types[eventTypeId].lineStyle.toLowerCase(); + } + + if (this._types == null || !this._types[eventTypeId] || this._types[eventTypeId].lineWidth === undefined) { + lineWidth = 1; //default line width + } else { + lineWidth = this._types[eventTypeId].lineWidth; + } + + let topOffset = xaxis.options.eventSectionHeight || 0; + topOffset = topOffset / 3; + + top = o.top + this._plot.height() + topOffset; + left = xaxis.p2c(event.min) + o.left; + + let line = $('
') + .css({ + position: 'absolute', + opacity: 0.8, + left: left + 'px', + top: 8, + width: lineWidth + 'px', + height: this._plot.height() + topOffset * 0.8, + 'border-left-width': lineWidth + 'px', + 'border-left-style': lineStyle, + 'border-left-color': color, + color: color, + }) + .appendTo(container); + + if (markerShow) { + let marker = $('
').css({ + position: 'absolute', + left: -markerSize - Math.round(lineWidth / 2) + 'px', + 'font-size': 0, + 'line-height': 0, + width: 0, + height: 0, + 'border-left': markerSize + 'px solid transparent', + 'border-right': markerSize + 'px solid transparent', + }); + + marker.appendTo(line); + + if ( + this._types[eventTypeId] && + this._types[eventTypeId].position && + this._types[eventTypeId].position.toUpperCase() === 'BOTTOM' + ) { + marker.css({ + top: top - markerSize - 8 + 'px', + 'border-top': 'none', + 'border-bottom': markerSize + 'px solid ' + color, + }); + } else { + marker.css({ + top: '0px', + 'border-top': markerSize + 'px solid ' + color, + 'border-bottom': 'none', + }); + } + + marker.data({ + event: event, + }); + + let mouseenter = function() { + createAnnotationToolip(marker, $(this).data('event'), that._plot); + }; + + if (event.editModel) { + createEditPopover(marker, event.editModel, that._plot); + } + + let mouseleave = function() { + that._plot.clearSelection(); + }; + + if (markerTooltip) { + marker.css({ cursor: 'help' }); + marker.hover(mouseenter, mouseleave); + } + } + + let drawableEvent = new DrawableEvent( + line, + function drawFunc(obj) { + obj.show(); + }, + function(obj) { + obj.remove(); + }, + function(obj, position) { + obj.css({ + top: position.top, + left: position.left, + }); + }, + left, + top, + line.width(), + line.height() + ); + + return drawableEvent; + }; + + /** + * create a DOM element for the given region + */ + this._buildRegDiv = function(event) { + let that = this; + + let container = this._plot.getPlaceholder(); + let o = this._plot.getPlotOffset(); + let axes = this._plot.getAxes(); + let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; + let yaxis, top, left, lineWidth, regionWidth, lineStyle, color, markerTooltip; + + // determine the y axis used + if (axes.yaxis && axes.yaxis.used) { + yaxis = axes.yaxis; + } + if (axes.yaxis2 && axes.yaxis2.used) { + yaxis = axes.yaxis2; + } + + // map the eventType to a types object + let eventTypeId = event.eventType; + + if (this._types === null || !this._types[eventTypeId] || !this._types[eventTypeId].color) { + color = '#666'; + } else { + color = this._types[eventTypeId].color; + } + + if (this._types === null || !this._types[eventTypeId] || this._types[eventTypeId].markerTooltip === undefined) { + markerTooltip = true; + } else { + markerTooltip = this._types[eventTypeId].markerTooltip; + } + + if (this._types == null || !this._types[eventTypeId] || this._types[eventTypeId].lineWidth === undefined) { + lineWidth = 1; //default line width + } else { + lineWidth = this._types[eventTypeId].lineWidth; + } + + if (this._types == null || !this._types[eventTypeId] || !this._types[eventTypeId].lineStyle) { + lineStyle = 'dashed'; //default line style + } else { + lineStyle = this._types[eventTypeId].lineStyle.toLowerCase(); + } + + let topOffset = 2; + top = o.top + this._plot.height() + topOffset; + + let timeFrom = Math.min(event.min, event.timeEnd); + let timeTo = Math.max(event.min, event.timeEnd); + left = xaxis.p2c(timeFrom) + o.left; + let right = xaxis.p2c(timeTo) + o.left; + regionWidth = right - left; + + _.each([left, right], function(position) { + let line = $('
').css({ + position: 'absolute', + opacity: 0.8, + left: position + 'px', + top: 8, + width: lineWidth + 'px', + height: that._plot.height() + topOffset, + 'border-left-width': lineWidth + 'px', + 'border-left-style': lineStyle, + 'border-left-color': color, + color: color, + }); + line.appendTo(container); + }); + + let region = $('
').css({ + position: 'absolute', + opacity: 0.5, + left: left + 'px', + top: top, + width: Math.round(regionWidth + lineWidth) + 'px', + height: '0.5rem', + 'border-left-color': color, + color: color, + 'background-color': color, + }); + region.appendTo(container); + + region.data({ + event: event, + }); + + let mouseenter = function() { + createAnnotationToolip(region, $(this).data('event'), that._plot); + }; + + if (event.editModel) { + createEditPopover(region, event.editModel, that._plot); + } + + let mouseleave = function() { + that._plot.clearSelection(); + }; + + if (markerTooltip) { + region.css({ cursor: 'help' }); + region.hover(mouseenter, mouseleave); + } + + let drawableEvent = new DrawableEvent( + region, + function drawFunc(obj) { + obj.show(); + }, + function(obj) { + obj.remove(); + }, + function(obj, position) { + obj.css({ + top: position.top, + left: position.left, + }); + }, + left, + top, + region.width(), + region.height() + ); + + return drawableEvent; + }; + + /** + * check if the event is inside visible range + */ + this._insidePlot = function(x) { + let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; + let xc = xaxis.p2c(x); + return xc > 0 && xc < xaxis.p2c(xaxis.max); + }; +}; + +/** + * initialize the plugin for the given plot + */ +function init(plot) { + /*jshint validthis:true */ + let that = this; + let eventMarkers = new EventMarkers(plot); + + plot.getEvents = function() { + return eventMarkers._events; + }; + + plot.hideEvents = function() { + $.each(eventMarkers._events, function(index, event) { + event + .visual() + .getObject() + .hide(); + }); + }; + + plot.showEvents = function() { + plot.hideEvents(); + $.each(eventMarkers._events, function(index, event) { + event.hide(); + }); + + that.eventMarkers.drawEvents(); + }; + + // change events on an existing plot + plot.setEvents = function(events) { + if (eventMarkers.eventsEnabled) { + eventMarkers.setupEvents(events); + } + }; + + plot.hooks.processOptions.push(function(plot, options) { + // enable the plugin + if (options.events.data != null) { + eventMarkers.eventsEnabled = true; + } + }); + + plot.hooks.draw.push(function(plot) { + let options = plot.getOptions(); + + if (eventMarkers.eventsEnabled) { + // check for first run + if (eventMarkers.getEvents().length < 1) { + eventMarkers.setTypes(options.events.types); + eventMarkers.setupEvents(options.events.data); + } else { + eventMarkers.updateEvents(); + } + } + + eventMarkers.drawEvents(); + }); +} + +let defaultOptions = { + events: { + data: null, + types: null, + xaxis: 1, + position: 'BOTTOM', + }, +}; + +$.plot.plugins.push({ + init: init, + options: defaultOptions, + name: 'events', + version: '0.2.5', +}); From b2027af4cb3cdca3b84e3bf7547bf90ab38a240e Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Tue, 10 Apr 2018 14:16:56 +0200 Subject: [PATCH 0049/1100] wrote classes --- .../plugins/panel/graph/jquery.flot.events.ts | 258 +++++++++--------- 1 file changed, 133 insertions(+), 125 deletions(-) diff --git a/public/app/plugins/panel/graph/jquery.flot.events.ts b/public/app/plugins/panel/graph/jquery.flot.events.ts index 642883ff75c..9dfe0a8573f 100644 --- a/public/app/plugins/panel/graph/jquery.flot.events.ts +++ b/public/app/plugins/panel/graph/jquery.flot.events.ts @@ -1,9 +1,10 @@ +import angular from 'angular'; import $ from 'jquery'; import _ from 'lodash'; -import angular from 'angular'; import Drop from 'tether-drop'; -function createAnnotationToolip(element, event, plot) { +/** @ngInject */ +export function createAnnotationToolip(element, event, plot) { let injector = angular.element(document).injector(); let content = document.createElement('div'); content.innerHTML = ''; @@ -48,7 +49,8 @@ function createAnnotationToolip(element, event, plot) { let markerElementToAttachTo = null; -function createEditPopover(element, event, plot) { +/** @ngInject */ +export function createEditPopover(element, event, plot) { let eventManager = plot.getOptions().events.manager; if (eventManager.editorOpen) { // update marker element to attach to (needed in case of legend on the right @@ -129,106 +131,130 @@ function createEditPopover(element, event, plot) { /** * A class that allows for the drawing an remove of some object */ -let DrawableEvent = function(object, drawFunc, clearFunc, moveFunc, left, top, width, height) { - let _object = object; - let _drawFunc = drawFunc; - let _clearFunc = clearFunc; - let _moveFunc = moveFunc; - let _position = { left: left, top: top }; - let _width = width; - let _height = height; +export class DrawableEvent { + _object: any; + _drawFunc: any; + _clearFunc: any; + _moveFunc: any; + _position: any; + _width: any; + _height: any; - this.width = function() { - return _width; - }; - this.height = function() { - return _height; - }; - this.position = function() { - return _position; - }; - this.draw = function() { - _drawFunc(_object); - }; - this.clear = function() { - _clearFunc(_object); - }; - this.getObject = function() { - return _object; - }; - this.moveTo = function(position) { - _position = position; - _moveFunc(_object, _position); - }; -}; + /** @ngInject */ + constructor(object, drawFunc, clearFunc, moveFunc, left, top, width, height) { + this._object = object; + this._drawFunc = drawFunc; + this._clearFunc = clearFunc; + this._moveFunc = moveFunc; + this._position = { left: left, top: top }; + this._width = width; + this._height = height; + } + + width() { + return this._width; + } + height() { + return this._height; + } + position() { + return this._position; + } + draw() { + this._drawFunc(this._object); + } + clear() { + this._clearFunc(this._object); + } + getObject() { + return this._object; + } + moveTo(position) { + this._position = position; + this._moveFunc(this._object, this._position); + } +} /** * Event class that stores options (eventType, min, max, title, description) and the object to draw. */ -let VisualEvent = function(options, drawableEvent) { - let _parent; - let _options = options; - let _drawableEvent = drawableEvent; - let _hidden = false; +export class VisualEvent { + _parent: any; + _options: any; + _drawableEvent: any; + _hidden: any; - this.visual = function() { - return _drawableEvent; - }; - this.getOptions = function() { - return _options; - }; - this.getParent = function() { - return _parent; - }; - this.isHidden = function() { - return _hidden; - }; - this.hide = function() { - _hidden = true; - }; - this.unhide = function() { - _hidden = false; - }; -}; + /** @ngInject */ + constructor(options, drawableEvent) { + this._options = options; + this._drawableEvent = drawableEvent; + this._hidden = false; + } + + visual() { + return this._drawableEvent; + } + getOptions() { + return this._options; + } + getParent() { + return this._parent; + } + isHidden() { + return this._hidden; + } + hide() { + this._hidden = true; + } + unhide() { + this._hidden = false; + } +} /** * A Class that handles the event-markers inside the given plot */ -let EventMarkers = function(plot) { - let _events = []; +export class EventMarkers { + _events: any; + _types: any; + _plot: any; + eventsEnabled: any; - this._types = []; - this._plot = plot; - this.eventsEnabled = false; + /** @ngInject */ + constructor(plot) { + this._events = []; + this._types = []; + this._plot = plot; + this.eventsEnabled = false; + } - this.getEvents = function() { - return _events; - }; + getEvents() { + return this._events; + } - this.setTypes = function(types) { + setTypes(types) { return (this._types = types); - }; + } /** * create internal objects for the given events */ - this.setupEvents = function(events) { - let that = this; + setupEvents(events) { let parts = _.partition(events, 'isRegion'); let regions = parts[0]; events = parts[1]; - $.each(events, function(index, event) { - let ve = new VisualEvent(event, that._buildDiv(event)); - _events.push(ve); + $.each(events, (index, event) => { + let ve = new VisualEvent(event, this._buildDiv(event)); + this._events.push(ve); }); - $.each(regions, function(index, event) { - let vre = new VisualEvent(event, that._buildRegDiv(event)); - _events.push(vre); + $.each(regions, (index, event) => { + let vre = new VisualEvent(event, this._buildRegDiv(event)); + this._events.push(vre); }); - _events.sort(function(a, b) { + this._events.sort((a, b) => { let ao = a.getOptions(), bo = b.getOptions(); if (ao.min > bo.min) { @@ -239,18 +265,17 @@ let EventMarkers = function(plot) { } return 0; }); - }; + } /** * draw the events to the plot */ - this.drawEvents = function() { - let that = this; - // let o = this._plot.getPlotOffset(); + drawEvents() { + // var o = this._plot.getPlotOffset(); - $.each(_events, function(index, event) { + $.each(this._events, (index, event) => { // check event is inside the graph range - if (that._insidePlot(event.getOptions().min) && !event.isHidden()) { + if (this._insidePlot(event.getOptions().min) && !event.isHidden()) { event.visual().draw(); } else { event @@ -259,56 +284,46 @@ let EventMarkers = function(plot) { .hide(); } }); - }; + } /** * update the position of the event-markers (e.g. after scrolling or zooming) */ - this.updateEvents = function() { - let that = this; + updateEvents() { let o = this._plot.getPlotOffset(), left, top; let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - $.each(_events, function(index, event) { - top = o.top + that._plot.height() - event.visual().height(); + $.each(this._events, (index, event) => { + top = o.top + this._plot.height() - event.visual().height(); left = xaxis.p2c(event.getOptions().min) + o.left - event.visual().width() / 2; event.visual().moveTo({ top: top, left: left }); }); - }; + } /** * remove all events from the plot */ - this._clearEvents = function() { - $.each(_events, function(index, val) { + _clearEvents() { + $.each(this._events, (index, val) => { val.visual().clear(); }); - _events = []; - }; + this._events = []; + } /** * create a DOM element for the given event */ - this._buildDiv = function(event) { + _buildDiv(event) { let that = this; let container = this._plot.getPlaceholder(); let o = this._plot.getPlotOffset(); - let axes = this._plot.getAxes(); let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - let yaxis, top, left, color, markerSize, markerShow, lineStyle, lineWidth; + let top, left, color, markerSize, markerShow, lineStyle, lineWidth; let markerTooltip; - // determine the y axis used - if (axes.yaxis && axes.yaxis.used) { - yaxis = axes.yaxis; - } - if (axes.yaxis2 && axes.yaxis2.used) { - yaxis = axes.yaxis2; - } - // map the eventType to a types object let eventTypeId = event.eventType; @@ -444,27 +459,18 @@ let EventMarkers = function(plot) { ); return drawableEvent; - }; + } /** * create a DOM element for the given region */ - this._buildRegDiv = function(event) { + _buildRegDiv(event) { let that = this; let container = this._plot.getPlaceholder(); let o = this._plot.getPlotOffset(); - let axes = this._plot.getAxes(); let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; - let yaxis, top, left, lineWidth, regionWidth, lineStyle, color, markerTooltip; - - // determine the y axis used - if (axes.yaxis && axes.yaxis.used) { - yaxis = axes.yaxis; - } - if (axes.yaxis2 && axes.yaxis2.used) { - yaxis = axes.yaxis2; - } + let top, left, lineWidth, regionWidth, lineStyle, color, markerTooltip; // map the eventType to a types object let eventTypeId = event.eventType; @@ -502,14 +508,14 @@ let EventMarkers = function(plot) { let right = xaxis.p2c(timeTo) + o.left; regionWidth = right - left; - _.each([left, right], function(position) { + _.each([left, right], position => { let line = $('
').css({ position: 'absolute', opacity: 0.8, left: position + 'px', top: 8, width: lineWidth + 'px', - height: that._plot.height() + topOffset, + height: this._plot.height() + topOffset, 'border-left-width': lineWidth + 'px', 'border-left-style': lineStyle, 'border-left-color': color, @@ -573,22 +579,24 @@ let EventMarkers = function(plot) { ); return drawableEvent; - }; + } /** * check if the event is inside visible range */ - this._insidePlot = function(x) { + _insidePlot(x) { let xaxis = this._plot.getXAxes()[this._plot.getOptions().events.xaxis - 1]; let xc = xaxis.p2c(x); return xc > 0 && xc < xaxis.p2c(xaxis.max); - }; -}; + } +} /** * initialize the plugin for the given plot */ -function init(plot) { + +/** @ngInject */ +export function init(plot) { /*jshint validthis:true */ let that = this; let eventMarkers = new EventMarkers(plot); @@ -598,7 +606,7 @@ function init(plot) { }; plot.hideEvents = function() { - $.each(eventMarkers._events, function(index, event) { + $.each(eventMarkers._events, (index, event) => { event .visual() .getObject() @@ -608,7 +616,7 @@ function init(plot) { plot.showEvents = function() { plot.hideEvents(); - $.each(eventMarkers._events, function(index, event) { + $.each(eventMarkers._events, (index, event) => { event.hide(); }); From 731c7520b393ae7e82603032860390771b35dc7d Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 19 May 2018 15:34:48 +0200 Subject: [PATCH 0050/1100] return values quotes for suggestions in where expression --- public/app/plugins/datasource/postgres/query_builder.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 275d15492fe..3dfbf99fefd 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -39,11 +39,10 @@ export class PostgresQueryBuilder { } buildValueQuery(column: string) { - var query = "SELECT DISTINCT " + this.queryModel.quoteIdentifier(column) + "::text"; + var query = "SELECT DISTINCT quote_literal(" + column + ")"; query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); query += "." + this.queryModel.quoteIdentifier(this.target.table); - query += " ORDER BY " + this.queryModel.quoteIdentifier(column); - query += " LIMIT 100"; + query += " ORDER BY 1 LIMIT 100"; return query; } From b12d049f6f1b24ff733b0e6f71f910977751e5a2 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 19 May 2018 17:21:53 +0200 Subject: [PATCH 0051/1100] quote column name in buildValueQuery --- public/app/plugins/datasource/postgres/query_builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 3dfbf99fefd..dd6f95b550c 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -39,7 +39,7 @@ export class PostgresQueryBuilder { } buildValueQuery(column: string) { - var query = "SELECT DISTINCT quote_literal(" + column + ")"; + var query = "SELECT DISTINCT quote_literal(" + this.queryModel.quoteIdentifier(column) + ")"; query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); query += "." + this.queryModel.quoteIdentifier(this.target.table); query += " ORDER BY 1 LIMIT 100"; From 181dfdba04d6ff31ae468eea11a5fe334290f617 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 09:37:04 +0200 Subject: [PATCH 0052/1100] add sql_part component --- .../app/core/components/sql_part/sql_part.ts | 118 +++++++++++ .../components/sql_part/sql_part_editor.ts | 185 ++++++++++++++++++ public/app/core/core.ts | 2 + 3 files changed, 305 insertions(+) create mode 100644 public/app/core/components/sql_part/sql_part.ts create mode 100644 public/app/core/components/sql_part/sql_part_editor.ts diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts new file mode 100644 index 00000000000..77366e6d09b --- /dev/null +++ b/public/app/core/components/sql_part/sql_part.ts @@ -0,0 +1,118 @@ +import _ from 'lodash'; + +export class SqlPartDef { + type: string; + params: any[]; + defaultParams: any[]; + renderer: any; + category: any; + addStrategy: any; + + constructor(options: any) { + this.type = options.type; + this.params = options.params; + this.defaultParams = options.defaultParams; + this.renderer = options.renderer; + this.category = options.category; + this.addStrategy = options.addStrategy; + } +} + +export class SqlPart { + part: any; + def: SqlPartDef; + params: any[]; + text: string; + + constructor(part: any, def: any) { + this.part = part; + this.def = def; + if (!this.def) { + throw { message: 'Could not find query part ' + part.type }; + } + + part.params = part.params || _.clone(this.def.defaultParams); + this.params = part.params; + this.updateText(); + } + + render(innerExpr: string) { + return this.def.renderer(this, innerExpr); + } + + hasMultipleParamsInString(strValue, index) { + if (strValue.indexOf(',') === -1) { + return false; + } + + return this.def.params[index + 1] && this.def.params[index + 1].optional; + } + + updateParam(strValue, index) { + // handle optional parameters + // if string contains ',' and next param is optional, split and update both + if (this.hasMultipleParamsInString(strValue, index)) { + _.each(strValue.split(','), (partVal, idx) => { + this.updateParam(partVal.trim(), idx); + }); + return; + } + + if (strValue === '' && this.def.params[index].optional) { + this.params.splice(index, 1); + } else { + this.params[index] = strValue; + } + + this.part.params = this.params; + this.updateText(); + } + + updateText() { + if (this.params.length === 0) { + this.text = this.def.type + '()'; + return; + } + + var text = this.def.type + '('; + text += this.params.join(', '); + text += ')'; + this.text = text; + } +} + +export function functionRenderer(part, innerExpr) { + var str = part.def.type + '('; + var parameters = _.map(part.params, (value, index) => { + var paramType = part.def.params[index]; + if (paramType.type === 'time') { + if (value === 'auto') { + value = '$__interval'; + } + } + if (paramType.quote === 'single') { + return "'" + value + "'"; + } else if (paramType.quote === 'double') { + return '"' + value + '"'; + } + + return value; + }); + + if (innerExpr) { + parameters.unshift(innerExpr); + } + return str + parameters.join(', ') + ')'; +} + +export function suffixRenderer(part, innerExpr) { + return innerExpr + ' ' + part.params[0]; +} + +export function identityRenderer(part, innerExpr) { + return part.params[0]; +} + +export function quotedIdentityRenderer(part, innerExpr) { + return '"' + part.params[0] + '"'; +} diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts new file mode 100644 index 00000000000..837f280e66d --- /dev/null +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -0,0 +1,185 @@ +import _ from 'lodash'; +import $ from 'jquery'; +import coreModule from 'app/core/core_module'; + +var template = ` +
+{{part.def.type}} +() + +`; + +/** @ngInject */ +export function sqlPartEditorDirective($compile, templateSrv) { + var paramTemplate = ''; + + return { + restrict: 'E', + template: template, + scope: { + part: '=', + handleEvent: '&', + debounce: '@', + }, + link: function postLink($scope, elem) { + var part = $scope.part; + var partDef = part.def; + var $paramsContainer = elem.find('.query-part-parameters'); + var debounceLookup = $scope.debounce; + + $scope.partActions = []; + + function clickFuncParam(paramIndex) { + /*jshint validthis:true */ + var $link = $(this); + var $input = $link.next(); + + $input.val(part.params[paramIndex]); + $input.css('width', $link.width() + 16 + 'px'); + + $link.hide(); + $input.show(); + $input.focus(); + $input.select(); + + var typeahead = $input.data('typeahead'); + if (typeahead) { + $input.val(''); + typeahead.lookup(); + } + } + + function inputBlur(paramIndex) { + /*jshint validthis:true */ + var $input = $(this); + var $link = $input.prev(); + var newValue = $input.val(); + + if (newValue !== '' || part.def.params[paramIndex].optional) { + $link.html(templateSrv.highlightVariablesAsHtml(newValue)); + + part.updateParam($input.val(), paramIndex); + $scope.$apply(() => { + $scope.handleEvent({ $event: { name: 'part-param-changed' } }); + }); + } + + $input.hide(); + $link.show(); + } + + function inputKeyPress(paramIndex, e) { + /*jshint validthis:true */ + if (e.which === 13) { + inputBlur.call(this, paramIndex); + } + } + + function inputKeyDown() { + /*jshint validthis:true */ + this.style.width = (3 + this.value.length) * 8 + 'px'; + } + + function addTypeahead($input, param, paramIndex) { + if (!param.options && !param.dynamicLookup) { + return; + } + + var typeaheadSource = function(query, callback) { + if (param.options) { + var options = param.options; + if (param.type === 'int') { + options = _.map(options, function(val) { + return val.toString(); + }); + } + return options; + } + + $scope.$apply(function() { + $scope.handleEvent({ $event: { name: 'get-param-options' } }).then(function(result) { + var dynamicOptions = _.map(result, function(op) { + return op.value; + }); + callback(dynamicOptions); + }); + }); + }; + + $input.attr('data-provide', 'typeahead'); + + $input.typeahead({ + source: typeaheadSource, + minLength: 0, + items: 1000, + updater: function(value) { + setTimeout(function() { + inputBlur.call($input[0], paramIndex); + }, 0); + return value; + }, + }); + + var typeahead = $input.data('typeahead'); + typeahead.lookup = function() { + this.query = this.$element.val() || ''; + var items = this.source(this.query, $.proxy(this.process, this)); + return items ? this.process(items) : items; + }; + + if (debounceLookup) { + typeahead.lookup = _.debounce(typeahead.lookup, 500, { leading: true }); + } + } + + $scope.showActionsMenu = function() { + $scope.handleEvent({ $event: { name: 'get-part-actions' } }).then(res => { + $scope.partActions = res; + }); + }; + + $scope.triggerPartAction = function(action) { + $scope.handleEvent({ $event: { name: 'action', action: action } }); + }; + + function addElementsAndCompile() { + _.each(partDef.params, function(param, index) { + if (param.optional && part.params.length <= index) { + return; + } + + if (index > 0) { + $(', ').appendTo($paramsContainer); + } + + var paramValue = templateSrv.highlightVariablesAsHtml(part.params[index]); + var $paramLink = $('' + paramValue + ''); + var $input = $(paramTemplate); + + $paramLink.appendTo($paramsContainer); + $input.appendTo($paramsContainer); + + $input.blur(_.partial(inputBlur, index)); + $input.keyup(inputKeyDown); + $input.keypress(_.partial(inputKeyPress, index)); + $paramLink.click(_.partial(clickFuncParam, index)); + + addTypeahead($input, param, index); + }); + } + + function relink() { + $paramsContainer.empty(); + addElementsAndCompile(); + } + + relink(); + }, + }; +} + +coreModule.directive('sqlPartEditor', sqlPartEditorDirective); diff --git a/public/app/core/core.ts b/public/app/core/core.ts index fb7021fe883..2bfe84a74df 100644 --- a/public/app/core/core.ts +++ b/public/app/core/core.ts @@ -31,6 +31,7 @@ import { layoutSelector } from './components/layout_selector/layout_selector'; import { switchDirective } from './components/switch'; import { dashboardSelector } from './components/dashboard_selector'; import { queryPartEditorDirective } from './components/query_part/query_part_editor'; +import { sqlPartEditorDirective } from './components/sql_part/sql_part_editor'; import { formDropdownDirective } from './components/form_dropdown/form_dropdown'; import 'app/core/controllers/all'; import 'app/core/services/all'; @@ -74,6 +75,7 @@ export { appEvents, dashboardSelector, queryPartEditorDirective, + sqlPartEditorDirective, colors, formDropdownDirective, assignModelProperties, From de9a117dc67aee70aff6cc108fea4981a8228559 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 11:04:52 +0200 Subject: [PATCH 0053/1100] use sql part component --- .../datasource/postgres/partials/query.editor.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 1833b845d41..b0614a3ee9e 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -36,8 +36,8 @@
- - + +
@@ -72,10 +72,10 @@ GROUP BY - - +
From e93276b1f822edb434b280662fb914a1e0e4a626 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 11:10:00 +0200 Subject: [PATCH 0054/1100] use sql part component --- public/app/plugins/datasource/postgres/query_part.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 0d2b365fe66..d54a66783eb 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/core/components/query_part/query_part'; +import { SqlPartDef, SqlPart, functionRenderer, suffixRenderer } from 'app/core/components/sql_part/sql_part'; var index = []; @@ -9,11 +9,11 @@ function createPart(part): any { throw { message: 'Could not find query part ' + part.type }; } - return new QueryPart(part, def); + return new SqlPart(part, def); } function register(options: any) { - index[options.type] = new QueryPartDef(options); + index[options.type] = new SqlPartDef(options); } function aliasRenderer(part, innerExpr) { From 3af4e4e0d696ba62731bcd9b527472b5c2acef68 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 11:44:37 +0200 Subject: [PATCH 0055/1100] separate label in template from type --- public/app/core/components/sql_part/sql_part.ts | 2 ++ public/app/core/components/sql_part/sql_part_editor.ts | 2 +- public/app/plugins/datasource/postgres/query_part.ts | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index 77366e6d09b..94dd0ff6ae8 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -2,6 +2,7 @@ import _ from 'lodash'; export class SqlPartDef { type: string; + label: string; params: any[]; defaultParams: any[]; renderer: any; @@ -10,6 +11,7 @@ export class SqlPartDef { constructor(options: any) { this.type = options.type; + this.label = options.label; this.params = options.params; this.defaultParams = options.defaultParams; this.renderer = options.renderer; diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index 837f280e66d..2ebeca5753a 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -4,7 +4,7 @@ import coreModule from 'app/core/core_module'; var template = `
-{{part.def.type}} +{{part.def.label}} ()
  • diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index d54a66783eb..029820da776 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -85,6 +85,7 @@ function addColumnStrategy(selectParts, partModel, query) { register({ type: 'column', + label: 'Column:', addStrategy: addColumnStrategy, params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], @@ -93,6 +94,7 @@ register({ register({ type: 'aggregate', + label: 'Aggregate:', addStrategy: replaceAggregationAddStrategy, params: [{name: 'name', type: 'string', dynamicLookup: true}], defaultParams: ['avg'], @@ -101,6 +103,7 @@ register({ register({ type: 'math', + label: 'Math:', addStrategy: addMathStrategy, params: [{ name: 'expr', type: 'string' }], defaultParams: [' / 100'], @@ -109,6 +112,7 @@ register({ register({ type: 'alias', + label: 'Alias:', addStrategy: addAliasStrategy, params: [{ name: 'name', type: 'string', quote: 'double' }], defaultParams: ['alias'], @@ -118,6 +122,7 @@ register({ register({ type: 'time', + label: 'time', params: [ { name: 'interval', From a5bcd8eac02f52900c0839dd2642a3dbd02ced23 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 12:10:18 +0200 Subject: [PATCH 0056/1100] fix metric column when using group by --- .../postgres/partials/query.editor.html | 28 +++++++++---------- .../datasource/postgres/postgres_query.ts | 20 ++++++------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index b0614a3ee9e..220f11fdfef 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -28,6 +28,20 @@
+
+
+ + + + +
+ +
+
+
+ +
+
-
-
- - - - -
- -
-
-
- -
-
+
+
Filter
+
+ Alert name + +
+
+ Dashboard title + +
+
+ + +
+
+ Dashboard tags + + +
+
State filter
diff --git a/public/app/plugins/panel/alertlist/module.ts b/public/app/plugins/panel/alertlist/module.ts index 35fbaead3b1..55869ce626d 100644 --- a/public/app/plugins/panel/alertlist/module.ts +++ b/public/app/plugins/panel/alertlist/module.ts @@ -21,6 +21,7 @@ class AlertListPanel extends PanelCtrl { currentAlerts: any = []; alertHistory: any = []; noAlertsMessage: string; + // Set and populate defaults panelDefaults = { show: 'current', @@ -28,6 +29,9 @@ class AlertListPanel extends PanelCtrl { stateFilter: [], onlyAlertsOnDashboard: false, sortOrder: 1, + dashboardFilter: '', + nameFilter: '', + folderId: null, }; /** @ngInject */ @@ -89,6 +93,11 @@ class AlertListPanel extends PanelCtrl { }); } + onFolderChange(folder: any) { + this.panel.folderId = folder.id; + this.refresh(); + } + getStateChanges() { var params: any = { limit: this.panel.limit, @@ -110,6 +119,7 @@ class AlertListPanel extends PanelCtrl { al.info = alertDef.getAlertAnnotationInfo(al); return al; }); + this.noAlertsMessage = this.alertHistory.length === 0 ? 'No alerts in current time range' : ''; return this.alertHistory; @@ -121,10 +131,26 @@ class AlertListPanel extends PanelCtrl { state: this.panel.stateFilter, }; + if (this.panel.nameFilter) { + params.query = this.panel.nameFilter; + } + + if (this.panel.folderId >= 0) { + params.folderId = this.panel.folderId; + } + + if (this.panel.dashboardFilter) { + params.dashboardQuery = this.panel.dashboardFilter; + } + if (this.panel.onlyAlertsOnDashboard) { params.dashboardId = this.dashboard.id; } + if (this.panel.dashboardTags) { + params.dashboardTag = this.panel.dashboardTags; + } + return this.backendSrv.get(`/api/alerts`, params).then(res => { this.currentAlerts = this.sortResult( _.map(res, al => { @@ -135,6 +161,9 @@ class AlertListPanel extends PanelCtrl { return al; }) ); + if (this.currentAlerts.length > this.panel.limit) { + this.currentAlerts = this.currentAlerts.slice(0, this.panel.limit); + } this.noAlertsMessage = this.currentAlerts.length === 0 ? 'No alerts' : ''; return this.currentAlerts; From b67872bc35c63eb6debf2ac121673442d0a3f948 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 1 Jun 2018 14:49:14 +0200 Subject: [PATCH 0091/1100] changelog: add notes about closing #11500, #8168, #6541 [skip ci] --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7e46d6cf4..7ef36a8796f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # 5.2.0 (unreleased) +### New Features + +* **Alert list panel**: Updated to support filtering alerts by name, dashboard title, folder, tags [#11500](https://github.com/grafana/grafana/issues/11500), [#8168](https://github.com/grafana/grafana/issues/8168), [#6541](https://github.com/grafana/grafana/issues/6541) + ### Minor * **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) From f5cf92636451ef2bb80f86606e6e8b03cb28c962 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 1 Jun 2018 15:23:26 +0200 Subject: [PATCH 0092/1100] changelog: add notes about closing #5893 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef36a8796f..76e538a8e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### New Features +* **Elasticsearch**: Alerting support [#5893](https://github.com/grafana/grafana/issues/5893), thx [@WPH95](https://github.com/WPH95) * **Alert list panel**: Updated to support filtering alerts by name, dashboard title, folder, tags [#11500](https://github.com/grafana/grafana/issues/11500), [#8168](https://github.com/grafana/grafana/issues/8168), [#6541](https://github.com/grafana/grafana/issues/6541) ### Minor From 75ee1e920890e2b7568407b0034cbddc01ebdce3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 08:13:20 +0200 Subject: [PATCH 0093/1100] renames intervalSeconds to updateIntervalSeconds --- docs/sources/administration/provisioning.md | 1 + .../provisioning/dashboards/config_reader.go | 4 +- .../dashboards/config_reader_test.go | 12 +-- .../provisioning/dashboards/file_reader.go | 2 +- .../dashboards-from-disk/dev-dashboards.yaml | 2 +- .../test-configs/version-0/version-0.yaml | 2 +- pkg/services/provisioning/dashboards/types.go | 80 +++++++++---------- 7 files changed, 53 insertions(+), 50 deletions(-) diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index 79b47aee9f6..888a0777796 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -197,6 +197,7 @@ providers: folder: '' type: file disableDeletion: false + updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards options: path: /var/lib/grafana/dashboards ``` diff --git a/pkg/services/provisioning/dashboards/config_reader.go b/pkg/services/provisioning/dashboards/config_reader.go index f8b6070c704..7508550838f 100644 --- a/pkg/services/provisioning/dashboards/config_reader.go +++ b/pkg/services/provisioning/dashboards/config_reader.go @@ -82,8 +82,8 @@ func (cr *configReader) readConfig() ([]*DashboardsAsConfig, error) { dashboards[i].OrgId = 1 } - if dashboards[i].IntervalSeconds == 0 { - dashboards[i].IntervalSeconds = 3 + if dashboards[i].UpdateIntervalSeconds == 0 { + dashboards[i].UpdateIntervalSeconds = 3 } } diff --git a/pkg/services/provisioning/dashboards/config_reader_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go index b49cd258005..df0d2ae038e 100644 --- a/pkg/services/provisioning/dashboards/config_reader_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -22,7 +22,7 @@ func TestDashboardsAsConfig(t *testing.T) { cfg, err := cfgProvider.readConfig() So(err, ShouldBeNil) - validateDashboardAsConfig(cfg) + validateDashboardAsConfig(t, cfg) }) Convey("Can read config file in version 0 format", func() { @@ -30,7 +30,7 @@ func TestDashboardsAsConfig(t *testing.T) { cfg, err := cfgProvider.readConfig() So(err, ShouldBeNil) - validateDashboardAsConfig(cfg) + validateDashboardAsConfig(t, cfg) }) Convey("Should skip invalid path", func() { @@ -56,7 +56,9 @@ func TestDashboardsAsConfig(t *testing.T) { }) }) } -func validateDashboardAsConfig(cfg []*DashboardsAsConfig) { +func validateDashboardAsConfig(t *testing.T, cfg []*DashboardsAsConfig) { + t.Helper() + So(len(cfg), ShouldEqual, 2) ds := cfg[0] @@ -68,7 +70,7 @@ func validateDashboardAsConfig(cfg []*DashboardsAsConfig) { So(len(ds.Options), ShouldEqual, 1) So(ds.Options["path"], ShouldEqual, "/var/lib/grafana/dashboards") So(ds.DisableDeletion, ShouldBeTrue) - So(ds.IntervalSeconds, ShouldEqual, 10) + So(ds.UpdateIntervalSeconds, ShouldEqual, 10) ds2 := cfg[1] So(ds2.Name, ShouldEqual, "default") @@ -79,5 +81,5 @@ func validateDashboardAsConfig(cfg []*DashboardsAsConfig) { So(len(ds2.Options), ShouldEqual, 1) So(ds2.Options["path"], ShouldEqual, "/var/lib/grafana/dashboards") So(ds2.DisableDeletion, ShouldBeFalse) - So(ds2.IntervalSeconds, ShouldEqual, 3) + So(ds2.UpdateIntervalSeconds, ShouldEqual, 3) } diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index a25b0208ad3..89416d2596c 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -66,7 +66,7 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { fr.log.Error("failed to search for dashboards", "error", err) } - ticker := time.NewTicker(time.Duration(int64(time.Second) * fr.Cfg.IntervalSeconds)) + ticker := time.NewTicker(time.Duration(int64(time.Second) * fr.Cfg.UpdateIntervalSeconds)) running := false diff --git a/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml index 5ea2a0a4f75..e26c329f87c 100644 --- a/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml +++ b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml @@ -6,7 +6,7 @@ providers: folder: 'developers' editable: true disableDeletion: true - intervalSeconds: 10 + updateIntervalSeconds: 10 type: file options: path: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml index bdbb06079fd..69a317fb396 100644 --- a/pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml +++ b/pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml @@ -3,7 +3,7 @@ folder: 'developers' editable: true disableDeletion: true - intervalSeconds: 10 + updateIntervalSeconds: 10 type: file options: path: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index 424e5e35f4a..a658b816c7d 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -10,25 +10,25 @@ import ( ) type DashboardsAsConfig struct { - Name string - Type string - OrgId int64 - Folder string - Editable bool - Options map[string]interface{} - DisableDeletion bool - IntervalSeconds int64 + Name string + Type string + OrgId int64 + Folder string + Editable bool + Options map[string]interface{} + DisableDeletion bool + UpdateIntervalSeconds int64 } type DashboardsAsConfigV0 struct { - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - OrgId int64 `json:"org_id" yaml:"org_id"` - Folder string `json:"folder" yaml:"folder"` - Editable bool `json:"editable" yaml:"editable"` - Options map[string]interface{} `json:"options" yaml:"options"` - DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` - IntervalSeconds int64 `json:"intervalSeconds" yaml:"intervalSeconds"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + OrgId int64 `json:"org_id" yaml:"org_id"` + Folder string `json:"folder" yaml:"folder"` + Editable bool `json:"editable" yaml:"editable"` + Options map[string]interface{} `json:"options" yaml:"options"` + DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` + UpdateIntervalSeconds int64 `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"` } type ConfigVersion struct { @@ -40,14 +40,14 @@ type DashboardAsConfigV1 struct { } type DashboardProviderConfigs struct { - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - OrgId int64 `json:"orgId" yaml:"orgId"` - Folder string `json:"folder" yaml:"folder"` - Editable bool `json:"editable" yaml:"editable"` - Options map[string]interface{} `json:"options" yaml:"options"` - DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` - IntervalSeconds int64 `json:"intervalSeconds" yaml:"intervalSeconds"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + OrgId int64 `json:"orgId" yaml:"orgId"` + Folder string `json:"folder" yaml:"folder"` + Editable bool `json:"editable" yaml:"editable"` + Options map[string]interface{} `json:"options" yaml:"options"` + DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` + UpdateIntervalSeconds int64 `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"` } func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) { @@ -71,14 +71,14 @@ func mapV0ToDashboardAsConfig(v0 []*DashboardsAsConfigV0) []*DashboardsAsConfig for _, v := range v0 { r = append(r, &DashboardsAsConfig{ - Name: v.Name, - Type: v.Type, - OrgId: v.OrgId, - Folder: v.Folder, - Editable: v.Editable, - Options: v.Options, - DisableDeletion: v.DisableDeletion, - IntervalSeconds: v.IntervalSeconds, + Name: v.Name, + Type: v.Type, + OrgId: v.OrgId, + Folder: v.Folder, + Editable: v.Editable, + Options: v.Options, + DisableDeletion: v.DisableDeletion, + UpdateIntervalSeconds: v.UpdateIntervalSeconds, }) } @@ -90,14 +90,14 @@ func (dc *DashboardAsConfigV1) mapToDashboardAsConfig() []*DashboardsAsConfig { for _, v := range dc.Providers { r = append(r, &DashboardsAsConfig{ - Name: v.Name, - Type: v.Type, - OrgId: v.OrgId, - Folder: v.Folder, - Editable: v.Editable, - Options: v.Options, - DisableDeletion: v.DisableDeletion, - IntervalSeconds: v.IntervalSeconds, + Name: v.Name, + Type: v.Type, + OrgId: v.OrgId, + Folder: v.Folder, + Editable: v.Editable, + Options: v.Options, + DisableDeletion: v.DisableDeletion, + UpdateIntervalSeconds: v.UpdateIntervalSeconds, }) } From 3f5078339c0193a416775e719fd5c8a0293229ab Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 08:27:03 +0200 Subject: [PATCH 0094/1100] tests: uses different paths depending on os --- .../provisioning/dashboards/file_reader_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 87e9ec6d226..bdc1e95aafe 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -49,13 +49,16 @@ func TestCreatingNewDashboardFileReader(t *testing.T) { }) Convey("using full path", func() { - cfg.Options["folder"] = "/var/lib/grafana/dashboards" + fullPath := "/var/lib/grafana/dashboards" + if runtime.GOOS == "windows" { + fullPath = `c:\var\lib\grafana` + } + + cfg.Options["folder"] = fullPath reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) So(err, ShouldBeNil) - if runtime.GOOS != "windows" { - So(reader.Path, ShouldEqual, "/var/lib/grafana/dashboards") - } + So(reader.Path, ShouldEqual, fullPath) So(filepath.IsAbs(reader.Path), ShouldBeTrue) }) From f606654c50239fbc4616bcdd50c0441dd810ed1f Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 09:04:33 +0200 Subject: [PATCH 0095/1100] provisioning: adds fallback if evalsymlink/abs fails --- pkg/services/provisioning/dashboards/file_reader.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index a1ba4dbf8e2..8af23980531 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -51,7 +51,6 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade path, err := filepath.Abs(path) if err != nil { log.Error("Could not create absolute path ", "path", path) - path = copy //if .Abs return an error we fallback to path } path, err = filepath.EvalSymlinks(path) @@ -59,6 +58,11 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade log.Error("Failed to read content of symlinked path: %s", path) } + if path == "" { + path = copy + log.Info("falling back to original path due to EvalSymlink/Abs failure") + } + return &fileReader{ Cfg: cfg, Path: path, From feb5e20779379863687e624e7ddf52e1c503061d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 4 Jun 2018 11:17:50 +0200 Subject: [PATCH 0096/1100] datasource: added option no-direct-access to ds-http-settings diretive, closes #12138 --- public/app/features/plugins/ds_edit_ctrl.ts | 4 ++++ public/app/features/plugins/partials/ds_http_settings.html | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/features/plugins/ds_edit_ctrl.ts b/public/app/features/plugins/ds_edit_ctrl.ts index b98f0f48910..f86cc694255 100644 --- a/public/app/features/plugins/ds_edit_ctrl.ts +++ b/public/app/features/plugins/ds_edit_ctrl.ts @@ -204,10 +204,14 @@ coreModule.directive('datasourceHttpSettings', function() { scope: { current: '=', suggestUrl: '@', + noDirectAccess: '@', }, templateUrl: 'public/app/features/plugins/partials/ds_http_settings.html', link: { pre: function($scope, elem, attrs) { + // do not show access option if direct access is disabled + $scope.showAccessOption = $scope.noDirectAccess !== 'true'; + $scope.getSuggestUrls = function() { return [$scope.suggestUrl]; }; diff --git a/public/app/features/plugins/partials/ds_http_settings.html b/public/app/features/plugins/partials/ds_http_settings.html index b9f5683129c..b35aab0c099 100644 --- a/public/app/features/plugins/partials/ds_http_settings.html +++ b/public/app/features/plugins/partials/ds_http_settings.html @@ -22,7 +22,7 @@
-
+
Access
From 13c6f37ea581db9ecb04c859618847425d7cba46 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Fri, 1 Jun 2018 13:39:44 +0300 Subject: [PATCH 0097/1100] alerting: show alerts for user with Viewer role changelog: add notes about closing #11167 remove changelog note reformat alert_test.go --- pkg/api/alerting.go | 2 +- pkg/services/sqlstore/alert.go | 2 +- pkg/services/sqlstore/alert_test.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 961fc11b2dc..60013fe2b10 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -79,7 +79,7 @@ func GetAlerts(c *m.ReqContext) Response { DashboardIds: dashboardIDs, Type: string(search.DashHitDB), FolderIds: folderIDs, - Permission: m.PERMISSION_EDIT, + Permission: m.PERMISSION_VIEW, } err := bus.Dispatch(&searchQuery) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 58ec7e2857a..531a70b2101 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -116,7 +116,7 @@ func HandleAlertsQuery(query *m.GetAlertsQuery) error { } if query.User.OrgRole != m.ROLE_ADMIN { - builder.writeDashboardPermissionFilter(query.User, m.PERMISSION_EDIT) + builder.writeDashboardPermissionFilter(query.User, m.PERMISSION_VIEW) } builder.Write(" ORDER BY name ASC") diff --git a/pkg/services/sqlstore/alert_test.go b/pkg/services/sqlstore/alert_test.go index be48c7b2f52..79fa99864e7 100644 --- a/pkg/services/sqlstore/alert_test.go +++ b/pkg/services/sqlstore/alert_test.go @@ -2,7 +2,6 @@ package sqlstore import ( "testing" - "time" "github.com/grafana/grafana/pkg/components/simplejson" @@ -110,11 +109,12 @@ func TestAlertingDataAccess(t *testing.T) { }) Convey("Viewer cannot read alerts", func() { - alertQuery := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_VIEWER}} + viewerUser := &m.SignedInUser{OrgRole: m.ROLE_VIEWER, OrgId: 1} + alertQuery := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, PanelId: 1, OrgId: 1, User: viewerUser} err2 := HandleAlertsQuery(&alertQuery) So(err2, ShouldBeNil) - So(alertQuery.Result, ShouldHaveLength, 0) + So(alertQuery.Result, ShouldHaveLength, 1) }) Convey("Alerts with same dashboard id and panel id should update", func() { From e562ae753b75210a56d98e3689179bebb318d0f7 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Mon, 4 Jun 2018 11:49:12 +0200 Subject: [PATCH 0098/1100] docs: docker secrets support. (#12141) Closes #12132 --- CHANGELOG.md | 1 + docs/sources/installation/docker.md | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76e538a8e32..ecbc99608c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ * **Singlestat**: Fix singlestat threshold tooltip [#11971](https://github.com/grafana/grafana/issues/11971) * **Dashboard**: Hide grid controls in fullscreen/low-activity views [#11771](https://github.com/grafana/grafana/issues/11771) * **Dashboard**: Validate uid when importing dashboards [#11515](https://github.com/grafana/grafana/issues/11515) +* **Docker**: Support for env variables ending with _FILE [grafana-docker #166](https://github.com/grafana/grafana-docker/pull/166), thx [@efrecon](https://github.com/efrecon) # 5.1.3 (2018-05-16) diff --git a/docs/sources/installation/docker.md b/docs/sources/installation/docker.md index e78796845c4..e7dee84b5f4 100644 --- a/docs/sources/installation/docker.md +++ b/docs/sources/installation/docker.md @@ -130,6 +130,18 @@ ID=$(id -u) # saves your user id in the ID variable docker run -d --user $ID --volume "$PWD/data:/var/lib/grafana" -p 3000:3000 grafana/grafana:5.1.0 ``` +## Reading secrets from files (support for Docker Secrets) + +It's possible to supply Grafana with configuration through files. This works well with [Docker Secrets](https://docs.docker.com/engine/swarm/secrets/) as the secrets by default gets mapped into `/run/secrets/` of the container. + +You can do this with any of the configuration options in conf/grafana.ini by setting `GF___FILE` to the path of the file holding the secret. + +Let's say you want to set the admin password this way. + +- Admin password secret: `/run/secrets/admin_password` +- Environment variable: `GF_SECURITY_ADMIN_PASSWORD_FILE=/run/secrets/admin_password` + + ## Migration from a previous version of the docker container to 5.1 or later The docker container for Grafana has seen a major rewrite for 5.1. From 7453df2662c569643e0d358c8e06ae99af89041e Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 4 Jun 2018 11:57:13 +0200 Subject: [PATCH 0099/1100] changelog: add notes about closing #11167 [skip ci] --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecbc99608c4..9eda912e86a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,8 @@ * **Singlestat**: Fix singlestat threshold tooltip [#11971](https://github.com/grafana/grafana/issues/11971) * **Dashboard**: Hide grid controls in fullscreen/low-activity views [#11771](https://github.com/grafana/grafana/issues/11771) * **Dashboard**: Validate uid when importing dashboards [#11515](https://github.com/grafana/grafana/issues/11515) -* **Docker**: Support for env variables ending with _FILE [grafana-docker #166](https://github.com/grafana/grafana-docker/pull/166), thx [@efrecon](https://github.com/efrecon) +* **Docker**: Support for env variables ending with _FILE [grafana-docker #166](https://github.com/grafana/grafana-docker/pull/166), thx [@efrecon](https://github.com/efrecon) +* **Alert list panel**: Show alerts for user with viewer role [#11167](https://github.com/grafana/grafana/issues/11167) # 5.1.3 (2018-05-16) From 08ee1da6b128b8a3191768448118aec2ed564ef2 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 30 May 2018 11:29:44 +0200 Subject: [PATCH 0100/1100] InfluxDB IFQL datasource --- package.json | 1 + pkg/api/frontendsettings.go | 11 + pkg/models/datasource.go | 1 + public/app/core/table_model.ts | 4 + .../app/features/plugins/built_in_plugins.ts | 2 + .../datasource/influxdb-ifql/README.md | 26 ++ .../datasource/influxdb-ifql/datasource.ts | 255 +++++++++++++ .../influxdb-ifql/img/influxdb_logo.svg | 26 ++ .../datasource/influxdb-ifql/module.ts | 17 + .../partials/annotations.editor.html | 24 ++ .../influxdb-ifql/partials/config.html | 24 ++ .../influxdb-ifql/partials/query.editor.html | 24 ++ .../datasource/influxdb-ifql/plugin.json | 24 ++ .../datasource/influxdb-ifql/query_ctrl.ts | 17 + .../influxdb-ifql/response_parser.ts | 88 +++++ .../specs/response_parser.jest.ts | 63 ++++ .../specs/sample_response_csv.ts | 349 ++++++++++++++++++ yarn.lock | 4 + 18 files changed, 960 insertions(+) create mode 100644 public/app/plugins/datasource/influxdb-ifql/README.md create mode 100644 public/app/plugins/datasource/influxdb-ifql/datasource.ts create mode 100644 public/app/plugins/datasource/influxdb-ifql/img/influxdb_logo.svg create mode 100644 public/app/plugins/datasource/influxdb-ifql/module.ts create mode 100644 public/app/plugins/datasource/influxdb-ifql/partials/annotations.editor.html create mode 100644 public/app/plugins/datasource/influxdb-ifql/partials/config.html create mode 100644 public/app/plugins/datasource/influxdb-ifql/partials/query.editor.html create mode 100644 public/app/plugins/datasource/influxdb-ifql/plugin.json create mode 100644 public/app/plugins/datasource/influxdb-ifql/query_ctrl.ts create mode 100644 public/app/plugins/datasource/influxdb-ifql/response_parser.ts create mode 100644 public/app/plugins/datasource/influxdb-ifql/specs/response_parser.jest.ts create mode 100644 public/app/plugins/datasource/influxdb-ifql/specs/sample_response_csv.ts diff --git a/package.json b/package.json index df3da5812c1..5fd72357f6f 100644 --- a/package.json +++ b/package.json @@ -157,6 +157,7 @@ "moment": "^2.18.1", "mousetrap": "^1.6.0", "mousetrap-global-bind": "^1.1.0", + "papaparse": "^4.4.0", "prismjs": "^1.6.0", "prop-types": "^15.6.0", "react": "^16.2.0", diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 5cd52122c3f..84524bad526 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -85,6 +85,13 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { dsMap["database"] = ds.Database dsMap["url"] = url } + + if ds.Type == m.DS_INFLUXDB_IFQL { + dsMap["username"] = ds.User + dsMap["password"] = ds.Password + dsMap["database"] = ds.Database + dsMap["url"] = url + } } if ds.Type == m.DS_ES { @@ -95,6 +102,10 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { dsMap["database"] = ds.Database } + if ds.Type == m.DS_INFLUXDB_IFQL { + dsMap["database"] = ds.Database + } + if ds.Type == m.DS_PROMETHEUS { // add unproxied server URL for link to Prometheus web UI dsMap["directUrl"] = ds.Url diff --git a/pkg/models/datasource.go b/pkg/models/datasource.go index b7e3e3eaa17..530f31242a9 100644 --- a/pkg/models/datasource.go +++ b/pkg/models/datasource.go @@ -12,6 +12,7 @@ const ( DS_GRAPHITE = "graphite" DS_INFLUXDB = "influxdb" DS_INFLUXDB_08 = "influxdb_08" + DS_INFLUXDB_IFQL = "influxdb-ifql" DS_ES = "elasticsearch" DS_OPENTSDB = "opentsdb" DS_CLOUDWATCH = "cloudwatch" diff --git a/public/app/core/table_model.ts b/public/app/core/table_model.ts index 57800b3e48d..5716aac2be6 100644 --- a/public/app/core/table_model.ts +++ b/public/app/core/table_model.ts @@ -44,4 +44,8 @@ export default class TableModel { this.columnMap[col.text] = col; } } + + addRow(row) { + this.rows.push(row); + } } diff --git a/public/app/features/plugins/built_in_plugins.ts b/public/app/features/plugins/built_in_plugins.ts index 6998321dd75..49be31e5474 100644 --- a/public/app/features/plugins/built_in_plugins.ts +++ b/public/app/features/plugins/built_in_plugins.ts @@ -4,6 +4,7 @@ import * as elasticsearchPlugin from 'app/plugins/datasource/elasticsearch/modul import * as opentsdbPlugin from 'app/plugins/datasource/opentsdb/module'; import * as grafanaPlugin from 'app/plugins/datasource/grafana/module'; import * as influxdbPlugin from 'app/plugins/datasource/influxdb/module'; +import * as influxdbIfqlPlugin from 'app/plugins/datasource/influxdb-ifql/module'; import * as mixedPlugin from 'app/plugins/datasource/mixed/module'; import * as mysqlPlugin from 'app/plugins/datasource/mysql/module'; import * as postgresPlugin from 'app/plugins/datasource/postgres/module'; @@ -30,6 +31,7 @@ const builtInPlugins = { 'app/plugins/datasource/opentsdb/module': opentsdbPlugin, 'app/plugins/datasource/grafana/module': grafanaPlugin, 'app/plugins/datasource/influxdb/module': influxdbPlugin, + 'app/plugins/datasource/influxdb-ifql/module': influxdbIfqlPlugin, 'app/plugins/datasource/mixed/module': mixedPlugin, 'app/plugins/datasource/mysql/module': mysqlPlugin, 'app/plugins/datasource/postgres/module': postgresPlugin, diff --git a/public/app/plugins/datasource/influxdb-ifql/README.md b/public/app/plugins/datasource/influxdb-ifql/README.md new file mode 100644 index 00000000000..91f82b2a89d --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/README.md @@ -0,0 +1,26 @@ +# InfluxDB (IFQL) Datasource [BETA] - Native Plugin + +Grafana ships with **built in** support for InfluxDB (>= 1.4.1). + +Use this datasource if you want to use IFQL to query your InfluxDB. +Feel free to run this datasource side-by-side with the non-IFQL datasource. +If you point both datasources to the same InfluxDB instance, you can switch query mode by switching the datasources. + +Read more about IFQL here: + +[https://github.com/influxdata/ifql](https://github.com/influxdata/ifql) + +Read more about InfluxDB here: + +[http://docs.grafana.org/datasources/influxdb/](http://docs.grafana.org/datasources/influxdb/) + +## Roadmap + +- Sync Grafana time ranges with `range()` +- Template variable expansion +- Syntax highlighting +- Tab completion (functions, values) +- Result helpers (result counts, table previews) +- Annotations support +- Alerting integration +- Explore UI integration diff --git a/public/app/plugins/datasource/influxdb-ifql/datasource.ts b/public/app/plugins/datasource/influxdb-ifql/datasource.ts new file mode 100644 index 00000000000..bd3cb7e2d5b --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/datasource.ts @@ -0,0 +1,255 @@ +import _ from 'lodash'; + +import * as dateMath from 'app/core/utils/datemath'; + +import { getTableModelFromResult, getTimeSeriesFromResult, parseResults } from './response_parser'; + +function serializeParams(params) { + if (!params) { + return ''; + } + + return _.reduce( + params, + (memo, value, key) => { + if (value === null || value === undefined) { + return memo; + } + memo.push(encodeURIComponent(key) + '=' + encodeURIComponent(value)); + return memo; + }, + [] + ).join('&'); +} + +const MAX_SERIES = 20; +export default class InfluxDatasource { + type: string; + urls: any; + username: string; + password: string; + name: string; + orgName: string; + database: any; + basicAuth: any; + withCredentials: any; + interval: any; + supportAnnotations: boolean; + supportMetrics: boolean; + + /** @ngInject */ + constructor(instanceSettings, private backendSrv, private templateSrv) { + this.type = 'influxdb-ifql'; + this.urls = instanceSettings.url.split(',').map(url => url.trim()); + + this.username = instanceSettings.username; + this.password = instanceSettings.password; + this.name = instanceSettings.name; + this.orgName = instanceSettings.orgName || 'defaultorgname'; + this.database = instanceSettings.database; + this.basicAuth = instanceSettings.basicAuth; + this.withCredentials = instanceSettings.withCredentials; + this.interval = (instanceSettings.jsonData || {}).timeInterval; + this.supportAnnotations = true; + this.supportMetrics = true; + } + + query(options) { + const targets = _.cloneDeep(options.targets); + const queryTargets = targets.filter(t => t.query); + if (queryTargets.length === 0) { + return Promise.resolve({ data: [] }); + } + + // replace grafana variables + const timeFilter = this.getTimeFilter(options); + options.scopedVars.timeFilter = { value: timeFilter }; + + const queries = queryTargets.map(target => { + const { query, resultFormat } = target; + + // TODO replace templated variables + // allQueries = this.templateSrv.replace(allQueries, scopedVars); + + if (resultFormat === 'table') { + return ( + this._seriesQuery(query, options) + .then(response => parseResults(response.data)) + // Keep only first result from each request + .then(results => results[0]) + .then(getTableModelFromResult) + ); + } else { + return this._seriesQuery(query, options) + .then(response => parseResults(response.data)) + .then(results => results.map(getTimeSeriesFromResult)); + } + }); + + return Promise.all(queries).then((series: any) => { + let seriesList = _.flattenDeep(series).slice(0, MAX_SERIES); + return { data: seriesList }; + }); + } + + annotationQuery(options) { + if (!options.annotation.query) { + return Promise.reject({ + message: 'Query missing in annotation definition', + }); + } + + var timeFilter = this.getTimeFilter({ rangeRaw: options.rangeRaw }); + var query = options.annotation.query.replace('$timeFilter', timeFilter); + query = this.templateSrv.replace(query, null, 'regex'); + + return {}; + } + + targetContainsTemplate(target) { + for (let group of target.groupBy) { + for (let param of group.params) { + if (this.templateSrv.variableExists(param)) { + return true; + } + } + } + + for (let i in target.tags) { + if (this.templateSrv.variableExists(target.tags[i].value)) { + return true; + } + } + + return false; + } + + metricFindQuery(query: string, options?: any) { + var interpolated = this.templateSrv.replace(query, null, 'regex'); + + return this._seriesQuery(interpolated, options).then(_.curry(parseResults)(query)); + } + + _seriesQuery(query: string, options?: any) { + if (!query) { + return Promise.resolve({ data: '' }); + } + return this._influxRequest('POST', '/v1/query', { q: query }, options); + } + + testDatasource() { + const query = `from(db:"${this.database}") |> last()`; + + return this._influxRequest('POST', '/v1/query', { q: query }) + .then(res => { + if (res && res.trim()) { + return { status: 'success', message: 'Data source connected and database found.' }; + } + return { + status: 'error', + message: + 'Data source connected, but has no data. Verify the "Database" field and make sure the database has data.', + }; + }) + .catch(err => { + return { status: 'error', message: err.message }; + }); + } + + _influxRequest(method: string, url: string, data: any, options?: any) { + // TODO reinstante Round-robin + // const currentUrl = this.urls.shift(); + // this.urls.push(currentUrl); + const currentUrl = this.urls[0]; + + let params: any = { + orgName: this.orgName, + }; + + if (this.username) { + params.u = this.username; + params.p = this.password; + } + + if (options && options.database) { + params.db = options.database; + } else if (this.database) { + params.db = this.database; + } + + // data sent as GET param + _.extend(params, data); + data = null; + + let req: any = { + method: method, + url: currentUrl + url, + params: params, + data: data, + precision: 'ms', + inspect: { type: this.type }, + paramSerializer: serializeParams, + }; + + req.headers = req.headers || {}; + if (this.basicAuth || this.withCredentials) { + req.withCredentials = true; + } + if (this.basicAuth) { + req.headers.Authorization = this.basicAuth; + } + + return this.backendSrv.datasourceRequest(req).then( + result => { + return result; + }, + function(err) { + if (err.status !== 0 || err.status >= 300) { + if (err.data && err.data.error) { + throw { + message: 'InfluxDB Error: ' + err.data.error, + data: err.data, + config: err.config, + }; + } else { + throw { + message: 'Network Error: ' + err.statusText + '(' + err.status + ')', + data: err.data, + config: err.config, + }; + } + } + } + ); + } + + getTimeFilter(options) { + var from = this.getInfluxTime(options.rangeRaw.from, false); + var until = this.getInfluxTime(options.rangeRaw.to, true); + var fromIsAbsolute = from[from.length - 1] === 'ms'; + + if (until === 'now()' && !fromIsAbsolute) { + return 'time >= ' + from; + } + + return 'time >= ' + from + ' and time <= ' + until; + } + + getInfluxTime(date, roundUp) { + if (_.isString(date)) { + if (date === 'now') { + return 'now()'; + } + + var parts = /^now-(\d+)([d|h|m|s])$/.exec(date); + if (parts) { + var amount = parseInt(parts[1]); + var unit = parts[2]; + return 'now() - ' + amount + unit; + } + date = dateMath.parse(date, roundUp); + } + + return date.valueOf() + 'ms'; + } +} diff --git a/public/app/plugins/datasource/influxdb-ifql/img/influxdb_logo.svg b/public/app/plugins/datasource/influxdb-ifql/img/influxdb_logo.svg new file mode 100644 index 00000000000..3c0e379e0d7 --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/img/influxdb_logo.svg @@ -0,0 +1,26 @@ + + + + + + diff --git a/public/app/plugins/datasource/influxdb-ifql/module.ts b/public/app/plugins/datasource/influxdb-ifql/module.ts new file mode 100644 index 00000000000..5997a7d061b --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/module.ts @@ -0,0 +1,17 @@ +import InfluxDatasource from './datasource'; +import { InfluxQueryCtrl } from './query_ctrl'; + +class InfluxConfigCtrl { + static templateUrl = 'partials/config.html'; +} + +class InfluxAnnotationsQueryCtrl { + static templateUrl = 'partials/annotations.editor.html'; +} + +export { + InfluxDatasource as Datasource, + InfluxQueryCtrl as QueryCtrl, + InfluxConfigCtrl as ConfigCtrl, + InfluxAnnotationsQueryCtrl as AnnotationsQueryCtrl, +}; diff --git a/public/app/plugins/datasource/influxdb-ifql/partials/annotations.editor.html b/public/app/plugins/datasource/influxdb-ifql/partials/annotations.editor.html new file mode 100644 index 00000000000..48991426c1e --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/partials/annotations.editor.html @@ -0,0 +1,24 @@ + +
+
+ +
+
+ +
Field mappings If your influxdb query returns more than one field you need to specify the column names below. An annotation event is composed of a title, tags, and an additional text field.
+
+
+
+ Text + +
+
+ Tags + +
+
+ Title (deprecated) + +
+
+
diff --git a/public/app/plugins/datasource/influxdb-ifql/partials/config.html b/public/app/plugins/datasource/influxdb-ifql/partials/config.html new file mode 100644 index 00000000000..be6f0438efd --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/partials/config.html @@ -0,0 +1,24 @@ + + + +

InfluxDB Details

+ +
+
+
+ Default Database + +
+
+ +
+
+ User + +
+
+ Password + +
+
+
\ No newline at end of file diff --git a/public/app/plugins/datasource/influxdb-ifql/partials/query.editor.html b/public/app/plugins/datasource/influxdb-ifql/partials/query.editor.html new file mode 100644 index 00000000000..31f5923cdb2 --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/partials/query.editor.html @@ -0,0 +1,24 @@ + + +
+ +
+
+
+ +
+ +
+
+
+ + +
+
+
+
+
+ + \ No newline at end of file diff --git a/public/app/plugins/datasource/influxdb-ifql/plugin.json b/public/app/plugins/datasource/influxdb-ifql/plugin.json new file mode 100644 index 00000000000..b4eb764d556 --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/plugin.json @@ -0,0 +1,24 @@ +{ + "type": "datasource", + "name": "InfluxDB (IFQL) [BETA]", + "id": "influxdb-ifql", + "defaultMatchFormat": "regex values", + "metrics": true, + "annotations": false, + "alerting": false, + "queryOptions": { + "minInterval": true + }, + "info": { + "description": "InfluxDB Data Source for IFQL Queries for Grafana", + "author": { + "name": "Grafana Project", + "url": "https://grafana.com" + }, + "logos": { + "small": "img/influxdb_logo.svg", + "large": "img/influxdb_logo.svg" + }, + "version": "5.1.0" + } +} \ No newline at end of file diff --git a/public/app/plugins/datasource/influxdb-ifql/query_ctrl.ts b/public/app/plugins/datasource/influxdb-ifql/query_ctrl.ts new file mode 100644 index 00000000000..950a3feb58e --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/query_ctrl.ts @@ -0,0 +1,17 @@ +import { QueryCtrl } from 'app/plugins/sdk'; + +export class InfluxQueryCtrl extends QueryCtrl { + static templateUrl = 'partials/query.editor.html'; + + resultFormats: any[]; + + /** @ngInject **/ + constructor($scope, $injector) { + super($scope, $injector); + this.resultFormats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; + } + + getCollapsedText() { + return this.target.query; + } +} diff --git a/public/app/plugins/datasource/influxdb-ifql/response_parser.ts b/public/app/plugins/datasource/influxdb-ifql/response_parser.ts new file mode 100644 index 00000000000..e2ef753392c --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/response_parser.ts @@ -0,0 +1,88 @@ +import Papa from 'papaparse'; +import groupBy from 'lodash/groupBy'; + +import TableModel from 'app/core/table_model'; + +const filterColumnKeys = key => key && key[0] !== '_' && key !== 'result' && key !== 'table'; + +const IGNORE_FIELDS_FOR_NAME = ['result', '', 'table']; +export const getNameFromRecord = record => { + // Measurement and field + const metric = [record._measurement, record._field]; + + // Add tags + const tags = Object.keys(record) + .filter(key => key[0] !== '_') + .filter(key => IGNORE_FIELDS_FOR_NAME.indexOf(key) === -1) + .map(key => `${key}=${record[key]}`); + + return [...metric, ...tags].join(' '); +}; + +const parseCSV = (input: string) => + Papa.parse(input, { + header: true, + comments: '#', + }).data; + +export const parseValue = (input: string) => { + const value = parseFloat(input); + return isNaN(value) ? null : value; +}; + +export const parseTime = (input: string) => Date.parse(input); + +export function parseResults(response: string): any[] { + return response.trim().split(/\n\s*\s/); +} + +export function getTableModelFromResult(result: string) { + const data = parseCSV(result); + + const table = new TableModel(); + if (data.length > 0) { + // First columns are fixed + const firstColumns = [ + { text: 'Time', id: '_time' }, + { text: 'Measurement', id: '_measurement' }, + { text: 'Field', id: '_field' }, + ]; + + // Dynamically add columns for tags + const firstRecord = data[0]; + const tags = Object.keys(firstRecord) + .filter(filterColumnKeys) + .map(key => ({ id: key, text: key })); + + const valueColumn = { id: '_value', text: 'Value' }; + const columns = [...firstColumns, ...tags, valueColumn]; + columns.forEach(c => table.addColumn(c)); + + // Add rows + data.forEach(record => { + const row = columns.map(c => record[c.id]); + table.addRow(row); + }); + } + + return table; +} + +export function getTimeSeriesFromResult(result: string) { + const data = parseCSV(result); + if (data.length === 0) { + return []; + } + + // Group results by table ID (assume one table per timeseries for now) + const tables = groupBy(data, 'table'); + const seriesList = Object.keys(tables) + .map(id => tables[id]) + .map(series => { + const datapoints = series.map(record => [parseValue(record._value), parseTime(record._time)]); + const alias = getNameFromRecord(series[0]); + return { datapoints, target: alias }; + }); + + return seriesList; +} diff --git a/public/app/plugins/datasource/influxdb-ifql/specs/response_parser.jest.ts b/public/app/plugins/datasource/influxdb-ifql/specs/response_parser.jest.ts new file mode 100644 index 00000000000..bac154c0760 --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/specs/response_parser.jest.ts @@ -0,0 +1,63 @@ +import { + getNameFromRecord, + getTableModelFromResult, + getTimeSeriesFromResult, + parseResults, + parseValue, +} from '../response_parser'; +import response from './sample_response_csv'; + +describe('influxdb ifql response parser', () => { + describe('parseResults()', () => { + it('expects three results', () => { + const results = parseResults(response); + expect(results.length).toBe(2); + }); + }); + + describe('getTableModelFromResult()', () => { + it('expects a table model', () => { + const results = parseResults(response); + const table = getTableModelFromResult(results[0]); + expect(table.columns.length).toBe(6); + expect(table.rows.length).toBe(300); + }); + }); + + describe('getTimeSeriesFromResult()', () => { + it('expects time series', () => { + const results = parseResults(response); + const series = getTimeSeriesFromResult(results[0]); + expect(series.length).toBe(50); + expect(series[0].datapoints.length).toBe(6); + }); + }); + + describe('getNameFromRecord()', () => { + it('expects name based on measurements and tags', () => { + const record = { + '': '', + result: '', + table: '0', + _start: '2018-06-02T06:35:25.651942602Z', + _stop: '2018-06-02T07:35:25.651942602Z', + _time: '2018-06-02T06:35:31Z', + _value: '0', + _field: 'usage_guest', + _measurement: 'cpu', + cpu: 'cpu-total', + host: 'kenobi-3.local', + }; + expect(getNameFromRecord(record)).toBe('cpu usage_guest cpu=cpu-total host=kenobi-3.local'); + }); + }); + + describe('parseValue()', () => { + it('parses a number', () => { + expect(parseValue('42.3')).toBe(42.3); + }); + it('parses a non-number to null', () => { + expect(parseValue('foo')).toBe(null); + }); + }); +}); diff --git a/public/app/plugins/datasource/influxdb-ifql/specs/sample_response_csv.ts b/public/app/plugins/datasource/influxdb-ifql/specs/sample_response_csv.ts new file mode 100644 index 00000000000..2c7c0194684 --- /dev/null +++ b/public/app/plugins/datasource/influxdb-ifql/specs/sample_response_csv.ts @@ -0,0 +1,349 @@ +const result = `#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string,string,string,string +#partition,false,false,true,true,false,false,true,true,true,true +#default,_result,,,,,,,,, +,result,table,_start,_stop,_time,_value,_field,_measurement,cpu,host +,,0,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest,cpu,cpu-total,kenobi-3.local +,,0,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest,cpu,cpu-total,kenobi-3.local +,,0,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest,cpu,cpu-total,kenobi-3.local +,,0,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest,cpu,cpu-total,kenobi-3.local +,,0,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest,cpu,cpu-total,kenobi-3.local +,,0,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest,cpu,cpu-total,kenobi-3.local +,,1,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest_nice,cpu,cpu-total,kenobi-3.local +,,1,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest_nice,cpu,cpu-total,kenobi-3.local +,,1,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest_nice,cpu,cpu-total,kenobi-3.local +,,1,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest_nice,cpu,cpu-total,kenobi-3.local +,,1,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest_nice,cpu,cpu-total,kenobi-3.local +,,1,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest_nice,cpu,cpu-total,kenobi-3.local +,,2,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,81.87046761690422,usage_idle,cpu,cpu-total,kenobi-3.local +,,2,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,82.03398300849575,usage_idle,cpu,cpu-total,kenobi-3.local +,,2,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,76.26186906546727,usage_idle,cpu,cpu-total,kenobi-3.local +,,2,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,79.65465465465465,usage_idle,cpu,cpu-total,kenobi-3.local +,,2,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,70.72195853110168,usage_idle,cpu,cpu-total,kenobi-3.local +,,2,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,69.86746686671668,usage_idle,cpu,cpu-total,kenobi-3.local +,,3,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_iowait,cpu,cpu-total,kenobi-3.local +,,3,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_iowait,cpu,cpu-total,kenobi-3.local +,,3,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_iowait,cpu,cpu-total,kenobi-3.local +,,3,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_iowait,cpu,cpu-total,kenobi-3.local +,,3,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_iowait,cpu,cpu-total,kenobi-3.local +,,3,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_iowait,cpu,cpu-total,kenobi-3.local +,,4,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_irq,cpu,cpu-total,kenobi-3.local +,,4,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_irq,cpu,cpu-total,kenobi-3.local +,,4,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_irq,cpu,cpu-total,kenobi-3.local +,,4,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_irq,cpu,cpu-total,kenobi-3.local +,,4,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_irq,cpu,cpu-total,kenobi-3.local +,,4,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_irq,cpu,cpu-total,kenobi-3.local +,,5,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_nice,cpu,cpu-total,kenobi-3.local +,,5,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_nice,cpu,cpu-total,kenobi-3.local +,,5,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_nice,cpu,cpu-total,kenobi-3.local +,,5,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_nice,cpu,cpu-total,kenobi-3.local +,,5,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_nice,cpu,cpu-total,kenobi-3.local +,,5,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_nice,cpu,cpu-total,kenobi-3.local +,,6,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_softirq,cpu,cpu-total,kenobi-3.local +,,6,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_softirq,cpu,cpu-total,kenobi-3.local +,,6,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_softirq,cpu,cpu-total,kenobi-3.local +,,6,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_softirq,cpu,cpu-total,kenobi-3.local +,,6,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_softirq,cpu,cpu-total,kenobi-3.local +,,6,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_softirq,cpu,cpu-total,kenobi-3.local +,,7,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_steal,cpu,cpu-total,kenobi-3.local +,,7,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_steal,cpu,cpu-total,kenobi-3.local +,,7,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_steal,cpu,cpu-total,kenobi-3.local +,,7,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_steal,cpu,cpu-total,kenobi-3.local +,,7,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_steal,cpu,cpu-total,kenobi-3.local +,,7,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_steal,cpu,cpu-total,kenobi-3.local +,,8,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,6.25156289072268,usage_system,cpu,cpu-total,kenobi-3.local +,,8,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,8.045977011494253,usage_system,cpu,cpu-total,kenobi-3.local +,,8,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,8.79560219890055,usage_system,cpu,cpu-total,kenobi-3.local +,,8,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,8.408408408408409,usage_system,cpu,cpu-total,kenobi-3.local +,,8,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,11.64126904821384,usage_system,cpu,cpu-total,kenobi-3.local +,,8,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,13.078269567391848,usage_system,cpu,cpu-total,kenobi-3.local +,,9,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,11.877969492373094,usage_user,cpu,cpu-total,kenobi-3.local +,,9,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,9.920039980009996,usage_user,cpu,cpu-total,kenobi-3.local +,,9,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,14.942528735632184,usage_user,cpu,cpu-total,kenobi-3.local +,,9,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,11.936936936936936,usage_user,cpu,cpu-total,kenobi-3.local +,,9,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,17.636772420684487,usage_user,cpu,cpu-total,kenobi-3.local +,,9,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,17.05426356589147,usage_user,cpu,cpu-total,kenobi-3.local +,,10,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest,cpu,cpu0,kenobi-3.local +,,10,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest,cpu,cpu0,kenobi-3.local +,,10,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest,cpu,cpu0,kenobi-3.local +,,10,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest,cpu,cpu0,kenobi-3.local +,,10,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest,cpu,cpu0,kenobi-3.local +,,10,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest,cpu,cpu0,kenobi-3.local +,,11,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest_nice,cpu,cpu0,kenobi-3.local +,,11,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest_nice,cpu,cpu0,kenobi-3.local +,,11,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest_nice,cpu,cpu0,kenobi-3.local +,,11,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest_nice,cpu,cpu0,kenobi-3.local +,,11,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest_nice,cpu,cpu0,kenobi-3.local +,,11,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest_nice,cpu,cpu0,kenobi-3.local +,,12,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,73.1,usage_idle,cpu,cpu0,kenobi-3.local +,,12,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,69.03096903096903,usage_idle,cpu,cpu0,kenobi-3.local +,,12,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,63.63636363636363,usage_idle,cpu,cpu0,kenobi-3.local +,,12,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,67.86786786786787,usage_idle,cpu,cpu0,kenobi-3.local +,,12,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,57.4,usage_idle,cpu,cpu0,kenobi-3.local +,,12,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,57.8,usage_idle,cpu,cpu0,kenobi-3.local +,,13,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_iowait,cpu,cpu0,kenobi-3.local +,,13,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_iowait,cpu,cpu0,kenobi-3.local +,,13,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_iowait,cpu,cpu0,kenobi-3.local +,,13,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_iowait,cpu,cpu0,kenobi-3.local +,,13,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_iowait,cpu,cpu0,kenobi-3.local +,,13,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_iowait,cpu,cpu0,kenobi-3.local +,,14,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_irq,cpu,cpu0,kenobi-3.local +,,14,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_irq,cpu,cpu0,kenobi-3.local +,,14,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_irq,cpu,cpu0,kenobi-3.local +,,14,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_irq,cpu,cpu0,kenobi-3.local +,,14,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_irq,cpu,cpu0,kenobi-3.local +,,14,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_irq,cpu,cpu0,kenobi-3.local +,,15,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_nice,cpu,cpu0,kenobi-3.local +,,15,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_nice,cpu,cpu0,kenobi-3.local +,,15,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_nice,cpu,cpu0,kenobi-3.local +,,15,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_nice,cpu,cpu0,kenobi-3.local +,,15,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_nice,cpu,cpu0,kenobi-3.local +,,15,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_nice,cpu,cpu0,kenobi-3.local +,,16,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_softirq,cpu,cpu0,kenobi-3.local +,,16,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_softirq,cpu,cpu0,kenobi-3.local +,,16,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_softirq,cpu,cpu0,kenobi-3.local +,,16,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_softirq,cpu,cpu0,kenobi-3.local +,,16,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_softirq,cpu,cpu0,kenobi-3.local +,,16,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_softirq,cpu,cpu0,kenobi-3.local +,,17,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_steal,cpu,cpu0,kenobi-3.local +,,17,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_steal,cpu,cpu0,kenobi-3.local +,,17,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_steal,cpu,cpu0,kenobi-3.local +,,17,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_steal,cpu,cpu0,kenobi-3.local +,,17,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_steal,cpu,cpu0,kenobi-3.local +,,17,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_steal,cpu,cpu0,kenobi-3.local +,,18,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,9.6,usage_system,cpu,cpu0,kenobi-3.local +,,18,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,14.985014985014985,usage_system,cpu,cpu0,kenobi-3.local +,,18,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,14.185814185814186,usage_system,cpu,cpu0,kenobi-3.local +,,18,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,13.813813813813814,usage_system,cpu,cpu0,kenobi-3.local +,,18,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,17.9,usage_system,cpu,cpu0,kenobi-3.local +,,18,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,20,usage_system,cpu,cpu0,kenobi-3.local +,,19,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,17.3,usage_user,cpu,cpu0,kenobi-3.local +,,19,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,15.984015984015985,usage_user,cpu,cpu0,kenobi-3.local +,,19,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,22.17782217782218,usage_user,cpu,cpu0,kenobi-3.local +,,19,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,18.31831831831832,usage_user,cpu,cpu0,kenobi-3.local +,,19,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,24.7,usage_user,cpu,cpu0,kenobi-3.local +,,19,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,22.2,usage_user,cpu,cpu0,kenobi-3.local +,,20,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest,cpu,cpu1,kenobi-3.local +,,20,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest,cpu,cpu1,kenobi-3.local +,,20,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest,cpu,cpu1,kenobi-3.local +,,20,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest,cpu,cpu1,kenobi-3.local +,,20,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest,cpu,cpu1,kenobi-3.local +,,20,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest,cpu,cpu1,kenobi-3.local +,,21,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest_nice,cpu,cpu1,kenobi-3.local +,,21,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest_nice,cpu,cpu1,kenobi-3.local +,,21,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest_nice,cpu,cpu1,kenobi-3.local +,,21,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest_nice,cpu,cpu1,kenobi-3.local +,,21,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest_nice,cpu,cpu1,kenobi-3.local +,,21,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest_nice,cpu,cpu1,kenobi-3.local +,,22,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,89.8,usage_idle,cpu,cpu1,kenobi-3.local +,,22,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,91.8,usage_idle,cpu,cpu1,kenobi-3.local +,,22,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,87.11288711288711,usage_idle,cpu,cpu1,kenobi-3.local +,,22,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,89.48948948948949,usage_idle,cpu,cpu1,kenobi-3.local +,,22,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,83,usage_idle,cpu,cpu1,kenobi-3.local +,,22,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,80.1,usage_idle,cpu,cpu1,kenobi-3.local +,,23,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_iowait,cpu,cpu1,kenobi-3.local +,,23,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_iowait,cpu,cpu1,kenobi-3.local +,,23,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_iowait,cpu,cpu1,kenobi-3.local +,,23,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_iowait,cpu,cpu1,kenobi-3.local +,,23,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_iowait,cpu,cpu1,kenobi-3.local +,,23,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_iowait,cpu,cpu1,kenobi-3.local +,,24,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_irq,cpu,cpu1,kenobi-3.local +,,24,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_irq,cpu,cpu1,kenobi-3.local +,,24,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_irq,cpu,cpu1,kenobi-3.local +,,24,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_irq,cpu,cpu1,kenobi-3.local +,,24,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_irq,cpu,cpu1,kenobi-3.local +,,24,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_irq,cpu,cpu1,kenobi-3.local +,,25,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_nice,cpu,cpu1,kenobi-3.local +,,25,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_nice,cpu,cpu1,kenobi-3.local +,,25,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_nice,cpu,cpu1,kenobi-3.local +,,25,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_nice,cpu,cpu1,kenobi-3.local +,,25,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_nice,cpu,cpu1,kenobi-3.local +,,25,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_nice,cpu,cpu1,kenobi-3.local +,,26,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_softirq,cpu,cpu1,kenobi-3.local +,,26,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_softirq,cpu,cpu1,kenobi-3.local +,,26,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_softirq,cpu,cpu1,kenobi-3.local +,,26,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_softirq,cpu,cpu1,kenobi-3.local +,,26,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_softirq,cpu,cpu1,kenobi-3.local +,,26,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_softirq,cpu,cpu1,kenobi-3.local +,,27,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_steal,cpu,cpu1,kenobi-3.local +,,27,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_steal,cpu,cpu1,kenobi-3.local +,,27,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_steal,cpu,cpu1,kenobi-3.local +,,27,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_steal,cpu,cpu1,kenobi-3.local +,,27,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_steal,cpu,cpu1,kenobi-3.local +,,27,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_steal,cpu,cpu1,kenobi-3.local +,,28,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,3.5,usage_system,cpu,cpu1,kenobi-3.local +,,28,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,4,usage_system,cpu,cpu1,kenobi-3.local +,,28,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,4.895104895104895,usage_system,cpu,cpu1,kenobi-3.local +,,28,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,4.504504504504505,usage_system,cpu,cpu1,kenobi-3.local +,,28,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,6.3,usage_system,cpu,cpu1,kenobi-3.local +,,28,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,7.9,usage_system,cpu,cpu1,kenobi-3.local +,,29,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,6.7,usage_user,cpu,cpu1,kenobi-3.local +,,29,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,4.2,usage_user,cpu,cpu1,kenobi-3.local +,,29,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,7.992007992007992,usage_user,cpu,cpu1,kenobi-3.local +,,29,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,6.006006006006006,usage_user,cpu,cpu1,kenobi-3.local +,,29,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,10.7,usage_user,cpu,cpu1,kenobi-3.local +,,29,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,12,usage_user,cpu,cpu1,kenobi-3.local +,,30,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest,cpu,cpu2,kenobi-3.local +,,30,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest,cpu,cpu2,kenobi-3.local +,,30,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest,cpu,cpu2,kenobi-3.local +,,30,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest,cpu,cpu2,kenobi-3.local +,,30,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest,cpu,cpu2,kenobi-3.local +,,30,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest,cpu,cpu2,kenobi-3.local +,,31,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest_nice,cpu,cpu2,kenobi-3.local +,,31,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest_nice,cpu,cpu2,kenobi-3.local +,,31,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest_nice,cpu,cpu2,kenobi-3.local +,,31,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest_nice,cpu,cpu2,kenobi-3.local +,,31,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest_nice,cpu,cpu2,kenobi-3.local +,,31,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest_nice,cpu,cpu2,kenobi-3.local +,,32,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,75.17517517517517,usage_idle,cpu,cpu2,kenobi-3.local +,,32,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,74.82517482517483,usage_idle,cpu,cpu2,kenobi-3.local +,,32,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,67.9,usage_idle,cpu,cpu2,kenobi-3.local +,,32,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,72.47247247247248,usage_idle,cpu,cpu2,kenobi-3.local +,,32,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,61.63836163836164,usage_idle,cpu,cpu2,kenobi-3.local +,,32,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,62,usage_idle,cpu,cpu2,kenobi-3.local +,,33,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_iowait,cpu,cpu2,kenobi-3.local +,,33,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_iowait,cpu,cpu2,kenobi-3.local +,,33,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_iowait,cpu,cpu2,kenobi-3.local +,,33,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_iowait,cpu,cpu2,kenobi-3.local +,,33,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_iowait,cpu,cpu2,kenobi-3.local +,,33,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_iowait,cpu,cpu2,kenobi-3.local +,,34,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_irq,cpu,cpu2,kenobi-3.local +,,34,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_irq,cpu,cpu2,kenobi-3.local +,,34,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_irq,cpu,cpu2,kenobi-3.local +,,34,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_irq,cpu,cpu2,kenobi-3.local +,,34,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_irq,cpu,cpu2,kenobi-3.local +,,34,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_irq,cpu,cpu2,kenobi-3.local +,,35,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_nice,cpu,cpu2,kenobi-3.local +,,35,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_nice,cpu,cpu2,kenobi-3.local +,,35,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_nice,cpu,cpu2,kenobi-3.local +,,35,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_nice,cpu,cpu2,kenobi-3.local +,,35,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_nice,cpu,cpu2,kenobi-3.local +,,35,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_nice,cpu,cpu2,kenobi-3.local +,,36,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_softirq,cpu,cpu2,kenobi-3.local +,,36,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_softirq,cpu,cpu2,kenobi-3.local +,,36,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_softirq,cpu,cpu2,kenobi-3.local +,,36,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_softirq,cpu,cpu2,kenobi-3.local +,,36,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_softirq,cpu,cpu2,kenobi-3.local +,,36,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_softirq,cpu,cpu2,kenobi-3.local +,,37,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_steal,cpu,cpu2,kenobi-3.local +,,37,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_steal,cpu,cpu2,kenobi-3.local +,,37,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_steal,cpu,cpu2,kenobi-3.local +,,37,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_steal,cpu,cpu2,kenobi-3.local +,,37,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_steal,cpu,cpu2,kenobi-3.local +,,37,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_steal,cpu,cpu2,kenobi-3.local +,,38,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,8.208208208208209,usage_system,cpu,cpu2,kenobi-3.local +,,38,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,9.99000999000999,usage_system,cpu,cpu2,kenobi-3.local +,,38,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,11.2,usage_system,cpu,cpu2,kenobi-3.local +,,38,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,10.81081081081081,usage_system,cpu,cpu2,kenobi-3.local +,,38,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,14.785214785214785,usage_system,cpu,cpu2,kenobi-3.local +,,38,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,16.2,usage_system,cpu,cpu2,kenobi-3.local +,,39,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,16.616616616616618,usage_user,cpu,cpu2,kenobi-3.local +,,39,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,15.184815184815184,usage_user,cpu,cpu2,kenobi-3.local +,,39,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,20.9,usage_user,cpu,cpu2,kenobi-3.local +,,39,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,16.716716716716718,usage_user,cpu,cpu2,kenobi-3.local +,,39,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,23.576423576423576,usage_user,cpu,cpu2,kenobi-3.local +,,39,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,21.8,usage_user,cpu,cpu2,kenobi-3.local +,,40,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest,cpu,cpu3,kenobi-3.local +,,40,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest,cpu,cpu3,kenobi-3.local +,,40,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest,cpu,cpu3,kenobi-3.local +,,40,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest,cpu,cpu3,kenobi-3.local +,,40,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest,cpu,cpu3,kenobi-3.local +,,40,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest,cpu,cpu3,kenobi-3.local +,,41,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_guest_nice,cpu,cpu3,kenobi-3.local +,,41,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_guest_nice,cpu,cpu3,kenobi-3.local +,,41,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_guest_nice,cpu,cpu3,kenobi-3.local +,,41,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_guest_nice,cpu,cpu3,kenobi-3.local +,,41,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_guest_nice,cpu,cpu3,kenobi-3.local +,,41,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_guest_nice,cpu,cpu3,kenobi-3.local +,,42,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,89.4,usage_idle,cpu,cpu3,kenobi-3.local +,,42,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,92.5,usage_idle,cpu,cpu3,kenobi-3.local +,,42,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,86.4,usage_idle,cpu,cpu3,kenobi-3.local +,,42,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,88.78878878878879,usage_idle,cpu,cpu3,kenobi-3.local +,,42,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,80.83832335329342,usage_idle,cpu,cpu3,kenobi-3.local +,,42,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,79.57957957957957,usage_idle,cpu,cpu3,kenobi-3.local +,,43,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_iowait,cpu,cpu3,kenobi-3.local +,,43,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_iowait,cpu,cpu3,kenobi-3.local +,,43,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_iowait,cpu,cpu3,kenobi-3.local +,,43,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_iowait,cpu,cpu3,kenobi-3.local +,,43,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_iowait,cpu,cpu3,kenobi-3.local +,,43,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_iowait,cpu,cpu3,kenobi-3.local +,,44,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_irq,cpu,cpu3,kenobi-3.local +,,44,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_irq,cpu,cpu3,kenobi-3.local +,,44,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_irq,cpu,cpu3,kenobi-3.local +,,44,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_irq,cpu,cpu3,kenobi-3.local +,,44,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_irq,cpu,cpu3,kenobi-3.local +,,44,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_irq,cpu,cpu3,kenobi-3.local +,,45,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_nice,cpu,cpu3,kenobi-3.local +,,45,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_nice,cpu,cpu3,kenobi-3.local +,,45,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_nice,cpu,cpu3,kenobi-3.local +,,45,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_nice,cpu,cpu3,kenobi-3.local +,,45,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_nice,cpu,cpu3,kenobi-3.local +,,45,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_nice,cpu,cpu3,kenobi-3.local +,,46,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_softirq,cpu,cpu3,kenobi-3.local +,,46,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_softirq,cpu,cpu3,kenobi-3.local +,,46,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_softirq,cpu,cpu3,kenobi-3.local +,,46,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_softirq,cpu,cpu3,kenobi-3.local +,,46,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_softirq,cpu,cpu3,kenobi-3.local +,,46,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_softirq,cpu,cpu3,kenobi-3.local +,,47,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,0,usage_steal,cpu,cpu3,kenobi-3.local +,,47,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,0,usage_steal,cpu,cpu3,kenobi-3.local +,,47,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,0,usage_steal,cpu,cpu3,kenobi-3.local +,,47,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,0,usage_steal,cpu,cpu3,kenobi-3.local +,,47,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,0,usage_steal,cpu,cpu3,kenobi-3.local +,,47,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,0,usage_steal,cpu,cpu3,kenobi-3.local +,,48,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,3.7,usage_system,cpu,cpu3,kenobi-3.local +,,48,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,3.2,usage_system,cpu,cpu3,kenobi-3.local +,,48,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,4.9,usage_system,cpu,cpu3,kenobi-3.local +,,48,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,4.504504504504505,usage_system,cpu,cpu3,kenobi-3.local +,,48,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,7.584830339321357,usage_system,cpu,cpu3,kenobi-3.local +,,48,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,8.208208208208209,usage_system,cpu,cpu3,kenobi-3.local +,,49,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,6.9,usage_user,cpu,cpu3,kenobi-3.local +,,49,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,4.3,usage_user,cpu,cpu3,kenobi-3.local +,,49,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,8.7,usage_user,cpu,cpu3,kenobi-3.local +,,49,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,6.706706706706707,usage_user,cpu,cpu3,kenobi-3.local +,,49,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,11.57684630738523,usage_user,cpu,cpu3,kenobi-3.local +,,49,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,12.212212212212211,usage_user,cpu,cpu3,kenobi-3.local + +#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,long,string,string,string,string,string,string,string +#partition,false,false,true,true,false,false,true,true,true,true,true,true,true +#default,_result,,,,,,,,,,,, +,result,table,_start,_stop,_time,_value,_field,_measurement,device,fstype,host,mode,path +,,50,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,9024180224,free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,50,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,9025056768,free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,50,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,9024774144,free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,50,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,9024638976,free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,50,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,9024299008,free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,50,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,9024036864,free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,51,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,4290025660,inodes_free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,51,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,4290025659,inodes_free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,51,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,4290025659,inodes_free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,51,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,4290025660,inodes_free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,51,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,4290025660,inodes_free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,51,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,4290025657,inodes_free,disk,disk1,hfs,kenobi-3.local,rw,/ +,,52,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,4294967279,inodes_total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,52,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,4294967279,inodes_total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,52,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,4294967279,inodes_total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,52,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,4294967279,inodes_total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,52,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,4294967279,inodes_total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,52,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,4294967279,inodes_total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,53,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,4941619,inodes_used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,53,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,4941620,inodes_used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,53,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,4941620,inodes_used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,53,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,4941619,inodes_used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,53,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,4941619,inodes_used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,53,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,4941622,inodes_used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,54,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,249804886016,total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,54,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,249804886016,total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,54,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,249804886016,total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,54,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,249804886016,total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,54,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,249804886016,total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,54,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,249804886016,total,disk,disk1,hfs,kenobi-3.local,rw,/ +,,55,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:23Z,240518561792,used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,55,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:33Z,240517685248,used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,55,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:43Z,240517967872,used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,55,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:54:53Z,240518103040,used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,55,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:03Z,240518443008,used,disk,disk1,hfs,kenobi-3.local,rw,/ +,,55,2018-06-01T12:54:13.516195939Z,2018-06-01T12:55:13.516195939Z,2018-06-01T12:55:13Z,240518705152,used,disk,disk1,hfs,kenobi-3.local,rw,/ + +`; + +export default result; diff --git a/yarn.lock b/yarn.lock index f58731040c6..97435f665fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7928,6 +7928,10 @@ pako@~1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.6.tgz#0101211baa70c4bca4a0f63f2206e97b7dfaf258" +papaparse@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-4.4.0.tgz#6bcdbda80873e00cfb0bdcd7a4571c72a9a40168" + parallel-transform@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.1.0.tgz#d410f065b05da23081fcd10f28854c29bda33b06" From e068be4c26dc2d969ca4b0cc70bb00e2ee4d85a1 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Sat, 12 May 2018 21:11:58 -0400 Subject: [PATCH 0101/1100] Feature for repeated alerting in grafana --- pkg/api/dtos/alerting.go | 3 +++ pkg/models/alert.go | 9 ++++++++ pkg/services/alerting/extractor.go | 2 ++ pkg/services/alerting/notifiers/base.go | 5 ++++- pkg/services/alerting/result_handler.go | 1 + pkg/services/alerting/rule.go | 6 +++++ pkg/services/sqlstore/alert.go | 22 ++++++++++++++++++- pkg/services/sqlstore/migrations/alert_mig.go | 3 +++ .../app/features/alerting/alert_tab_ctrl.ts | 3 +++ .../features/alerting/partials/alert_tab.html | 3 +++ 10 files changed, 55 insertions(+), 2 deletions(-) diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index d30f2697f3f..64dd619a4eb 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -21,6 +21,9 @@ type AlertRule struct { ExecutionError string `json:"executionError"` Url string `json:"url"` CanEdit bool `json:"canEdit"` + NotifyOnce bool `json:"notifyOnce"` + NotifyEval uint64 `json:"notifyEval"` + NotifyFreq uint64 `json:"notifyFrequency"` } type AlertNotification struct { diff --git a/pkg/models/alert.go b/pkg/models/alert.go index fba2aa63df9..56ceeb2cbf7 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -72,6 +72,9 @@ type Alert struct { Silenced bool ExecutionError string Frequency int64 + NotifyOnce bool + NotifyFreq uint64 + NotifyEval uint64 EvalData *simplejson.Json NewStateDate time.Time @@ -95,6 +98,8 @@ func (this *Alert) ContainsUpdates(other *Alert) bool { result := false result = result || this.Name != other.Name result = result || this.Message != other.Message + result = result || this.NotifyOnce != other.NotifyOnce + result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq) if this.Settings != nil && other.Settings != nil { json1, err1 := this.Settings.Encode() @@ -159,6 +164,10 @@ type SetAlertStateCommand struct { Timestamp time.Time } +type IncAlertEvalCommand struct { + AlertId int64 +} + //Queries type GetAlertsQuery struct { OrgId int64 diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index e1c1bfacb2e..f820e546a93 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -122,6 +122,8 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, Handler: jsonAlert.Get("handler").MustInt64(), Message: jsonAlert.Get("message").MustString(), Frequency: frequency, + NotifyOnce: jsonAlert.Get("notifyOnce").MustBool(), + NotifyFreq: jsonAlert.Get("notifyFrequency").MustUint64(), } for _, condition := range jsonAlert.Get("conditions").MustArray() { diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 51676efdfd5..498bae3a6e6 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -32,7 +32,10 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model func defaultShouldNotify(context *alerting.EvalContext) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State { + if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce { + return false + } + if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 { return false } // Do not notify when we become OK for the first time. diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index c57b28c7c3e..56d299001f0 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,6 +88,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } + bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id}) handler.notifier.SendIfNeeded(evalContext) return nil diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index 018d138dbe4..0003fe791e3 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -23,6 +23,9 @@ type Rule struct { State m.AlertStateType Conditions []Condition Notifications []int64 + NotifyOnce bool + NotifyFreq uint64 + NotifyEval uint64 } type ValidationError struct { @@ -97,6 +100,9 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.Name = ruleDef.Name model.Message = ruleDef.Message model.Frequency = ruleDef.Frequency + model.NotifyOnce = ruleDef.NotifyOnce + model.NotifyFreq = ruleDef.NotifyFreq + model.NotifyEval = ruleDef.NotifyEval model.State = ruleDef.State model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 58ec7e2857a..9ab28be84ee 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -22,6 +22,7 @@ func init() { bus.AddHandler("sql", GetAlertStatesForDashboard) bus.AddHandler("sql", PauseAlert) bus.AddHandler("sql", PauseAllAlerts) + bus.AddHandler("sql", IncAlertEval) } func GetAlertById(query *m.GetAlertByIdQuery) error { @@ -188,7 +189,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS if alertToUpdate.ContainsUpdates(alert) { alert.Updated = timeNow() alert.State = alertToUpdate.State - sess.MustCols("message") + sess.MustCols("message", "notify_freq", "notify_once") _, err := sess.Id(alert.Id).Update(alert) if err != nil { return err @@ -343,3 +344,22 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error return err } + +func IncAlertEval(cmd *m.IncAlertEvalCommand) error { + return inTransaction(func(sess *DBSession) error { + alert := m.Alert{} + + if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil { + return err + } + + alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq + + sess.MustCols("notify_eval") + if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil { + return err + } + + return nil + }) +} diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 2a364d5f464..3452e5710cc 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -29,6 +29,9 @@ func addAlertMigrations(mg *Migrator) { {Name: "state_changes", Type: DB_Int, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, + {Name: "notify_once", Type: DB_Bool, Nullable: false}, + {Name: "notify_freq", Type: DB_Int, Nullable: false}, + {Name: "notify_eval", Type: DB_Int, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"org_id", "id"}, Type: IndexType}, diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index 79baa1e3f5a..f0d965ae81e 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -167,6 +167,9 @@ export class AlertTabCtrl { alert.noDataState = alert.noDataState || 'no_data'; alert.executionErrorState = alert.executionErrorState || 'alerting'; alert.frequency = alert.frequency || '60s'; + alert.notifyFrequency = alert.notifyFrequency || 10; + alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce; + alert.frequency = alert.frequency || '60s'; alert.handler = alert.handler || 1; alert.notifications = alert.notifications || []; diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html index cb101672aa4..084aeb2036a 100644 --- a/public/app/features/alerting/partials/alert_tab.html +++ b/public/app/features/alerting/partials/alert_tab.html @@ -31,6 +31,9 @@ Evaluate every + {{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }} + + evaluations
From 3cb0e27e1c474e5d203eb32428b2f39ee5fb3216 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Sat, 19 May 2018 16:21:00 -0400 Subject: [PATCH 0102/1100] Revert changes post code review and move them to notification page --- pkg/api/dtos/alerting.go | 25 +++---- pkg/models/alert.go | 9 --- pkg/models/alert_notifications.go | 71 ++++++++++++++----- pkg/services/alerting/eval_context.go | 15 ++++ pkg/services/alerting/extractor.go | 2 - pkg/services/alerting/interfaces.go | 2 + pkg/services/alerting/notifier.go | 12 +++- .../alerting/notifiers/alertmanager.go | 2 +- pkg/services/alerting/notifiers/base.go | 25 +++++-- pkg/services/alerting/notifiers/dingding.go | 2 +- pkg/services/alerting/notifiers/discord.go | 2 +- pkg/services/alerting/notifiers/email.go | 2 +- pkg/services/alerting/notifiers/hipchat.go | 2 +- pkg/services/alerting/notifiers/kafka.go | 2 +- pkg/services/alerting/notifiers/line.go | 2 +- pkg/services/alerting/notifiers/opsgenie.go | 2 +- pkg/services/alerting/notifiers/pagerduty.go | 2 +- pkg/services/alerting/notifiers/pushover.go | 2 +- pkg/services/alerting/notifiers/sensu.go | 2 +- pkg/services/alerting/notifiers/slack.go | 2 +- pkg/services/alerting/notifiers/teams.go | 2 +- pkg/services/alerting/notifiers/telegram.go | 2 +- pkg/services/alerting/notifiers/threema.go | 2 +- pkg/services/alerting/notifiers/victorops.go | 2 +- pkg/services/alerting/notifiers/webhook.go | 2 +- pkg/services/alerting/result_handler.go | 1 - pkg/services/alerting/rule.go | 6 -- pkg/services/sqlstore/alert.go | 22 +----- pkg/services/sqlstore/alert_notification.go | 58 +++++++++++++-- pkg/services/sqlstore/migrations/alert_mig.go | 27 ++++++- .../app/features/alerting/alert_tab_ctrl.ts | 3 - .../alerting/notification_edit_ctrl.ts | 2 + .../features/alerting/partials/alert_tab.html | 3 - .../alerting/partials/notification_edit.html | 5 ++ 34 files changed, 215 insertions(+), 107 deletions(-) diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 64dd619a4eb..5e0196c20d1 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -21,18 +21,17 @@ type AlertRule struct { ExecutionError string `json:"executionError"` Url string `json:"url"` CanEdit bool `json:"canEdit"` - NotifyOnce bool `json:"notifyOnce"` - NotifyEval uint64 `json:"notifyEval"` - NotifyFreq uint64 `json:"notifyFrequency"` } type AlertNotification struct { - Id int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + IsDefault bool `json:"isDefault"` + NotifyOnce bool `json:"notifyOnce"` + Frequency bool `json:"frequency"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type AlertTestCommand struct { @@ -62,9 +61,11 @@ type EvalMatch struct { } type NotificationTestCommand struct { - Name string `json:"name"` - Type string `json:"type"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name"` + Type string `json:"type"` + NotifyOnce bool `json:"notifyOnce"` + Frequency time.Duration `json:"frequency"` + Settings *simplejson.Json `json:"settings"` } type PauseAlertCommand struct { diff --git a/pkg/models/alert.go b/pkg/models/alert.go index 56ceeb2cbf7..fba2aa63df9 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -72,9 +72,6 @@ type Alert struct { Silenced bool ExecutionError string Frequency int64 - NotifyOnce bool - NotifyFreq uint64 - NotifyEval uint64 EvalData *simplejson.Json NewStateDate time.Time @@ -98,8 +95,6 @@ func (this *Alert) ContainsUpdates(other *Alert) bool { result := false result = result || this.Name != other.Name result = result || this.Message != other.Message - result = result || this.NotifyOnce != other.NotifyOnce - result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq) if this.Settings != nil && other.Settings != nil { json1, err1 := this.Settings.Encode() @@ -164,10 +159,6 @@ type SetAlertStateCommand struct { Timestamp time.Time } -type IncAlertEvalCommand struct { - AlertId int64 -} - //Queries type GetAlertsQuery struct { OrgId int64 diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 87b515f370c..cba62a51527 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -7,32 +7,38 @@ import ( ) type AlertNotification struct { - Id int64 `json:"id"` - OrgId int64 `json:"-"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + OrgId int64 `json:"-"` + Name string `json:"name"` + Type string `json:"type"` + NotifyOnce bool `json:"notifyOnce"` + Frequency time.Duration `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type CreateAlertNotificationCommand struct { - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + NotifyOnce bool `json:"notifyOnce" binding:"Required"` + Frequency time.Duration `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` OrgId int64 `json:"-"` Result *AlertNotification } type UpdateAlertNotificationCommand struct { - Id int64 `json:"id" binding:"Required"` - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings" binding:"Required"` + Id int64 `json:"id" binding:"Required"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + NotifyOnce string `json:"notifyOnce" binding:"Required"` + Frequency string `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings" binding:"Required"` OrgId int64 `json:"-"` Result *AlertNotification @@ -63,3 +69,34 @@ type GetAllAlertNotificationsQuery struct { Result []*AlertNotification } + +type NotificationJournal struct { + Id int64 + OrgId int64 + AlertId int64 + NotifierId int64 + SentAt time.Time + Success bool +} + +type RecordNotificationJournalCommand struct { + OrgId int64 + AlertId int64 + NotifierId int64 + SentAt time.Time + Success bool +} + +type GetLatestNotificationQuery struct { + OrgId int64 + AlertId int64 + NotifierId int64 + + Result *NotificationJournal +} + +type CleanNotificationJournalCommand struct { + OrgId int64 + AlertId int64 + NotifierId int64 +} diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index d0441d379b7..b451d188a64 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -143,3 +143,18 @@ func (c *EvalContext) GetNewState() m.AlertStateType { return m.AlertStateOK } + +func (c *EvalContext) LastNotify(notifierId int64) *time.Time { + cmd := &m.GetLatestNotificationQuery{ + OrgId: c.Rule.OrgId, + AlertId: c.Rule.Id, + NotifierId: notifierId, + } + if err := bus.Dispatch(cmd); err != nil { + c.log.Warn("Could not determine last time alert", + c.Rule.Name, "notified") + return nil + } + + return &cmd.Result.SentAt +} diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index f820e546a93..e1c1bfacb2e 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -122,8 +122,6 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, Handler: jsonAlert.Get("handler").MustInt64(), Message: jsonAlert.Get("message").MustString(), Frequency: frequency, - NotifyOnce: jsonAlert.Get("notifyOnce").MustBool(), - NotifyFreq: jsonAlert.Get("notifyFrequency").MustUint64(), } for _, condition := range jsonAlert.Get("conditions").MustArray() { diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 18f969ba1b9..8842b35fba2 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -19,6 +19,8 @@ type Notifier interface { GetNotifierId() int64 GetIsDefault() bool + GetNotifyOnce() bool + GetFrequency() time.Duration } type NotifierSlice []Notifier diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 2ea68cf5085..53923a420fe 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -66,7 +66,17 @@ func (n *notificationService) sendNotifications(context *EvalContext, notifiers not := notifier //avoid updating scope variable in go routine n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() - g.Go(func() error { return not.Notify(context) }) + g.Go(func() error { + success := not.Notify(context) == nil + cmd := &m.RecordNotificationJournalCommand{ + OrgId: context.Rule.OrgId, + AlertId: context.Rule.Id, + NotifierId: not.GetNotifierId(), + SentAt: time.Now(), + Success: success, + } + return bus.Dispatch(cmd) + }) } return g.Wait() diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index d449167de13..3eeb25986e0 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -33,7 +33,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err } return &AlertmanagerNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, log: log.New("alerting.notifier.prometheus-alertmanager"), }, nil diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 498bae3a6e6..e9e32020c6d 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -1,6 +1,8 @@ package notifiers import ( + "time" + "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" @@ -12,9 +14,11 @@ type NotifierBase struct { Id int64 IsDeault bool UploadImage bool + NotifyOnce bool + Frequency time.Duration } -func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model *simplejson.Json) NotifierBase { +func NewNotifierBase(id int64, isDefault bool, name, notifierType string, notifyOnce bool, frequency time.Duration, model *simplejson.Json) NotifierBase { uploadImage := true value, exist := model.CheckGet("uploadImage") if exist { @@ -27,15 +31,17 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model IsDeault: isDefault, Type: notifierType, UploadImage: uploadImage, + NotifyOnce: notifyOnce, + Frequency: frequency, } } -func defaultShouldNotify(context *alerting.EvalContext) bool { +func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequency time.Duration, lastNotify *time.Time) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce { + if context.PrevAlertState == context.Rule.State && notifyOnce { return false } - if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 { + if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } // Do not notify when we become OK for the first time. @@ -46,7 +52,8 @@ func defaultShouldNotify(context *alerting.EvalContext) bool { } func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool { - return defaultShouldNotify(context) + lastNotify := context.LastNotify(n.Id) + return defaultShouldNotify(context, n.NotifyOnce, n.Frequency, lastNotify) } func (n *NotifierBase) GetType() string { @@ -64,3 +71,11 @@ func (n *NotifierBase) GetNotifierId() int64 { func (n *NotifierBase) GetIsDefault() bool { return n.IsDeault } + +func (n *NotifierBase) GetNotifyOnce() bool { + return n.NotifyOnce +} + +func (n *NotifierBase) GetFrequency() time.Duration { + return n.Frequency +} diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go index 14eacef5831..78446c56f88 100644 --- a/pkg/services/alerting/notifiers/dingding.go +++ b/pkg/services/alerting/notifiers/dingding.go @@ -32,7 +32,7 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &DingDingNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, log: log.New("alerting.notifier.dingding"), }, nil diff --git a/pkg/services/alerting/notifiers/discord.go b/pkg/services/alerting/notifiers/discord.go index 3ffa7484870..693ed31e206 100644 --- a/pkg/services/alerting/notifiers/discord.go +++ b/pkg/services/alerting/notifiers/discord.go @@ -39,7 +39,7 @@ func NewDiscordNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &DiscordNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), WebhookURL: url, log: log.New("alerting.notifier.discord"), }, nil diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go index 562ffbe1269..234a4f8e756 100644 --- a/pkg/services/alerting/notifiers/email.go +++ b/pkg/services/alerting/notifiers/email.go @@ -52,7 +52,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) { }) return &EmailNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Addresses: addresses, log: log.New("alerting.notifier.email"), }, nil diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go index 58e1b7bd71e..4eb5b78811e 100644 --- a/pkg/services/alerting/notifiers/hipchat.go +++ b/pkg/services/alerting/notifiers/hipchat.go @@ -59,7 +59,7 @@ func NewHipChatNotifier(model *models.AlertNotification) (alerting.Notifier, err roomId := model.Settings.Get("roomid").MustString() return &HipChatNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, ApiKey: apikey, RoomId: roomId, diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go index 92f6489106b..0dab556d5e1 100644 --- a/pkg/services/alerting/notifiers/kafka.go +++ b/pkg/services/alerting/notifiers/kafka.go @@ -43,7 +43,7 @@ func NewKafkaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &KafkaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Endpoint: endpoint, Topic: topic, log: log.New("alerting.notifier.kafka"), diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go index 4814662f3a9..0ee252e6447 100644 --- a/pkg/services/alerting/notifiers/line.go +++ b/pkg/services/alerting/notifiers/line.go @@ -39,7 +39,7 @@ func NewLINENotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &LineNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Token: token, log: log.New("alerting.notifier.line"), }, nil diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go index f0f5142cf05..991afd5ce9b 100644 --- a/pkg/services/alerting/notifiers/opsgenie.go +++ b/pkg/services/alerting/notifiers/opsgenie.go @@ -56,7 +56,7 @@ func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &OpsGenieNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), ApiKey: apiKey, ApiUrl: apiUrl, AutoClose: autoClose, diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index 02219b2203d..afa0ba63eca 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -51,7 +51,7 @@ func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &PagerdutyNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Key: key, AutoResolve: autoResolve, log: log.New("alerting.notifier.pagerduty"), diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go index cbe9e16801a..09dfd6f0f9b 100644 --- a/pkg/services/alerting/notifiers/pushover.go +++ b/pkg/services/alerting/notifiers/pushover.go @@ -99,7 +99,7 @@ func NewPushoverNotifier(model *m.AlertNotification) (alerting.Notifier, error) return nil, alerting.ValidationError{Reason: "API token not given"} } return &PushoverNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), UserKey: userKey, ApiToken: apiToken, Priority: priority, diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go index 9f77801d458..e6b94d3223e 100644 --- a/pkg/services/alerting/notifiers/sensu.go +++ b/pkg/services/alerting/notifiers/sensu.go @@ -51,7 +51,7 @@ func NewSensuNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &SensuNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, User: model.Settings.Get("username").MustString(), Source: model.Settings.Get("source").MustString(), diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go index a8139b62726..fbbe4b3e59d 100644 --- a/pkg/services/alerting/notifiers/slack.go +++ b/pkg/services/alerting/notifiers/slack.go @@ -78,7 +78,7 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) { uploadImage := model.Settings.Get("uploadImage").MustBool(true) return &SlackNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, Recipient: recipient, Mention: mention, diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go index 7f62340d0e1..362a367e1f2 100644 --- a/pkg/services/alerting/notifiers/teams.go +++ b/pkg/services/alerting/notifiers/teams.go @@ -33,7 +33,7 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &TeamsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, log: log.New("alerting.notifier.teams"), }, nil diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go index ca24c996914..97696b2290c 100644 --- a/pkg/services/alerting/notifiers/telegram.go +++ b/pkg/services/alerting/notifiers/telegram.go @@ -78,7 +78,7 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &TelegramNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), BotToken: botToken, ChatID: chatId, UploadImage: uploadImage, diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go index e4ffffc9108..e7fb39f27db 100644 --- a/pkg/services/alerting/notifiers/threema.go +++ b/pkg/services/alerting/notifiers/threema.go @@ -106,7 +106,7 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &ThreemaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), GatewayID: gatewayID, RecipientID: recipientID, APISecret: apiSecret, diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index a753ca3cbf6..c6c1cf76047 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -51,7 +51,7 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e } return &VictoropsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), URL: url, AutoResolve: autoResolve, log: log.New("alerting.notifier.victorops"), diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go index 4c97ed2b75e..26989873e9e 100644 --- a/pkg/services/alerting/notifiers/webhook.go +++ b/pkg/services/alerting/notifiers/webhook.go @@ -47,7 +47,7 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &WebhookNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, User: model.Settings.Get("username").MustString(), Password: model.Settings.Get("password").MustString(), diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 56d299001f0..c57b28c7c3e 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,7 +88,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } - bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id}) handler.notifier.SendIfNeeded(evalContext) return nil diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index 0003fe791e3..018d138dbe4 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -23,9 +23,6 @@ type Rule struct { State m.AlertStateType Conditions []Condition Notifications []int64 - NotifyOnce bool - NotifyFreq uint64 - NotifyEval uint64 } type ValidationError struct { @@ -100,9 +97,6 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.Name = ruleDef.Name model.Message = ruleDef.Message model.Frequency = ruleDef.Frequency - model.NotifyOnce = ruleDef.NotifyOnce - model.NotifyFreq = ruleDef.NotifyFreq - model.NotifyEval = ruleDef.NotifyEval model.State = ruleDef.State model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 9ab28be84ee..58ec7e2857a 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -22,7 +22,6 @@ func init() { bus.AddHandler("sql", GetAlertStatesForDashboard) bus.AddHandler("sql", PauseAlert) bus.AddHandler("sql", PauseAllAlerts) - bus.AddHandler("sql", IncAlertEval) } func GetAlertById(query *m.GetAlertByIdQuery) error { @@ -189,7 +188,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS if alertToUpdate.ContainsUpdates(alert) { alert.Updated = timeNow() alert.State = alertToUpdate.State - sess.MustCols("message", "notify_freq", "notify_once") + sess.MustCols("message") _, err := sess.Id(alert.Id).Update(alert) if err != nil { return err @@ -344,22 +343,3 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error return err } - -func IncAlertEval(cmd *m.IncAlertEvalCommand) error { - return inTransaction(func(sess *DBSession) error { - alert := m.Alert{} - - if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil { - return err - } - - alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq - - sess.MustCols("notify_eval") - if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil { - return err - } - - return nil - }) -} diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 651241f7714..8bb17143042 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -17,6 +17,9 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) + bus.AddHandler("sql", RecordNotificationJournal) + bus.AddHandler("sql", GetLatestNotification) + bus.AddHandler("sql", CleanNotificationJournal) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -138,13 +141,15 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error } alertNotification := &m.AlertNotification{ - OrgId: cmd.OrgId, - Name: cmd.Name, - Type: cmd.Type, - Settings: cmd.Settings, - Created: time.Now(), - Updated: time.Now(), - IsDefault: cmd.IsDefault, + OrgId: cmd.OrgId, + Name: cmd.Name, + Type: cmd.Type, + Settings: cmd.Settings, + NotifyOnce: cmd.NotifyOnce, + Frequency: cmd.Frequency, + Created: time.Now(), + Updated: time.Now(), + IsDefault: cmd.IsDefault, } if _, err = sess.Insert(alertNotification); err != nil { @@ -192,3 +197,42 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return nil }) } + +func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { + return inTransaction(func(sess *DBSession) error { + journalEntry := &m.NotificationJournal{ + OrgId: cmd.OrgId, + AlertId: cmd.AlertId, + NotifierId: cmd.NotifierId, + SentAt: cmd.SentAt, + Success: cmd.Success, + } + + if _, err := sess.Insert(journalEntry); err != nil { + return err + } + + return nil + }) +} + +func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { + return inTransaction(func(sess *DBSession) error { + notificationJournal := &m.NotificationJournal{} + _, err := sess.OrderBy("notification_journal.sent_at").Desc().Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + if err != nil { + return err + } + + cmd.Result = notificationJournal + return nil + }) +} + +func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error { + return inTransaction(func(sess *DBSession) error { + sql := "DELETE FROM notification_journal WHERE notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?" + _, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId) + return err + }) +} diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 3452e5710cc..d045f611fb2 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -29,9 +29,6 @@ func addAlertMigrations(mg *Migrator) { {Name: "state_changes", Type: DB_Int, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, - {Name: "notify_once", Type: DB_Bool, Nullable: false}, - {Name: "notify_freq", Type: DB_Int, Nullable: false}, - {Name: "notify_eval", Type: DB_Int, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"org_id", "id"}, Type: IndexType}, @@ -68,8 +65,32 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("Add column is_default", NewAddColumnMigration(alert_notification, &Column{ Name: "is_default", Type: DB_Bool, Nullable: false, Default: "0", })) + mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{ + Name: "frequency", Type: DB_BigInt, Nullable: true, + })) + mg.AddMigration("Add column notify_once", NewAddColumnMigration(alert_notification, &Column{ + Name: "notify_once", Type: DB_Bool, Nullable: false, Default: "1", + })) mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) + notification_journal := Table{ + Name: "notification_journal", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "alert_id", Type: DB_BigInt, Nullable: false}, + {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, + {Name: "sent_at", Type: DB_DateTime, Nullable: false}, + {Name: "success", Type: DB_Bool, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, + }, + } + + mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal)) + mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0])) + mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{ {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "message", Type: DB_Text, Nullable: false}, diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index f0d965ae81e..79baa1e3f5a 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -167,9 +167,6 @@ export class AlertTabCtrl { alert.noDataState = alert.noDataState || 'no_data'; alert.executionErrorState = alert.executionErrorState || 'alerting'; alert.frequency = alert.frequency || '60s'; - alert.notifyFrequency = alert.notifyFrequency || 10; - alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce; - alert.frequency = alert.frequency || '60s'; alert.handler = alert.handler || 1; alert.notifications = alert.notifications || []; diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 18b1c4d1d55..2fd185bee29 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -11,6 +11,7 @@ export class AlertNotificationEditCtrl { model: any; defaults: any = { type: 'email', + notifyOnce: true, settings: { httpMethod: 'POST', autoResolve: true, @@ -102,6 +103,7 @@ export class AlertNotificationEditCtrl { var payload = { name: this.model.name, type: this.model.type, + frequency: this.model.frequency, settings: this.model.settings, }; diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html index 084aeb2036a..cb101672aa4 100644 --- a/public/app/features/alerting/partials/alert_tab.html +++ b/public/app/features/alerting/partials/alert_tab.html @@ -31,9 +31,6 @@ Evaluate every - {{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }} - - evaluations
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index d20b9031a8f..ccdb9ef1073 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -18,6 +18,11 @@
+
+ {{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }} + + seconds +
Date: Sun, 20 May 2018 12:12:10 -0400 Subject: [PATCH 0103/1100] Fix multiple bugs --- pkg/api/alerting.go | 57 ++++++++++++++++--- pkg/api/dtos/alerting.go | 19 ++++--- pkg/models/alert_notifications.go | 6 +- pkg/services/alerting/eval_context.go | 4 +- pkg/services/alerting/notifiers/base.go | 1 + pkg/services/alerting/notifiers/base_test.go | 13 +++-- pkg/services/sqlstore/alert_notification.go | 28 +++++++-- .../alerting/partials/notification_edit.html | 3 +- 8 files changed, 95 insertions(+), 36 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 961fc11b2dc..c5b47270f4d 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -193,12 +193,15 @@ func GetAlertNotifications(c *m.ReqContext) Response { for _, notification := range query.Result { result = append(result, &dtos.AlertNotification{ - Id: notification.Id, - Name: notification.Name, - Type: notification.Type, - IsDefault: notification.IsDefault, - Created: notification.Created, - Updated: notification.Updated, + Id: notification.Id, + Name: notification.Name, + Type: notification.Type, + IsDefault: notification.IsDefault, + Created: notification.Created, + Updated: notification.Updated, + Frequency: notification.Frequency.String(), + NotifyOnce: notification.NotifyOnce, + Settings: notification.Settings, }) } @@ -215,7 +218,19 @@ func GetAlertNotificationByID(c *m.ReqContext) Response { return Error(500, "Failed to get alert notifications", err) } - return JSON(200, query.Result) + result := &dtos.AlertNotification{ + Id: query.Result.Id, + Name: query.Result.Name, + Type: query.Result.Type, + IsDefault: query.Result.IsDefault, + Created: query.Result.Created, + Updated: query.Result.Updated, + Frequency: query.Result.Frequency.String(), + NotifyOnce: query.Result.NotifyOnce, + Settings: query.Result.Settings, + } + + return JSON(200, result) } func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationCommand) Response { @@ -225,7 +240,19 @@ func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationComma return Error(500, "Failed to create alert notification", err) } - return JSON(200, cmd.Result) + result := &dtos.AlertNotification{ + Id: cmd.Result.Id, + Name: cmd.Result.Name, + Type: cmd.Result.Type, + IsDefault: cmd.Result.IsDefault, + Created: cmd.Result.Created, + Updated: cmd.Result.Updated, + Frequency: cmd.Result.Frequency.String(), + NotifyOnce: cmd.Result.NotifyOnce, + Settings: cmd.Result.Settings, + } + + return JSON(200, result) } func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationCommand) Response { @@ -235,7 +262,19 @@ func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationComma return Error(500, "Failed to update alert notification", err) } - return JSON(200, cmd.Result) + result := &dtos.AlertNotification{ + Id: cmd.Result.Id, + Name: cmd.Result.Name, + Type: cmd.Result.Type, + IsDefault: cmd.Result.IsDefault, + Created: cmd.Result.Created, + Updated: cmd.Result.Updated, + Frequency: cmd.Result.Frequency.String(), + NotifyOnce: cmd.Result.NotifyOnce, + Settings: cmd.Result.Settings, + } + + return JSON(200, result) } func DeleteAlertNotification(c *m.ReqContext) Response { diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 5e0196c20d1..7d4201fba87 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -24,14 +24,15 @@ type AlertRule struct { } type AlertNotification struct { - Id int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - NotifyOnce bool `json:"notifyOnce"` - Frequency bool `json:"frequency"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + IsDefault bool `json:"isDefault"` + NotifyOnce bool `json:"notifyOnce"` + Frequency string `json:"frequency"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Settings *simplejson.Json `json:"settings"` } type AlertTestCommand struct { @@ -64,7 +65,7 @@ type NotificationTestCommand struct { Name string `json:"name"` Type string `json:"type"` NotifyOnce bool `json:"notifyOnce"` - Frequency time.Duration `json:"frequency"` + Frequency string `json:"frequency"` Settings *simplejson.Json `json:"settings"` } diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index cba62a51527..6715eb21395 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -22,8 +22,8 @@ type AlertNotification struct { type CreateAlertNotificationCommand struct { Name string `json:"name" binding:"Required"` Type string `json:"type" binding:"Required"` - NotifyOnce bool `json:"notifyOnce" binding:"Required"` - Frequency time.Duration `json:"frequency"` + NotifyOnce bool `json:"notifyOnce"` + Frequency string `json:"frequency"` IsDefault bool `json:"isDefault"` Settings *simplejson.Json `json:"settings"` @@ -35,7 +35,7 @@ type UpdateAlertNotificationCommand struct { Id int64 `json:"id" binding:"Required"` Name string `json:"name" binding:"Required"` Type string `json:"type" binding:"Required"` - NotifyOnce string `json:"notifyOnce" binding:"Required"` + NotifyOnce bool `json:"notifyOnce"` Frequency string `json:"frequency"` IsDefault bool `json:"isDefault"` Settings *simplejson.Json `json:"settings" binding:"Required"` diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index b451d188a64..3817f4b4a3c 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -151,8 +151,8 @@ func (c *EvalContext) LastNotify(notifierId int64) *time.Time { NotifierId: notifierId, } if err := bus.Dispatch(cmd); err != nil { - c.log.Warn("Could not determine last time alert", - c.Rule.Name, "notified") + c.log.Warn("Could not determine last time alert notifier fired", + "Alert name", c.Rule.Name, "Error", err) return nil } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index e9e32020c6d..734b5e56b28 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -41,6 +41,7 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen if context.PrevAlertState == context.Rule.State && notifyOnce { return false } + // Do not notify if interval has not elapsed if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index b7142d144cc..5f2d4989063 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -3,6 +3,7 @@ package notifiers import ( "context" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" @@ -18,19 +19,19 @@ func TestBaseNotifier(t *testing.T) { Convey("can parse false value", func() { bJson.Set("uploadImage", false) - base := NewNotifierBase(1, false, "name", "email", bJson) + base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) So(base.UploadImage, ShouldBeFalse) }) Convey("can parse true value", func() { bJson.Set("uploadImage", true) - base := NewNotifierBase(1, false, "name", "email", bJson) + base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) So(base.UploadImage, ShouldBeTrue) }) Convey("default value should be true for backwards compatibility", func() { - base := NewNotifierBase(1, false, "name", "email", bJson) + base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) So(base.UploadImage, ShouldBeTrue) }) }) @@ -41,7 +42,8 @@ func TestBaseNotifier(t *testing.T) { State: m.AlertStatePending, }) context.Rule.State = m.AlertStateOK - So(defaultShouldNotify(context), ShouldBeFalse) + timeNow := time.Now() + So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeFalse) }) Convey("ok -> alerting", func() { @@ -49,7 +51,8 @@ func TestBaseNotifier(t *testing.T) { State: m.AlertStateOK, }) context.Rule.State = m.AlertStateAlerting - So(defaultShouldNotify(context), ShouldBeTrue) + timeNow := time.Now() + So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeTrue) }) }) }) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 8bb17143042..a2cfa37ce5c 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -56,7 +56,9 @@ func GetAlertNotificationsToSend(query *m.GetAlertNotificationsToSendQuery) erro alert_notification.created, alert_notification.updated, alert_notification.settings, - alert_notification.is_default + alert_notification.is_default, + alert_notification.notify_once, + alert_notification.frequency FROM alert_notification `) @@ -94,7 +96,9 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS alert_notification.created, alert_notification.updated, alert_notification.settings, - alert_notification.is_default + alert_notification.is_default, + alert_notification.notify_once, + alert_notification.frequency FROM alert_notification `) @@ -140,19 +144,24 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification name %s already exists", cmd.Name) } + frequency, err_convert := time.ParseDuration(cmd.Frequency) + if err_convert != nil { + return err + } + alertNotification := &m.AlertNotification{ OrgId: cmd.OrgId, Name: cmd.Name, Type: cmd.Type, Settings: cmd.Settings, NotifyOnce: cmd.NotifyOnce, - Frequency: cmd.Frequency, + Frequency: frequency, Created: time.Now(), Updated: time.Now(), IsDefault: cmd.IsDefault, } - if _, err = sess.Insert(alertNotification); err != nil { + if _, err = sess.MustCols("notify_once").Insert(alertNotification); err != nil { return err } @@ -184,8 +193,15 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.Name = cmd.Name current.Type = cmd.Type current.IsDefault = cmd.IsDefault + current.NotifyOnce = cmd.NotifyOnce - sess.UseBool("is_default") + frequency, err_convert := time.ParseDuration(cmd.Frequency) + if err_convert != nil { + return err + } + current.Frequency = frequency + + sess.UseBool("is_default", "notify_once") if affected, err := sess.ID(cmd.Id).Update(current); err != nil { return err @@ -219,7 +235,7 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { return inTransaction(func(sess *DBSession) error { notificationJournal := &m.NotificationJournal{} - _, err := sess.OrderBy("notification_journal.sent_at").Desc().Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + _, err := sess.Desc("notification_journal.sent_at").Limit(1).Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) if err != nil { return err } diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index ccdb9ef1073..dd56564cb95 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -20,8 +20,7 @@
{{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }} - - seconds +
Date: Sun, 20 May 2018 16:08:42 -0400 Subject: [PATCH 0104/1100] Fix tests --- pkg/services/sqlstore/alert_notification.go | 8 +++++ .../sqlstore/alert_notification_test.go | 32 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index a2cfa37ce5c..0ecd6a18818 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -144,6 +144,10 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification name %s already exists", cmd.Name) } + if cmd.Frequency == "" { + return fmt.Errorf("Alert notification frequency required") + } + frequency, err_convert := time.ParseDuration(cmd.Frequency) if err_convert != nil { return err @@ -195,6 +199,10 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.IsDefault = cmd.IsDefault current.NotifyOnce = cmd.NotifyOnce + if cmd.Frequency == "" { + return fmt.Errorf("Alert notification frequency required") + } + frequency, err_convert := time.ParseDuration(cmd.Frequency) if err_convert != nil { return err diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 2dbf9de5ca8..01c6c3aebd6 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -26,10 +26,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can save Alert Notification", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", - OrgId: 1, - Settings: simplejson.New(), + Name: "ops", + Type: "email", + OrgId: 1, + NotifyOnce: true, + Frequency: "10s", + Settings: simplejson.New(), } err = CreateAlertNotificationCommand(cmd) @@ -45,11 +47,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can update alert notification", func() { newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: "webhook", - OrgId: cmd.Result.OrgId, - Settings: simplejson.New(), - Id: cmd.Result.Id, + Name: "NewName", + Type: "webhook", + OrgId: cmd.Result.OrgId, + NotifyOnce: true, + Frequency: "10s", + Settings: simplejson.New(), + Id: cmd.Result.Id, } err := UpdateAlertNotification(newCmd) So(err, ShouldBeNil) @@ -58,12 +62,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can search using an array of ids", func() { - cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, Settings: simplejson.New()} - cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, Settings: simplejson.New()} - cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, Settings: simplejson.New()} - cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, Settings: simplejson.New()} + cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} - otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, Settings: simplejson.New()} + otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil) So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil) From 5c5951bc4274f3b4ff1ea3b41507e394faaeb22f Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Sun, 20 May 2018 19:01:10 -0400 Subject: [PATCH 0105/1100] Bug fix for repeated alerting even on OK state and add notification_journal cleanup when alert resolves --- pkg/services/alerting/engine.go | 14 ++++++++++++++ pkg/services/alerting/notifiers/base.go | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 0f8e24bcef5..43f6db66771 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -10,7 +10,9 @@ import ( tlog "github.com/opentracing/opentracing-go/log" "github.com/benbjohnson/clock" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -205,6 +207,18 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel } evalContext.Rule.State = evalContext.GetNewState() + if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { + for _, notifierId := range evalContext.Rule.Notifications { + cmd := &m.CleanNotificationJournalCommand{ + AlertId: evalContext.Rule.Id, + NotifierId: notifierId, + OrgId: evalContext.Rule.OrgId, + } + if err := bus.Dispatch(cmd); err != nil { + e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) + } + } + } e.resultHandler.Handle(evalContext) span.Finish() e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 734b5e56b28..7672d397491 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -45,6 +45,10 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } + // Do not notify if alert state if OK or pending even on repeated notify + if !notifyOnce && (context.Rule.State == m.AlertStateOK || context.Rule.State == m.AlertStatePending) { + return false + } // Do not notify when we become OK for the first time. if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) { return false From bdf433594add113b05b2bbd4f0381c1090fd2d6b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 25 May 2018 14:14:33 -0400 Subject: [PATCH 0106/1100] Implement code review changes --- pkg/models/alert_notifications.go | 5 +++++ pkg/services/alerting/engine.go | 14 -------------- pkg/services/alerting/result_handler.go | 12 ++++++++++++ pkg/services/sqlstore/alert_notification.go | 7 ++++--- .../features/alerting/notification_edit_ctrl.ts | 1 + .../alerting/partials/notification_edit.html | 15 +++++++++++---- 6 files changed, 33 insertions(+), 21 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 6715eb21395..ed6b8f372d1 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -1,11 +1,16 @@ package models import ( + "errors" "time" "github.com/grafana/grafana/pkg/components/simplejson" ) +var ( + ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") +) + type AlertNotification struct { Id int64 `json:"id"` OrgId int64 `json:"-"` diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 43f6db66771..0f8e24bcef5 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -10,9 +10,7 @@ import ( tlog "github.com/opentracing/opentracing-go/log" "github.com/benbjohnson/clock" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" - m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -207,18 +205,6 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel } evalContext.Rule.State = evalContext.GetNewState() - if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { - for _, notifierId := range evalContext.Rule.Notifications { - cmd := &m.CleanNotificationJournalCommand{ - AlertId: evalContext.Rule.Id, - NotifierId: notifierId, - OrgId: evalContext.Rule.OrgId, - } - if err := bus.Dispatch(cmd); err != nil { - e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) - } - } - } e.resultHandler.Handle(evalContext) span.Finish() e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index c57b28c7c3e..c4c20bd8beb 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,6 +88,18 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } + if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { + for _, notifierId := range evalContext.Rule.Notifications { + cmd := &m.CleanNotificationJournalCommand{ + AlertId: evalContext.Rule.Id, + NotifierId: notifierId, + OrgId: evalContext.Rule.OrgId, + } + if err := bus.Dispatch(cmd); err != nil { + handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) + } + } + } handler.notifier.SendIfNeeded(evalContext) return nil diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 0ecd6a18818..6913009a163 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -148,8 +148,9 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification frequency required") } - frequency, err_convert := time.ParseDuration(cmd.Frequency) - if err_convert != nil { + var frequency time.Duration + frequency, err = time.ParseDuration(cmd.Frequency) + if err != nil { return err } @@ -200,7 +201,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.NotifyOnce = cmd.NotifyOnce if cmd.Frequency == "" { - return fmt.Errorf("Alert notification frequency required") + return m.ErrNotificationFrequencyNotFound } frequency, err_convert := time.ParseDuration(cmd.Frequency) diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 2fd185bee29..9d20e871c7c 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -12,6 +12,7 @@ export class AlertNotificationEditCtrl { defaults: any = { type: 'email', notifyOnce: true, + frequency: '15m', settings: { httpMethod: 'POST', autoResolve: true, diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index dd56564cb95..48d44b74581 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -18,10 +18,6 @@ -
- {{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }} - -
+ + +
+ Notify every + +
From 0d5579b4c04fa7c04c3ae59950f962775a3f0777 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 4 Jun 2018 14:31:43 +0200 Subject: [PATCH 0107/1100] docs: what's new in v5.2 --- docs/sources/guides/whats-new-in-v5-2.md | 70 ++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 docs/sources/guides/whats-new-in-v5-2.md diff --git a/docs/sources/guides/whats-new-in-v5-2.md b/docs/sources/guides/whats-new-in-v5-2.md new file mode 100644 index 00000000000..8cff353ff45 --- /dev/null +++ b/docs/sources/guides/whats-new-in-v5-2.md @@ -0,0 +1,70 @@ ++++ +title = "What's New in Grafana v5.2" +description = "Feature & improvement highlights for Grafana v5.2" +keywords = ["grafana", "new", "documentation", "5.2"] +type = "docs" +[menu.docs] +name = "Version 5.2" +identifier = "v5.2" +parent = "whatsnew" +weight = -8 ++++ + +# What's New in Grafana v5.2 + +Grafana v5.2 brings new features, many enhancements and bug fixes. This article will detail the major new features and enhancements. + +* [Elasticsearch alerting]({{< relref "#elasticsearch-alerting" >}}) it's finally here! +* [Cross platform build support]({{< relref "#cross-platform-build-support" >}}) enables native builds of Grafana for many more platforms! +* [Improved Docker image]({{< relref "#improved-docker-image" >}}) with support for docker secrets +* [Prometheus]({{< relref "#prometheus" >}}) with alignment enhancements +* [Alerting]({{< relref "#alerting" >}}) with alert notification channel type for Discord +* [Dashboards & Panels]({{< relref "#dashboards-panels" >}}) + +## Elasticsearch alerting + +{{< docs-imagebox img="/img/docs/v52/elasticsearch_alerting.png" max-width="800px" class="docs-image--right" >}} + +Grafana v5.2 ships with an updated Elasticsearch datasource with support for alerting. Alerting support for Elasticsearch has been one of +the most requested features by our community and now it's finally here. Please try it out and let us know what you think. + +
+ +## Cross platform build support + +Grafana v5.2 brings an improved build pipeline with cross platform support. This enables native builds of Grafana for ARMv7 (x32), ARM64 (x64), +MacOS/Darwin (x64) and Windows (x64) in both stable and nightly builds. + +We've been longing for native ARM build support for a long time. With the help from our amazing community this is now finally available. + +## Improved Docker image + +The Grafana docker image now includes support for Docker secrets which enables you to supply Grafana with configuration through files. More +information in the [Installing using Docker documentation](/installation/docker/#reading-secrets-from-files-support-for-docker-secrets). + +## Prometheus + +The Prometheus datasource now aligns the start/end of the query sent to Prometheus with the step, which ensures PromQL expressions with *rate* +functions get consistent results, and thus avoid graphs jumping around on reload. + +## Alerting + +By popular demand Grafana now includes support for an alert notification channel type for [Discord](https://discordapp.com/). + +## Dashboards & Panels + +### Modified time range and variables are no longer saved by default + +{{< docs-imagebox img="/img/docs/v52/dashboard_save_modal.png" max-width="800px" class="docs-image--right" >}} + +Starting from Grafana v5.2 a modified time range or variable are no longer saved by default. To save a modified +time range or variable you'll need to actively select that when saving a dashboard, see screenshot. +This should hopefully make it easier to have sane defaults of time and variables in dashboards and make it more explicit +when you actually want to overwrite those settings. + +
+ +## Changelog + +Checkout the [CHANGELOG.md](https://github.com/grafana/grafana/blob/master/CHANGELOG.md) file for a complete list +of new features, changes, and bug fixes. From 38906acda98a43302f3f688042dc40e757284495 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 4 Jun 2018 15:15:47 +0200 Subject: [PATCH 0108/1100] elasticsearch: sort bucket keys to fix issue wth response parser tests --- pkg/tsdb/elasticsearch/response_parser.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 4a45d6271b9..7bdab60389c 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -113,15 +113,22 @@ func (rp *responseParser) processBuckets(aggs map[string]interface{}, target *Qu } } - for k, v := range esAgg.Get("buckets").MustMap() { - bucket := simplejson.NewFromAny(v) + buckets := esAgg.Get("buckets").MustMap() + bucketKeys := make([]string, 0) + for k := range buckets { + bucketKeys = append(bucketKeys, k) + } + sort.Strings(bucketKeys) + + for _, bucketKey := range bucketKeys { + bucket := simplejson.NewFromAny(buckets[bucketKey]) newProps := make(map[string]string, 0) for k, v := range props { newProps[k] = v } - newProps["filter"] = k + newProps["filter"] = bucketKey err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) if err != nil { From c138ff2c903c4cb7b5844529dae70037e651a15e Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 15:16:05 +0200 Subject: [PATCH 0109/1100] changelog: adds note about closing #11670 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eda912e86a..f11d06e990a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ * **Dashboard**: Validate uid when importing dashboards [#11515](https://github.com/grafana/grafana/issues/11515) * **Docker**: Support for env variables ending with _FILE [grafana-docker #166](https://github.com/grafana/grafana-docker/pull/166), thx [@efrecon](https://github.com/efrecon) * **Alert list panel**: Show alerts for user with viewer role [#11167](https://github.com/grafana/grafana/issues/11167) +* **Provisioning**: Verify checksum of dashboards before updating to reduce load on database [#11670](https://github.com/grafana/grafana/issues/11670) # 5.1.3 (2018-05-16) From d089b5e05dccfd60d49b802be3a28ec3530fb0e8 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 15:20:26 +0200 Subject: [PATCH 0110/1100] provisioning: turn relative symlinked path into absolut paths --- pkg/services/provisioning/dashboards/file_reader.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 8af23980531..3196c3a35af 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -48,16 +48,25 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } copy := path + + // get absolut path of config file path, err := filepath.Abs(path) if err != nil { log.Error("Could not create absolute path ", "path", path) } + // follow the symlink to get the real path path, err = filepath.EvalSymlinks(path) if err != nil { log.Error("Failed to read content of symlinked path: %s", path) } + // get the absolut path in case the symlink is relative + path, err = filepath.Abs(path) + if err != nil { + log.Error("Could not create absolute path ", "path", path) + } + if path == "" { path = copy log.Info("falling back to original path due to EvalSymlink/Abs failure") From cd4026da6b60967dee2c51d626715913d1fa9914 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 15:38:37 +0200 Subject: [PATCH 0111/1100] Revert "provisioning: turn relative symlinked path into absolut paths" This reverts commit d089b5e05dccfd60d49b802be3a28ec3530fb0e8. --- pkg/services/provisioning/dashboards/file_reader.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 3196c3a35af..8af23980531 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -48,25 +48,16 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } copy := path - - // get absolut path of config file path, err := filepath.Abs(path) if err != nil { log.Error("Could not create absolute path ", "path", path) } - // follow the symlink to get the real path path, err = filepath.EvalSymlinks(path) if err != nil { log.Error("Failed to read content of symlinked path: %s", path) } - // get the absolut path in case the symlink is relative - path, err = filepath.Abs(path) - if err != nil { - log.Error("Could not create absolute path ", "path", path) - } - if path == "" { path = copy log.Info("falling back to original path due to EvalSymlink/Abs failure") From 829af9425f4e1f6d0d3cea9f8d5fa78e46bc4a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 4 Jun 2018 15:45:29 +0200 Subject: [PATCH 0112/1100] revert: reverted singlestat panel position change PR #12004 --- public/sass/components/_panel_singlestat.scss | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/public/sass/components/_panel_singlestat.scss b/public/sass/components/_panel_singlestat.scss index af11de3b835..d680941bfb1 100644 --- a/public/sass/components/_panel_singlestat.scss +++ b/public/sass/components/_panel_singlestat.scss @@ -7,14 +7,13 @@ .singlestat-panel-value-container { line-height: 1; - position: absolute; + display: table-cell; + vertical-align: middle; + text-align: center; + position: relative; z-index: 1; font-size: 3em; - font-weight: bold; - margin: 0; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); + font-weight: $font-weight-semi-bold; } .singlestat-panel-prefix { From 574e92e1d8497f2be17d781b2eeb3e98867d2b39 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 15:23:17 +0200 Subject: [PATCH 0113/1100] changelog: adds note about closing #11958 [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f11d06e990a..22e2c29c91b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ * **Docker**: Support for env variables ending with _FILE [grafana-docker #166](https://github.com/grafana/grafana-docker/pull/166), thx [@efrecon](https://github.com/efrecon) * **Alert list panel**: Show alerts for user with viewer role [#11167](https://github.com/grafana/grafana/issues/11167) * **Provisioning**: Verify checksum of dashboards before updating to reduce load on database [#11670](https://github.com/grafana/grafana/issues/11670) +* **Provisioning**: Support symlinked files in dashboard provisioning config files [#11958](https://github.com/grafana/grafana/issues/11958) # 5.1.3 (2018-05-16) From cb6c6c817234b59cce137f071ea31ccc58f1896d Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Wed, 23 May 2018 11:34:22 +0200 Subject: [PATCH 0114/1100] change admin password after first login --- public/app/core/controllers/login_ctrl.ts | 66 +++++++++-- public/app/partials/login.html | 135 +++++++++++++--------- public/sass/components/_gf-form.scss | 4 + public/sass/pages/_login.scss | 38 ++++++ 4 files changed, 184 insertions(+), 59 deletions(-) diff --git a/public/app/core/controllers/login_ctrl.ts b/public/app/core/controllers/login_ctrl.ts index 313fc2efa1a..0a66f83d08a 100644 --- a/public/app/core/controllers/login_ctrl.ts +++ b/public/app/core/controllers/login_ctrl.ts @@ -11,10 +11,15 @@ export class LoginCtrl { password: '', }; + $scope.command = {}; + $scope.result = ''; + contextSrv.sidemenu = false; $scope.oauth = config.oauth; $scope.oauthEnabled = _.keys(config.oauth).length > 0; + $scope.ldapEnabled = config.ldapEnabled; + $scope.authProxyEnabled = config.authProxyEnabled; $scope.disableLoginForm = config.disableLoginForm; $scope.disableUserSignUp = config.disableUserSignUp; @@ -39,6 +44,43 @@ export class LoginCtrl { } }; + $scope.changeView = function() { + let loginView = document.querySelector('#login-view'); + let changePasswordView = document.querySelector('#change-password-view'); + + loginView.className += ' add'; + setTimeout(() => { + loginView.className += ' hidden'; + }, 250); + setTimeout(() => { + changePasswordView.classList.remove('hidden'); + }, 251); + setTimeout(() => { + changePasswordView.classList.remove('remove'); + }, 301); + + setTimeout(() => { + document.getElementById('newPassword').focus(); + }, 400); + }; + + $scope.changePassword = function() { + $scope.command.oldPassword = 'admin'; + + if ($scope.command.newPassword !== $scope.command.confirmNew) { + $scope.appEvent('alert-warning', ['New passwords do not match', '']); + return; + } + + backendSrv.put('/api/user/password', $scope.command).then(function() { + $scope.toGrafana(); + }); + }; + + $scope.skip = function() { + $scope.toGrafana(); + }; + $scope.loginModeChanged = function(newValue) { $scope.submitBtnText = newValue ? 'Log in' : 'Sign up'; }; @@ -65,18 +107,28 @@ export class LoginCtrl { } backendSrv.post('/login', $scope.formModel).then(function(result) { - var params = $location.search(); + $scope.result = result; - if (params.redirect && params.redirect[0] === '/') { - window.location.href = config.appSubUrl + params.redirect; - } else if (result.redirectUrl) { - window.location.href = result.redirectUrl; - } else { - window.location.href = config.appSubUrl + '/'; + if ($scope.formModel.password !== 'admin' || $scope.ldapEnabled || $scope.authProxyEnabled) { + $scope.toGrafana(); + return; } + $scope.changeView(); }); }; + $scope.toGrafana = function() { + var params = $location.search(); + + if (params.redirect && params.redirect[0] === '/') { + window.location.href = config.appSubUrl + params.redirect; + } else if ($scope.result.redirectUrl) { + window.location.href = $scope.result.redirectUrl; + } else { + window.location.href = config.appSubUrl + '/'; + } + }; + $scope.init(); } } diff --git a/public/app/partials/login.html b/public/app/partials/login.html index 8680924977f..8be9e777b9f 100644 --- a/public/app/partials/login.html +++ b/public/app/partials/login.html @@ -4,70 +4,101 @@ Grafana
-