From 8087af9c89c1f87b1e8d9269f0594cdb42b3dd97 Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Mon, 27 Feb 2017 15:27:00 +0530 Subject: [PATCH 01/30] ES Nested type bucket aggregation (#4694) (#4694) * (elasticsearch): add nested agg (use bucket aggs). fixes #4693 * (elasticsearch): rebased after merge of #6043 refactored from #4527 --- .../datasource/elasticsearch/bucket_agg.js | 32 ++++++++ .../datasource/elasticsearch/datasource.js | 3 +- .../elasticsearch/elastic_response.js | 74 ++++++++++++----- .../elasticsearch/partials/bucket_agg.html | 13 +++ .../datasource/elasticsearch/query_builder.js | 80 +++++++++++++------ .../datasource/elasticsearch/query_def.js | 1 + .../specs/query_builder_specs.ts | 31 +++++++ 7 files changed, 190 insertions(+), 44 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/bucket_agg.js b/public/app/plugins/datasource/elasticsearch/bucket_agg.js index 28f1df08251..7b3685fca5a 100644 --- a/public/app/plugins/datasource/elasticsearch/bucket_agg.js +++ b/public/app/plugins/datasource/elasticsearch/bucket_agg.js @@ -17,6 +17,7 @@ function (angular, _, queryDef) { target: "=", index: "=", onChange: "&", + // getNestedKeys: "&", getFields: "&", } }; @@ -52,14 +53,25 @@ function (angular, _, queryDef) { case 'date_histogram': case 'terms': { delete $scope.agg.query; + delete $scope.agg.settings.nested.path; $scope.agg.field = 'select field'; break; } case 'filters': { delete $scope.agg.field; + delete $scope.agg.settings.nested.path; $scope.agg.query = '*'; break; } + case 'nested': { + delete $scope.agg.field; + delete $scope.agg.query; + $scope.agg.settings.nested = {}; + $scope.agg.settings.nested.path = 'select field (type: nested)'; + $scope.agg.settings.nested.term = 'select nested term path'; + $scope.agg.settings.nested.query = 'select query for Nested Term'; + break; + } case 'geohash_grid': { $scope.agg.settings.precision = 3; break; @@ -79,6 +91,12 @@ function (angular, _, queryDef) { var settings = $scope.agg.settings || {}; switch($scope.agg.type) { + case 'nested': { + if (settingsLinkText === '') { + settingsLinkText = 'Options'; + } + break; + } case 'terms': { settings.order = settings.order || "asc"; settings.size = settings.size || "10"; @@ -170,6 +188,20 @@ function (angular, _, queryDef) { } }; + $scope.getFieldsNestedPath = function() { + return $scope.getFields({$fieldType: 'nested'}); + }; + + $scope.getFieldsNestedTerm = function() { + return $scope.getFields({$fieldType: 'string'}); + }; + + // FIX THIS: add a method getNestedKeys to be called here + // for getting nested key string ids, based on term/path supplied. + // $scope.getFieldsNestedQuery = function() { + // return $scope.getNestedKeys(); + // }; + $scope.getIntervalOptions = function() { return $q.when(uiSegmentSrv.transformToSegments(true, 'interval')(queryDef.intervalOptions)); }; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 4377d634982..14601314118 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -290,9 +290,10 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes } // transform to array - return _.map(fields, function(value) { + var resp = _.map(fields, function(value) { return value; }); + return resp; }); }; diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index 27285852e33..125abc8690a 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -97,6 +97,33 @@ function (_, queryDef) { } }; + ElasticResponse.prototype.processNestedAggregationDocs = function(esAgg, aggDef, target, seriesList, props) { + var metric, y, i, newSeries, bucket, value; + + for (y = 0; y < target.metrics.length; y++) { + metric = target.metrics[y]; + if (metric.hide) { + continue; + } + + newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; + for (i = 0; i < esAgg.buckets.length; i++) { + bucket = esAgg.buckets[i][aggDef.id]['nested_aggs']; + if (bucket !== undefined) { + value = bucket[metric.id]; + if (value !== undefined) { + if (value.normalized_value) { + newSeries.datapoints.push([value.normalized_value, esAgg.buckets[i].key]); + } else { + newSeries.datapoints.push([value.value, esAgg.buckets[i].key]); + } + } + } + } + seriesList.push(newSeries); + } + }; + ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, docs, props) { var metric, y, i, bucket, metricName, doc; @@ -145,31 +172,39 @@ function (_, queryDef) { // This is quite complex // neeed to recurise down the nested buckets to build series ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, docs, props, depth) { - var bucket, aggDef, esAgg, aggId; + var bucket, aggDef, aggDefNested, esAgg, aggId; var maxDepth = target.bucketAggs.length-1; + aggDefNested = _.find(target.bucketAggs, {type: "nested"}); + for (aggId in aggs) { - aggDef = _.find(target.bucketAggs, {id: aggId}); esAgg = aggs[aggId]; - if (!aggDef) { - continue; - } - - if (depth === maxDepth) { - if (aggDef.type === 'date_histogram') { - this.processMetrics(esAgg, target, seriesList, props); - } else { - this.processAggregationDocs(esAgg, aggDef, target, docs, props); - } + if (aggDefNested) { + this.processNestedAggregationDocs(esAgg, aggDefNested, target, seriesList, props); } else { - for (var nameIndex in esAgg.buckets) { - bucket = esAgg.buckets[nameIndex]; - props = _.clone(props); - if (bucket.key !== void 0) { - props[aggDef.field] = bucket.key; + aggDef = _.find(target.bucketAggs, {id: aggId}); + + if (!aggDef) { + continue; + } + + if (depth === maxDepth) { + if (aggDef.type === 'date_histogram') { + this.processMetrics(esAgg, target, seriesList, props); } else { - props["filter"] = nameIndex; + this.processAggregationDocs(esAgg, aggDef, target, docs, props); + } + } else { + for (var nameIndex in esAgg.buckets) { + bucket = esAgg.buckets[nameIndex]; + props = _.clone(props); + if (bucket.key) { + props[aggDef.field] = bucket.key; + } else { + props["filter"] = nameIndex; + } + this.processBuckets(bucket, target, seriesList, docs, props, depth+1); } if (bucket.key_as_string) { props[aggDef.field] = bucket.key_as_string; @@ -293,7 +328,7 @@ function (_, queryDef) { if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) { result.message = err.root_cause[0].reason; } else { - result.message = err.reason || 'Unkown elatic error response'; + result.message = err.reason || 'Unknown elastic error response'; } if (response.$$config) { @@ -325,7 +360,6 @@ function (_, queryDef) { this.processBuckets(aggregations, target, tmpSeriesList, docs, {}, 0); this.trimDatapoints(tmpSeriesList, target); this.nameSeries(tmpSeriesList, target); - for (var y = 0; y < tmpSeriesList.length; y++) { seriesList.push(tmpSeriesList[y]); } diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index 562f0b86e56..bc013cf519e 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -7,6 +7,7 @@ +
@@ -52,6 +53,18 @@
+
+
+ + +
+ +
+ + +
+
+
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 11e8b69ddaa..51aa748e6dc 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -59,6 +59,33 @@ function (queryDef) { return queryNode; }; + ElasticQueryBuilder.prototype.buildNestedAgg = function(aggDef, queryNode, target) { + var metric, y; + if (!aggDef.settings) { + return queryNode; + } + var settings = aggDef.settings; + queryNode.nested = {}; + queryNode.nested.path = settings.nested.path; + + queryNode.aggs = { + nested_aggs: { + filter: {}, + aggs: {} + } + }; + queryNode.aggs.nested_aggs.filter.term = {}; + queryNode.aggs.nested_aggs.filter.term[settings.nested.term] = settings.nested.query; + + queryNode.aggs.nested_aggs.aggs = {}; + for (y = 0; y < target.metrics.length; y++) { + metric = target.metrics[y]; + queryNode.aggs.nested_aggs.aggs[metric.id] = {}; + queryNode.aggs.nested_aggs.aggs[metric.id][metric.type] = {field: metric.field}; + } + return queryNode; + }; + ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) { var esAgg = {}; var settings = aggDef.settings || {}; @@ -183,6 +210,7 @@ function (queryDef) { nestedAggs = query; + var foundNested = false; for (i = 0; i < target.bucketAggs.length; i++) { var aggDef = target.bucketAggs[i]; var esAgg = {}; @@ -204,6 +232,10 @@ function (queryDef) { esAgg['geohash_grid'] = {field: aggDef.field, precision: aggDef.settings.precision}; break; } + case 'nested': { + foundNested = true; + this.buildNestedAgg(aggDef, esAgg, target); + } } nestedAggs.aggs = nestedAggs.aggs || {}; @@ -211,35 +243,37 @@ function (queryDef) { nestedAggs = esAgg; } - nestedAggs.aggs = {}; + if (!foundNested) { + nestedAggs.aggs = {}; - for (i = 0; i < target.metrics.length; i++) { - metric = target.metrics[i]; - if (metric.type === 'count') { - continue; - } - - var aggField = {}; - var metricAgg = null; - - if (queryDef.isPipelineAgg(metric.type)) { - if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { - metricAgg = { buckets_path: metric.pipelineAgg }; - } else { + for (i = 0; i < target.metrics.length; i++) { + metric = target.metrics[i]; + if (metric.type === 'count') { continue; } - } else { - metricAgg = {field: metric.field}; - } - for (var prop in metric.settings) { - if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) { - metricAgg[prop] = metric.settings[prop]; + var aggField = {}; + var metricAgg = null; + + if (queryDef.isPipelineAgg(metric.type)) { + if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { + metricAgg = { buckets_path: metric.pipelineAgg }; + } else { + continue; + } + } else { + metricAgg = {field: metric.field}; } - } - aggField[metric.type] = metricAgg; - nestedAggs.aggs[metric.id] = aggField; + for (var prop in metric.settings) { + if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) { + metricAgg[prop] = metric.settings[prop]; + } + } + + aggField[metric.type] = metricAgg; + nestedAggs.aggs[metric.id] = aggField; + } } return query; diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index 5afc908edba..ec7dc444437 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -22,6 +22,7 @@ function (_) { bucketAggTypes: [ {text: "Terms", value: 'terms', requiresField: true}, {text: "Filters", value: 'filters' }, + {text: "Nested", value: 'nested', requiresField: true}, {text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true}, {text: "Date Histogram", value: 'date_histogram', requiresField: true}, ], diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index e60cde7a163..4fa3e6daf19 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -102,6 +102,37 @@ describe('ElasticQueryBuilder', function() { expect(firstLevel.aggs["1"].percentiles.percents).to.eql([1,2,3,4]); }); + it('with nested aggs', function(){ + var query = builder.build({ + metrics: [{type: 'avg', field: 'disk.wr_sec', id: '1'}], + bucketAggs: [ + {type: 'date_histogram', field: '@timestamp', id: '2'}, + { + id: '3', + type: 'nested', + settings: { + nested: { + path: 'disk' , + term: 'disk.disk-device', + query: 'dev8-0', + } + } + } + ], + }); + + var firstLevel = query.aggs["2"]; + var secondLevel = firstLevel.aggs["3"]; + var thirdLevel = secondLevel.aggs["nested_aggs"]; + var fourthLevel = thirdLevel.aggs["1"]; + + expect(firstLevel.date_histogram.field).to.be("@timestamp"); + expect(secondLevel.nested.path).to.be("disk"); + expect(Object.keys(thirdLevel.filter.term)[0]).to.be("disk.disk-device"); + expect(thirdLevel.filter.term["disk.disk-device"]).to.be("dev8-0"); + expect(fourthLevel.avg.field).to.be("disk.wr_sec"); + }); + it('with filters aggs', function() { var query = builder.build({ metrics: [{type: 'count', id: '1'}], From d2dac02a2844a68a8c46bccd928183601c796946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 27 Feb 2017 10:57:44 +0100 Subject: [PATCH 02/30] Revert "ES Nested type bucket aggregation (#4694) (#4694)" This reverts commit 8087af9c89c1f87b1e8d9269f0594cdb42b3dd97. --- .../datasource/elasticsearch/bucket_agg.js | 32 ------- .../datasource/elasticsearch/datasource.js | 3 +- .../elasticsearch/elastic_response.js | 74 +++++----------- .../elasticsearch/partials/bucket_agg.html | 13 --- .../datasource/elasticsearch/query_builder.js | 84 ++++++------------- .../datasource/elasticsearch/query_def.js | 1 - .../specs/query_builder_specs.ts | 31 ------- 7 files changed, 46 insertions(+), 192 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/bucket_agg.js b/public/app/plugins/datasource/elasticsearch/bucket_agg.js index 7b3685fca5a..28f1df08251 100644 --- a/public/app/plugins/datasource/elasticsearch/bucket_agg.js +++ b/public/app/plugins/datasource/elasticsearch/bucket_agg.js @@ -17,7 +17,6 @@ function (angular, _, queryDef) { target: "=", index: "=", onChange: "&", - // getNestedKeys: "&", getFields: "&", } }; @@ -53,25 +52,14 @@ function (angular, _, queryDef) { case 'date_histogram': case 'terms': { delete $scope.agg.query; - delete $scope.agg.settings.nested.path; $scope.agg.field = 'select field'; break; } case 'filters': { delete $scope.agg.field; - delete $scope.agg.settings.nested.path; $scope.agg.query = '*'; break; } - case 'nested': { - delete $scope.agg.field; - delete $scope.agg.query; - $scope.agg.settings.nested = {}; - $scope.agg.settings.nested.path = 'select field (type: nested)'; - $scope.agg.settings.nested.term = 'select nested term path'; - $scope.agg.settings.nested.query = 'select query for Nested Term'; - break; - } case 'geohash_grid': { $scope.agg.settings.precision = 3; break; @@ -91,12 +79,6 @@ function (angular, _, queryDef) { var settings = $scope.agg.settings || {}; switch($scope.agg.type) { - case 'nested': { - if (settingsLinkText === '') { - settingsLinkText = 'Options'; - } - break; - } case 'terms': { settings.order = settings.order || "asc"; settings.size = settings.size || "10"; @@ -188,20 +170,6 @@ function (angular, _, queryDef) { } }; - $scope.getFieldsNestedPath = function() { - return $scope.getFields({$fieldType: 'nested'}); - }; - - $scope.getFieldsNestedTerm = function() { - return $scope.getFields({$fieldType: 'string'}); - }; - - // FIX THIS: add a method getNestedKeys to be called here - // for getting nested key string ids, based on term/path supplied. - // $scope.getFieldsNestedQuery = function() { - // return $scope.getNestedKeys(); - // }; - $scope.getIntervalOptions = function() { return $q.when(uiSegmentSrv.transformToSegments(true, 'interval')(queryDef.intervalOptions)); }; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 14601314118..4377d634982 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -290,10 +290,9 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes } // transform to array - var resp = _.map(fields, function(value) { + return _.map(fields, function(value) { return value; }); - return resp; }); }; diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.js b/public/app/plugins/datasource/elasticsearch/elastic_response.js index 125abc8690a..27285852e33 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.js +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.js @@ -97,33 +97,6 @@ function (_, queryDef) { } }; - ElasticResponse.prototype.processNestedAggregationDocs = function(esAgg, aggDef, target, seriesList, props) { - var metric, y, i, newSeries, bucket, value; - - for (y = 0; y < target.metrics.length; y++) { - metric = target.metrics[y]; - if (metric.hide) { - continue; - } - - newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; - for (i = 0; i < esAgg.buckets.length; i++) { - bucket = esAgg.buckets[i][aggDef.id]['nested_aggs']; - if (bucket !== undefined) { - value = bucket[metric.id]; - if (value !== undefined) { - if (value.normalized_value) { - newSeries.datapoints.push([value.normalized_value, esAgg.buckets[i].key]); - } else { - newSeries.datapoints.push([value.value, esAgg.buckets[i].key]); - } - } - } - } - seriesList.push(newSeries); - } - }; - ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, docs, props) { var metric, y, i, bucket, metricName, doc; @@ -172,39 +145,31 @@ function (_, queryDef) { // This is quite complex // neeed to recurise down the nested buckets to build series ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, docs, props, depth) { - var bucket, aggDef, aggDefNested, esAgg, aggId; + var bucket, aggDef, esAgg, aggId; var maxDepth = target.bucketAggs.length-1; - aggDefNested = _.find(target.bucketAggs, {type: "nested"}); - for (aggId in aggs) { + aggDef = _.find(target.bucketAggs, {id: aggId}); esAgg = aggs[aggId]; - if (aggDefNested) { - this.processNestedAggregationDocs(esAgg, aggDefNested, target, seriesList, props); - } else { - aggDef = _.find(target.bucketAggs, {id: aggId}); + if (!aggDef) { + continue; + } - if (!aggDef) { - continue; - } - - if (depth === maxDepth) { - if (aggDef.type === 'date_histogram') { - this.processMetrics(esAgg, target, seriesList, props); - } else { - this.processAggregationDocs(esAgg, aggDef, target, docs, props); - } + if (depth === maxDepth) { + if (aggDef.type === 'date_histogram') { + this.processMetrics(esAgg, target, seriesList, props); } else { - for (var nameIndex in esAgg.buckets) { - bucket = esAgg.buckets[nameIndex]; - props = _.clone(props); - if (bucket.key) { - props[aggDef.field] = bucket.key; - } else { - props["filter"] = nameIndex; - } - this.processBuckets(bucket, target, seriesList, docs, props, depth+1); + this.processAggregationDocs(esAgg, aggDef, target, docs, props); + } + } else { + for (var nameIndex in esAgg.buckets) { + bucket = esAgg.buckets[nameIndex]; + props = _.clone(props); + if (bucket.key !== void 0) { + props[aggDef.field] = bucket.key; + } else { + props["filter"] = nameIndex; } if (bucket.key_as_string) { props[aggDef.field] = bucket.key_as_string; @@ -328,7 +293,7 @@ function (_, queryDef) { if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) { result.message = err.root_cause[0].reason; } else { - result.message = err.reason || 'Unknown elastic error response'; + result.message = err.reason || 'Unkown elatic error response'; } if (response.$$config) { @@ -360,6 +325,7 @@ function (_, queryDef) { this.processBuckets(aggregations, target, tmpSeriesList, docs, {}, 0); this.trimDatapoints(tmpSeriesList, target); this.nameSeries(tmpSeriesList, target); + for (var y = 0; y < tmpSeriesList.length; y++) { seriesList.push(tmpSeriesList[y]); } diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index bc013cf519e..562f0b86e56 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -7,7 +7,6 @@ -
@@ -53,18 +52,6 @@
-
-
- - -
- -
- - -
-
-
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.js index 51aa748e6dc..11e8b69ddaa 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.js @@ -59,33 +59,6 @@ function (queryDef) { return queryNode; }; - ElasticQueryBuilder.prototype.buildNestedAgg = function(aggDef, queryNode, target) { - var metric, y; - if (!aggDef.settings) { - return queryNode; - } - var settings = aggDef.settings; - queryNode.nested = {}; - queryNode.nested.path = settings.nested.path; - - queryNode.aggs = { - nested_aggs: { - filter: {}, - aggs: {} - } - }; - queryNode.aggs.nested_aggs.filter.term = {}; - queryNode.aggs.nested_aggs.filter.term[settings.nested.term] = settings.nested.query; - - queryNode.aggs.nested_aggs.aggs = {}; - for (y = 0; y < target.metrics.length; y++) { - metric = target.metrics[y]; - queryNode.aggs.nested_aggs.aggs[metric.id] = {}; - queryNode.aggs.nested_aggs.aggs[metric.id][metric.type] = {field: metric.field}; - } - return queryNode; - }; - ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) { var esAgg = {}; var settings = aggDef.settings || {}; @@ -210,7 +183,6 @@ function (queryDef) { nestedAggs = query; - var foundNested = false; for (i = 0; i < target.bucketAggs.length; i++) { var aggDef = target.bucketAggs[i]; var esAgg = {}; @@ -232,10 +204,6 @@ function (queryDef) { esAgg['geohash_grid'] = {field: aggDef.field, precision: aggDef.settings.precision}; break; } - case 'nested': { - foundNested = true; - this.buildNestedAgg(aggDef, esAgg, target); - } } nestedAggs.aggs = nestedAggs.aggs || {}; @@ -243,37 +211,35 @@ function (queryDef) { nestedAggs = esAgg; } - if (!foundNested) { - nestedAggs.aggs = {}; + nestedAggs.aggs = {}; - for (i = 0; i < target.metrics.length; i++) { - metric = target.metrics[i]; - if (metric.type === 'count') { + for (i = 0; i < target.metrics.length; i++) { + metric = target.metrics[i]; + if (metric.type === 'count') { + continue; + } + + var aggField = {}; + var metricAgg = null; + + if (queryDef.isPipelineAgg(metric.type)) { + if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { + metricAgg = { buckets_path: metric.pipelineAgg }; + } else { continue; } - - var aggField = {}; - var metricAgg = null; - - if (queryDef.isPipelineAgg(metric.type)) { - if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { - metricAgg = { buckets_path: metric.pipelineAgg }; - } else { - continue; - } - } else { - metricAgg = {field: metric.field}; - } - - for (var prop in metric.settings) { - if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) { - metricAgg[prop] = metric.settings[prop]; - } - } - - aggField[metric.type] = metricAgg; - nestedAggs.aggs[metric.id] = aggField; + } else { + metricAgg = {field: metric.field}; } + + for (var prop in metric.settings) { + if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) { + metricAgg[prop] = metric.settings[prop]; + } + } + + aggField[metric.type] = metricAgg; + nestedAggs.aggs[metric.id] = aggField; } return query; diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js index ec7dc444437..5afc908edba 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ b/public/app/plugins/datasource/elasticsearch/query_def.js @@ -22,7 +22,6 @@ function (_) { bucketAggTypes: [ {text: "Terms", value: 'terms', requiresField: true}, {text: "Filters", value: 'filters' }, - {text: "Nested", value: 'nested', requiresField: true}, {text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true}, {text: "Date Histogram", value: 'date_histogram', requiresField: true}, ], diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index 4fa3e6daf19..e60cde7a163 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -102,37 +102,6 @@ describe('ElasticQueryBuilder', function() { expect(firstLevel.aggs["1"].percentiles.percents).to.eql([1,2,3,4]); }); - it('with nested aggs', function(){ - var query = builder.build({ - metrics: [{type: 'avg', field: 'disk.wr_sec', id: '1'}], - bucketAggs: [ - {type: 'date_histogram', field: '@timestamp', id: '2'}, - { - id: '3', - type: 'nested', - settings: { - nested: { - path: 'disk' , - term: 'disk.disk-device', - query: 'dev8-0', - } - } - } - ], - }); - - var firstLevel = query.aggs["2"]; - var secondLevel = firstLevel.aggs["3"]; - var thirdLevel = secondLevel.aggs["nested_aggs"]; - var fourthLevel = thirdLevel.aggs["1"]; - - expect(firstLevel.date_histogram.field).to.be("@timestamp"); - expect(secondLevel.nested.path).to.be("disk"); - expect(Object.keys(thirdLevel.filter.term)[0]).to.be("disk.disk-device"); - expect(thirdLevel.filter.term["disk.disk-device"]).to.be("dev8-0"); - expect(fourthLevel.avg.field).to.be("disk.wr_sec"); - }); - it('with filters aggs', function() { var query = builder.build({ metrics: [{type: 'count', id: '1'}], From 86789c0b69a815087f7ca0016cd4b48d783d941c Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 27 Feb 2017 10:31:51 +0100 Subject: [PATCH 03/30] docs: update docs for 4.2.0 beta release --- CHANGELOG.md | 2 +- docs/sources/guides/whats-new-in-v4-1.md | 2 +- docs/sources/guides/whats-new-in-v4-2.md | 88 ++++++++++++++++++++++++ docs/sources/installation/debian.md | 9 +++ docs/sources/installation/rpm.md | 1 + docs/sources/installation/windows.md | 1 + 6 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 docs/sources/guides/whats-new-in-v4-2.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c540f02035..d45e8ac2852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ * **Alerting**: Uploading images for alert notifications is now optional [#7419](https://github.com/grafana/grafana/issues/7419) * **Dashboard**: Adds shortcut for collapsing/expanding all rows [#552](https://github.com/grafana/grafana/issues/552), thx [@mtanda](https://github.com/mtanda) * **Alerting**: Adds de duping of alert notifications [#7632](https://github.com/grafana/grafana/pull/7632) -* **Orgs**: Sharing dashboards using Grafana share feature will not redirect to correct org. [#1613](https://github.com/grafana/grafana/issues/1613) +* **Orgs**: Sharing dashboards using Grafana share feature will now redirect to correct org. [#1613](https://github.com/grafana/grafana/issues/1613) * **Pushover**: Add Pushover alert notifications [#7526](https://github.com/grafana/grafana/pull/7526) thx [@devkid](https://github.com/devkid) * **Threema**: Add Threema Gateway alert notification integration [#7482](https://github.com/grafana/grafana/pull/7482) thx [@dbrgn](https://github.com/dbrgn) diff --git a/docs/sources/guides/whats-new-in-v4-1.md b/docs/sources/guides/whats-new-in-v4-1.md index 59e6236af40..bd2b0f1b75f 100644 --- a/docs/sources/guides/whats-new-in-v4-1.md +++ b/docs/sources/guides/whats-new-in-v4-1.md @@ -7,7 +7,7 @@ type = "docs" name = "Version 4.1" identifier = "v4.1" parent = "whatsnew" -weight = -1 +weight = 3 +++ diff --git a/docs/sources/guides/whats-new-in-v4-2.md b/docs/sources/guides/whats-new-in-v4-2.md new file mode 100644 index 00000000000..44aa3a45dc0 --- /dev/null +++ b/docs/sources/guides/whats-new-in-v4-2.md @@ -0,0 +1,88 @@ ++++ +title = "What's New in Grafana v4.2" +description = "Feature & improvement highlights for Grafana v4.2" +keywords = ["grafana", "new", "documentation", "4.2.0"] +type = "docs" +[menu.docs] +name = "Version 4.2" +identifier = "v4.2" +parent = "whatsnew" +weight = -1 ++++ + +## Whats new in Grafana v4.2 + +Grafana v4.2 Beta is now [available for download](/download/4_2_0/). +Just like the last release this one contains lots bug fixes and minor improvements. +We are very happy to say that 27 of 40 issues was closed by pull requests from the community. +Big thumbs up! + +## Release Highlights + +- **Hipchat**: Adds support for sending alert notifications to hipchat [#6451](https://github.com/grafana/grafana/issues/6451), thx [@jregovic](https://github.com/jregovic) +- **Telegram**: Added Telegram alert notifier [#7098](https://github.com/grafana/grafana/pull/7098), thx [@leonoff](https://github.com/leonoff) +- **LINE**: Add LINE as alerting notification channel [#7301](https://github.com/grafana/grafana/pull/7301), thx [@huydx](https://github.com/huydx) +- **Templating**: Make $__interval and $__interval_ms global built in variables that can be used in by any datasource (in panel queries), closes [#7190](https://github.com/grafana/grafana/issues/7190), closes [#6582](https://github.com/grafana/grafana/issues/6582) +- **Alerting**: Adds deduping of alert notifications [#7632](https://github.com/grafana/grafana/pull/7632) +- **Alerting**: Better information about why an alert triggered [#7035](https://github.com/grafana/grafana/issues/7035) +- **Orgs**: Sharing dashboards using Grafana share feature will now redirect to correct org. [#6948](https://github.com/grafana/grafana/issues/6948) +- [Full changelog](https://github.com/grafana/grafana/blob/master/CHANGELOG.md) + +### New alert notification channels + +This release adds **five** new alert notifications channels, all of them contributed by the community. + +* Hipchat +* Telegram +* LINE +* Pushover +* Threema + +### Templating + +We added two new global built in variables in grafana. `$__interval` and `$__interval_ms` are now reserved template names in grafana and can be used by any datasource. +We might add more global built in variables in the future and if we do we will prefix them with `$__`. So please avoid using that in your template variables. + +### Dedupe alert notifications when running multiple servers + +In this release we will dedupe alert notificiations when you are running multiple servers. +This makes it possible to run alerting on multiple servers and only get one notification. + +We currently solve this with sql transactions which puts some limitations for how many servers you can use to execute the same rules. +3-5 servers should not be a problem but as always, it depends on how many alerts you have and how frequently they execute. + +Next up for a better HA situation is to add support for workload balancing between Grafana servers. + +### Alerting more info + +You can now see the reason why an alert triggered in the alert history. Its also easier to detect when an alert is set to `alerting` due to the `no_data` option. + +### Improved support for multi-org setup + +When loading dashboards we now set an query parameter called orgId. So we can detect from which org an user shared a dashboard. +This makes it possible for users to share dashboards between orgs without changing org first. + +We aim to introduce [dashboard groups](https://github.com/grafana/grafana/issues/1611) sometime in the future which will introduce access control and user groups within one org. +Making it possible to have users in multiple groups and have detailed access control. + +## Upgrade & Breaking changes + +If your using https in grafana we now force you to use tls 1.2 and the most secure ciphers. +We think its better to be secure by default rather then making it configurable. +If you want to run https with lower versions of tls we suggest you put a reserve proxy in front of grafana. + +If you have template variables name `$__interval` or `$__interval_ms` they will no longer work since these keywords +are reserved as global built in variables. We might add more global built in variables in the future and if we do, we will prefix them with `$__`. So please avoid using that in your template variables. + +## Changelog + +Checkout the [CHANGELOG.md](https://github.com/grafana/grafana/blob/master/CHANGELOG.md) file for a complete list +of new features, changes, and bug fixes. + +## Download + +Head to [v4.2-beta download page](/download/4_2_0/) for download links & instructions. + +## Thanks + +A big thanks to all the Grafana users who contribute by submitting PRs, bug reports & feedback! diff --git a/docs/sources/installation/debian.md b/docs/sources/installation/debian.md index fb5c649ad1d..607d683bbb2 100644 --- a/docs/sources/installation/debian.md +++ b/docs/sources/installation/debian.md @@ -16,6 +16,7 @@ weight = 1 Description | Download ------------ | ------------- Stable for Debian-based Linux | [4.1.2 (x86-64 deb)](https://grafanarel.s3.amazonaws.com/builds/grafana_4.1.2-1486989747_amd64.deb) +Beta for Debian-based Linux | [4.2.0-beta1 (x86-64 deb)](https://grafanarel.s3.amazonaws.com/builds/grafana_4.2.0-beta1_amd64.deb) ## Install Stable @@ -25,6 +26,14 @@ $ sudo apt-get install -y adduser libfontconfig $ sudo dpkg -i grafana_4.1.2-1486989747_amd64.deb ``` +## Install Beta + +``` +$ wget https://grafanarel.s3.amazonaws.com/builds/grafana_4.2.0-beta1_amd64.deb +$ sudo apt-get install -y adduser libfontconfig +$ sudo dpkg -i grafana_4.2.0-beta1_amd64.deb +``` + ## APT Repository Add the following line to your `/etc/apt/sources.list` file. diff --git a/docs/sources/installation/rpm.md b/docs/sources/installation/rpm.md index 56125a4951a..d329b7032c4 100644 --- a/docs/sources/installation/rpm.md +++ b/docs/sources/installation/rpm.md @@ -16,6 +16,7 @@ weight = 2 Description | Download ------------ | ------------- Stable for CentOS / Fedora / OpenSuse / Redhat Linux | [4.1.2 (x86-64 rpm)](https://grafanarel.s3.amazonaws.com/builds/grafana-4.1.2-1486989747.x86_64.rpm) +Beta for CentOS / Fedora / OpenSuse / Redhat Linux | [4.2.0-beta1 (x86-64 rpm)](https://grafanarel.s3.amazonaws.com/builds/grafana-4.2.0-beta1.x86_64.rpm) ## Install Stable diff --git a/docs/sources/installation/windows.md b/docs/sources/installation/windows.md index 73816d98a40..b495ad81e98 100644 --- a/docs/sources/installation/windows.md +++ b/docs/sources/installation/windows.md @@ -14,6 +14,7 @@ weight = 3 Description | Download ------------ | ------------- Latest stable package for Windows | [grafana.4.1.2.windows-x64.zip](https://grafanarel.s3.amazonaws.com/builds/grafana-4.1.2.windows-x64.zip) +Latest beta package for Windows | [grafana-4.2.0-beta1.windows-x64.zip](https://grafanarel.s3.amazonaws.com/builds/grafana-4.2.0-beta1.windows-x64.zip) ## Configure From 7d897c0980ba4e46570ed49a9794d35865dfd7da Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 27 Feb 2017 13:26:53 +0100 Subject: [PATCH 04/30] build: add docker trigger for release builds --- circle.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/circle.yml b/circle.yml index faa50cb5c81..6db2b3e99f7 100644 --- a/circle.yml +++ b/circle.yml @@ -53,4 +53,5 @@ deployment: - go run build.go sha1-dist - aws s3 sync ./dist s3://$BUCKET_NAME/release - ./scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} release + - ./scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} ${CIRCLE_TAG} From 1422655e9791230a38bf388d082b1a27b563fa7b Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Mon, 27 Feb 2017 17:10:52 +0100 Subject: [PATCH 05/30] feat(alerting): Add emoji to Threema alert notifications This commit prepends emoji to Threema alert notifications to make it easier to discern various notification types (e.g. alert, no data, ok). --- pkg/services/alerting/notifiers/threema.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go index a710d686d28..80c84c26a24 100644 --- a/pkg/services/alerting/notifiers/threema.go +++ b/pkg/services/alerting/notifiers/threema.go @@ -126,9 +126,21 @@ func (notifier *ThreemaNotifier) Notify(evalContext *alerting.EvalContext) error data.Set("to", notifier.RecipientID) data.Set("secret", notifier.APISecret) + // Determine emoji + stateEmoji := "" + switch evalContext.Rule.State { + case m.AlertStateOK: + stateEmoji = "\u2705 " // White Heavy Check Mark + case m.AlertStateNoData: + stateEmoji = "\u2753 " // Black Question Mark Ornament + case m.AlertStateAlerting: + stateEmoji = "\u26A0 " // Warning sign + } + // Build message - message := fmt.Sprintf("%s\n\n*State:* %s\n*Message:* %s\n", - evalContext.GetNotificationTitle(), evalContext.Rule.Name, evalContext.Rule.Message) + message := fmt.Sprintf("%s%s\n\n*State:* %s\n*Message:* %s\n", + stateEmoji, evalContext.GetNotificationTitle(), + evalContext.Rule.Name, evalContext.Rule.Message) ruleURL, err := evalContext.GetRuleUrl() if err == nil { message = message + fmt.Sprintf("*URL:* %s\n", ruleURL) From 3ef077c18026d78b1243e38c54282dd39a897ff9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 28 Feb 2017 09:09:49 +0100 Subject: [PATCH 06/30] build: use correct version for tar.gz files --- build.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/build.go b/build.go index 73f641bfec3..5c04b78cecf 100644 --- a/build.go +++ b/build.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" "time" + "path" ) var ( @@ -120,14 +121,24 @@ func main() { } func makeLatestDistCopies() { - rpmIteration := "-1" - if linuxPackageIteration != "" { - rpmIteration = linuxPackageIteration + files, err := ioutil.ReadDir("dist") + if err != nil { + log.Fatalf("failed to create latest copies. Cannot read from /dist") } - runError("cp", fmt.Sprintf("dist/grafana_%v-%v_amd64.deb", linuxPackageVersion, linuxPackageIteration), "dist/grafana_latest_amd64.deb") - runError("cp", fmt.Sprintf("dist/grafana-%v-%v.x86_64.rpm", linuxPackageVersion, rpmIteration), "dist/grafana-latest-1.x86_64.rpm") - runError("cp", fmt.Sprintf("dist/grafana-%v-%v.linux-x64.tar.gz", linuxPackageVersion, linuxPackageIteration), "dist/grafana-latest.linux-x64.tar.gz") + latestMapping := map[string]string { + ".deb": "dist/grafana_latest_amd64.deb", + ".rpm": "dist/grafana-latest-1.x86_64.rpm", + ".tar.gz": "dist/grafana-latest.linux-x64.tar.gz", + } + + for _, file := range files { + for extension, fullName := range latestMapping { + if strings.HasSuffix(file.Name(), extension) { + runError("cp", path.Join("dist", file.Name()), fullName) + } + } + } } func readVersionFromPackageJson() { @@ -332,9 +343,9 @@ func grunt(params ...string) { func gruntBuildArg(task string) []string { args := []string{task} if includeBuildNumber { - args = append(args, fmt.Sprintf("--pkgVer=%v-%v", linuxPackageVersion, linuxPackageIteration)) + args = append(args, fmt.Sprintf("--pkgVer=%v-%v", version, linuxPackageIteration)) } else { - args = append(args, fmt.Sprintf("--pkgVer=%v", linuxPackageVersion)) + args = append(args, fmt.Sprintf("--pkgVer=%v", version)) } if pkgArch != "" { args = append(args, fmt.Sprintf("--arch=%v", pkgArch)) From 9b6571fab16781fe6f86a17677c973cfb8c8d5a6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 28 Feb 2017 17:04:47 +0100 Subject: [PATCH 07/30] template: dont allow template variables to begin with '__' closes #7678 --- public/app/features/templating/editor_ctrl.ts | 1 + .../features/templating/partials/editor.html | 367 +++++++++--------- public/sass/components/_gf-form.scss | 19 + 3 files changed, 205 insertions(+), 182 deletions(-) diff --git a/public/app/features/templating/editor_ctrl.ts b/public/app/features/templating/editor_ctrl.ts index c5e24741a96..7bd745bed53 100644 --- a/public/app/features/templating/editor_ctrl.ts +++ b/public/app/features/templating/editor_ctrl.ts @@ -10,6 +10,7 @@ export class VariableEditorCtrl { constructor(private $scope, private datasourceSrv, private variableSrv, templateSrv) { $scope.variableTypes = variableTypes; $scope.ctrl = {}; + $scope.namePattern = /^((?!__).)*$/; $scope.refreshOptions = [ {value: 0, text: "Never"}, diff --git a/public/app/features/templating/partials/editor.html b/public/app/features/templating/partials/editor.html index 4fc2553244c..5f844931b4c 100644 --- a/public/app/features/templating/partials/editor.html +++ b/public/app/features/templating/partials/editor.html @@ -1,155 +1,158 @@
-
-

- Templating -

+
+

+ Templating +

- + - -
+ +
-
+
-
-
- No template variables defined -
- - - - +
+
+ No template variables defined +
+
- - ${{variable.name}} - - - {{variable.query}} -
+ + + - - - + + - - - -
+ + ${{variable.name}} + + + {{variable.query}} + + Duplicate - - - Edit - - - - - -
-
+ + + + Edit + + + + + + + + + +
-
-
-   New -
-
+
+
+   New +
+
-
-
Variable
-
-
-
- Name - -
-
- + +
Variable
+
+
+ Template names cannot begin with '__' that's reserved for Grafanas global variables +
+
+
+ Name + +
+
+ Type - {{variableTypes[current.type].description}} + {{variableTypes[current.type].description}} -
- -
-
+
+ +
+
-
-
- Label - -
-
- Hide +
+
+ Label + +
+
+ Hide
- -
-
-
-
+ +
+
+
+
-
+
Interval Options
-
- Values - -
+
+ Values + +
-
- - +
+ + -
- - Step count How many times should the current time range be divided to calculate the value - -
- -
-
-
- - Min interval The calculated value will not go below this threshold - - -
-
-
+
+ + Step count How many times should the current time range be divided to calculate the value + +
+ +
+
+
+ + Min interval The calculated value will not go below this threshold + + +
+
+
-
+
Custom Options
-
- Values separated by comma - -
-
+
+ Values separated by comma + +
+
-
+
Constant options
-
- Value - -
-
+
+ Value + +
+
-
+
Query Options
@@ -170,8 +173,8 @@
-
-
+
+
Query
@@ -184,26 +187,26 @@
-
- - Sort - - How to sort the values of this variable. - - -
- -
-
-
+
+ + Sort + + How to sort the values of this variable. + + +
+ +
+
+
-
-
Data source options
+
+
Data source options
-
- -
- +
+ +
+
@@ -222,18 +225,18 @@
-
+
Options
-
- Data source -
- -
-
-
+
+ Data source +
+ +
+
+
-
-
Selection Options
+
+
Selection Options
Value groups/tags (Experimental feature)
- - -
- Tags query - -
-
-
  • Tag values query
    • - -
    -
    + + +
    + Tags query + +
    +
    +
  • Tag values query
    • + +
    +
    -
    -
    Preview of values (shows max 20)
    -
    -
    - {{option.text}} -
    -
    -
    +
    +
    Preview of values (shows max 20)
    +
    +
    + {{option.text}} +
    +
    +
    -
    - {{infoText}} -
    +
    + {{infoText}} +
    -
    - - -
    +
    + + +
    - -
    + +
    diff --git a/public/sass/components/_gf-form.scss b/public/sass/components/_gf-form.scss index 004ab86fb76..19588634dd3 100644 --- a/public/sass/components/_gf-form.scss +++ b/public/sass/components/_gf-form.scss @@ -66,6 +66,25 @@ $gf-form-margin: 0.25rem; } } +.gf-form-error { + padding: $input-padding-y $input-padding-x; + margin-right: $gf-form-margin; + flex-shrink: 0; + + background-color: $input-label-bg; + display: block; + font-size: $font-size-sm; + margin-right: $gf-form-margin; + + border: $input-btn-border-width solid $red; + @include border-radius($label-border-radius-sm); + + &--grow { + flex-grow: 1; + min-height: 2.60rem; + } +} + .gf-form-checkbox { flex-shrink: 0; padding: $input-padding-y $input-padding-x; From 6b1dd1c7fc53d87fadf6eae735bda6451afabdf1 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 28 Feb 2017 17:08:17 +0100 Subject: [PATCH 08/30] changelog: adds note about closing #7678 --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d45e8ac2852..fc247b9d29b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ -# 4.2.0 (unreleased) +# 4.3.0 (unreleased) + + +# 4.2.0-beta2 (unreleased) +## Minor Enhancements +* **Templates**: Prevent use of the prefix `__` for templates in web UI [#7678](https://github.com/grafana/grafana/issues/7678) + +# 4.2.0-beta1 (2017-02-27) ## Enhancements * **Telegram**: Added Telegram alert notifier [#7098](https://github.com/grafana/grafana/pull/7098), thx [@leonoff](https://github.com/leonoff) From 573bcdde12a6b77a8fc608950c9c14f7ca73d9f7 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Wed, 1 Mar 2017 11:35:12 +0100 Subject: [PATCH 09/30] Added some details about Sessions in Postgres --- docs/sources/installation/configuration.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 4a2b60cb48d..4020aa9b900 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -457,7 +457,7 @@ session provider you have configured. - **file:** session file path, e.g. `data/sessions` - **mysql:** go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` -- **postgres:** ex: user=a password=b host=localhost port=5432 dbname=c sslmode=disable +- **postgres:** ex: user=a password=b host=localhost port=5432 dbname=c sslmode=require - **memcache:** ex: 127.0.0.1:11211 - **redis:** ex: `addr=127.0.0.1:6379,pool_size=100,prefix=grafana` @@ -472,6 +472,17 @@ Mysql Example: `expiry` INT(11) UNSIGNED NOT NULL, PRIMARY KEY (`key`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8; + +Postgres Example: + + CREATE TABLE session ( + key CHAR(16) NOT NULL, + data BYTEA, + expiry INTEGER NOT NULL, + PRIMARY KEY (key) + ); + +Postgres valid `sslmode` are `disable`, `require` (default), `verify-ca`, and `verify-full`. ### cookie_name From 06146b801c2b3e3055f535c612ee7f0ef63bd4f8 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Feb 2017 20:48:31 +0100 Subject: [PATCH 10/30] build: replace sha1 files with sha256 due to security reasons. https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html --- appveyor.yml | 2 +- build.go | 18 +++++++++--------- circle.yml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 25e4181ed80..303c3abca9e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,7 +30,7 @@ install: build_script: - go run build.go build - grunt release - - go run build.go sha1-dist + - go run build.go sha-dist - cp dist/* . artifacts: diff --git a/build.go b/build.go index 5c04b78cecf..7943bd8716c 100644 --- a/build.go +++ b/build.go @@ -5,7 +5,7 @@ package main import ( "bytes" "crypto/md5" - "crypto/sha1" + "crypto/sha256" "encoding/json" "flag" "fmt" @@ -105,8 +105,8 @@ func main() { grunt(gruntBuildArg("release")...) createDebPackages() - case "sha1-dist": - sha1FilesInDist() + case "sha-dist": + shaFilesInDist() case "latest": makeLatestDistCopies() @@ -522,14 +522,14 @@ func md5File(file string) error { return out.Close() } -func sha1FilesInDist() { +func shaFilesInDist() { filepath.Walk("./dist", func(path string, f os.FileInfo, err error) error { if path == "./dist" { return nil } - if strings.Contains(path, ".sha1") == false { - err := sha1File(path) + if strings.Contains(path, ".sha256") == false { + err := shaFile(path) if err != nil { log.Printf("Failed to create sha file. error: %v\n", err) } @@ -538,20 +538,20 @@ func sha1FilesInDist() { }) } -func sha1File(file string) error { +func shaFile(file string) error { fd, err := os.Open(file) if err != nil { return err } defer fd.Close() - h := sha1.New() + h := sha256.New() _, err = io.Copy(h, fd) if err != nil { return err } - out, err := os.Create(file + ".sha1") + out, err := os.Create(file + ".sha256") if err != nil { return err } diff --git a/circle.yml b/circle.yml index 6db2b3e99f7..75dae71d0ab 100644 --- a/circle.yml +++ b/circle.yml @@ -41,7 +41,7 @@ deployment: commands: - ./scripts/build/deploy.sh - ./scripts/build/sign_packages.sh - - go run build.go sha1-dist + - go run build.go sha-dist - aws s3 sync ./dist s3://$BUCKET_NAME/master - ./scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} master - ./scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} @@ -50,7 +50,7 @@ deployment: commands: - ./scripts/build/deploy.sh - ./scripts/build/sign_packages.sh - - go run build.go sha1-dist + - go run build.go sha-dist - aws s3 sync ./dist s3://$BUCKET_NAME/release - ./scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} release - ./scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} ${CIRCLE_TAG} From 473513aa2f3d1445318cba9c46e35d4e8848d4e5 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 1 Mar 2017 17:19:25 +0100 Subject: [PATCH 11/30] docs: how to configure alert notification links --- docs/sources/alerting/notifications.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index 9f8d3c36120..d0dd19cbcdc 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -101,4 +101,9 @@ config file. This is an optional requirement, you can get slack and email notifications without setting this up. +# Configure the link back to Grafana from alert notifications + +All alert notifications contains a link back to the triggered alert in the Grafana instance. +This url is based on the [domain](/installation/configuration/#domain) setting in Grafana. + From 412b8998a8e5e6812e0a3814398628dd8d445095 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 2 Mar 2017 13:57:28 +0100 Subject: [PATCH 12/30] webhooks: get proxy settings from ini file closes #7710 --- pkg/services/notifications/webhook.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/services/notifications/webhook.go b/pkg/services/notifications/webhook.go index 25d015e2db4..c74804ab828 100644 --- a/pkg/services/notifications/webhook.go +++ b/pkg/services/notifications/webhook.go @@ -25,6 +25,7 @@ type Webhook struct { } var netTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, Dial: (&net.Dialer{ Timeout: 30 * time.Second, }).Dial, From 55e0df7896d947685b34bf3ecc322674989cc9a1 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 2 Mar 2017 14:01:06 +0100 Subject: [PATCH 13/30] docs: adds note about closing #7710 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc247b9d29b..c10b63cb485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ ## Minor Enhancements * **Templates**: Prevent use of the prefix `__` for templates in web UI [#7678](https://github.com/grafana/grafana/issues/7678) +## Bugfixes +* **Webhook**: Use proxy settings from environment variables [#7710](https://github.com/grafana/grafana/issues/7710) + # 4.2.0-beta1 (2017-02-27) ## Enhancements From 26bb9ad399627ea5fc6fd6f116559a2860e711df Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 24 Feb 2017 09:22:57 +0100 Subject: [PATCH 14/30] build: use initial checkout within container speedup and simplify the build within the container --- circle.yml | 3 +-- scripts/build/build.sh | 19 +------------------ scripts/build/deploy.sh | 4 +++- scripts/circle-test.sh | 5 +++-- 4 files changed, 8 insertions(+), 23 deletions(-) diff --git a/circle.yml b/circle.yml index 75dae71d0ab..9c5f0f1ccbd 100644 --- a/circle.yml +++ b/circle.yml @@ -33,7 +33,7 @@ dependencies: test: override: - - bash scripts/circle-test.sh + - bash scripts/circle-test.sh deployment: gh_branch: @@ -54,4 +54,3 @@ deployment: - aws s3 sync ./dist s3://$BUCKET_NAME/release - ./scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} release - ./scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} ${CIRCLE_TAG} - diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 6e1b2cbe5b9..4f16ee240c4 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -7,20 +7,7 @@ GOPATH=/go REPO_PATH=$GOPATH/src/github.com/grafana/grafana -mkdir -p /go/src/github.com/grafana -cd /go/src/github.com/grafana - -if [ "$CIRCLE_TAG" != "" ]; then - echo "Builing from tag $CIRCLE_TAG" - git clone https://github.com/grafana/grafana.git - cd $REPO_PATH - git checkout $CIRCLE_TAG -else - echo "Building from branch $CIRCLE_BRANCH" - git clone --depth 1 https://github.com/grafana/grafana.git -b $CIRCLE_BRANCH - cd $REPO_PATH -fi - +cd /go/src/github.com/grafana/grafana echo "current dir: $(pwd)" if [ "$CIRCLE_TAG" != "" ]; then @@ -47,7 +34,3 @@ else echo "Packaging incremental build for $CIRCLE_BRANCH" go run build.go -buildNumber=${CIRCLE_BUILD_NUM} package latest fi - -cp dist/* /tmp/dist/ - - diff --git a/scripts/build/deploy.sh b/scripts/build/deploy.sh index bfc735e4c79..49b2a9e3a7c 100755 --- a/scripts/build/deploy.sh +++ b/scripts/build/deploy.sh @@ -5,8 +5,10 @@ mkdir -p dist echo "Circle branch: ${CIRCLE_BRANCH}" echo "Circle tag: ${CIRCLE_TAG}" docker run -i -t --name gfbuild \ - -v $(pwd)/dist:/tmp/dist \ + -v $(pwd):/go/src/github.com/grafana/grafana \ -e "CIRCLE_BRANCH=${CIRCLE_BRANCH}" \ -e "CIRCLE_TAG=${CIRCLE_TAG}" \ -e "CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM}" \ grafana/buildcontainer + +sudo chown -R ${USER:=$(/usr/bin/id -run)}:$USER dist diff --git a/scripts/circle-test.sh b/scripts/circle-test.sh index e11007d8e76..a3a2790bcdc 100755 --- a/scripts/circle-test.sh +++ b/scripts/circle-test.sh @@ -24,7 +24,8 @@ exit_if_fail test -z "$(gofmt -s -l ./pkg | tee /dev/stderr)" echo "running go vet" exit_if_fail test -z "$(go vet ./pkg/... | tee /dev/stderr)" +echo "building binaries" exit_if_fail go run build.go build + +echo "running go test" exit_if_fail go test -v ./pkg/... - - From fe970f66298f9135ae119dd0c603dcd66d633028 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 3 Mar 2017 11:09:05 +0100 Subject: [PATCH 15/30] docs: adds note about closing #7676 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c10b63cb485..46973702d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # 4.3.0 (unreleased) +## Minor Enchancements +* **Threema**: Add emoji to Threema alert notifications [#7676](https://github.com/grafana/grafana/pull/7676) thx [@dbrgn](https://github.com/dbrgn) # 4.2.0-beta2 (unreleased) ## Minor Enhancements From 7ace2463a471ac09884a4597ea883f31c034d113 Mon Sep 17 00:00:00 2001 From: Mitja Zivkovic Date: Fri, 3 Mar 2017 14:07:30 +0100 Subject: [PATCH 16/30] added cubic decimetre - dm3 #7695 --- public/app/core/utils/kbn.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/public/app/core/utils/kbn.js b/public/app/core/utils/kbn.js index 819206bf94a..f5255b52697 100644 --- a/public/app/core/utils/kbn.js +++ b/public/app/core/utils/kbn.js @@ -488,6 +488,7 @@ function($, _) { kbn.valueFormats.litre = kbn.formatBuilders.decimalSIPrefix('L'); kbn.valueFormats.mlitre = kbn.formatBuilders.decimalSIPrefix('L', -1); kbn.valueFormats.m3 = kbn.formatBuilders.decimalSIPrefix('m3'); + kbn.valueFormats.dm3 = kbn.formatBuilders.decimalSIPrefix('dm3'); kbn.valueFormats.gallons = kbn.formatBuilders.fixedUnit('gal'); // Flow @@ -805,10 +806,11 @@ function($, _) { { text: 'volume', submenu: [ - {text: 'millilitre', value: 'mlitre' }, - {text: 'litre', value: 'litre' }, - {text: 'cubic metre', value: 'm3' }, - {text: 'gallons', value: 'gallons'}, + {text: 'millilitre', value: 'mlitre' }, + {text: 'litre', value: 'litre' }, + {text: 'cubic metre', value: 'm3' }, + {text: 'cubic decimetre', value: 'dm3' }, + {text: 'gallons', value: 'gallons'}, ] }, { From e71b13d9fa183f856cf2c0e26d9acb5329d08ca3 Mon Sep 17 00:00:00 2001 From: Denis Doria Date: Mon, 6 Mar 2017 07:47:50 +0100 Subject: [PATCH 17/30] Improve regex detection for influxdb measurement #7276 (#7734) --- public/app/plugins/datasource/influxdb/influx_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/influxdb/influx_query.ts b/public/app/plugins/datasource/influxdb/influx_query.ts index 065b85cf175..074931b5d23 100644 --- a/public/app/plugins/datasource/influxdb/influx_query.ts +++ b/public/app/plugins/datasource/influxdb/influx_query.ts @@ -167,7 +167,7 @@ export default class InfluxQuery { var policy = this.target.policy; var measurement = this.target.measurement || 'measurement'; - if (!measurement.match('^/.*/')) { + if (!measurement.match('^/.*/$')) { measurement = '"' + measurement+ '"'; } else if (interpolate) { measurement = this.templateSrv.replace(measurement, this.scopedVars, 'regex'); From 0264fcc66ca3e650ec64de4d86398f322eae5351 Mon Sep 17 00:00:00 2001 From: Denis Doria Date: Mon, 6 Mar 2017 07:49:28 +0100 Subject: [PATCH 18/30] Remove unsaved dialog when removing a dashboard #7591 (#7733) --- public/app/features/dashboard/dashnav/dashnav.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/features/dashboard/dashnav/dashnav.ts b/public/app/features/dashboard/dashnav/dashnav.ts index 33bb4411640..faaac986ee7 100644 --- a/public/app/features/dashboard/dashnav/dashnav.ts +++ b/public/app/features/dashboard/dashnav/dashnav.ts @@ -106,6 +106,7 @@ export class DashNavCtrl { confirmText: confirmText, yesText: 'Delete', onConfirm: function() { + $scope.dashboardMeta.canSave = false; $scope.deleteDashboardConfirmed(); } }); From 6af62abd41c4f2b752772a9b816d4c68b734ef28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Mar 2017 08:26:36 +0100 Subject: [PATCH 19/30] docs(): added router_logging to configuration options, closes #7723 --- docs/sources/installation/configuration.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 4020aa9b900..ba44dd30658 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -135,6 +135,10 @@ Path to the certificate file (if `protocol` is set to `https`). Path to the certificate key file (if `protocol` is set to `https`). +### router_logging + +Set to true for Grafana to log all HTTP requests (not just errors). These are logged as Info level events +to grafana log.
    @@ -472,7 +476,7 @@ Mysql Example: `expiry` INT(11) UNSIGNED NOT NULL, PRIMARY KEY (`key`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8; - + Postgres Example: CREATE TABLE session ( @@ -481,7 +485,7 @@ Postgres Example: expiry INTEGER NOT NULL, PRIMARY KEY (key) ); - + Postgres valid `sslmode` are `disable`, `require` (default), `verify-ca`, and `verify-full`. ### cookie_name @@ -613,7 +617,7 @@ You can choose between (s3, webdav). If left empty Grafana will ignore the uploa ## [external_image_storage.s3] ### bucket_url -Bucket URL for S3. AWS region can be specified within URL or defaults to 'us-east-1', e.g. +Bucket URL for S3. AWS region can be specified within URL or defaults to 'us-east-1', e.g. - http://grafana.s3.amazonaws.com/ - https://grafana.s3-ap-southeast-2.amazonaws.com/ - https://grafana.s3-cn-north-1.amazonaws.com.cn From 31866b5e57cfe39a79a044e6ab1350b6e31cc1bb Mon Sep 17 00:00:00 2001 From: Denis Doria Date: Mon, 6 Mar 2017 08:37:49 +0100 Subject: [PATCH 20/30] Allow commas on template variable #7681 (#7732) This improvement allows to wrap an "expression" when using single or double quotes. So now you can have time interval with offset for influxdb. --- public/app/features/templating/interval_variable.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/app/features/templating/interval_variable.ts b/public/app/features/templating/interval_variable.ts index ab1b0e59442..10977596057 100644 --- a/public/app/features/templating/interval_variable.ts +++ b/public/app/features/templating/interval_variable.ts @@ -59,8 +59,9 @@ export class IntervalVariable implements Variable { } updateOptions() { - // extract options in comma separated string - this.options = _.map(this.query.split(/[,]+/), function(text) { + // extract options between quotes and/or comma + this.options = _.map(this.query.match(/(["'])(.*?)\1|\w+/g), function(text) { + text = text.replace(/["']+/g, ''); return {text: text.trim(), value: text.trim()}; }); From 9f1c6a73f0bb0f88be6c4aa9db7b4120114d30d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Mar 2017 11:51:35 +0100 Subject: [PATCH 21/30] tech: added sql logger to log all sql statements sent to sql server --- pkg/services/sqlstore/logger.go | 123 ++++++++++++++++++++++++++++++ pkg/services/sqlstore/sqlstore.go | 3 + 2 files changed, 126 insertions(+) create mode 100644 pkg/services/sqlstore/logger.go diff --git a/pkg/services/sqlstore/logger.go b/pkg/services/sqlstore/logger.go new file mode 100644 index 00000000000..ae1145c21b0 --- /dev/null +++ b/pkg/services/sqlstore/logger.go @@ -0,0 +1,123 @@ +package sqlstore + +import ( + "fmt" + + glog "github.com/grafana/grafana/pkg/log" + + "github.com/go-xorm/core" +) + +type XormLogger struct { + grafanaLog glog.Logger + level glog.Lvl + showSQL bool +} + +func NewXormLogger(level glog.Lvl, grafanaLog glog.Logger) *XormLogger { + return &XormLogger{ + grafanaLog: grafanaLog, + level: level, + showSQL: true, + } +} + +// Error implement core.ILogger +func (s *XormLogger) Err(v ...interface{}) error { + if s.level <= glog.LvlError { + s.grafanaLog.Error(fmt.Sprint(v...)) + } + return nil +} + +// Errorf implement core.ILogger +func (s *XormLogger) Errf(format string, v ...interface{}) error { + if s.level <= glog.LvlError { + s.grafanaLog.Error(fmt.Sprintf(format, v...)) + } + return nil +} + +// Debug implement core.ILogger +func (s *XormLogger) Debug(v ...interface{}) error { + if s.level <= glog.LvlDebug { + s.grafanaLog.Debug(fmt.Sprint(v...)) + } + return nil +} + +// Debugf implement core.ILogger +func (s *XormLogger) Debugf(format string, v ...interface{}) error { + if s.level <= glog.LvlDebug { + s.grafanaLog.Debug(fmt.Sprintf(format, v...)) + } + return nil +} + +// Info implement core.ILogger +func (s *XormLogger) Info(v ...interface{}) error { + if s.level <= glog.LvlInfo { + s.grafanaLog.Info(fmt.Sprint(v...)) + } + return nil +} + +// Infof implement core.ILogger +func (s *XormLogger) Infof(format string, v ...interface{}) error { + if s.level <= glog.LvlInfo { + s.grafanaLog.Info(fmt.Sprintf(format, v...)) + } + return nil +} + +// Warn implement core.ILogger +func (s *XormLogger) Warning(v ...interface{}) error { + if s.level <= glog.LvlWarn { + s.grafanaLog.Warn(fmt.Sprint(v...)) + } + return nil +} + +// Warnf implement core.ILogger +func (s *XormLogger) Warningf(format string, v ...interface{}) error { + if s.level <= glog.LvlWarn { + s.grafanaLog.Warn(fmt.Sprintf(format, v...)) + } + return nil +} + +// Level implement core.ILogger +func (s *XormLogger) Level() core.LogLevel { + switch s.level { + case glog.LvlError: + return core.LOG_ERR + case glog.LvlWarn: + return core.LOG_WARNING + case glog.LvlInfo: + return core.LOG_INFO + case glog.LvlDebug: + return core.LOG_DEBUG + default: + return core.LOG_ERR + } +} + +// SetLevel implement core.ILogger +func (s *XormLogger) SetLevel(l core.LogLevel) error { + return nil +} + +// ShowSQL implement core.ILogger +func (s *XormLogger) ShowSQL(show ...bool) { + s.grafanaLog.Error("ShowSQL", "show", "show") + if len(show) == 0 { + s.showSQL = true + return + } + s.showSQL = show[0] +} + +// IsShowSQL implement core.ILogger +func (s *XormLogger) IsShowSQL() bool { + return s.showSQL +} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 0d16d57bcde..ef22dd9b6c6 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -160,6 +160,9 @@ func getEngine() (*xorm.Engine, error) { engine.SetMaxConns(DbCfg.MaxConn) engine.SetMaxOpenConns(DbCfg.MaxOpenConn) engine.SetMaxIdleConns(DbCfg.MaxIdleConn) + // engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm"))) + // engine.ShowSQL = true + // engine.ShowInfo = true } return engine, nil } From c3202d3f99a264e9a7f74a060104b62c866f481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 6 Mar 2017 12:23:28 +0100 Subject: [PATCH 22/30] sessions: start session gc at startup but only after between 10 - 180 seconds --- pkg/middleware/session.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/middleware/session.go b/pkg/middleware/session.go index 0700612ecc2..dad2ad20efc 100644 --- a/pkg/middleware/session.go +++ b/pkg/middleware/session.go @@ -1,6 +1,7 @@ package middleware import ( + "math/rand" "time" "github.com/go-macaron/session" @@ -8,6 +9,7 @@ import ( _ "github.com/go-macaron/session/mysql" _ "github.com/go-macaron/session/postgres" _ "github.com/go-macaron/session/redis" + "github.com/grafana/grafana/pkg/log" "gopkg.in/macaron.v1" ) @@ -22,10 +24,12 @@ var sessionManager *session.Manager var sessionOptions *session.Options var startSessionGC func() var getSessionCount func() int +var sessionLogger = log.New("session") func init() { startSessionGC = func() { sessionManager.GC() + sessionLogger.Debug("Session GC") time.AfterFunc(time.Duration(sessionOptions.Gclifetime)*time.Second, startSessionGC) } getSessionCount = func() int { @@ -67,7 +71,9 @@ func Sessioner(options *session.Options) macaron.Handler { panic(err) } - go startSessionGC() + // start GC threads after some random seconds + rndSeconds := 10 + rand.Int63n(180) + time.AfterFunc(time.Duration(rndSeconds)*time.Second, startSessionGC) return func(ctx *Context) { ctx.Next() From 6ab90425c4bfd1f9041fa28541835e9af72f92fe Mon Sep 17 00:00:00 2001 From: Pranay Kanwar Date: Mon, 6 Mar 2017 20:49:37 +0530 Subject: [PATCH 23/30] Pass dropcounter rate option if no max counter and no reset value or reset value as 0 is specified (#7743) --- pkg/tsdb/opentsdb/opentsdb.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index 684f48bfc59..c0ba6603b20 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -190,6 +190,10 @@ func (e *OpenTsdbExecutor) buildMetric(query *tsdb.Query) map[string]interface{} rateOptions["resetValue"] = resetValue.MustFloat64() } + if !counterMaxCheck && (!resetValueCheck || resetValue.MustFloat64() == 0) { + rateOptions["dropcounter"] = true + } + metric["rateOptions"] = rateOptions } From b387a8759e0c88367b8c7a75115f26b6267ffbc3 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 7 Mar 2017 05:08:16 +0900 Subject: [PATCH 24/30] use session.NewSession() (#7745) --- pkg/api/cloudwatch/cloudwatch.go | 58 ++++++++++++++++++++---- pkg/api/cloudwatch/metrics.go | 7 ++- pkg/components/imguploader/s3uploader.go | 11 ++++- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 1b796be874e..1cf4b671e2d 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -114,7 +114,10 @@ func getCredentials(dsInfo *datasourceInfo) (*credentials.Credentials, error) { DurationSeconds: aws.Int64(900), } - stsSess := session.New() + stsSess, err := session.NewSession() + if err != nil { + return nil, err + } stsCreds := credentials.NewChainCredentials( []credentials.Provider{ &credentials.EnvProvider{}, @@ -126,7 +129,11 @@ func getCredentials(dsInfo *datasourceInfo) (*credentials.Credentials, error) { Credentials: stsCreds, } - svc := sts.New(session.New(stsConfig), stsConfig) + sess, err := session.NewSession(stsConfig) + if err != nil { + return nil, err + } + svc := sts.New(sess, stsConfig) resp, err := svc.AssumeRole(params) if err != nil { return nil, err @@ -139,7 +146,10 @@ func getCredentials(dsInfo *datasourceInfo) (*credentials.Credentials, error) { } } - sess := session.New() + sess, err := session.NewSession() + if err != nil { + return nil, err + } creds := credentials.NewChainCredentials( []credentials.Provider{ &credentials.StaticProvider{Value: credentials.Value{ @@ -185,7 +195,12 @@ func handleGetMetricStatistics(req *cwRequest, c *middleware.Context) { c.JsonApiErr(500, "Unable to call AWS API", err) return } - svc := cloudwatch.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + c.JsonApiErr(500, "Unable to call AWS API", err) + return + } + svc := cloudwatch.New(sess, cfg) reqParam := &struct { Parameters struct { @@ -232,7 +247,12 @@ func handleListMetrics(req *cwRequest, c *middleware.Context) { c.JsonApiErr(500, "Unable to call AWS API", err) return } - svc := cloudwatch.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + c.JsonApiErr(500, "Unable to call AWS API", err) + return + } + svc := cloudwatch.New(sess, cfg) reqParam := &struct { Parameters struct { @@ -273,7 +293,12 @@ func handleDescribeAlarms(req *cwRequest, c *middleware.Context) { c.JsonApiErr(500, "Unable to call AWS API", err) return } - svc := cloudwatch.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + c.JsonApiErr(500, "Unable to call AWS API", err) + return + } + svc := cloudwatch.New(sess, cfg) reqParam := &struct { Parameters struct { @@ -316,7 +341,12 @@ func handleDescribeAlarmsForMetric(req *cwRequest, c *middleware.Context) { c.JsonApiErr(500, "Unable to call AWS API", err) return } - svc := cloudwatch.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + c.JsonApiErr(500, "Unable to call AWS API", err) + return + } + svc := cloudwatch.New(sess, cfg) reqParam := &struct { Parameters struct { @@ -360,7 +390,12 @@ func handleDescribeAlarmHistory(req *cwRequest, c *middleware.Context) { c.JsonApiErr(500, "Unable to call AWS API", err) return } - svc := cloudwatch.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + c.JsonApiErr(500, "Unable to call AWS API", err) + return + } + svc := cloudwatch.New(sess, cfg) reqParam := &struct { Parameters struct { @@ -396,7 +431,12 @@ func handleDescribeInstances(req *cwRequest, c *middleware.Context) { c.JsonApiErr(500, "Unable to call AWS API", err) return } - svc := ec2.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + c.JsonApiErr(500, "Unable to call AWS API", err) + return + } + svc := ec2.New(sess, cfg) reqParam := &struct { Parameters struct { diff --git a/pkg/api/cloudwatch/metrics.go b/pkg/api/cloudwatch/metrics.go index 6b953bdf0fc..b41bb20d6c8 100644 --- a/pkg/api/cloudwatch/metrics.go +++ b/pkg/api/cloudwatch/metrics.go @@ -258,8 +258,11 @@ func getAllMetrics(cwData *datasourceInfo) (cloudwatch.ListMetricsOutput, error) Region: aws.String(cwData.Region), Credentials: creds, } - - svc := cloudwatch.New(session.New(cfg), cfg) + sess, err := session.NewSession(cfg) + if err != nil { + return cloudwatch.ListMetricsOutput{}, err + } + svc := cloudwatch.New(sess, cfg) params := &cloudwatch.ListMetricsInput{ Namespace: aws.String(cwData.Namespace), diff --git a/pkg/components/imguploader/s3uploader.go b/pkg/components/imguploader/s3uploader.go index 5f476b9e366..4f8632f965c 100644 --- a/pkg/components/imguploader/s3uploader.go +++ b/pkg/components/imguploader/s3uploader.go @@ -35,7 +35,10 @@ func NewS3Uploader(region, bucket, acl, accessKey, secretKey string) *S3Uploader } func (u *S3Uploader) Upload(imageDiskPath string) (string, error) { - sess := session.New() + sess, err := session.NewSession() + if err != nil { + return "", err + } creds := credentials.NewChainCredentials( []credentials.Provider{ &credentials.StaticProvider{Value: credentials.Value{ @@ -58,7 +61,11 @@ func (u *S3Uploader) Upload(imageDiskPath string) (string, error) { return "", err } - svc := s3.New(session.New(cfg), cfg) + sess, err = session.NewSession(cfg) + if err != nil { + return "", err + } + svc := s3.New(sess, cfg) params := &s3.PutObjectInput{ Bucket: aws.String(u.bucket), Key: aws.String(key), From 125ee865b6df6558b2419681b1fbdde21c5abf9c Mon Sep 17 00:00:00 2001 From: Wouter Smit Date: Mon, 6 Mar 2017 21:08:45 +0100 Subject: [PATCH 25/30] Spelling mistake (#7739) --- pkg/services/alerting/notifiers/email.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go index 4058d3860b5..dcf71f0f99e 100644 --- a/pkg/services/alerting/notifiers/email.go +++ b/pkg/services/alerting/notifiers/email.go @@ -16,7 +16,7 @@ func init() { alerting.RegisterNotifier(&alerting.NotifierPlugin{ Type: "email", Name: "Email", - Description: "Sends notifications using Grafana server configured STMP settings", + Description: "Sends notifications using Grafana server configured SMTP settings", Factory: NewEmailNotifier, OptionsTemplate: `

    Email addresses

    From 3735a1ace7bd24b527571d927c889567bfce0649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patric=20Kanngie=C3=9Fer?= Date: Tue, 7 Mar 2017 08:19:34 +0100 Subject: [PATCH 26/30] remember scroll position (https://github.com/grafana/grafana/issues/7680) (#7728) --- public/app/plugins/panel/graph/legend.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/app/plugins/panel/graph/legend.js b/public/app/plugins/panel/graph/legend.js index fd5abfb2db2..2cdb1821793 100644 --- a/public/app/plugins/panel/graph/legend.js +++ b/public/app/plugins/panel/graph/legend.js @@ -65,7 +65,9 @@ function (angular, _, $) { var el = $(e.currentTarget); var index = getSeriesIndexForElement(el); var seriesInfo = seriesList[index]; + var scrollPosition = $($container.children('tbody')).scrollTop(); ctrl.toggleSeries(seriesInfo, e); + $($container.children('tbody')).scrollTop(scrollPosition); } function sortLegend(e) { From a24ac012c4acc942a65baa8fb700753893b62f29 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 7 Mar 2017 16:35:29 +0900 Subject: [PATCH 27/30] support full resolution for $interval variable (#7696) --- public/app/features/templating/partials/editor.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/templating/partials/editor.html b/public/app/features/templating/partials/editor.html index 5f844931b4c..2d514060cb7 100644 --- a/public/app/features/templating/partials/editor.html +++ b/public/app/features/templating/partials/editor.html @@ -124,7 +124,7 @@ Step count How many times should the current time range be divided to calculate the value
    - +
    From a37a2259b3a0442fcc7250eb0cdca28a61c41920 Mon Sep 17 00:00:00 2001 From: Joseph Pintozzi Date: Tue, 7 Mar 2017 00:36:33 -0700 Subject: [PATCH 28/30] Allowing "Unique Count"s of any data type (#7704) --- public/app/plugins/datasource/elasticsearch/metric_agg.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.js b/public/app/plugins/datasource/elasticsearch/metric_agg.js index 9971c084882..c0c4ffc08d4 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.js +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.js @@ -162,6 +162,9 @@ function (angular, _, queryDef) { }; $scope.getFieldsInternal = function() { + if ($scope.agg.type === 'cardinality') { + return $scope.getFields(); + } return $scope.getFields({$fieldType: 'number'}); }; From 70c2586c805dd073f355fd0e58b276db6d17643a Mon Sep 17 00:00:00 2001 From: Ross Lodge Date: Mon, 6 Mar 2017 23:39:19 -0800 Subject: [PATCH 29/30] Use other variable dependencies in regex filter for datasource variable (#7547) --- public/app/features/templating/datasource_variable.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/public/app/features/templating/datasource_variable.ts b/public/app/features/templating/datasource_variable.ts index bfd4d965029..41f2262ab4a 100644 --- a/public/app/features/templating/datasource_variable.ts +++ b/public/app/features/templating/datasource_variable.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import kbn from 'app/core/utils/kbn'; -import {Variable, assignModelProperties, variableTypes} from './variable'; +import {Variable, containsVariable, assignModelProperties, variableTypes} from './variable'; import {VariableSrv} from './variable_srv'; export class DatasourceVariable implements Variable { @@ -25,7 +25,7 @@ export class DatasourceVariable implements Variable { }; /** @ngInject **/ - constructor(private model, private datasourceSrv, private variableSrv) { + constructor(private model, private datasourceSrv, private variableSrv, private templateSrv) { assignModelProperties(this, model, this.defaults); this.refresh = 1; } @@ -48,7 +48,8 @@ export class DatasourceVariable implements Variable { var regex; if (this.regex) { - regex = kbn.stringToJsRegex(this.regex); + regex = this.templateSrv.replace(this.regex, null, 'regex'); + regex = kbn.stringToJsRegex(regex); } for (var i = 0; i < sources.length; i++) { @@ -74,6 +75,9 @@ export class DatasourceVariable implements Variable { } dependsOn(variable) { + if (this.regex) { + return containsVariable(this.regex, variable.name); + } return false; } From 8e3f22d307cdfcf721bb2b249a76bcf1b854fc2f Mon Sep 17 00:00:00 2001 From: Daniel Lee Date: Tue, 7 Mar 2017 11:23:57 +0100 Subject: [PATCH 30/30] docs: adds note about closing #7695 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46973702d6e..b078bb892ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Minor Enchancements * **Threema**: Add emoji to Threema alert notifications [#7676](https://github.com/grafana/grafana/pull/7676) thx [@dbrgn](https://github.com/dbrgn) +* **Panels**: Support dm3 unit [#7695](https://github.com/grafana/grafana/issues/7695) thx [@mitjaziv](https://github.com/mitjaziv) # 4.2.0-beta2 (unreleased) ## Minor Enhancements