From 2b20402d7d446275cdce7b2991a1e630ad4998a0 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 19 Jun 2018 17:06:56 +0900 Subject: [PATCH 1/9] get region list from ec2:DescribeRegions --- pkg/tsdb/cloudwatch/metric_find_query.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index ee9d9583c4e..64e5118e579 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -234,10 +234,22 @@ func parseMultiSelectValue(input string) []string { // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { regions := []string{ - "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-south-1", "ca-central-1", "cn-north-1", "cn-northwest-1", - "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-gov-west-1", "us-west-1", "us-west-2", "us-isob-east-1", "us-iso-east-1", + "cn-north-1", "cn-northwest-1", "us-gov-west-1", "us-isob-east-1", "us-iso-east-1", } + err := e.ensureClientSession("us-east-1") + if err != nil { + return nil, err + } + r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{}) + if err != nil { + return nil, err + } + for _, region := range r.Regions { + regions = append(regions, *region.RegionName) + } + sort.Strings(regions) + result := make([]suggestData, 0) for _, region := range regions { result = append(result, suggestData{Text: region, Value: region}) From f6cb01d38a0b406e6c6318f295dc77c61ccefdd5 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 19 Jun 2018 17:41:26 +0900 Subject: [PATCH 2/9] show all CloudWatch regions --- pkg/tsdb/cloudwatch/metric_find_query.go | 2 +- .../datasource/cloudwatch/config_ctrl.ts | 24 ++++++++++++++++++- .../cloudwatch/partials/config.html | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 64e5118e579..214b6ccbc67 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -234,7 +234,7 @@ func parseMultiSelectValue(input string) []string { // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { regions := []string{ - "cn-north-1", "cn-northwest-1", "us-gov-west-1", "us-isob-east-1", "us-iso-east-1", + "cn-north-1", "cn-northwest-1", "us-gov-east-1", "us-gov-west-1", "us-isob-east-1", "us-iso-east-1", } err := e.ensureClientSession("us-east-1") diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts index ff0d39944ca..1ff744854f5 100644 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts @@ -1,17 +1,20 @@ export class CloudWatchConfigCtrl { static templateUrl = 'partials/config.html'; current: any; + $http: any; accessKeyExist = false; secretKeyExist = false; /** @ngInject */ - constructor($scope) { + constructor($scope, $http) { this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp'; this.current.jsonData.authType = this.current.jsonData.authType || 'credentials'; this.accessKeyExist = this.current.secureJsonFields.accessKey; this.secretKeyExist = this.current.secureJsonFields.secretKey; + this.$http = $http; + this.getRegions(); } resetAccessKey() { @@ -36,4 +39,23 @@ export class CloudWatchConfigCtrl { { name: 'Monthly', value: 'Monthly', example: '[logstash-]YYYY.MM' }, { name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' }, ]; + + regions = []; + + getRegions() { + this.$http.get('https://ip-ranges.amazonaws.com/ip-ranges.json').then(ip_ranges => { + let regions = {}; + ip_ranges.data.prefixes + .map(p => { + return p.region; + }) + .filter(r => { + return r !== 'GLOBAL'; + }) + .forEach(r => { + regions[r] = true; + }); + this.regions = Object.keys(regions).sort(); + }); + } } diff --git a/public/app/plugins/datasource/cloudwatch/partials/config.html b/public/app/plugins/datasource/cloudwatch/partials/config.html index e5ab0910cba..40249d32b7e 100644 --- a/public/app/plugins/datasource/cloudwatch/partials/config.html +++ b/public/app/plugins/datasource/cloudwatch/partials/config.html @@ -39,7 +39,7 @@
- + Specify the region, such as for US West (Oregon) use ` us-west-2 ` as the region. From 6564abd5f3984d56ca533672a33ba4c5f4eaf5ea Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 24 Jul 2018 22:09:33 +0900 Subject: [PATCH 3/9] get regions from after datasource save --- .../datasource/cloudwatch/config_ctrl.ts | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts index 1ff744854f5..7ee6b9dea43 100644 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts @@ -1,19 +1,20 @@ +import _ from 'lodash'; export class CloudWatchConfigCtrl { static templateUrl = 'partials/config.html'; current: any; - $http: any; + datasourceSrv: any; accessKeyExist = false; secretKeyExist = false; /** @ngInject */ - constructor($scope, $http) { + constructor($scope, datasourceSrv) { this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp'; this.current.jsonData.authType = this.current.jsonData.authType || 'credentials'; this.accessKeyExist = this.current.secureJsonFields.accessKey; this.secretKeyExist = this.current.secureJsonFields.secretKey; - this.$http = $http; + this.datasourceSrv = datasourceSrv; this.getRegions(); } @@ -40,22 +41,41 @@ export class CloudWatchConfigCtrl { { name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' }, ]; - regions = []; + regions = [ + 'ap-northeast-1', + 'ap-northeast-2', + 'ap-northeast-3', + 'ap-south-1', + 'ap-southeast-1', + 'ap-southeast-2', + 'ca-central-1', + 'cn-north-1', + 'cn-northwest-1', + 'eu-central-1', + 'eu-north-1', + 'eu-west-1', + 'eu-west-2', + 'eu-west-3', + 'me-south-1', + 'sa-east-1', + 'us-east-1', + 'us-east-2', + 'us-gov-east-1', + 'us-gov-west-1', + 'us-iso-east-1', + 'us-isob-east-1', + 'us-west-1', + 'us-west-2', + ]; getRegions() { - this.$http.get('https://ip-ranges.amazonaws.com/ip-ranges.json').then(ip_ranges => { - let regions = {}; - ip_ranges.data.prefixes - .map(p => { - return p.region; - }) - .filter(r => { - return r !== 'GLOBAL'; - }) - .forEach(r => { - regions[r] = true; - }); - this.regions = Object.keys(regions).sort(); - }); + this.datasourceSrv + .loadDatasource(this.current.name) + .then(ds => { + return ds.getRegions(); + }) + .then(regions => { + this.regions = _.uniq(this.regions.concat(_.map(regions, 'value'))); + }); } } From 61e27109614533a9c1b2188bc528abac4fe93a49 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sat, 13 Oct 2018 15:48:13 +0900 Subject: [PATCH 4/9] add error message --- public/app/plugins/datasource/cloudwatch/config_ctrl.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts index 7ee6b9dea43..345315be0d2 100644 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts @@ -76,6 +76,8 @@ export class CloudWatchConfigCtrl { }) .then(regions => { this.regions = _.uniq(this.regions.concat(_.map(regions, 'value'))); + }, err => { + console.error('failed to call ec2:DescribeRegions, please check iam setting'); }); } } From a417d6fa2139ec654097a10d0a49f74d5abd8edb Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sat, 13 Oct 2018 15:54:12 +0900 Subject: [PATCH 5/9] don't merge hard coded region list --- public/app/plugins/datasource/cloudwatch/config_ctrl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts index 345315be0d2..02a171f3f15 100644 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts @@ -75,7 +75,7 @@ export class CloudWatchConfigCtrl { return ds.getRegions(); }) .then(regions => { - this.regions = _.uniq(this.regions.concat(_.map(regions, 'value'))); + this.regions = _.map(regions, 'value'); }, err => { console.error('failed to call ec2:DescribeRegions, please check iam setting'); }); From fce50cf1b8cc05fdc7fae97572c675e916f54604 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 16 Oct 2018 12:06:57 +0900 Subject: [PATCH 6/9] return default region list from backend --- pkg/tsdb/cloudwatch/metric_find_query.go | 12 ++++-- .../datasource/cloudwatch/config_ctrl.ts | 40 +++++-------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 214b6ccbc67..f2633bd6be6 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -234,6 +234,8 @@ func parseMultiSelectValue(input string) []string { // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { regions := []string{ + "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", + "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2", "cn-north-1", "cn-northwest-1", "us-gov-east-1", "us-gov-west-1", "us-isob-east-1", "us-iso-east-1", } @@ -243,10 +245,12 @@ func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s } r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{}) if err != nil { - return nil, err - } - for _, region := range r.Regions { - regions = append(regions, *region.RegionName) + // ignore error for backward compatibility + plog.Error("Failed to get regions", "error", err) + } else { + for _, region := range r.Regions { + regions = append(regions, *region.RegionName) + } } sort.Strings(regions) diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts index 02a171f3f15..d46f5cf1580 100644 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts @@ -41,32 +41,7 @@ export class CloudWatchConfigCtrl { { name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' }, ]; - regions = [ - 'ap-northeast-1', - 'ap-northeast-2', - 'ap-northeast-3', - 'ap-south-1', - 'ap-southeast-1', - 'ap-southeast-2', - 'ca-central-1', - 'cn-north-1', - 'cn-northwest-1', - 'eu-central-1', - 'eu-north-1', - 'eu-west-1', - 'eu-west-2', - 'eu-west-3', - 'me-south-1', - 'sa-east-1', - 'us-east-1', - 'us-east-2', - 'us-gov-east-1', - 'us-gov-west-1', - 'us-iso-east-1', - 'us-isob-east-1', - 'us-west-1', - 'us-west-2', - ]; + regions = []; getRegions() { this.datasourceSrv @@ -74,10 +49,13 @@ export class CloudWatchConfigCtrl { .then(ds => { return ds.getRegions(); }) - .then(regions => { - this.regions = _.map(regions, 'value'); - }, err => { - console.error('failed to call ec2:DescribeRegions, please check iam setting'); - }); + .then( + regions => { + this.regions = _.map(regions, 'value'); + }, + err => { + console.error('failed to get latest regions'); + } + ); } } From c64a5a6e0ed12fd8714d8a4d60c432f2911ad79c Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 17 Oct 2018 14:02:02 +0900 Subject: [PATCH 7/9] re-add hard coded region list --- .../datasource/cloudwatch/config_ctrl.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts index d46f5cf1580..6fe48cb1715 100644 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts @@ -41,7 +41,32 @@ export class CloudWatchConfigCtrl { { name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' }, ]; - regions = []; + regions = [ + 'ap-northeast-1', + 'ap-northeast-2', + 'ap-northeast-3', + 'ap-south-1', + 'ap-southeast-1', + 'ap-southeast-2', + 'ca-central-1', + 'cn-north-1', + 'cn-northwest-1', + 'eu-central-1', + 'eu-north-1', + 'eu-west-1', + 'eu-west-2', + 'eu-west-3', + 'me-south-1', + 'sa-east-1', + 'us-east-1', + 'us-east-2', + 'us-gov-east-1', + 'us-gov-west-1', + 'us-iso-east-1', + 'us-isob-east-1', + 'us-west-1', + 'us-west-2', + ]; getRegions() { this.datasourceSrv From 518e485e366231641bcb3290177efbc5b56b1f38 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 17 Oct 2018 10:52:46 +0200 Subject: [PATCH 8/9] docs: update cloudwatch iam policy description --- docs/sources/features/datasources/cloudwatch.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/sources/features/datasources/cloudwatch.md b/docs/sources/features/datasources/cloudwatch.md index 7adc6ebe4fb..be36d108475 100644 --- a/docs/sources/features/datasources/cloudwatch.md +++ b/docs/sources/features/datasources/cloudwatch.md @@ -46,7 +46,7 @@ Checkout AWS docs on [IAM Roles](http://docs.aws.amazon.com/AWSEC2/latest/UserGu ## IAM Policies Grafana needs permissions granted via IAM to be able to read CloudWatch metrics -and EC2 tags/instances. You can attach these permissions to IAM roles and +and EC2 tags/instances/regions. You can attach these permissions to IAM roles and utilize Grafana's built-in support for assuming roles. Here is a minimal policy example: @@ -65,11 +65,12 @@ Here is a minimal policy example: "Resource": "*" }, { - "Sid": "AllowReadingTagsFromEC2", + "Sid": "AllowReadingTagsInstancesRegionsFromEC2", "Effect": "Allow", "Action": [ "ec2:DescribeTags", - "ec2:DescribeInstances" + "ec2:DescribeInstances", + "ec2:DescribeRegions" ], "Resource": "*" } From 52f398e7f794215375c767b87149951297287d23 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 17 Oct 2018 10:58:06 +0200 Subject: [PATCH 9/9] cloudwatch: return a distinct list of regions --- pkg/tsdb/cloudwatch/metric_find_query.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index f2633bd6be6..b74af76f09a 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -249,7 +249,18 @@ func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s plog.Error("Failed to get regions", "error", err) } else { for _, region := range r.Regions { - regions = append(regions, *region.RegionName) + exists := false + + for _, existingRegion := range regions { + if existingRegion == *region.RegionName { + exists = true + break + } + } + + if !exists { + regions = append(regions, *region.RegionName) + } } } sort.Strings(regions)