From 39607d09d7accab372be5e15dfd3ee389378931e Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Mon, 3 Apr 2017 21:50:40 +0900 Subject: [PATCH 01/44] (cloudwatch) alerting --- pkg/api/cloudwatch/cloudwatch.go | 10 +- pkg/api/cloudwatch/metrics.go | 8 +- pkg/api/cloudwatch/metrics_test.go | 8 +- pkg/cmd/grafana-server/main.go | 1 + pkg/tsdb/cloudwatch/cloudwatch.go | 350 ++++++++++++++++++ pkg/tsdb/cloudwatch/cloudwatch_test.go | 181 +++++++++ pkg/tsdb/cloudwatch/types.go | 16 + .../datasource/cloudwatch/datasource.js | 151 +++----- .../plugins/datasource/cloudwatch/plugin.json | 1 + .../cloudwatch/specs/datasource_specs.ts | 107 +++--- 10 files changed, 659 insertions(+), 174 deletions(-) create mode 100644 pkg/tsdb/cloudwatch/cloudwatch.go create mode 100644 pkg/tsdb/cloudwatch/cloudwatch_test.go create mode 100644 pkg/tsdb/cloudwatch/types.go diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index e0076db40c5..98ed1eabae2 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -36,7 +36,7 @@ type cwRequest struct { DataSource *m.DataSource } -type datasourceInfo struct { +type DatasourceInfo struct { Profile string Region string AuthType string @@ -47,7 +47,7 @@ type datasourceInfo struct { SecretKey string } -func (req *cwRequest) GetDatasourceInfo() *datasourceInfo { +func (req *cwRequest) GetDatasourceInfo() *DatasourceInfo { authType := req.DataSource.JsonData.Get("authType").MustString() assumeRoleArn := req.DataSource.JsonData.Get("assumeRoleArn").MustString() accessKey := "" @@ -62,7 +62,7 @@ func (req *cwRequest) GetDatasourceInfo() *datasourceInfo { } } - return &datasourceInfo{ + return &DatasourceInfo{ AuthType: authType, AssumeRoleArn: assumeRoleArn, Region: req.Region, @@ -95,7 +95,7 @@ type cache struct { var awsCredentialCache map[string]cache = make(map[string]cache) var credentialCacheLock sync.RWMutex -func getCredentials(dsInfo *datasourceInfo) (*credentials.Credentials, error) { +func GetCredentials(dsInfo *DatasourceInfo) (*credentials.Credentials, error) { cacheKey := dsInfo.Profile + ":" + dsInfo.AssumeRoleArn credentialCacheLock.RLock() if _, ok := awsCredentialCache[cacheKey]; ok { @@ -207,7 +207,7 @@ func ec2RoleProvider(sess *session.Session) credentials.Provider { } func getAwsConfig(req *cwRequest) (*aws.Config, error) { - creds, err := getCredentials(req.GetDatasourceInfo()) + creds, err := GetCredentials(req.GetDatasourceInfo()) if err != nil { return nil, err } diff --git a/pkg/api/cloudwatch/metrics.go b/pkg/api/cloudwatch/metrics.go index 16b496d6be6..0d471efad68 100644 --- a/pkg/api/cloudwatch/metrics.go +++ b/pkg/api/cloudwatch/metrics.go @@ -253,8 +253,8 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) { c.JSON(200, result) } -func getAllMetrics(cwData *datasourceInfo) (cloudwatch.ListMetricsOutput, error) { - creds, err := getCredentials(cwData) +func getAllMetrics(cwData *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + creds, err := GetCredentials(cwData) if err != nil { return cloudwatch.ListMetricsOutput{}, err } @@ -291,7 +291,7 @@ func getAllMetrics(cwData *datasourceInfo) (cloudwatch.ListMetricsOutput, error) var metricsCacheLock sync.Mutex -func getMetricsForCustomMetrics(dsInfo *datasourceInfo, getAllMetrics func(*datasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +func getMetricsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { metricsCacheLock.Lock() defer metricsCacheLock.Unlock() @@ -328,7 +328,7 @@ func getMetricsForCustomMetrics(dsInfo *datasourceInfo, getAllMetrics func(*data var dimensionsCacheLock sync.Mutex -func getDimensionsForCustomMetrics(dsInfo *datasourceInfo, getAllMetrics func(*datasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +func getDimensionsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { dimensionsCacheLock.Lock() defer dimensionsCacheLock.Unlock() diff --git a/pkg/api/cloudwatch/metrics_test.go b/pkg/api/cloudwatch/metrics_test.go index 4ac8a70a273..238e815fac1 100644 --- a/pkg/api/cloudwatch/metrics_test.go +++ b/pkg/api/cloudwatch/metrics_test.go @@ -11,13 +11,13 @@ import ( func TestCloudWatchMetrics(t *testing.T) { Convey("When calling getMetricsForCustomMetrics", t, func() { - dsInfo := &datasourceInfo{ + dsInfo := &DatasourceInfo{ Region: "us-east-1", Namespace: "Foo", Profile: "default", AssumeRoleArn: "", } - f := func(dsInfo *datasourceInfo) (cloudwatch.ListMetricsOutput, error) { + f := func(dsInfo *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { return cloudwatch.ListMetricsOutput{ Metrics: []*cloudwatch.Metric{ { @@ -39,13 +39,13 @@ func TestCloudWatchMetrics(t *testing.T) { }) Convey("When calling getDimensionsForCustomMetrics", t, func() { - dsInfo := &datasourceInfo{ + dsInfo := &DatasourceInfo{ Region: "us-east-1", Namespace: "Foo", Profile: "default", AssumeRoleArn: "", } - f := func(dsInfo *datasourceInfo) (cloudwatch.ListMetricsOutput, error) { + f := func(dsInfo *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { return cloudwatch.ListMetricsOutput{ Metrics: []*cloudwatch.Metric{ { diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index 8f90da93177..6545987152d 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/grafana/grafana/pkg/services/alerting/conditions" _ "github.com/grafana/grafana/pkg/services/alerting/notifiers" + _ "github.com/grafana/grafana/pkg/tsdb/cloudwatch" _ "github.com/grafana/grafana/pkg/tsdb/graphite" _ "github.com/grafana/grafana/pkg/tsdb/influxdb" _ "github.com/grafana/grafana/pkg/tsdb/mysql" diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go new file mode 100644 index 00000000000..19a5f19ef3b --- /dev/null +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -0,0 +1,350 @@ +package cloudwatch + +import ( + "context" + "errors" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatch" + cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" +) + +type CloudWatchExecutor struct { + *models.DataSource +} + +func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.Executor, error) { + return &CloudWatchExecutor{ + DataSource: dsInfo, + }, nil +} + +var ( + plog log.Logger + standardStatistics map[string]bool + aliasFormat *regexp.Regexp +) + +func init() { + plog = log.New("tsdb.cloudwatch") + tsdb.RegisterExecutor("cloudwatch", NewCloudWatchExecutor) + standardStatistics = map[string]bool{ + "Average": true, + "Maximum": true, + "Minimum": true, + "Sum": true, + "SampleCount": true, + } + aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) +} + +func (e *CloudWatchExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { + result := &tsdb.BatchResult{ + QueryResults: make(map[string]*tsdb.QueryResult), + } + + errCh := make(chan error, 1) + resCh := make(chan *tsdb.QueryResult, 1) + + currentlyExecuting := 0 + for _, model := range queries { + currentlyExecuting++ + go func(refId string) { + queryRes, err := e.executeQuery(ctx, model, queryContext) + currentlyExecuting-- + if err != nil { + errCh <- err + } else { + queryRes.RefId = refId + resCh <- queryRes + } + }(model.RefId) + } + + for currentlyExecuting != 0 { + select { + case res := <-resCh: + result.QueryResults[res.RefId] = res + case err := <-errCh: + return result.WithError(err) + case <-ctx.Done(): + return result.WithError(ctx.Err()) + } + } + + return result +} + +func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, error) { + assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() + + accessKey := "" + secretKey := "" + for key, value := range e.DataSource.SecureJsonData.Decrypt() { + if key == "accessKey" { + accessKey = value + } + if key == "secretKey" { + secretKey = value + } + } + + datasourceInfo := &cwapi.DatasourceInfo{ + Region: region, + Profile: e.DataSource.Database, + AssumeRoleArn: assumeRoleArn, + AccessKey: accessKey, + SecretKey: secretKey, + } + + credentials, err := cwapi.GetCredentials(datasourceInfo) + if err != nil { + return nil, err + } + + cfg := &aws.Config{ + Region: aws.String(region), + Credentials: credentials, + } + + sess, err := session.NewSession(cfg) + if err != nil { + return nil, err + } + + client := cloudwatch.New(sess, cfg) + return client, nil +} + +func (e *CloudWatchExecutor) executeQuery(ctx context.Context, model *tsdb.Query, queryContext *tsdb.QueryContext) (*tsdb.QueryResult, error) { + query, err := parseQuery(model.Model) + if err != nil { + return nil, err + } + + client, err := e.getClient(query.Region) + if err != nil { + return nil, err + } + + startTime, err := queryContext.TimeRange.ParseFrom() + if err != nil { + return nil, err + } + + endTime, err := queryContext.TimeRange.ParseTo() + if err != nil { + return nil, err + } + + params := &cloudwatch.GetMetricStatisticsInput{ + Namespace: aws.String(query.Namespace), + MetricName: aws.String(query.MetricName), + Dimensions: query.Dimensions, + Period: aws.Int64(int64(query.Period)), + StartTime: aws.Time(startTime.Add(-time.Minute * 15)), + EndTime: aws.Time(endTime), + } + if len(query.Statistics) > 0 { + params.Statistics = query.Statistics + } + if len(query.ExtendedStatistics) > 0 { + params.ExtendedStatistics = query.ExtendedStatistics + } + + resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second)) + if err != nil { + return nil, err + } + + queryRes, err := parseResponse(resp, query) + if err != nil { + return nil, err + } + + return queryRes, nil +} + +func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) { + var result []*cloudwatch.Dimension + + for k, v := range model.Get("dimensions").MustMap() { + kk := k + if vv, ok := v.(string); ok { + result = append(result, &cloudwatch.Dimension{ + Name: &kk, + Value: &vv, + }) + } else { + return nil, errors.New("failed to parse") + } + } + + sort.Slice(result, func(i, j int) bool { + return *result[i].Name < *result[j].Name + }) + return result, nil +} + +func parseStatistics(model *simplejson.Json) ([]*string, []*string, error) { + var statistics []*string + var extendedStatistics []*string + + for _, s := range model.Get("statistics").MustArray() { + if ss, ok := s.(string); ok { + if _, isStandard := standardStatistics[ss]; isStandard { + statistics = append(statistics, &ss) + } else { + extendedStatistics = append(extendedStatistics, &ss) + } + } else { + return nil, nil, errors.New("failed to parse") + } + } + + return statistics, extendedStatistics, nil +} + +func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { + region, err := model.Get("region").String() + if err != nil { + return nil, err + } + + namespace, err := model.Get("namespace").String() + if err != nil { + return nil, err + } + + metricName, err := model.Get("metricName").String() + if err != nil { + return nil, err + } + + dimensions, err := parseDimensions(model) + if err != nil { + return nil, err + } + + statistics, extendedStatistics, err := parseStatistics(model) + if err != nil { + return nil, err + } + + p := model.Get("period").MustString("") + if p == "" { + if namespace == "AWS/EC2" { + p = "300" + } else { + p = "60" + } + } + period, err := strconv.Atoi(p) + if err != nil { + return nil, err + } + + alias := model.Get("alias").MustString("{{metric}}_{{stat}}") + + return &CloudWatchQuery{ + Region: region, + Namespace: namespace, + MetricName: metricName, + Dimensions: dimensions, + Statistics: statistics, + ExtendedStatistics: extendedStatistics, + Period: period, + Alias: alias, + }, nil +} + +func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string { + data := map[string]string{} + data["region"] = query.Region + data["namespace"] = query.Namespace + data["metric"] = query.MetricName + data["stat"] = stat + for k, v := range dimensions { + data[k] = v + } + + result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte { + labelName := strings.Replace(string(in), "{{", "", 1) + labelName = strings.Replace(labelName, "}}", "", 1) + labelName = strings.TrimSpace(labelName) + if val, exists := data[labelName]; exists { + return []byte(val) + } + + return in + }) + + return string(result) +} + +func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) { + queryRes := tsdb.NewQueryResult() + + var value float64 + for _, s := range append(query.Statistics, query.ExtendedStatistics...) { + series := tsdb.TimeSeries{ + Tags: map[string]string{}, + } + for _, d := range query.Dimensions { + series.Tags[*d.Name] = *d.Value + } + series.Name = formatAlias(query, *s, series.Tags) + + lastTimestamp := make(map[string]time.Time) + sort.Slice(resp.Datapoints, func(i, j int) bool { + return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp) + }) + for _, v := range resp.Datapoints { + switch *s { + case "Average": + value = *v.Average + case "Maximum": + value = *v.Maximum + case "Minimum": + value = *v.Minimum + case "Sum": + value = *v.Sum + case "SampleCount": + value = *v.SampleCount + default: + if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil { + value = *v.ExtendedStatistics[*s] + } + } + + // terminate gap of data points + timestamp := *v.Timestamp + if _, ok := lastTimestamp[*s]; ok { + nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second) + if timestamp.After(nextTimestampFromLast) { + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000))) + } + } + lastTimestamp[*s] = timestamp + + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000))) + } + + queryRes.Series = append(queryRes.Series, &series) + } + + return queryRes, nil +} diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go new file mode 100644 index 00000000000..5c322a44d56 --- /dev/null +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -0,0 +1,181 @@ +package cloudwatch + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + . "github.com/smartystreets/goconvey/convey" +) + +func TestCloudWatch(t *testing.T) { + Convey("CloudWatch", t, func() { + + Convey("can parse cloudwatch json model", func() { + json := ` + { + "region": "us-east-1", + "namespace": "AWS/ApplicationELB", + "metricName": "TargetResponseTime", + "dimensions": { + "LoadBalancer": "lb", + "TargetGroup": "tg" + }, + "statistics": [ + "Average", + "Maximum", + "p50.00", + "p90.00" + ], + "period": "60", + "alias": "{{metric}}_{{stat}}" + } + ` + modelJson, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + res, err := parseQuery(modelJson) + So(err, ShouldBeNil) + So(res.Region, ShouldEqual, "us-east-1") + So(res.Namespace, ShouldEqual, "AWS/ApplicationELB") + So(res.MetricName, ShouldEqual, "TargetResponseTime") + So(len(res.Dimensions), ShouldEqual, 2) + So(*res.Dimensions[0].Name, ShouldEqual, "LoadBalancer") + So(*res.Dimensions[0].Value, ShouldEqual, "lb") + So(*res.Dimensions[1].Name, ShouldEqual, "TargetGroup") + So(*res.Dimensions[1].Value, ShouldEqual, "tg") + So(len(res.Statistics), ShouldEqual, 2) + So(*res.Statistics[0], ShouldEqual, "Average") + So(*res.Statistics[1], ShouldEqual, "Maximum") + So(len(res.ExtendedStatistics), ShouldEqual, 2) + So(*res.ExtendedStatistics[0], ShouldEqual, "p50.00") + So(*res.ExtendedStatistics[1], ShouldEqual, "p90.00") + So(res.Period, ShouldEqual, 60) + So(res.Alias, ShouldEqual, "{{metric}}_{{stat}}") + }) + + Convey("can parse cloudwatch response", func() { + timestamp := time.Unix(0, 0) + resp := &cloudwatch.GetMetricStatisticsOutput{ + Label: aws.String("TargetResponseTime"), + Datapoints: []*cloudwatch.Datapoint{ + { + Timestamp: aws.Time(timestamp), + Average: aws.Float64(10.0), + Maximum: aws.Float64(20.0), + ExtendedStatistics: map[string]*float64{ + "p50.00": aws.Float64(30.0), + "p90.00": aws.Float64(40.0), + }, + }, + }, + } + query := &CloudWatchQuery{ + Region: "us-east-1", + Namespace: "AWS/ApplicationELB", + MetricName: "TargetResponseTime", + Dimensions: []*cloudwatch.Dimension{ + { + Name: aws.String("LoadBalancer"), + Value: aws.String("lb"), + }, + { + Name: aws.String("TargetGroup"), + Value: aws.String("tg"), + }, + }, + Statistics: []*string{aws.String("Average"), aws.String("Maximum")}, + ExtendedStatistics: []*string{aws.String("p50.00"), aws.String("p90.00")}, + Period: 60, + Alias: "{{namespace}}_{{metric}}_{{stat}}", + } + + queryRes, err := parseResponse(resp, query) + So(err, ShouldBeNil) + So(queryRes.Series[0].Name, ShouldEqual, "AWS/ApplicationELB_TargetResponseTime_Average") + So(queryRes.Series[0].Tags["LoadBalancer"], ShouldEqual, "lb") + So(queryRes.Series[0].Tags["TargetGroup"], ShouldEqual, "tg") + So(queryRes.Series[0].Points[0][0].String(), ShouldEqual, null.FloatFrom(10.0).String()) + So(queryRes.Series[1].Points[0][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) + So(queryRes.Series[2].Points[0][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) + So(queryRes.Series[3].Points[0][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) + }) + + Convey("terminate gap of data points", func() { + timestamp := time.Unix(0, 0) + resp := &cloudwatch.GetMetricStatisticsOutput{ + Label: aws.String("TargetResponseTime"), + Datapoints: []*cloudwatch.Datapoint{ + { + Timestamp: aws.Time(timestamp), + Average: aws.Float64(10.0), + Maximum: aws.Float64(20.0), + ExtendedStatistics: map[string]*float64{ + "p50.00": aws.Float64(30.0), + "p90.00": aws.Float64(40.0), + }, + }, + { + Timestamp: aws.Time(timestamp.Add(60 * time.Second)), + Average: aws.Float64(20.0), + Maximum: aws.Float64(30.0), + ExtendedStatistics: map[string]*float64{ + "p50.00": aws.Float64(40.0), + "p90.00": aws.Float64(50.0), + }, + }, + { + Timestamp: aws.Time(timestamp.Add(180 * time.Second)), + Average: aws.Float64(30.0), + Maximum: aws.Float64(40.0), + ExtendedStatistics: map[string]*float64{ + "p50.00": aws.Float64(50.0), + "p90.00": aws.Float64(60.0), + }, + }, + }, + } + query := &CloudWatchQuery{ + Region: "us-east-1", + Namespace: "AWS/ApplicationELB", + MetricName: "TargetResponseTime", + Dimensions: []*cloudwatch.Dimension{ + { + Name: aws.String("LoadBalancer"), + Value: aws.String("lb"), + }, + { + Name: aws.String("TargetGroup"), + Value: aws.String("tg"), + }, + }, + Statistics: []*string{aws.String("Average"), aws.String("Maximum")}, + ExtendedStatistics: []*string{aws.String("p50.00"), aws.String("p90.00")}, + Period: 60, + Alias: "{{namespace}}_{{metric}}_{{stat}}", + } + + queryRes, err := parseResponse(resp, query) + So(err, ShouldBeNil) + So(queryRes.Series[0].Points[0][0].String(), ShouldEqual, null.FloatFrom(10.0).String()) + So(queryRes.Series[1].Points[0][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) + So(queryRes.Series[2].Points[0][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) + So(queryRes.Series[3].Points[0][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) + So(queryRes.Series[0].Points[1][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) + So(queryRes.Series[1].Points[1][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) + So(queryRes.Series[2].Points[1][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) + So(queryRes.Series[3].Points[1][0].String(), ShouldEqual, null.FloatFrom(50.0).String()) + So(queryRes.Series[0].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) + So(queryRes.Series[1].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) + So(queryRes.Series[2].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) + So(queryRes.Series[3].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) + So(queryRes.Series[0].Points[3][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) + So(queryRes.Series[1].Points[3][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) + So(queryRes.Series[2].Points[3][0].String(), ShouldEqual, null.FloatFrom(50.0).String()) + So(queryRes.Series[3].Points[3][0].String(), ShouldEqual, null.FloatFrom(60.0).String()) + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/types.go b/pkg/tsdb/cloudwatch/types.go new file mode 100644 index 00000000000..c2a5ab8c3d7 --- /dev/null +++ b/pkg/tsdb/cloudwatch/types.go @@ -0,0 +1,16 @@ +package cloudwatch + +import ( + "github.com/aws/aws-sdk-go/service/cloudwatch" +) + +type CloudWatchQuery struct { + Region string + Namespace string + MetricName string + Dimensions []*cloudwatch.Dimension + Statistics []*string + ExtendedStatistics []*string + Period int + Alias string +} diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 0f0406a219c..74f93874519 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -17,6 +17,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot this.supportMetrics = true; this.proxyUrl = instanceSettings.url; this.defaultRegion = instanceSettings.jsonData.defaultRegion; + this.instanceSettings = instanceSettings; this.standardStatistics = [ 'Average', 'Maximum', @@ -27,31 +28,29 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var self = this; this.query = function(options) { - var start = self.convertToCloudWatchTime(options.range.from, false); - var end = self.convertToCloudWatchTime(options.range.to, true); - - var queries = []; options = angular.copy(options); options.targets = this.expandTemplateVariable(options.targets, options.scopedVars, templateSrv); - _.each(options.targets, function(target) { - if (target.hide || !target.namespace || !target.metricName || _.isEmpty(target.statistics)) { - return; - } - var query = {}; - query.region = templateSrv.replace(target.region, options.scopedVars); - query.namespace = templateSrv.replace(target.namespace, options.scopedVars); - query.metricName = templateSrv.replace(target.metricName, options.scopedVars); - query.dimensions = self.convertDimensionFormat(target.dimensions, options.scopedVars); - query.statistics = target.statistics; + var queries = _.filter(options.targets, function (item) { + return item.hide !== true || !item.namespace || !item.metricName || _.isEmpty(item.statistics); + }).map(function (item) { + item.region = templateSrv.replace(item.region, options.scopedVars); + item.namespace = templateSrv.replace(item.namespace, options.scopedVars); + item.metricName = templateSrv.replace(item.metricName, options.scopedVars); + var dimensions = {}; + _.each(item.dimensions, function (value, key) { + dimensions[templateSrv.replace(key, options.scopedVars)] = templateSrv.replace(value, options.scopedVars); + }); + item.dimensions = dimensions; + item.period = self.getPeriod(item, options); - var now = Math.round(Date.now() / 1000); - var period = this.getPeriod(target, query, options, start, end, now); - target.period = period; - query.period = period; - - queries.push(query); - }.bind(this)); + return _.extend({ + refId: item.refId, + intervalMs: options.intervalMs, + maxDataPoints: options.maxDataPoints, + datasourceId: self.instanceSettings.id, + }, item); + }); // No valid targets, return the empty result to save a round trip. if (_.isEmpty(queries)) { @@ -60,23 +59,20 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot return d.promise; } - var allQueryPromise = _.map(queries, function(query) { - return this.performTimeSeriesQuery(query, start, end); - }.bind(this)); + var request = { + from: options.rangeRaw.from, + to: options.rangeRaw.to, + queries: queries + }; - return $q.all(allQueryPromise).then(function(allResponse) { - var result = []; - - _.each(allResponse, function(response, index) { - var metrics = transformMetricData(response, options.targets[index], options.scopedVars); - result = result.concat(metrics); - }); - - return {data: result}; - }); + return this.performTimeSeriesQuery(request); }; - this.getPeriod = function(target, query, options, start, end, now) { + this.getPeriod = function(target, options) { + var start = this.convertToCloudWatchTime(options.range.from, false); + var end = this.convertToCloudWatchTime(options.range.to, true); + var now = Math.round(Date.now() / 1000); + var period; var range = end - start; @@ -85,7 +81,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var periodUnit = 60; if (!target.period) { if (now - start <= (daySec * 15)) { // until 15 days ago - if (query.namespace === 'AWS/EC2') { + if (target.namespace === 'AWS/EC2') { periodUnit = period = 300; } else { periodUnit = period = 60; @@ -114,22 +110,19 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot return period; }; - this.performTimeSeriesQuery = function(query, start, end) { - var statistics = _.filter(query.statistics, function(s) { return _.includes(self.standardStatistics, s); }); - var extendedStatistics = _.reject(query.statistics, function(s) { return _.includes(self.standardStatistics, s); }); - return this.awsRequest({ - region: query.region, - action: 'GetMetricStatistics', - parameters: { - namespace: query.namespace, - metricName: query.metricName, - dimensions: query.dimensions, - statistics: statistics, - extendedStatistics: extendedStatistics, - startTime: start, - endTime: end, - period: query.period + this.performTimeSeriesQuery = function(request) { + return backendSrv.post('/api/tsdb/query', request).then(function (res) { + var data = []; + + if (res.results) { + _.forEach(res.results, function (queryRes) { + _.forEach(queryRes.series, function (series) { + data.push({target: series.name, datapoints: series.points}); + }); + }); } + + return {data: data}; }); }; @@ -355,62 +348,6 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot return this.defaultRegion; }; - function transformMetricData(md, options, scopedVars) { - var aliasRegex = /\{\{(.+?)\}\}/g; - var aliasPattern = options.alias || '{{metric}}_{{stat}}'; - var aliasData = { - region: templateSrv.replace(options.region, scopedVars), - namespace: templateSrv.replace(options.namespace, scopedVars), - metric: templateSrv.replace(options.metricName, scopedVars), - }; - - var aliasDimensions = {}; - - _.each(_.keys(options.dimensions), function(origKey) { - var key = templateSrv.replace(origKey, scopedVars); - var value = templateSrv.replace(options.dimensions[origKey], scopedVars); - aliasDimensions[key] = value; - }); - - _.extend(aliasData, aliasDimensions); - - var periodMs = options.period * 1000; - - return _.map(options.statistics, function(stat) { - var extended = !_.includes(self.standardStatistics, stat); - var dps = []; - var lastTimestamp = null; - _.chain(md.Datapoints) - .sortBy(function(dp) { - return dp.Timestamp; - }) - .each(function(dp) { - var timestamp = new Date(dp.Timestamp).getTime(); - while (lastTimestamp && (timestamp - lastTimestamp) > periodMs) { - dps.push([null, lastTimestamp + periodMs]); - lastTimestamp = lastTimestamp + periodMs; - } - lastTimestamp = timestamp; - if (!extended) { - dps.push([dp[stat], timestamp]); - } else { - dps.push([dp.ExtendedStatistics[stat], timestamp]); - } - }) - .value(); - - aliasData.stat = stat; - var seriesName = aliasPattern.replace(aliasRegex, function(match, g1) { - if (aliasData[g1]) { - return aliasData[g1]; - } - return g1; - }); - - return {target: seriesName, datapoints: dps}; - }); - } - this.getExpandedVariables = function(target, dimensionKey, variable, templateSrv) { /* if the all checkbox is marked we should add all values to the targets */ var allSelected = _.find(variable.options, {'selected': true, 'text': 'All'}); diff --git a/public/app/plugins/datasource/cloudwatch/plugin.json b/public/app/plugins/datasource/cloudwatch/plugin.json index bac2ddca8b9..3af7d8ccb6e 100644 --- a/public/app/plugins/datasource/cloudwatch/plugin.json +++ b/public/app/plugins/datasource/cloudwatch/plugin.json @@ -4,6 +4,7 @@ "id": "cloudwatch", "metrics": true, + "alerting": true, "annotations": true, "info": { diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 48824f051c7..87acb40028c 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -28,6 +28,7 @@ describe('CloudWatchDatasource', function() { var query = { range: { from: 'now-1h', to: 'now' }, + rangeRaw: { from: 1483228800, to: 1483232400 }, targets: [ { region: 'us-east-1', @@ -43,37 +44,41 @@ describe('CloudWatchDatasource', function() { }; var response = { - Datapoints: [ - { - Average: 1, - Timestamp: 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' - }, - { - Average: 2, - Timestamp: 'Wed Dec 31 1969 16:05:00 GMT-0800 (PST)' - }, - { - Average: 5, - Timestamp: 'Wed Dec 31 1969 16:15:00 GMT-0800 (PST)' + timings: [null], + results: { + A: { + error: '', + refId: 'A', + series: [ + { + name: 'CPUUtilization_Average', + points: [ + [1, 1483228800000], + [2, 1483229100000], + [5, 1483229700000], + ], + tags: { + InstanceId: 'i-12345678' + } + } + ] } - ], - Label: 'CPUUtilization' + } }; beforeEach(function() { - ctx.backendSrv.datasourceRequest = function(params) { + ctx.backendSrv.post = function(path, params) { requestParams = params; - return ctx.$q.when({data: response}); + return ctx.$q.when(response); }; }); it('should generate the correct query', function(done) { ctx.ds.query(query).then(function() { - var params = requestParams.data.parameters; + var params = requestParams.queries[0]; expect(params.namespace).to.be(query.targets[0].namespace); expect(params.metricName).to.be(query.targets[0].metricName); - expect(params.dimensions[0].Name).to.be(Object.keys(query.targets[0].dimensions)[0]); - expect(params.dimensions[0].Value).to.be(query.targets[0].dimensions[Object.keys(query.targets[0].dimensions)[0]]); + expect(params.dimensions['InstanceId']).to.be('i-12345678'); expect(params.statistics).to.eql(query.targets[0].statistics); expect(params.period).to.be(query.targets[0].period); done(); @@ -88,6 +93,7 @@ describe('CloudWatchDatasource', function() { var query = { range: { from: 'now-1h', to: 'now' }, + rangeRaw: { from: 1483228800, to: 1483232400 }, targets: [ { region: 'us-east-1', @@ -103,7 +109,7 @@ describe('CloudWatchDatasource', function() { }; ctx.ds.query(query).then(function() { - var params = requestParams.data.parameters; + var params = requestParams.queries[0]; expect(params.period).to.be(600); done(); }); @@ -112,16 +118,8 @@ describe('CloudWatchDatasource', function() { it('should return series list', function(done) { ctx.ds.query(query).then(function(result) { - expect(result.data[0].target).to.be('CPUUtilization_Average'); - expect(result.data[0].datapoints[0][0]).to.be(response.Datapoints[0]['Average']); - done(); - }); - ctx.$rootScope.$apply(); - }); - - it('should return null for missing data point', function(done) { - ctx.ds.query(query).then(function(result) { - expect(result.data[0].datapoints[2][0]).to.be(null); + expect(result.data[0].target).to.be(response.results.A.series[0].name); + expect(result.data[0].datapoints[0][0]).to.be(response.results.A.series[0].points[0][0]); done(); }); ctx.$rootScope.$apply(); @@ -173,6 +171,7 @@ describe('CloudWatchDatasource', function() { var query = { range: { from: 'now-1h', to: 'now' }, + rangeRaw: { from: 1483228800, to: 1483232400 }, targets: [ { region: 'us-east-1', @@ -189,40 +188,40 @@ describe('CloudWatchDatasource', function() { }; var response = { - Datapoints: [ - { - ExtendedStatistics: { - 'p90.00': 1 - }, - Timestamp: 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' - }, - { - ExtendedStatistics: { - 'p90.00': 2 - }, - Timestamp: 'Wed Dec 31 1969 16:05:00 GMT-0800 (PST)' - }, - { - ExtendedStatistics: { - 'p90.00': 5 - }, - Timestamp: 'Wed Dec 31 1969 16:15:00 GMT-0800 (PST)' + timings: [null], + results: { + A: { + error: '', + refId: 'A', + series: [ + { + name: 'TargetResponseTime_p90.00', + points: [ + [1, 1483228800000], + [2, 1483229100000], + [5, 1483229700000], + ], + tags: { + LoadBalancer: 'lb', + TargetGroup: 'tg' + } + } + ] } - ], - Label: 'TargetResponseTime' + } }; beforeEach(function() { - ctx.backendSrv.datasourceRequest = function(params) { + ctx.backendSrv.post = function(path, params) { requestParams = params; - return ctx.$q.when({data: response}); + return ctx.$q.when(response); }; }); it('should return series list', function(done) { ctx.ds.query(query).then(function(result) { - expect(result.data[0].target).to.be('TargetResponseTime_p90.00'); - expect(result.data[0].datapoints[0][0]).to.be(response.Datapoints[0].ExtendedStatistics['p90.00']); + expect(result.data[0].target).to.be(response.results.A.series[0].name); + expect(result.data[0].datapoints[0][0]).to.be(response.results.A.series[0].points[0][0]); done(); }); ctx.$rootScope.$apply(); From dcb5ea58cec6cae9f948f9f0c0d262f4bfd55fba Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Thu, 13 Apr 2017 19:22:00 +0900 Subject: [PATCH 02/44] count up metrics --- pkg/tsdb/cloudwatch/cloudwatch.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 19a5f19ef3b..445ec94aef2 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -20,6 +20,7 @@ import ( cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/metrics" ) type CloudWatchExecutor struct { @@ -169,6 +170,7 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, model *tsdb.Query if err != nil { return nil, err } + metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc(1) queryRes, err := parseResponse(resp, query) if err != nil { From 728e96e13442ed10f97488995d0f274193d9a6e2 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Mon, 26 Jun 2017 16:13:30 +0900 Subject: [PATCH 03/44] fix invalid query filter --- public/app/plugins/datasource/cloudwatch/datasource.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 74f93874519..c8fd9e71237 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -32,7 +32,11 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot options.targets = this.expandTemplateVariable(options.targets, options.scopedVars, templateSrv); var queries = _.filter(options.targets, function (item) { - return item.hide !== true || !item.namespace || !item.metricName || _.isEmpty(item.statistics); + return item.hide !== true && + !!item.region && + !!item.namespace && + !!item.metricName && + !_.isEmpty(item.statistics); }).map(function (item) { item.region = templateSrv.replace(item.region, options.scopedVars); item.namespace = templateSrv.replace(item.namespace, options.scopedVars); From 83b79dd624938374c77f6a82f19a9f0d87522659 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 16 Aug 2017 16:40:46 +0900 Subject: [PATCH 04/44] fix test --- .../datasource/cloudwatch/datasource.js | 4 +- .../cloudwatch/specs/datasource_specs.ts | 85 ++++++++++++++----- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index c8fd9e71237..0a8791f489c 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -72,10 +72,10 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot return this.performTimeSeriesQuery(request); }; - this.getPeriod = function(target, options) { + this.getPeriod = function(target, options, now) { var start = this.convertToCloudWatchTime(options.range.from, false); var end = this.convertToCloudWatchTime(options.range.to, true); - var now = Math.round(Date.now() / 1000); + now = Math.round((now || Date.now()) / 1000); var period; var range = end - start; diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 87acb40028c..1150dd9c58b 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -320,30 +320,75 @@ describe('CloudWatchDatasource', function() { it('should caclculate the correct period', function () { var hourSec = 60 * 60; var daySec = hourSec * 24; - var start = 1483196400; + var start = 1483196400 * 1000; var testData: any[] = [ - [{ period: 60 }, { namespace: 'AWS/EC2' }, {}, start, start + 3600, (hourSec * 3), 60], - [{ period: null }, { namespace: 'AWS/EC2' }, {}, start, start + 3600, (hourSec * 3), 300], - [{ period: 60 }, { namespace: 'AWS/ELB' }, {}, start, start + 3600, (hourSec * 3), 60], - [{ period: null }, { namespace: 'AWS/ELB' }, {}, start, start + 3600, (hourSec * 3), 60], - [{ period: 1 }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 1440 - 1, (hourSec * 3 - 1), 1], - [{ period: 1 }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (hourSec * 3 - 1), 60], - [{ period: 60 }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (hourSec * 3), 60], - [{ period: null }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (hourSec * 3 - 1), 60], - [{ period: null }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (hourSec * 3), 60], - [{ period: null }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (daySec * 15), 60], - [{ period: null }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (daySec * 63), 300], - [{ period: null }, { namespace: 'CustomMetricsNamespace' }, {}, start, start + 3600, (daySec * 455), 3600] + [ + { period: 60, namespace: 'AWS/EC2' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3), 60 + ], + [ + { period: null, namespace: 'AWS/EC2' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3), 300 + ], + [ + { period: 60, namespace: 'AWS/ELB' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3), 60 + ], + [ + { period: null, namespace: 'AWS/ELB' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3), 60 + ], + [ + { period: 1, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + (1440 - 1) * 1000) } }, + (hourSec * 3 - 1), 1 + ], + [ + { period: 1, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3 - 1), 60 + ], + [ + { period: 60, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3), 60 + ], + [ + { period: null, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3 - 1), 60 + ], + [ + { period: null, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (hourSec * 3), 60 + ], + [ + { period: null, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (daySec * 15), 60 + ], + [ + { period: null, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (daySec * 63), 300 + ], + [ + { period: null, namespace: 'CustomMetricsNamespace' }, + { range: { from: new Date(start), to: new Date(start + 3600 * 1000) } }, + (daySec * 455), 3600 + ] ]; for (let t of testData) { let target = t[0]; - let query = t[1]; - let options = t[2]; - let start = t[3]; - let end = t[4]; - let now = start + t[5]; - let expected = t[6]; - let actual = ctx.ds.getPeriod(target, query, options, start, end, now); + let options = t[1]; + let now = new Date(options.range.from.valueOf() + t[2] * 1000); + let expected = t[3]; + let actual = ctx.ds.getPeriod(target, options, now); expect(actual).to.be(expected); } }); From d31f264576404d40663c567364cf8ea4ab3b267b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Thu, 24 Aug 2017 15:33:31 +0900 Subject: [PATCH 05/44] cache creds for keys/credentials auth type --- pkg/api/cloudwatch/cloudwatch.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 98ed1eabae2..8188e3913e9 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -96,7 +96,7 @@ var awsCredentialCache map[string]cache = make(map[string]cache) var credentialCacheLock sync.RWMutex func GetCredentials(dsInfo *DatasourceInfo) (*credentials.Credentials, error) { - cacheKey := dsInfo.Profile + ":" + dsInfo.AssumeRoleArn + cacheKey := dsInfo.AccessKey + ":" + dsInfo.Profile + ":" + dsInfo.AssumeRoleArn credentialCacheLock.RLock() if _, ok := awsCredentialCache[cacheKey]; ok { if awsCredentialCache[cacheKey].expiration != nil && @@ -150,6 +150,10 @@ func GetCredentials(dsInfo *DatasourceInfo) (*credentials.Credentials, error) { sessionToken = *resp.Credentials.SessionToken expiration = resp.Credentials.Expiration } + } else { + now := time.Now() + e := now.Add(5 * time.Minute) + expiration = &e } sess, err := session.NewSession() From c6607f3fa78782871805230122d20292f4e6b7dc Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Fri, 1 Sep 2017 21:34:50 +0900 Subject: [PATCH 06/44] remove offset for startTime --- pkg/tsdb/cloudwatch/cloudwatch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 445ec94aef2..58c14685d96 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -156,7 +156,7 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, model *tsdb.Query MetricName: aws.String(query.MetricName), Dimensions: query.Dimensions, Period: aws.Int64(int64(query.Period)), - StartTime: aws.Time(startTime.Add(-time.Minute * 15)), + StartTime: aws.Time(startTime), EndTime: aws.Time(endTime), } if len(query.Statistics) > 0 { From 110f157621b873fed8f529f7024be3005efe219d Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Thu, 7 Sep 2017 15:56:16 +0900 Subject: [PATCH 07/44] parse duration --- pkg/tsdb/cloudwatch/cloudwatch.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 58c14685d96..d86acfa6ee4 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -254,9 +254,19 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { p = "60" } } - period, err := strconv.Atoi(p) - if err != nil { - return nil, err + + period := 300 + if regexp.MustCompile(`^\d+$`).Match([]byte(p)) { + period, err = strconv.Atoi(p) + if err != nil { + return nil, err + } + } else { + d, err := time.ParseDuration(p) + if err != nil { + return nil, err + } + period = int(d.Seconds()) } alias := model.Get("alias").MustString("{{metric}}_{{stat}}") From 62d84c1e14386f74e8ce8035913fec6cb497c28b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sun, 10 Sep 2017 01:36:40 +0900 Subject: [PATCH 08/44] (cloudwatch) move query parameter to 'parameters' --- pkg/tsdb/cloudwatch/cloudwatch.go | 10 +++++++--- public/app/plugins/datasource/cloudwatch/datasource.js | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index d86acfa6ee4..a388bb0d9ee 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -62,9 +62,13 @@ func (e *CloudWatchExecutor) Execute(ctx context.Context, queries tsdb.QuerySlic currentlyExecuting := 0 for _, model := range queries { + queryType := model.Model.Get("type").MustString() + if queryType != "timeSeriesQuery" { + continue + } currentlyExecuting++ go func(refId string) { - queryRes, err := e.executeQuery(ctx, model, queryContext) + queryRes, err := e.executeQuery(ctx, model.Model.Get("parameters"), queryContext) currentlyExecuting-- if err != nil { errCh <- err @@ -130,8 +134,8 @@ func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, e return client, nil } -func (e *CloudWatchExecutor) executeQuery(ctx context.Context, model *tsdb.Query, queryContext *tsdb.QueryContext) (*tsdb.QueryResult, error) { - query, err := parseQuery(model.Model) +func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) (*tsdb.QueryResult, error) { + query, err := parseQuery(parameters) if err != nil { return nil, err } diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 0a8791f489c..40094003456 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -48,12 +48,14 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot item.dimensions = dimensions; item.period = self.getPeriod(item, options); - return _.extend({ + return { refId: item.refId, intervalMs: options.intervalMs, maxDataPoints: options.maxDataPoints, datasourceId: self.instanceSettings.id, - }, item); + type: 'timeSeriesQuery', + parameters: item + }; }); // No valid targets, return the empty result to save a round trip. From 0c951484864a80c92cfc702340ab130b2e8d1b3b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sun, 10 Sep 2017 04:14:27 +0900 Subject: [PATCH 09/44] move the metric find query code --- .../metrics.go => tsdb/cloudwatch/metric_find_query.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/{api/cloudwatch/metrics.go => tsdb/cloudwatch/metric_find_query.go} (100%) diff --git a/pkg/api/cloudwatch/metrics.go b/pkg/tsdb/cloudwatch/metric_find_query.go similarity index 100% rename from pkg/api/cloudwatch/metrics.go rename to pkg/tsdb/cloudwatch/metric_find_query.go From feed90c0e2d770da21762623d80eb94c09cddef0 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sun, 10 Sep 2017 04:24:39 +0900 Subject: [PATCH 10/44] re-implement get regions --- pkg/api/cloudwatch/cloudwatch.go | 4 - pkg/tsdb/cloudwatch/cloudwatch.go | 14 + pkg/tsdb/cloudwatch/metric_find_query.go | 504 ++++++++++-------- .../datasource/cloudwatch/datasource.js | 28 +- 4 files changed, 311 insertions(+), 239 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 8188e3913e9..86161c756e3 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -80,10 +80,6 @@ func init() { "DescribeAlarmsForMetric": handleDescribeAlarmsForMetric, "DescribeAlarmHistory": handleDescribeAlarmHistory, "DescribeInstances": handleDescribeInstances, - "__GetRegions": handleGetRegions, - "__GetNamespaces": handleGetNamespaces, - "__GetMetrics": handleGetMetrics, - "__GetDimensions": handleGetDimensions, } } diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index a388bb0d9ee..2e897f86140 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -53,6 +53,20 @@ func init() { } func (e *CloudWatchExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { + var result *tsdb.BatchResult + queryType := queries[0].Model.Get("type").MustString() + switch queryType { + case "timeSeriesQuery": + result = e.executeTimeSeriesQuery(ctx, queries, queryContext) + break + case "metricFindQuery": + result = e.executeMetricFindQuery(ctx, queries, queryContext) + break + } + return result +} + +func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { result := &tsdb.BatchResult{ QueryResults: make(map[string]*tsdb.QueryResult), } diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 0d471efad68..9cc6974fb21 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -1,24 +1,21 @@ package cloudwatch import ( - "encoding/json" - "sort" - "strings" - "sync" + "context" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awsutil" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/middleware" - "github.com/grafana/grafana/pkg/util" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" ) var metricsMap map[string][]string var dimensionsMap map[string][]string +type suggestData struct { + Text string + Value string +} + type CustomMetricsCache struct { Expire time.Time Cache []string @@ -144,236 +141,279 @@ func init() { customMetricsDimensionsMap = make(map[string]map[string]map[string]*CustomMetricsCache) } +func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { + result := &tsdb.BatchResult{ + QueryResults: make(map[string]*tsdb.QueryResult), + } + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: queries[0].RefId} + + parameters := queries[0].Model.Get("parameters") + subType := queries[0].Model.Get("subtype").MustString() + var data []suggestData + var err error + switch subType { + case "regions": + data, err = e.handleGetRegions(ctx, parameters, queryContext) + if err != nil { + queryResult.Error = err + } + break + } + transformToTable(data, queryResult) + result.QueryResults[queries[0].RefId] = queryResult + return result +} + +func transformToTable(data []suggestData, result *tsdb.QueryResult) { + table := &tsdb.Table{ + Columns: make([]tsdb.TableColumn, 2), + Rows: make([]tsdb.RowValues, 0), + } + table.Columns[0].Text = "text" + table.Columns[1].Text = "value" + + for _, r := range data { + values := make([]interface{}, 2) + values[0] = r.Text + values[1] = r.Value + table.Rows = append(table.Rows, values) + } + result.Tables = append(result.Tables, table) + result.Meta.Set("rowCount", len(data)) +} + // Whenever this list is updated, frontend list should also be updated. // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html -func handleGetRegions(req *cwRequest, c *middleware.Context) { +func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { regions := []string{ "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-south-1", "ca-central-1", "cn-north-1", "eu-central-1", "eu-west-1", "eu-west-2", "sa-east-1", "us-east-1", "us-east-2", "us-gov-west-1", "us-west-1", "us-west-2", } - result := []interface{}{} + result := make([]suggestData, 0) for _, region := range regions { - result = append(result, util.DynMap{"text": region, "value": region}) + result = append(result, suggestData{Text: region, Value: region}) } - c.JSON(200, result) + return result, nil } -func handleGetNamespaces(req *cwRequest, c *middleware.Context) { - keys := []string{} - for key := range metricsMap { - keys = append(keys, key) - } - - customNamespaces := req.DataSource.JsonData.Get("customMetricsNamespaces").MustString() - if customNamespaces != "" { - keys = append(keys, strings.Split(customNamespaces, ",")...) - } - - sort.Sort(sort.StringSlice(keys)) - - result := []interface{}{} - for _, key := range keys { - result = append(result, util.DynMap{"text": key, "value": key}) - } - - c.JSON(200, result) -} - -func handleGetMetrics(req *cwRequest, c *middleware.Context) { - reqParam := &struct { - Parameters struct { - Namespace string `json:"namespace"` - } `json:"parameters"` - }{} - - json.Unmarshal(req.Body, reqParam) - - var namespaceMetrics []string - if !isCustomMetrics(reqParam.Parameters.Namespace) { - var exists bool - if namespaceMetrics, exists = metricsMap[reqParam.Parameters.Namespace]; !exists { - c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil) - return - } - } else { - var err error - cwData := req.GetDatasourceInfo() - cwData.Namespace = reqParam.Parameters.Namespace - - if namespaceMetrics, err = getMetricsForCustomMetrics(cwData, getAllMetrics); err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - } - sort.Sort(sort.StringSlice(namespaceMetrics)) - - result := []interface{}{} - for _, name := range namespaceMetrics { - result = append(result, util.DynMap{"text": name, "value": name}) - } - - c.JSON(200, result) -} - -func handleGetDimensions(req *cwRequest, c *middleware.Context) { - reqParam := &struct { - Parameters struct { - Namespace string `json:"namespace"` - } `json:"parameters"` - }{} - - json.Unmarshal(req.Body, reqParam) - - var dimensionValues []string - if !isCustomMetrics(reqParam.Parameters.Namespace) { - var exists bool - if dimensionValues, exists = dimensionsMap[reqParam.Parameters.Namespace]; !exists { - c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil) - return - } - } else { - var err error - dsInfo := req.GetDatasourceInfo() - dsInfo.Namespace = reqParam.Parameters.Namespace - - if dimensionValues, err = getDimensionsForCustomMetrics(dsInfo, getAllMetrics); err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - } - sort.Sort(sort.StringSlice(dimensionValues)) - - result := []interface{}{} - for _, name := range dimensionValues { - result = append(result, util.DynMap{"text": name, "value": name}) - } - - c.JSON(200, result) -} - -func getAllMetrics(cwData *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { - creds, err := GetCredentials(cwData) - if err != nil { - return cloudwatch.ListMetricsOutput{}, err - } - cfg := &aws.Config{ - Region: aws.String(cwData.Region), - Credentials: creds, - } - sess, err := session.NewSession(cfg) - if err != nil { - return cloudwatch.ListMetricsOutput{}, err - } - svc := cloudwatch.New(sess, cfg) - - params := &cloudwatch.ListMetricsInput{ - Namespace: aws.String(cwData.Namespace), - } - - var resp cloudwatch.ListMetricsOutput - err = svc.ListMetricsPages(params, - func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { - metrics.M_Aws_CloudWatch_ListMetrics.Inc() - metrics, _ := awsutil.ValuesAtPath(page, "Metrics") - for _, metric := range metrics { - resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) - } - return !lastPage - }) - if err != nil { - return resp, err - } - - return resp, nil -} - -var metricsCacheLock sync.Mutex - -func getMetricsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { - metricsCacheLock.Lock() - defer metricsCacheLock.Unlock() - - if _, ok := customMetricsMetricsMap[dsInfo.Profile]; !ok { - customMetricsMetricsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) - } - if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region]; !ok { - customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) - } - if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { - customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} - customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) - } - - if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { - return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil - } - result, err := getAllMetrics(dsInfo) - if err != nil { - return []string{}, err - } - customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) - customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) - - for _, metric := range result.Metrics { - if isDuplicate(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) { - continue - } - customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) - } - - return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil -} - -var dimensionsCacheLock sync.Mutex - -func getDimensionsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { - dimensionsCacheLock.Lock() - defer dimensionsCacheLock.Unlock() - - if _, ok := customMetricsDimensionsMap[dsInfo.Profile]; !ok { - customMetricsDimensionsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) - } - if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region]; !ok { - customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) - } - if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { - customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} - customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) - } - - if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { - return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil - } - result, err := getAllMetrics(dsInfo) - if err != nil { - return []string{}, err - } - customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) - customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) - - for _, metric := range result.Metrics { - for _, dimension := range metric.Dimensions { - if isDuplicate(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) { - continue - } - customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) - } - } - - return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil -} - -func isDuplicate(nameList []string, target string) bool { - for _, name := range nameList { - if name == target { - return true - } - } - return false -} - -func isCustomMetrics(namespace string) bool { - return strings.Index(namespace, "AWS/") != 0 -} +//func handleGetNamespaces(req *cwRequest, c *middleware.Context) { +// keys := []string{} +// for key := range metricsMap { +// keys = append(keys, key) +// } +// +// customNamespaces := req.DataSource.JsonData.Get("customMetricsNamespaces").MustString() +// if customNamespaces != "" { +// for _, key := range strings.Split(customNamespaces, ",") { +// keys = append(keys, key) +// } +// } +// +// sort.Sort(sort.StringSlice(keys)) +// +// result := []interface{}{} +// for _, key := range keys { +// result = append(result, util.DynMap{"text": key, "value": key}) +// } +// +// c.JSON(200, result) +//} +// +//func handleGetMetrics(req *cwRequest, c *middleware.Context) { +// reqParam := &struct { +// Parameters struct { +// Namespace string `json:"namespace"` +// } `json:"parameters"` +// }{} +// +// json.Unmarshal(req.Body, reqParam) +// +// var namespaceMetrics []string +// if !isCustomMetrics(reqParam.Parameters.Namespace) { +// var exists bool +// if namespaceMetrics, exists = metricsMap[reqParam.Parameters.Namespace]; !exists { +// c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil) +// return +// } +// } else { +// var err error +// cwData := req.GetDatasourceInfo() +// cwData.Namespace = reqParam.Parameters.Namespace +// +// if namespaceMetrics, err = getMetricsForCustomMetrics(cwData, getAllMetrics); err != nil { +// c.JsonApiErr(500, "Unable to call AWS API", err) +// return +// } +// } +// sort.Sort(sort.StringSlice(namespaceMetrics)) +// +// result := []interface{}{} +// for _, name := range namespaceMetrics { +// result = append(result, util.DynMap{"text": name, "value": name}) +// } +// +// c.JSON(200, result) +//} +// +//func handleGetDimensions(req *cwRequest, c *middleware.Context) { +// reqParam := &struct { +// Parameters struct { +// Namespace string `json:"namespace"` +// } `json:"parameters"` +// }{} +// +// json.Unmarshal(req.Body, reqParam) +// +// var dimensionValues []string +// if !isCustomMetrics(reqParam.Parameters.Namespace) { +// var exists bool +// if dimensionValues, exists = dimensionsMap[reqParam.Parameters.Namespace]; !exists { +// c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil) +// return +// } +// } else { +// var err error +// dsInfo := req.GetDatasourceInfo() +// dsInfo.Namespace = reqParam.Parameters.Namespace +// +// if dimensionValues, err = getDimensionsForCustomMetrics(dsInfo, getAllMetrics); err != nil { +// c.JsonApiErr(500, "Unable to call AWS API", err) +// return +// } +// } +// sort.Sort(sort.StringSlice(dimensionValues)) +// +// result := []interface{}{} +// for _, name := range dimensionValues { +// result = append(result, util.DynMap{"text": name, "value": name}) +// } +// +// c.JSON(200, result) +//} +// +//func getAllMetrics(cwData *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { +// creds, err := GetCredentials(cwData) +// if err != nil { +// return cloudwatch.ListMetricsOutput{}, err +// } +// cfg := &aws.Config{ +// Region: aws.String(cwData.Region), +// Credentials: creds, +// } +// sess, err := session.NewSession(cfg) +// if err != nil { +// return cloudwatch.ListMetricsOutput{}, err +// } +// svc := cloudwatch.New(sess, cfg) +// +// params := &cloudwatch.ListMetricsInput{ +// Namespace: aws.String(cwData.Namespace), +// } +// +// var resp cloudwatch.ListMetricsOutput +// err = svc.ListMetricsPages(params, +// func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { +// metrics.M_Aws_CloudWatch_ListMetrics.Inc(1) +// metrics, _ := awsutil.ValuesAtPath(page, "Metrics") +// for _, metric := range metrics { +// resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) +// } +// return !lastPage +// }) +// if err != nil { +// return resp, err +// } +// +// return resp, nil +//} +// +//var metricsCacheLock sync.Mutex +// +//func getMetricsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +// metricsCacheLock.Lock() +// defer metricsCacheLock.Unlock() +// +// if _, ok := customMetricsMetricsMap[dsInfo.Profile]; !ok { +// customMetricsMetricsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) +// } +// if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region]; !ok { +// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) +// } +// if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { +// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} +// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) +// } +// +// if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { +// return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil +// } +// result, err := getAllMetrics(dsInfo) +// if err != nil { +// return []string{}, err +// } +// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) +// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) +// +// for _, metric := range result.Metrics { +// if isDuplicate(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) { +// continue +// } +// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) +// } +// +// return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil +//} +// +//var dimensionsCacheLock sync.Mutex +// +//func getDimensionsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +// dimensionsCacheLock.Lock() +// defer dimensionsCacheLock.Unlock() +// +// if _, ok := customMetricsDimensionsMap[dsInfo.Profile]; !ok { +// customMetricsDimensionsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) +// } +// if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region]; !ok { +// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) +// } +// if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { +// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} +// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) +// } +// +// if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { +// return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil +// } +// result, err := getAllMetrics(dsInfo) +// if err != nil { +// return []string{}, err +// } +// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) +// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) +// +// for _, metric := range result.Metrics { +// for _, dimension := range metric.Dimensions { +// if isDuplicate(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) { +// continue +// } +// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) +// } +// } +// +// return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil +//} +// +//func isDuplicate(nameList []string, target string) bool { +// for _, name := range nameList { +// if name == target { +// return true +// } +// } +// return false +//} +// +//func isCustomMetrics(namespace string) bool { +// return strings.Index(namespace, "AWS/") != 0 +//} diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 40094003456..ae0225e6877 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -11,7 +11,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot 'use strict'; /** @ngInject */ - function CloudWatchDatasource(instanceSettings, $q, backendSrv, templateSrv) { + function CloudWatchDatasource(instanceSettings, $q, backendSrv, templateSrv, timeSrv) { this.type = 'cloudwatch'; this.name = instanceSettings.name; this.supportMetrics = true; @@ -133,7 +133,21 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }; this.getRegions = function() { - return this.awsRequest({action: '__GetRegions'}); + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'regions' + } + ] + }); }; this.getNamespaces = function() { @@ -200,6 +214,14 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var namespace; var metricName; + var transformSuggestDataFromTable = function(suggestData) { + return _.map(suggestData.results['metricFindQuery'].tables[0].rows, function (v) { + return { + text: v[0], + value: v[1] + }; + }); + }; var transformSuggestData = function(suggestData) { return _.map(suggestData, function(v) { return { text: v }; @@ -208,7 +230,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var regionQuery = query.match(/^regions\(\)/); if (regionQuery) { - return this.getRegions(); + return this.getRegions().then(function (r) { return transformSuggestDataFromTable(r); }); } var namespaceQuery = query.match(/^namespaces\(\)/); From fe3d3bc384d094a6b954d5ba9c4a63a5f1874b1b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 13 Sep 2017 19:35:05 +0900 Subject: [PATCH 11/44] porting other suggestion --- pkg/tsdb/cloudwatch/metric_find_query.go | 473 ++++++++++-------- .../datasource/cloudwatch/datasource.js | 93 +++- 2 files changed, 317 insertions(+), 249 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 9cc6974fb21..694599a5e83 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -2,9 +2,19 @@ package cloudwatch import ( "context" + "errors" + "sort" + "strings" + "sync" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awsutil" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatch" + cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/tsdb" ) @@ -154,10 +164,19 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries switch subType { case "regions": data, err = e.handleGetRegions(ctx, parameters, queryContext) - if err != nil { - queryResult.Error = err - } break + case "namespaces": + data, err = e.handleGetNamespaces(ctx, parameters, queryContext) + break + case "metrics": + data, err = e.handleGetMetrics(ctx, parameters, queryContext) + break + case "dimension_keys": + data, err = e.handleGetDimensions(ctx, parameters, queryContext) + break + } + if err != nil { + queryResult.Error = err } transformToTable(data, queryResult) result.QueryResults[queries[0].RefId] = queryResult @@ -182,6 +201,30 @@ func transformToTable(data []suggestData, result *tsdb.QueryResult) { result.Meta.Set("rowCount", len(data)) } +func (e *CloudWatchExecutor) getDsInfo(region string) *cwapi.DatasourceInfo { + assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() + accessKey := "" + secretKey := "" + for key, value := range e.DataSource.SecureJsonData.Decrypt() { + if key == "accessKey" { + accessKey = value + } + if key == "secretKey" { + secretKey = value + } + } + + datasourceInfo := &cwapi.DatasourceInfo{ + Region: region, + Profile: e.DataSource.Database, + AssumeRoleArn: assumeRoleArn, + AccessKey: accessKey, + SecretKey: secretKey, + } + + return datasourceInfo +} + // Whenever this list is updated, frontend list should also be updated. // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { @@ -198,222 +241,208 @@ func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s return result, nil } -//func handleGetNamespaces(req *cwRequest, c *middleware.Context) { -// keys := []string{} -// for key := range metricsMap { -// keys = append(keys, key) -// } -// -// customNamespaces := req.DataSource.JsonData.Get("customMetricsNamespaces").MustString() -// if customNamespaces != "" { -// for _, key := range strings.Split(customNamespaces, ",") { -// keys = append(keys, key) -// } -// } -// -// sort.Sort(sort.StringSlice(keys)) -// -// result := []interface{}{} -// for _, key := range keys { -// result = append(result, util.DynMap{"text": key, "value": key}) -// } -// -// c.JSON(200, result) -//} -// -//func handleGetMetrics(req *cwRequest, c *middleware.Context) { -// reqParam := &struct { -// Parameters struct { -// Namespace string `json:"namespace"` -// } `json:"parameters"` -// }{} -// -// json.Unmarshal(req.Body, reqParam) -// -// var namespaceMetrics []string -// if !isCustomMetrics(reqParam.Parameters.Namespace) { -// var exists bool -// if namespaceMetrics, exists = metricsMap[reqParam.Parameters.Namespace]; !exists { -// c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil) -// return -// } -// } else { -// var err error -// cwData := req.GetDatasourceInfo() -// cwData.Namespace = reqParam.Parameters.Namespace -// -// if namespaceMetrics, err = getMetricsForCustomMetrics(cwData, getAllMetrics); err != nil { -// c.JsonApiErr(500, "Unable to call AWS API", err) -// return -// } -// } -// sort.Sort(sort.StringSlice(namespaceMetrics)) -// -// result := []interface{}{} -// for _, name := range namespaceMetrics { -// result = append(result, util.DynMap{"text": name, "value": name}) -// } -// -// c.JSON(200, result) -//} -// -//func handleGetDimensions(req *cwRequest, c *middleware.Context) { -// reqParam := &struct { -// Parameters struct { -// Namespace string `json:"namespace"` -// } `json:"parameters"` -// }{} -// -// json.Unmarshal(req.Body, reqParam) -// -// var dimensionValues []string -// if !isCustomMetrics(reqParam.Parameters.Namespace) { -// var exists bool -// if dimensionValues, exists = dimensionsMap[reqParam.Parameters.Namespace]; !exists { -// c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil) -// return -// } -// } else { -// var err error -// dsInfo := req.GetDatasourceInfo() -// dsInfo.Namespace = reqParam.Parameters.Namespace -// -// if dimensionValues, err = getDimensionsForCustomMetrics(dsInfo, getAllMetrics); err != nil { -// c.JsonApiErr(500, "Unable to call AWS API", err) -// return -// } -// } -// sort.Sort(sort.StringSlice(dimensionValues)) -// -// result := []interface{}{} -// for _, name := range dimensionValues { -// result = append(result, util.DynMap{"text": name, "value": name}) -// } -// -// c.JSON(200, result) -//} -// -//func getAllMetrics(cwData *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { -// creds, err := GetCredentials(cwData) -// if err != nil { -// return cloudwatch.ListMetricsOutput{}, err -// } -// cfg := &aws.Config{ -// Region: aws.String(cwData.Region), -// Credentials: creds, -// } -// sess, err := session.NewSession(cfg) -// if err != nil { -// return cloudwatch.ListMetricsOutput{}, err -// } -// svc := cloudwatch.New(sess, cfg) -// -// params := &cloudwatch.ListMetricsInput{ -// Namespace: aws.String(cwData.Namespace), -// } -// -// var resp cloudwatch.ListMetricsOutput -// err = svc.ListMetricsPages(params, -// func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { -// metrics.M_Aws_CloudWatch_ListMetrics.Inc(1) -// metrics, _ := awsutil.ValuesAtPath(page, "Metrics") -// for _, metric := range metrics { -// resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) -// } -// return !lastPage -// }) -// if err != nil { -// return resp, err -// } -// -// return resp, nil -//} -// -//var metricsCacheLock sync.Mutex -// -//func getMetricsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { -// metricsCacheLock.Lock() -// defer metricsCacheLock.Unlock() -// -// if _, ok := customMetricsMetricsMap[dsInfo.Profile]; !ok { -// customMetricsMetricsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) -// } -// if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region]; !ok { -// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) -// } -// if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { -// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} -// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) -// } -// -// if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { -// return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil -// } -// result, err := getAllMetrics(dsInfo) -// if err != nil { -// return []string{}, err -// } -// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) -// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) -// -// for _, metric := range result.Metrics { -// if isDuplicate(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) { -// continue -// } -// customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) -// } -// -// return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil -//} -// -//var dimensionsCacheLock sync.Mutex -// -//func getDimensionsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { -// dimensionsCacheLock.Lock() -// defer dimensionsCacheLock.Unlock() -// -// if _, ok := customMetricsDimensionsMap[dsInfo.Profile]; !ok { -// customMetricsDimensionsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) -// } -// if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region]; !ok { -// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) -// } -// if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { -// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} -// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) -// } -// -// if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { -// return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil -// } -// result, err := getAllMetrics(dsInfo) -// if err != nil { -// return []string{}, err -// } -// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) -// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) -// -// for _, metric := range result.Metrics { -// for _, dimension := range metric.Dimensions { -// if isDuplicate(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) { -// continue -// } -// customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) -// } -// } -// -// return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil -//} -// -//func isDuplicate(nameList []string, target string) bool { -// for _, name := range nameList { -// if name == target { -// return true -// } -// } -// return false -//} -// -//func isCustomMetrics(namespace string) bool { -// return strings.Index(namespace, "AWS/") != 0 -//} +func (e *CloudWatchExecutor) handleGetNamespaces(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { + keys := []string{} + for key := range metricsMap { + keys = append(keys, key) + } + + customNamespaces := e.DataSource.JsonData.Get("customMetricsNamespaces").MustString() + if customNamespaces != "" { + for _, key := range strings.Split(customNamespaces, ",") { + keys = append(keys, key) + } + } + + sort.Sort(sort.StringSlice(keys)) + + result := make([]suggestData, 0) + for _, key := range keys { + result = append(result, suggestData{Text: key, Value: key}) + } + + return result, nil +} + +func (e *CloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { + region := parameters.Get("region").MustString() + namespace := parameters.Get("namespace").MustString() + + var namespaceMetrics []string + if !isCustomMetrics(namespace) { + var exists bool + if namespaceMetrics, exists = metricsMap[namespace]; !exists { + return nil, errors.New("Unable to find namespace " + namespace) + } + } else { + var err error + dsInfo := e.getDsInfo(region) + dsInfo.Namespace = namespace + + if namespaceMetrics, err = getMetricsForCustomMetrics(dsInfo, getAllMetrics); err != nil { + return nil, errors.New("Unable to call AWS API") + } + } + sort.Sort(sort.StringSlice(namespaceMetrics)) + + result := make([]suggestData, 0) + for _, name := range namespaceMetrics { + result = append(result, suggestData{Text: name, Value: name}) + } + + return result, nil +} + +func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { + region := parameters.Get("region").MustString() + namespace := parameters.Get("namespace").MustString() + + var dimensionValues []string + if !isCustomMetrics(namespace) { + var exists bool + if dimensionValues, exists = dimensionsMap[namespace]; !exists { + return nil, errors.New("Unable to find dimension " + namespace) + } + } else { + var err error + dsInfo := e.getDsInfo(region) + dsInfo.Namespace = namespace + + if dimensionValues, err = getDimensionsForCustomMetrics(dsInfo, getAllMetrics); err != nil { + return nil, errors.New("Unable to call AWS API") + } + } + sort.Sort(sort.StringSlice(dimensionValues)) + + result := make([]suggestData, 0) + for _, name := range dimensionValues { + result = append(result, suggestData{Text: name, Value: name}) + } + + return result, nil +} + +func getAllMetrics(cwData *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + creds, err := cwapi.GetCredentials(cwData) + if err != nil { + return cloudwatch.ListMetricsOutput{}, err + } + cfg := &aws.Config{ + Region: aws.String(cwData.Region), + Credentials: creds, + } + sess, err := session.NewSession(cfg) + if err != nil { + return cloudwatch.ListMetricsOutput{}, err + } + svc := cloudwatch.New(sess, cfg) + + params := &cloudwatch.ListMetricsInput{ + Namespace: aws.String(cwData.Namespace), + } + + var resp cloudwatch.ListMetricsOutput + err = svc.ListMetricsPages(params, + func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { + metrics.M_Aws_CloudWatch_ListMetrics.Inc(1) + metrics, _ := awsutil.ValuesAtPath(page, "Metrics") + for _, metric := range metrics { + resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) + } + return !lastPage + }) + if err != nil { + return resp, err + } + + return resp, nil +} + +var metricsCacheLock sync.Mutex + +func getMetricsForCustomMetrics(dsInfo *cwapi.DatasourceInfo, getAllMetrics func(*cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { + metricsCacheLock.Lock() + defer metricsCacheLock.Unlock() + + if _, ok := customMetricsMetricsMap[dsInfo.Profile]; !ok { + customMetricsMetricsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) + } + if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region]; !ok { + customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) + } + if _, ok := customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { + customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} + customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) + } + + if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { + return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil + } + result, err := getAllMetrics(dsInfo) + if err != nil { + return []string{}, err + } + customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) + customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) + + for _, metric := range result.Metrics { + if isDuplicate(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) { + continue + } + customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *metric.MetricName) + } + + return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil +} + +var dimensionsCacheLock sync.Mutex + +func getDimensionsForCustomMetrics(dsInfo *cwapi.DatasourceInfo, getAllMetrics func(*cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { + dimensionsCacheLock.Lock() + defer dimensionsCacheLock.Unlock() + + if _, ok := customMetricsDimensionsMap[dsInfo.Profile]; !ok { + customMetricsDimensionsMap[dsInfo.Profile] = make(map[string]map[string]*CustomMetricsCache) + } + if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region]; !ok { + customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region] = make(map[string]*CustomMetricsCache) + } + if _, ok := customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace]; !ok { + customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace] = &CustomMetricsCache{} + customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) + } + + if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { + return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil + } + result, err := getAllMetrics(dsInfo) + if err != nil { + return []string{}, err + } + customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = make([]string, 0) + customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire = time.Now().Add(5 * time.Minute) + + for _, metric := range result.Metrics { + for _, dimension := range metric.Dimensions { + if isDuplicate(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) { + continue + } + customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache = append(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, *dimension.Name) + } + } + + return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil +} + +func isDuplicate(nameList []string, target string) bool { + for _, name := range nameList { + if name == target { + return true + } + } + return false +} + +func isCustomMetrics(namespace string) bool { + return strings.Index(namespace, "AWS/") != 0 +} diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index ae0225e6877..916c95fbd77 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -132,7 +132,16 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }); }; - this.getRegions = function() { + function transformSuggestDataFromTable(suggestData) { + return _.map(suggestData.results['metricFindQuery'].tables[0].rows, function (v) { + return { + text: v[0], + value: v[1] + }; + }); + } + + this.getRegions = function () { var range = timeSrv.timeRange(); return backendSrv.post('/api/tsdb/query', { from: range.from, @@ -147,31 +156,69 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot subtype: 'regions' } ] - }); + }).then(function (r) { return transformSuggestDataFromTable(r); }); }; this.getNamespaces = function() { - return this.awsRequest({action: '__GetNamespaces'}); + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'namespaces' + } + ] + }).then(function (r) { return transformSuggestDataFromTable(r); }); }; - this.getMetrics = function(namespace, region) { - return this.awsRequest({ - action: '__GetMetrics', - region: region, - parameters: { - namespace: templateSrv.replace(namespace) - } - }); + this.getMetrics = function (namespace, region) { + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'metrics', + parameters: { + region: region, + namespace: templateSrv.replace(namespace) + } + } + ] + }).then(function (r) { return transformSuggestDataFromTable(r); }); }; this.getDimensionKeys = function(namespace, region) { - return this.awsRequest({ - action: '__GetDimensions', - region: region, - parameters: { - namespace: templateSrv.replace(namespace) - } - }); + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'dimension_keys', + parameters: { + region: region, + namespace: templateSrv.replace(namespace) + } + } + ] + }).then(function (r) { return transformSuggestDataFromTable(r); }); }; this.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { @@ -214,14 +261,6 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var namespace; var metricName; - var transformSuggestDataFromTable = function(suggestData) { - return _.map(suggestData.results['metricFindQuery'].tables[0].rows, function (v) { - return { - text: v[0], - value: v[1] - }; - }); - }; var transformSuggestData = function(suggestData) { return _.map(suggestData, function(v) { return { text: v }; @@ -230,7 +269,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var regionQuery = query.match(/^regions\(\)/); if (regionQuery) { - return this.getRegions().then(function (r) { return transformSuggestDataFromTable(r); }); + return this.getRegions(); } var namespaceQuery = query.match(/^namespaces\(\)/); From 1d265e05c9935912aa676468aefa9955741f0b10 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 13 Sep 2017 20:34:05 +0900 Subject: [PATCH 12/44] fix conflict --- pkg/tsdb/cloudwatch/metric_find_query.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 694599a5e83..da6c31aa1dc 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -249,9 +249,7 @@ func (e *CloudWatchExecutor) handleGetNamespaces(ctx context.Context, parameters customNamespaces := e.DataSource.JsonData.Get("customMetricsNamespaces").MustString() if customNamespaces != "" { - for _, key := range strings.Split(customNamespaces, ",") { - keys = append(keys, key) - } + keys = append(keys, strings.Split(customNamespaces, ",")...) } sort.Sort(sort.StringSlice(keys)) From f590db1b785bb56c916c7a70dee3b2d17141adf6 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 13 Sep 2017 20:38:17 +0900 Subject: [PATCH 13/44] move test code --- .../metrics_test.go => tsdb/cloudwatch/metric_find_query_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/{api/cloudwatch/metrics_test.go => tsdb/cloudwatch/metric_find_query_test.go} (100%) diff --git a/pkg/api/cloudwatch/metrics_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go similarity index 100% rename from pkg/api/cloudwatch/metrics_test.go rename to pkg/tsdb/cloudwatch/metric_find_query_test.go From 01d2aa8af0daa8dca57f3057fcc1938180eee02c Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 13 Sep 2017 21:11:25 +0900 Subject: [PATCH 14/44] fix test --- .../cloudwatch/specs/datasource_specs.ts | 70 +++++++++++++++---- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 1150dd9c58b..a326a49e1de 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -76,11 +76,11 @@ describe('CloudWatchDatasource', function() { it('should generate the correct query', function(done) { ctx.ds.query(query).then(function() { var params = requestParams.queries[0]; - expect(params.namespace).to.be(query.targets[0].namespace); - expect(params.metricName).to.be(query.targets[0].metricName); - expect(params.dimensions['InstanceId']).to.be('i-12345678'); - expect(params.statistics).to.eql(query.targets[0].statistics); - expect(params.period).to.be(query.targets[0].period); + expect(params.parameters.namespace).to.be(query.targets[0].namespace); + expect(params.parameters.metricName).to.be(query.targets[0].metricName); + expect(params.parameters.dimensions['InstanceId']).to.be('i-12345678'); + expect(params.parameters.statistics).to.eql(query.targets[0].statistics); + expect(params.parameters.period).to.be(query.targets[0].period); done(); }); ctx.$rootScope.$apply(); @@ -110,7 +110,7 @@ describe('CloudWatchDatasource', function() { ctx.ds.query(query).then(function() { var params = requestParams.queries[0]; - expect(params.period).to.be(600); + expect(params.parameters.period).to.be(600); done(); }); ctx.$rootScope.$apply(); @@ -236,7 +236,11 @@ describe('CloudWatchDatasource', function() { setupCallback(); ctx.backendSrv.datasourceRequest = args => { scenario.request = args; - return ctx.$q.when({data: scenario.requestResponse }); + return ctx.$q.when({ data: scenario.requestResponse }); + }; + ctx.backendSrv.post = (path, args) => { + scenario.request = args; + return ctx.$q.when(scenario.requestResponse); }; ctx.ds.metricFindQuery(query).then(args => { scenario.result = args; @@ -251,45 +255,81 @@ describe('CloudWatchDatasource', function() { describeMetricFindQuery('regions()', scenario => { scenario.setup(() => { - scenario.requestResponse = [{text: 'us-east-1'}]; + scenario.requestResponse = { + results: { + metricFindQuery: { + tables: [ + { rows: [['us-east-1', 'us-east-1']] } + ] + } + } + }; }); it('should call __GetRegions and return result', () => { expect(scenario.result[0].text).to.contain('us-east-1'); - expect(scenario.request.data.action).to.be('__GetRegions'); + expect(scenario.request.queries[0].type).to.be('metricFindQuery'); + expect(scenario.request.queries[0].subtype).to.be('regions'); }); }); describeMetricFindQuery('namespaces()', scenario => { scenario.setup(() => { - scenario.requestResponse = [{text: 'AWS/EC2'}]; + scenario.requestResponse = { + results: { + metricFindQuery: { + tables: [ + { rows: [['AWS/EC2', 'AWS/EC2']] } + ] + } + } + }; }); it('should call __GetNamespaces and return result', () => { expect(scenario.result[0].text).to.contain('AWS/EC2'); - expect(scenario.request.data.action).to.be('__GetNamespaces'); + expect(scenario.request.queries[0].type).to.be('metricFindQuery'); + expect(scenario.request.queries[0].subtype).to.be('namespaces'); }); }); describeMetricFindQuery('metrics(AWS/EC2)', scenario => { scenario.setup(() => { - scenario.requestResponse = [{text: 'CPUUtilization'}]; + scenario.requestResponse = { + results: { + metricFindQuery: { + tables: [ + { rows: [['CPUUtilization', 'CPUUtilization']] } + ] + } + } + }; }); it('should call __GetMetrics and return result', () => { expect(scenario.result[0].text).to.be('CPUUtilization'); - expect(scenario.request.data.action).to.be('__GetMetrics'); + expect(scenario.request.queries[0].type).to.be('metricFindQuery'); + expect(scenario.request.queries[0].subtype).to.be('metrics'); }); }); describeMetricFindQuery('dimension_keys(AWS/EC2)', scenario => { scenario.setup(() => { - scenario.requestResponse = [{text: 'InstanceId'}]; + scenario.requestResponse = { + results: { + metricFindQuery: { + tables: [ + { rows: [['InstanceId', 'InstanceId']] } + ] + } + } + }; }); it('should call __GetDimensions and return result', () => { expect(scenario.result[0].text).to.be('InstanceId'); - expect(scenario.request.data.action).to.be('__GetDimensions'); + expect(scenario.request.queries[0].type).to.be('metricFindQuery'); + expect(scenario.request.queries[0].subtype).to.be('dimension_keys'); }); }); From f66e1c02a64fa838667795e01d702fc49d9b2fb9 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 13 Sep 2017 21:17:45 +0900 Subject: [PATCH 15/44] remove obsolete GetMetricStatistics() --- pkg/api/cloudwatch/cloudwatch.go | 53 -------------------------------- 1 file changed, 53 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 86161c756e3..caeb14a2848 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -74,7 +74,6 @@ func (req *cwRequest) GetDatasourceInfo() *DatasourceInfo { func init() { actionHandlers = map[string]actionHandler{ - "GetMetricStatistics": handleGetMetricStatistics, "ListMetrics": handleListMetrics, "DescribeAlarms": handleDescribeAlarms, "DescribeAlarmsForMetric": handleDescribeAlarmsForMetric, @@ -219,58 +218,6 @@ func getAwsConfig(req *cwRequest) (*aws.Config, error) { return cfg, nil } -func handleGetMetricStatistics(req *cwRequest, c *middleware.Context) { - cfg, err := getAwsConfig(req) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - sess, err := session.NewSession(cfg) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - svc := cloudwatch.New(sess, cfg) - - reqParam := &struct { - Parameters struct { - Namespace string `json:"namespace"` - MetricName string `json:"metricName"` - Dimensions []*cloudwatch.Dimension `json:"dimensions"` - Statistics []*string `json:"statistics"` - ExtendedStatistics []*string `json:"extendedStatistics"` - StartTime int64 `json:"startTime"` - EndTime int64 `json:"endTime"` - Period int64 `json:"period"` - } `json:"parameters"` - }{} - json.Unmarshal(req.Body, reqParam) - - params := &cloudwatch.GetMetricStatisticsInput{ - Namespace: aws.String(reqParam.Parameters.Namespace), - MetricName: aws.String(reqParam.Parameters.MetricName), - Dimensions: reqParam.Parameters.Dimensions, - StartTime: aws.Time(time.Unix(reqParam.Parameters.StartTime, 0)), - EndTime: aws.Time(time.Unix(reqParam.Parameters.EndTime, 0)), - Period: aws.Int64(reqParam.Parameters.Period), - } - if len(reqParam.Parameters.Statistics) != 0 { - params.Statistics = reqParam.Parameters.Statistics - } - if len(reqParam.Parameters.ExtendedStatistics) != 0 { - params.ExtendedStatistics = reqParam.Parameters.ExtendedStatistics - } - - resp, err := svc.GetMetricStatistics(params) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc() - - c.JSON(200, resp) -} - func handleListMetrics(req *cwRequest, c *middleware.Context) { cfg, err := getAwsConfig(req) if err != nil { From fa074249e4f7ed1ebddf65f22e326fed21d6237a Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 13 Sep 2017 23:45:39 +0900 Subject: [PATCH 16/44] fix test --- pkg/tsdb/cloudwatch/metric_find_query_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index 238e815fac1..2b5dcaec247 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -5,19 +5,20 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" + cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" . "github.com/smartystreets/goconvey/convey" ) func TestCloudWatchMetrics(t *testing.T) { Convey("When calling getMetricsForCustomMetrics", t, func() { - dsInfo := &DatasourceInfo{ + dsInfo := &cwapi.DatasourceInfo{ Region: "us-east-1", Namespace: "Foo", Profile: "default", AssumeRoleArn: "", } - f := func(dsInfo *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + f := func(dsInfo *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { return cloudwatch.ListMetricsOutput{ Metrics: []*cloudwatch.Metric{ { @@ -39,13 +40,13 @@ func TestCloudWatchMetrics(t *testing.T) { }) Convey("When calling getDimensionsForCustomMetrics", t, func() { - dsInfo := &DatasourceInfo{ + dsInfo := &cwapi.DatasourceInfo{ Region: "us-east-1", Namespace: "Foo", Profile: "default", AssumeRoleArn: "", } - f := func(dsInfo *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + f := func(dsInfo *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { return cloudwatch.ListMetricsOutput{ Metrics: []*cloudwatch.Metric{ { From 36a537a3ce139e90b7d4ca3d9ef866641c3fba85 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Fri, 15 Sep 2017 23:59:47 +0900 Subject: [PATCH 17/44] fix conflict --- pkg/tsdb/cloudwatch/cloudwatch.go | 2 +- pkg/tsdb/cloudwatch/metric_find_query.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 2e897f86140..ef5b8271c3c 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -188,7 +188,7 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl if err != nil { return nil, err } - metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc(1) + metrics.M_Aws_CloudWatch_GetMetricStatistics.Inc() queryRes, err := parseResponse(resp, query) if err != nil { diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index da6c31aa1dc..faf0b8b7ce2 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -342,7 +342,7 @@ func getAllMetrics(cwData *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, var resp cloudwatch.ListMetricsOutput err = svc.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { - metrics.M_Aws_CloudWatch_ListMetrics.Inc(1) + metrics.M_Aws_CloudWatch_ListMetrics.Inc() metrics, _ := awsutil.ValuesAtPath(page, "Metrics") for _, metric := range metrics { resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) From e588b682fb91b51ee7aaa8a2e51acb108de56af8 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 19 Sep 2017 00:01:41 +0900 Subject: [PATCH 18/44] import the change, https://github.com/grafana/grafana/pull/9268 --- pkg/tsdb/cloudwatch/cloudwatch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index ef5b8271c3c..62e79f4fab9 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -364,8 +364,9 @@ func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatch timestamp := *v.Timestamp if _, ok := lastTimestamp[*s]; ok { nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second) - if timestamp.After(nextTimestampFromLast) { + for timestamp.After(nextTimestampFromLast) { series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000))) + nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second) } } lastTimestamp[*s] = timestamp From cf23734d7deb8779859ed88e2b2d5e4a598c995b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 19 Sep 2017 18:09:57 +0900 Subject: [PATCH 19/44] re-implement ebs_volume_ids() --- pkg/tsdb/cloudwatch/metric_find_query.go | 68 +++++++++++++++++++ .../datasource/cloudwatch/datasource.js | 34 +++++++--- 2 files changed, 91 insertions(+), 11 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index faf0b8b7ce2..42f5eed510e 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/ec2" cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/metrics" @@ -174,6 +175,9 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries case "dimension_keys": data, err = e.handleGetDimensions(ctx, parameters, queryContext) break + case "ebs_volume_ids": + data, err = e.handleGetEbsVolumeIds(ctx, parameters, queryContext) + break } if err != nil { queryResult.Error = err @@ -320,6 +324,70 @@ func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters return result, nil } +func (e *CloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { + region := parameters.Get("region").MustString() + instanceId := parameters.Get("instanceId").MustString() + + instanceIds := []*string{aws.String(instanceId)} + instances, err := e.ec2DescribeInstances(region, nil, instanceIds) + if err != nil { + return nil, err + } + + result := make([]suggestData, 0) + for _, mapping := range instances.Reservations[0].Instances[0].BlockDeviceMappings { + result = append(result, suggestData{Text: *mapping.Ebs.VolumeId, Value: *mapping.Ebs.VolumeId}) + } + + return result, nil +} + +func getAwsConfig(dsInfo *cwapi.DatasourceInfo) (*aws.Config, error) { + creds, err := cwapi.GetCredentials(dsInfo) + if err != nil { + return nil, err + } + + cfg := &aws.Config{ + Region: aws.String(dsInfo.Region), + Credentials: creds, + } + return cfg, nil +} + +func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.Filter, instanceIds []*string) (*ec2.DescribeInstancesOutput, error) { + dsInfo := e.getDsInfo(region) + cfg, err := getAwsConfig(dsInfo) + if err != nil { + return nil, errors.New("Failed to call describe instances") + } + sess, err := session.NewSession(cfg) + if err != nil { + return nil, errors.New("Failed to call describe instances") + } + svc := ec2.New(sess, cfg) + + params := &ec2.DescribeInstancesInput{ + Filters: filters, + InstanceIds: instanceIds, + } + + var resp ec2.DescribeInstancesOutput + err = svc.DescribeInstancesPages(params, + func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { + reservations, _ := awsutil.ValuesAtPath(page, "Reservations") + for _, reservation := range reservations { + resp.Reservations = append(resp.Reservations, reservation.(*ec2.Reservation)) + } + return !lastPage + }) + if err != nil { + return nil, errors.New("Failed to call describe instances") + } + + return &resp, nil +} + func getAllMetrics(cwData *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { creds, err := cwapi.GetCredentials(cwData) if err != nil { diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 916c95fbd77..40fba8af36f 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -248,6 +248,28 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }); }; + this.getEbsVolumeIds = function(region, instanceId) { + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'ebs_volume_ids', + parameters: { + region: region, + instanceId: instanceId + } + } + ] + }).then(function (r) { return transformSuggestDataFromTable(r); }); + }; + this.performEC2DescribeInstances = function(region, filters, instanceIds) { return this.awsRequest({ region: region, @@ -301,17 +323,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot if (ebsVolumeIdsQuery) { region = templateSrv.replace(ebsVolumeIdsQuery[1]); var instanceId = templateSrv.replace(ebsVolumeIdsQuery[2]); - var instanceIds = [ - instanceId - ]; - - return this.performEC2DescribeInstances(region, [], instanceIds).then(function(result) { - var volumeIds = _.map(result.Reservations[0].Instances[0].BlockDeviceMappings, function(mapping) { - return mapping.Ebs.VolumeId; - }); - - return transformSuggestData(volumeIds); - }); + return this.getEbsVolumeIds(region, instanceId); } var ec2InstanceAttributeQuery = query.match(/^ec2_instance_attribute\(([^,]+?),\s?([^,]+?),\s?(.+?)\)/); From 8fba6dcb0d26e8a46de607c67a8048b1af9406c5 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 19 Sep 2017 18:55:11 +0900 Subject: [PATCH 20/44] re-implement ec2_instance_attribute() --- pkg/tsdb/cloudwatch/metric_find_query.go | 73 +++++++++++++++++++ .../datasource/cloudwatch/datasource.js | 57 +++++++-------- 2 files changed, 98 insertions(+), 32 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 42f5eed510e..80ca06113b2 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -3,6 +3,7 @@ package cloudwatch import ( "context" "errors" + "reflect" "sort" "strings" "sync" @@ -178,6 +179,9 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries case "ebs_volume_ids": data, err = e.handleGetEbsVolumeIds(ctx, parameters, queryContext) break + case "ec2_instance_attribute": + data, err = e.handleGetEc2InstanceAttribute(ctx, parameters, queryContext) + break } if err != nil { queryResult.Error = err @@ -342,6 +346,75 @@ func (e *CloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, paramete return result, nil } +func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { + region := parameters.Get("region").MustString() + attributeName := parameters.Get("attributeName").MustString() + filterJson := parameters.Get("filters").MustMap() + + var filters []*ec2.Filter + for k, v := range filterJson { + if vv, ok := v.([]string); ok { + var vvvv []*string + for _, vvv := range vv { + vvvv = append(vvvv, &vvv) + } + filters = append(filters, &ec2.Filter{ + Name: aws.String(k), + Values: vvvv, + }) + } + } + + instances, err := e.ec2DescribeInstances(region, filters, nil) + if err != nil { + return nil, err + } + + result := make([]suggestData, 0) + dupCheck := make(map[string]bool) + for _, instance := range instances.Reservations[0].Instances { + tags := make(map[string]string) + for _, tag := range instance.Tags { + tags[*tag.Key] = *tag.Value + } + + var data string + if strings.Index(attributeName, "Tags.") == 0 { + tagName := attributeName[5:] + data = tags[tagName] + } else { + attributePath := strings.Split(attributeName, ".") + v := reflect.ValueOf(instance) + for _, key := range attributePath { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Struct { + return nil, errors.New("invalid attribute path") + } + v = v.FieldByName(key) + } + if attr, ok := v.Interface().(*string); ok { + data = *attr + } else { + return nil, errors.New("invalid attribute path") + } + } + + if _, exists := dupCheck[data]; exists { + continue + } + dupCheck[data] = true + result = append(result, suggestData{Text: data, Value: data}) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].Text < result[j].Text + }) + + return result, nil +} + func getAwsConfig(dsInfo *cwapi.DatasourceInfo) (*aws.Config, error) { creds, err := cwapi.GetCredentials(dsInfo) if err != nil { diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 40fba8af36f..12562a1fb88 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -270,6 +270,29 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }).then(function (r) { return transformSuggestDataFromTable(r); }); }; + this.getEc2InstanceAttribute = function(region, attributeName, filters) { + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'ec2_instance_attribute', + parameters: { + region: region, + attributeName: attributeName, + filters: filters + } + } + ] + }).then(function (r) { return transformSuggestDataFromTable(r); }); + }; + this.performEC2DescribeInstances = function(region, filters, instanceIds) { return this.awsRequest({ region: region, @@ -283,12 +306,6 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var namespace; var metricName; - var transformSuggestData = function(suggestData) { - return _.map(suggestData, function(v) { - return { text: v }; - }); - }; - var regionQuery = query.match(/^regions\(\)/); if (regionQuery) { return this.getRegions(); @@ -329,33 +346,9 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var ec2InstanceAttributeQuery = query.match(/^ec2_instance_attribute\(([^,]+?),\s?([^,]+?),\s?(.+?)\)/); if (ec2InstanceAttributeQuery) { region = templateSrv.replace(ec2InstanceAttributeQuery[1]); - var filterJson = JSON.parse(templateSrv.replace(ec2InstanceAttributeQuery[3])); - var filters = _.map(filterJson, function(values, name) { - return { - Name: name, - Values: values - }; - }); var targetAttributeName = templateSrv.replace(ec2InstanceAttributeQuery[2]); - - return this.performEC2DescribeInstances(region, filters, null).then(function(result) { - var attributes = _.chain(result.Reservations) - .map(function(reservations) { - return _.map(reservations.Instances, function(instance) { - var tags = {}; - _.each(instance.Tags, function(tag) { - tags[tag.Key] = tag.Value; - }); - instance.Tags = tags; - return instance; - }); - }) - .map(function(instances) { - return _.map(instances, targetAttributeName); - }) - .flatten().uniq().sortBy().value(); - return transformSuggestData(attributes); - }); + var filterJson = JSON.parse(templateSrv.replace(ec2InstanceAttributeQuery[3])); + return this.getEc2InstanceAttribute(region, targetAttributeName, filterJson); } return $q.when([]); From 78e3556e95deac0bde4eaf323338c7c14a51392d Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 19 Sep 2017 23:32:40 +0900 Subject: [PATCH 21/44] remove performEC2DescribeInstances() --- pkg/api/cloudwatch/cloudwatch.go | 48 ------------------- .../datasource/cloudwatch/datasource.js | 8 ---- 2 files changed, 56 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index caeb14a2848..6f8a17ab69d 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -18,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/sts" "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" @@ -78,7 +77,6 @@ func init() { "DescribeAlarms": handleDescribeAlarms, "DescribeAlarmsForMetric": handleDescribeAlarmsForMetric, "DescribeAlarmHistory": handleDescribeAlarmHistory, - "DescribeInstances": handleDescribeInstances, } } @@ -402,52 +400,6 @@ func handleDescribeAlarmHistory(req *cwRequest, c *middleware.Context) { c.JSON(200, resp) } -func handleDescribeInstances(req *cwRequest, c *middleware.Context) { - cfg, err := getAwsConfig(req) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - sess, err := session.NewSession(cfg) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - svc := ec2.New(sess, cfg) - - reqParam := &struct { - Parameters struct { - Filters []*ec2.Filter `json:"filters"` - InstanceIds []*string `json:"instanceIds"` - } `json:"parameters"` - }{} - json.Unmarshal(req.Body, reqParam) - - params := &ec2.DescribeInstancesInput{} - if len(reqParam.Parameters.Filters) > 0 { - params.Filters = reqParam.Parameters.Filters - } - if len(reqParam.Parameters.InstanceIds) > 0 { - params.InstanceIds = reqParam.Parameters.InstanceIds - } - - var resp ec2.DescribeInstancesOutput - err = svc.DescribeInstancesPages(params, - func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { - reservations, _ := awsutil.ValuesAtPath(page, "Reservations") - for _, reservation := range reservations { - resp.Reservations = append(resp.Reservations, reservation.(*ec2.Reservation)) - } - return !lastPage - }) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - - c.JSON(200, resp) -} - func HandleRequest(c *middleware.Context, ds *m.DataSource) { var req cwRequest req.Body, _ = ioutil.ReadAll(c.Req.Request.Body) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 12562a1fb88..1f89b88b5ff 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -293,14 +293,6 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }).then(function (r) { return transformSuggestDataFromTable(r); }); }; - this.performEC2DescribeInstances = function(region, filters, instanceIds) { - return this.awsRequest({ - region: region, - action: 'DescribeInstances', - parameters: { filters: filters, instanceIds: instanceIds } - }); - }; - this.metricFindQuery = function(query) { var region; var namespace; From ec632bb9eda483db13a89024c9f330aabadfe093 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 20 Sep 2017 00:08:00 +0900 Subject: [PATCH 22/44] fix error message --- pkg/tsdb/cloudwatch/metric_find_query.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 80ca06113b2..db20ccd7dc5 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -432,11 +432,11 @@ func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2. dsInfo := e.getDsInfo(region) cfg, err := getAwsConfig(dsInfo) if err != nil { - return nil, errors.New("Failed to call describe instances") + return nil, errors.New("Failed to call ec2:DescribeInstances") } sess, err := session.NewSession(cfg) if err != nil { - return nil, errors.New("Failed to call describe instances") + return nil, errors.New("Failed to call ec2:DescribeInstances") } svc := ec2.New(sess, cfg) @@ -455,7 +455,7 @@ func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2. return !lastPage }) if err != nil { - return nil, errors.New("Failed to call describe instances") + return nil, errors.New("Failed to call ec2:DescribeInstances") } return &resp, nil From 1dcc51adce701df2b1f2a74de938105d8cc53693 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 20 Sep 2017 00:10:21 +0900 Subject: [PATCH 23/44] re-implement dimension_values() --- pkg/api/cloudwatch/cloudwatch.go | 49 ----------- pkg/tsdb/cloudwatch/metric_find_query.go | 81 +++++++++++++++++++ .../datasource/cloudwatch/datasource.js | 46 +++++------ 3 files changed, 103 insertions(+), 73 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 6f8a17ab69d..7b6a189cfde 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -11,7 +11,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" @@ -19,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/sts" - "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" ) @@ -73,7 +71,6 @@ func (req *cwRequest) GetDatasourceInfo() *DatasourceInfo { func init() { actionHandlers = map[string]actionHandler{ - "ListMetrics": handleListMetrics, "DescribeAlarms": handleDescribeAlarms, "DescribeAlarmsForMetric": handleDescribeAlarmsForMetric, "DescribeAlarmHistory": handleDescribeAlarmHistory, @@ -216,52 +213,6 @@ func getAwsConfig(req *cwRequest) (*aws.Config, error) { return cfg, nil } -func handleListMetrics(req *cwRequest, c *middleware.Context) { - cfg, err := getAwsConfig(req) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - sess, err := session.NewSession(cfg) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - svc := cloudwatch.New(sess, cfg) - - reqParam := &struct { - Parameters struct { - Namespace string `json:"namespace"` - MetricName string `json:"metricName"` - Dimensions []*cloudwatch.DimensionFilter `json:"dimensions"` - } `json:"parameters"` - }{} - json.Unmarshal(req.Body, reqParam) - - params := &cloudwatch.ListMetricsInput{ - Namespace: aws.String(reqParam.Parameters.Namespace), - MetricName: aws.String(reqParam.Parameters.MetricName), - Dimensions: reqParam.Parameters.Dimensions, - } - - var resp cloudwatch.ListMetricsOutput - err = svc.ListMetricsPages(params, - func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { - metrics.M_Aws_CloudWatch_ListMetrics.Inc() - metrics, _ := awsutil.ValuesAtPath(page, "Metrics") - for _, metric := range metrics { - resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) - } - return !lastPage - }) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - - c.JSON(200, resp) -} - func handleDescribeAlarms(req *cwRequest, c *middleware.Context) { cfg, err := getAwsConfig(req) if err != nil { diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index db20ccd7dc5..2f3abd4b36b 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -176,6 +176,9 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries case "dimension_keys": data, err = e.handleGetDimensions(ctx, parameters, queryContext) break + case "dimension_values": + data, err = e.handleGetDimensionValues(ctx, parameters, queryContext) + break case "ebs_volume_ids": data, err = e.handleGetEbsVolumeIds(ctx, parameters, queryContext) break @@ -328,6 +331,49 @@ func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters return result, nil } +func (e *CloudWatchExecutor) handleGetDimensionValues(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { + region := parameters.Get("region").MustString() + namespace := parameters.Get("namespace").MustString() + metricName := parameters.Get("metricName").MustString() + dimensionKey := parameters.Get("dimensionKey").MustString() + dimensionsJson := parameters.Get("dimensionKey").MustMap() + + var dimensions []*cloudwatch.DimensionFilter + for _, d := range dimensionsJson { + if dd, ok := d.(map[string]string); ok { + dimensions = append(dimensions, &cloudwatch.DimensionFilter{ + Name: aws.String(dd["Name"]), + Value: aws.String(dd["Value"]), + }) + } + } + + metrics, err := e.cloudwatchListMetrics(region, namespace, metricName, dimensions) + if err != nil { + return nil, err + } + + result := make([]suggestData, 0) + dupCheck := make(map[string]bool) + for _, metric := range metrics.Metrics { + for _, dim := range metric.Dimensions { + if *dim.Name == dimensionKey { + if _, exists := dupCheck[*dim.Value]; exists { + continue + } + dupCheck[*dim.Value] = true + result = append(result, suggestData{Text: *dim.Value, Value: *dim.Value}) + } + } + } + + sort.Slice(result, func(i, j int) bool { + return result[i].Text < result[j].Text + }) + + return result, nil +} + func (e *CloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { region := parameters.Get("region").MustString() instanceId := parameters.Get("instanceId").MustString() @@ -428,6 +474,41 @@ func getAwsConfig(dsInfo *cwapi.DatasourceInfo) (*aws.Config, error) { return cfg, nil } +func (e *CloudWatchExecutor) cloudwatchListMetrics(region string, namespace string, metricName string, dimensions []*cloudwatch.DimensionFilter) (*cloudwatch.ListMetricsOutput, error) { + dsInfo := e.getDsInfo(region) + cfg, err := getAwsConfig(dsInfo) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:ListMetrics") + } + sess, err := session.NewSession(cfg) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:ListMetrics") + } + svc := cloudwatch.New(sess, cfg) + + params := &cloudwatch.ListMetricsInput{ + Namespace: aws.String(namespace), + MetricName: aws.String(metricName), + Dimensions: dimensions, + } + + var resp cloudwatch.ListMetricsOutput + err = svc.ListMetricsPages(params, + func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { + metrics.M_Aws_CloudWatch_ListMetrics.Inc() + metrics, _ := awsutil.ValuesAtPath(page, "Metrics") + for _, metric := range metrics { + resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric)) + } + return !lastPage + }) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:ListMetrics") + } + + return &resp, nil +} + func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.Filter, instanceIds []*string) (*ec2.DescribeInstancesOutput, error) { dsInfo := e.getDsInfo(region) cfg, err := getAwsConfig(dsInfo) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 1f89b88b5ff..31999cf5afe 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -222,30 +222,28 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }; this.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { - var request = { - region: templateSrv.replace(region), - action: 'ListMetrics', - parameters: { - namespace: templateSrv.replace(namespace), - metricName: templateSrv.replace(metricName), - dimensions: this.convertDimensionFormat(filterDimensions, {}), - } - }; - - return this.awsRequest(request).then(function(result) { - return _.chain(result.Metrics) - .map('Dimensions') - .flatten() - .filter(function(dimension) { - return dimension !== null && dimension.Name === dimensionKey; - }) - .map('Value') - .uniq() - .sortBy() - .map(function(value) { - return {value: value, text: value}; - }).value(); - }); + var range = timeSrv.timeRange(); + return backendSrv.post('/api/tsdb/query', { + from: range.from, + to: range.to, + queries: [ + { + refId: 'metricFindQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'metricFindQuery', + subtype: 'dimension_values', + parameters: { + region: region, + namespace: templateSrv.replace(namespace), + metricName: templateSrv.replace(metricName), + dimensionKey: templateSrv.replace(dimensionKey), + dimensions: this.convertDimensionFormat(filterDimensions, {}), + } + } + ] + }).then(function (r) { return transformSuggestDataFromTable(r); }); }; this.getEbsVolumeIds = function(region, instanceId) { From ea704306a0abeb68b381f95f85c228c2690374f6 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 20 Sep 2017 00:45:07 +0900 Subject: [PATCH 24/44] fix test --- .../cloudwatch/specs/datasource_specs.ts | 54 +++---------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index a326a49e1de..15eafadbb8f 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -336,24 +336,20 @@ describe('CloudWatchDatasource', function() { describeMetricFindQuery('dimension_values(us-east-1,AWS/EC2,CPUUtilization,InstanceId)', scenario => { scenario.setup(() => { scenario.requestResponse = { - Metrics: [ - { - Namespace: 'AWS/EC2', - MetricName: 'CPUUtilization', - Dimensions: [ - { - Name: 'InstanceId', - Value: 'i-12345678' - } + results: { + metricFindQuery: { + tables: [ + { rows: [['i-12345678', 'i-12345678']] } ] } - ] + } }; }); it('should call __ListMetrics and return result', () => { - expect(scenario.result[0].text).to.be('i-12345678'); - expect(scenario.request.data.action).to.be('ListMetrics'); + expect(scenario.result[0].text).to.contain('i-12345678'); + expect(scenario.request.queries[0].type).to.be('metricFindQuery'); + expect(scenario.request.queries[0].subtype).to.be('dimension_values'); }); }); @@ -433,38 +429,4 @@ describe('CloudWatchDatasource', function() { } }); - describeMetricFindQuery('ec2_instance_attribute(us-east-1, Tags.Name, { "tag:team": [ "sysops" ] })', scenario => { - scenario.setup(() => { - scenario.requestResponse = { - Reservations: [ - { - Instances: [ - { - Tags: [ - { Key: 'InstanceId', Value: 'i-123456' }, - { Key: 'Name', Value: 'Sysops Dev Server' }, - { Key: 'env', Value: 'dev' }, - { Key: 'team', Value: 'sysops' } - ] - }, - { - Tags: [ - { Key: 'InstanceId', Value: 'i-789012' }, - { Key: 'Name', Value: 'Sysops Staging Server' }, - { Key: 'env', Value: 'staging' }, - { Key: 'team', Value: 'sysops' } - ] - } - ] - } - ] - }; - }); - - it('should return the "Name" tag for each instance', function() { - expect(scenario.result[0].text).to.be('Sysops Dev Server'); - expect(scenario.result[1].text).to.be('Sysops Staging Server'); - }); - }); - }); From e4de6332de140c2dce335cc7d78643c0ee8ae203 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 20 Sep 2017 12:36:48 +0900 Subject: [PATCH 25/44] refactor cloudwatch frontend code --- .../datasource/cloudwatch/datasource.js | 148 ++++-------------- 1 file changed, 32 insertions(+), 116 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 31999cf5afe..a444f2e65a0 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -141,7 +141,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot }); } - this.getRegions = function () { + this.doMetricQueryRequest = function (subtype, parameters) { var range = timeSrv.timeRange(); return backendSrv.post('/api/tsdb/query', { from: range.from, @@ -153,142 +153,58 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot maxDataPoints: 1, // dummy datasourceId: this.instanceSettings.id, type: 'metricFindQuery', - subtype: 'regions' + subtype: subtype, + parameters: parameters } ] }).then(function (r) { return transformSuggestDataFromTable(r); }); }; + this.getRegions = function () { + return this.doMetricQueryRequest('regions', null); + }; + this.getNamespaces = function() { - var range = timeSrv.timeRange(); - return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, - queries: [ - { - refId: 'metricFindQuery', - intervalMs: 1, // dummy - maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, - type: 'metricFindQuery', - subtype: 'namespaces' - } - ] - }).then(function (r) { return transformSuggestDataFromTable(r); }); + return this.doMetricQueryRequest('namespaces', null); }; this.getMetrics = function (namespace, region) { - var range = timeSrv.timeRange(); - return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, - queries: [ - { - refId: 'metricFindQuery', - intervalMs: 1, // dummy - maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, - type: 'metricFindQuery', - subtype: 'metrics', - parameters: { - region: region, - namespace: templateSrv.replace(namespace) - } - } - ] - }).then(function (r) { return transformSuggestDataFromTable(r); }); + return this.doMetricQueryRequest('metrics', { + region: region, + namespace: templateSrv.replace(namespace) + }); }; this.getDimensionKeys = function(namespace, region) { - var range = timeSrv.timeRange(); - return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, - queries: [ - { - refId: 'metricFindQuery', - intervalMs: 1, // dummy - maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, - type: 'metricFindQuery', - subtype: 'dimension_keys', - parameters: { - region: region, - namespace: templateSrv.replace(namespace) - } - } - ] - }).then(function (r) { return transformSuggestDataFromTable(r); }); + return this.doMetricQueryRequest('dimension_keys', { + region: region, + namespace: templateSrv.replace(namespace) + }); }; this.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { - var range = timeSrv.timeRange(); - return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, - queries: [ - { - refId: 'metricFindQuery', - intervalMs: 1, // dummy - maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, - type: 'metricFindQuery', - subtype: 'dimension_values', - parameters: { - region: region, - namespace: templateSrv.replace(namespace), - metricName: templateSrv.replace(metricName), - dimensionKey: templateSrv.replace(dimensionKey), - dimensions: this.convertDimensionFormat(filterDimensions, {}), - } - } - ] - }).then(function (r) { return transformSuggestDataFromTable(r); }); + return this.doMetricQueryRequest('dimension_values', { + region: region, + namespace: templateSrv.replace(namespace), + metricName: templateSrv.replace(metricName), + dimensionKey: templateSrv.replace(dimensionKey), + dimensions: this.convertDimensionFormat(filterDimensions, {}), + }); }; this.getEbsVolumeIds = function(region, instanceId) { - var range = timeSrv.timeRange(); - return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, - queries: [ - { - refId: 'metricFindQuery', - intervalMs: 1, // dummy - maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, - type: 'metricFindQuery', - subtype: 'ebs_volume_ids', - parameters: { - region: region, - instanceId: instanceId - } - } - ] - }).then(function (r) { return transformSuggestDataFromTable(r); }); + return this.doMetricQueryRequest('ebs_volume_ids', { + region: region, + instanceId: instanceId + }); }; this.getEc2InstanceAttribute = function(region, attributeName, filters) { - var range = timeSrv.timeRange(); - return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, - queries: [ - { - refId: 'metricFindQuery', - intervalMs: 1, // dummy - maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, - type: 'metricFindQuery', - subtype: 'ec2_instance_attribute', - parameters: { - region: region, - attributeName: attributeName, - filters: filters - } - } - ] - }).then(function (r) { return transformSuggestDataFromTable(r); }); + return this.doMetricQueryRequest('ec2_instance_attribute', { + region: region, + attributeName: attributeName, + filters: filters + }); }; this.metricFindQuery = function(query) { From 4b34ff5b83e507e104ce482d101a93c6fe8dca3a Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 20 Sep 2017 12:47:36 +0900 Subject: [PATCH 26/44] refactor cloudwatch frontend code --- .../datasource/cloudwatch/datasource.js | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index a444f2e65a0..52074cf9a8c 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -170,21 +170,21 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot this.getMetrics = function (namespace, region) { return this.doMetricQueryRequest('metrics', { - region: region, + region: templateSrv.replace(region), namespace: templateSrv.replace(namespace) }); }; this.getDimensionKeys = function(namespace, region) { return this.doMetricQueryRequest('dimension_keys', { - region: region, + region: templateSrv.replace(region), namespace: templateSrv.replace(namespace) }); }; this.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { return this.doMetricQueryRequest('dimension_values', { - region: region, + region: templateSrv.replace(region), namespace: templateSrv.replace(namespace), metricName: templateSrv.replace(metricName), dimensionKey: templateSrv.replace(dimensionKey), @@ -194,15 +194,15 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot this.getEbsVolumeIds = function(region, instanceId) { return this.doMetricQueryRequest('ebs_volume_ids', { - region: region, - instanceId: instanceId + region: templateSrv.replace(region), + instanceId: templateSrv.replace(instanceId) }); }; this.getEc2InstanceAttribute = function(region, attributeName, filters) { return this.doMetricQueryRequest('ec2_instance_attribute', { - region: region, - attributeName: attributeName, + region: templateSrv.replace(region), + attributeName: templateSrv.replace(attributeName), filters: filters }); }; @@ -224,35 +224,39 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot var metricNameQuery = query.match(/^metrics\(([^\)]+?)(,\s?([^,]+?))?\)/); if (metricNameQuery) { - return this.getMetrics(templateSrv.replace(metricNameQuery[1]), templateSrv.replace(metricNameQuery[3])); + namespace = metricNameQuery[1]; + region = metricNameQuery[3]; + return this.getMetrics(namespace, region); } var dimensionKeysQuery = query.match(/^dimension_keys\(([^\)]+?)(,\s?([^,]+?))?\)/); if (dimensionKeysQuery) { - return this.getDimensionKeys(templateSrv.replace(dimensionKeysQuery[1]), templateSrv.replace(dimensionKeysQuery[3])); + namespace = dimensionKeysQuery[1]; + region = dimensionKeysQuery[3]; + return this.getDimensionKeys(namespace, region); } var dimensionValuesQuery = query.match(/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?),\s?([^,]+?)\)/); if (dimensionValuesQuery) { - region = templateSrv.replace(dimensionValuesQuery[1]); - namespace = templateSrv.replace(dimensionValuesQuery[2]); - metricName = templateSrv.replace(dimensionValuesQuery[3]); - var dimensionKey = templateSrv.replace(dimensionValuesQuery[4]); + region = dimensionValuesQuery[1]; + namespace = dimensionValuesQuery[2]; + metricName = dimensionValuesQuery[3]; + var dimensionKey = dimensionValuesQuery[4]; return this.getDimensionValues(region, namespace, metricName, dimensionKey, {}); } var ebsVolumeIdsQuery = query.match(/^ebs_volume_ids\(([^,]+?),\s?([^,]+?)\)/); if (ebsVolumeIdsQuery) { - region = templateSrv.replace(ebsVolumeIdsQuery[1]); - var instanceId = templateSrv.replace(ebsVolumeIdsQuery[2]); + region = ebsVolumeIdsQuery[1]; + var instanceId = ebsVolumeIdsQuery[2]; return this.getEbsVolumeIds(region, instanceId); } var ec2InstanceAttributeQuery = query.match(/^ec2_instance_attribute\(([^,]+?),\s?([^,]+?),\s?(.+?)\)/); if (ec2InstanceAttributeQuery) { - region = templateSrv.replace(ec2InstanceAttributeQuery[1]); - var targetAttributeName = templateSrv.replace(ec2InstanceAttributeQuery[2]); + region = ec2InstanceAttributeQuery[1]; + var targetAttributeName = ec2InstanceAttributeQuery[2]; var filterJson = JSON.parse(templateSrv.replace(ec2InstanceAttributeQuery[3])); return this.getEc2InstanceAttribute(region, targetAttributeName, filterJson); } From fe1d395d79c0c974e44c120b40f0655f4f00b96e Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 22 Sep 2017 11:07:10 +0200 Subject: [PATCH 27/44] refactor cloudwatch to support new tsdb interface --- pkg/tsdb/cloudwatch/cloudwatch.go | 51 +++++++++++++----------- pkg/tsdb/cloudwatch/metric_find_query.go | 35 ++++++++-------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 62e79f4fab9..08f62c691e7 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -3,6 +3,7 @@ package cloudwatch import ( "context" "errors" + "fmt" "regexp" "sort" "strconv" @@ -27,10 +28,8 @@ type CloudWatchExecutor struct { *models.DataSource } -func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.Executor, error) { - return &CloudWatchExecutor{ - DataSource: dsInfo, - }, nil +func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + return &CloudWatchExecutor{}, nil } var ( @@ -41,7 +40,7 @@ var ( func init() { plog = log.New("tsdb.cloudwatch") - tsdb.RegisterExecutor("cloudwatch", NewCloudWatchExecutor) + tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor) standardStatistics = map[string]bool{ "Average": true, "Maximum": true, @@ -52,37 +51,43 @@ func init() { aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) } -func (e *CloudWatchExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { - var result *tsdb.BatchResult - queryType := queries[0].Model.Get("type").MustString() +func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { + var result *tsdb.Response + e.DataSource = dsInfo + queryType := queryContext.Queries[0].Model.Get("type").MustString("") + var err error + switch queryType { case "timeSeriesQuery": - result = e.executeTimeSeriesQuery(ctx, queries, queryContext) + result, err = e.executeTimeSeriesQuery(ctx, queryContext) break case "metricFindQuery": - result = e.executeMetricFindQuery(ctx, queries, queryContext) + result, err = e.executeMetricFindQuery(ctx, queryContext) break + default: + err = fmt.Errorf("missing querytype") } - return result + + return result, err } -func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { - result := &tsdb.BatchResult{ - QueryResults: make(map[string]*tsdb.QueryResult), +func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), } errCh := make(chan error, 1) resCh := make(chan *tsdb.QueryResult, 1) currentlyExecuting := 0 - for _, model := range queries { + for i, model := range queryContext.Queries { queryType := model.Model.Get("type").MustString() if queryType != "timeSeriesQuery" { continue } currentlyExecuting++ - go func(refId string) { - queryRes, err := e.executeQuery(ctx, model.Model.Get("parameters"), queryContext) + go func(refId string, index int) { + queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model.Get("parameters"), queryContext) currentlyExecuting-- if err != nil { errCh <- err @@ -90,21 +95,21 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queries queryRes.RefId = refId resCh <- queryRes } - }(model.RefId) + }(model.RefId, i) } for currentlyExecuting != 0 { select { case res := <-resCh: - result.QueryResults[res.RefId] = res + result.Results[res.RefId] = res case err := <-errCh: - return result.WithError(err) + return result, err case <-ctx.Done(): - return result.WithError(ctx.Err()) + return result, ctx.Err() } } - return result + return result, nil } func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, error) { @@ -148,7 +153,7 @@ func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, e return client, nil } -func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) (*tsdb.QueryResult, error) { +func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) { query, err := parseQuery(parameters) if err != nil { return nil, err diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 2f3abd4b36b..4e24ec2660e 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -153,14 +153,15 @@ func init() { customMetricsDimensionsMap = make(map[string]map[string]map[string]*CustomMetricsCache) } -func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult { - result := &tsdb.BatchResult{ - QueryResults: make(map[string]*tsdb.QueryResult), +func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), } - queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: queries[0].RefId} + firstQuery := queryContext.Queries[0] + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: firstQuery.RefId} - parameters := queries[0].Model.Get("parameters") - subType := queries[0].Model.Get("subtype").MustString() + parameters := firstQuery.Model.Get("parameters") + subType := firstQuery.Model.Get("subtype").MustString() var data []suggestData var err error switch subType { @@ -186,12 +187,10 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queries data, err = e.handleGetEc2InstanceAttribute(ctx, parameters, queryContext) break } - if err != nil { - queryResult.Error = err - } + transformToTable(data, queryResult) - result.QueryResults[queries[0].RefId] = queryResult - return result + result.Results[firstQuery.RefId] = queryResult + return result, err } func transformToTable(data []suggestData, result *tsdb.QueryResult) { @@ -238,7 +237,7 @@ func (e *CloudWatchExecutor) getDsInfo(region string) *cwapi.DatasourceInfo { // Whenever this list is updated, frontend list should also be updated. // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html -func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { regions := []string{ "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-south-1", "ca-central-1", "cn-north-1", "eu-central-1", "eu-west-1", "eu-west-2", "sa-east-1", "us-east-1", "us-east-2", "us-gov-west-1", "us-west-1", "us-west-2", @@ -252,7 +251,7 @@ func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s return result, nil } -func (e *CloudWatchExecutor) handleGetNamespaces(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetNamespaces(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { keys := []string{} for key := range metricsMap { keys = append(keys, key) @@ -273,7 +272,7 @@ func (e *CloudWatchExecutor) handleGetNamespaces(ctx context.Context, parameters return result, nil } -func (e *CloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() namespace := parameters.Get("namespace").MustString() @@ -302,7 +301,7 @@ func (e *CloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s return result, nil } -func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() namespace := parameters.Get("namespace").MustString() @@ -331,7 +330,7 @@ func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters return result, nil } -func (e *CloudWatchExecutor) handleGetDimensionValues(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetDimensionValues(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() namespace := parameters.Get("namespace").MustString() metricName := parameters.Get("metricName").MustString() @@ -374,7 +373,7 @@ func (e *CloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param return result, nil } -func (e *CloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() instanceId := parameters.Get("instanceId").MustString() @@ -392,7 +391,7 @@ func (e *CloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, paramete return result, nil } -func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.QueryContext) ([]suggestData, error) { +func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() attributeName := parameters.Get("attributeName").MustString() filterJson := parameters.Get("filters").MustMap() From 8243ac39c235278800db564fac911fb3987bf031 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sun, 24 Sep 2017 12:25:52 +0900 Subject: [PATCH 28/44] fix parameter format --- pkg/tsdb/cloudwatch/cloudwatch.go | 2 +- .../app/plugins/datasource/cloudwatch/datasource.js | 12 +++++------- .../datasource/cloudwatch/specs/datasource_specs.ts | 13 ++++++------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 08f62c691e7..1890e538d4a 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -87,7 +87,7 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo } currentlyExecuting++ go func(refId string, index int) { - queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model.Get("parameters"), queryContext) + queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext) currentlyExecuting-- if err != nil { errCh <- err diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 52074cf9a8c..370f1f971d5 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -48,14 +48,13 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot item.dimensions = dimensions; item.period = self.getPeriod(item, options); - return { + return _.extend({ refId: item.refId, intervalMs: options.intervalMs, maxDataPoints: options.maxDataPoints, datasourceId: self.instanceSettings.id, type: 'timeSeriesQuery', - parameters: item - }; + }, item); }); // No valid targets, return the empty result to save a round trip. @@ -147,15 +146,14 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot from: range.from, to: range.to, queries: [ - { + _.extend({ refId: 'metricFindQuery', intervalMs: 1, // dummy maxDataPoints: 1, // dummy datasourceId: this.instanceSettings.id, type: 'metricFindQuery', - subtype: subtype, - parameters: parameters - } + subtype: subtype + }, parameters) ] }).then(function (r) { return transformSuggestDataFromTable(r); }); }; diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index 15eafadbb8f..8642cc871f6 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -1,4 +1,3 @@ - import "../datasource"; import {describe, beforeEach, it, expect, angularMocks} from 'test/lib/common'; import helpers from 'test/specs/helpers'; @@ -76,11 +75,11 @@ describe('CloudWatchDatasource', function() { it('should generate the correct query', function(done) { ctx.ds.query(query).then(function() { var params = requestParams.queries[0]; - expect(params.parameters.namespace).to.be(query.targets[0].namespace); - expect(params.parameters.metricName).to.be(query.targets[0].metricName); - expect(params.parameters.dimensions['InstanceId']).to.be('i-12345678'); - expect(params.parameters.statistics).to.eql(query.targets[0].statistics); - expect(params.parameters.period).to.be(query.targets[0].period); + expect(params.namespace).to.be(query.targets[0].namespace); + expect(params.metricName).to.be(query.targets[0].metricName); + expect(params.dimensions['InstanceId']).to.be('i-12345678'); + expect(params.statistics).to.eql(query.targets[0].statistics); + expect(params.period).to.be(query.targets[0].period); done(); }); ctx.$rootScope.$apply(); @@ -110,7 +109,7 @@ describe('CloudWatchDatasource', function() { ctx.ds.query(query).then(function() { var params = requestParams.queries[0]; - expect(params.parameters.period).to.be(600); + expect(params.period).to.be(600); done(); }); ctx.$rootScope.$apply(); From 17445e0c98be2b9471ef957a6d5a02f2bbcc6e15 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Sun, 24 Sep 2017 12:30:34 +0900 Subject: [PATCH 29/44] fix alert feature --- pkg/tsdb/cloudwatch/cloudwatch.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 1890e538d4a..122cb868fac 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -3,7 +3,6 @@ package cloudwatch import ( "context" "errors" - "fmt" "regexp" "sort" "strconv" @@ -58,14 +57,14 @@ func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSourc var err error switch queryType { - case "timeSeriesQuery": - result, err = e.executeTimeSeriesQuery(ctx, queryContext) - break case "metricFindQuery": result, err = e.executeMetricFindQuery(ctx, queryContext) break + case "timeSeriesQuery": + fallthrough default: - err = fmt.Errorf("missing querytype") + result, err = e.executeTimeSeriesQuery(ctx, queryContext) + break } return result, err From 8f3b060946067f77729617dae12c9343e517d546 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Mon, 25 Sep 2017 12:58:47 +0900 Subject: [PATCH 30/44] fix parameter format --- pkg/tsdb/cloudwatch/metric_find_query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 4e24ec2660e..f09c7118137 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -160,7 +160,7 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queryCo firstQuery := queryContext.Queries[0] queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: firstQuery.RefId} - parameters := firstQuery.Model.Get("parameters") + parameters := firstQuery.Model subType := firstQuery.Model.Get("subtype").MustString() var data []suggestData var err error From c140d7aa066473cca13bf070c35eabae0e722307 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Mon, 25 Sep 2017 18:16:40 +0900 Subject: [PATCH 31/44] re-implement annotation query --- pkg/tsdb/cloudwatch/annotation_query.go | 218 ++++++++++++++++++ pkg/tsdb/cloudwatch/cloudwatch.go | 3 + .../cloudwatch/annotation_query.d.ts | 2 - .../datasource/cloudwatch/annotation_query.js | 106 --------- .../datasource/cloudwatch/datasource.js | 77 ++++--- .../specs/annotation_query_specs.ts | 81 ------- 6 files changed, 263 insertions(+), 224 deletions(-) create mode 100644 pkg/tsdb/cloudwatch/annotation_query.go delete mode 100644 public/app/plugins/datasource/cloudwatch/annotation_query.d.ts delete mode 100644 public/app/plugins/datasource/cloudwatch/annotation_query.js delete mode 100644 public/app/plugins/datasource/cloudwatch/specs/annotation_query_specs.ts diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go new file mode 100644 index 00000000000..e9680e9ff84 --- /dev/null +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -0,0 +1,218 @@ +package cloudwatch + +import ( + "context" + "errors" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + firstQuery := queryContext.Queries[0] + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: firstQuery.RefId} + + parameters := firstQuery.Model + usePrefixMatch := parameters.Get("prefixMatching").MustBool() + region := parameters.Get("region").MustString("") + namespace := parameters.Get("namespace").MustString("") + metricName := parameters.Get("metricName").MustString("") + dimensions := parameters.Get("dimensions").MustMap() + statistics := parameters.Get("statistics").MustStringArray() + extendedStatistics := parameters.Get("extendedStatistics").MustStringArray() + period := int64(300) + if usePrefixMatch { + period = int64(parameters.Get("period").MustInt(0)) + } + actionPrefix := parameters.Get("actionPrefix").MustString("") + alarmNamePrefix := parameters.Get("alarmNamePrefix").MustString("") + + dsInfo := e.getDsInfo(region) + cfg, err := getAwsConfig(dsInfo) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:ListMetrics") + } + sess, err := session.NewSession(cfg) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:ListMetrics") + } + svc := cloudwatch.New(sess, cfg) + + var alarmNames []*string + if usePrefixMatch { + params := &cloudwatch.DescribeAlarmsInput{ + MaxRecords: aws.Int64(100), + ActionPrefix: aws.String(actionPrefix), + AlarmNamePrefix: aws.String(alarmNamePrefix), + } + resp, err := svc.DescribeAlarms(params) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:DescribeAlarms") + } + alarmNames = filterAlarms(resp, namespace, metricName, dimensions, statistics, extendedStatistics, period) + } else { + if region == "" || namespace == "" || metricName == "" || len(statistics) == 0 { + return result, nil + } + + var qd []*cloudwatch.Dimension + for k, v := range dimensions { + if vv, ok := v.(string); ok { + qd = append(qd, &cloudwatch.Dimension{ + Name: aws.String(k), + Value: aws.String(vv), + }) + } + } + for _, s := range statistics { + params := &cloudwatch.DescribeAlarmsForMetricInput{ + Namespace: aws.String(namespace), + MetricName: aws.String(metricName), + Period: aws.Int64(int64(period)), + Dimensions: qd, + Statistic: aws.String(s), + } + resp, err := svc.DescribeAlarmsForMetric(params) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:DescribeAlarmsForMetric") + } + for _, alarm := range resp.MetricAlarms { + alarmNames = append(alarmNames, alarm.AlarmName) + } + } + for _, s := range extendedStatistics { + params := &cloudwatch.DescribeAlarmsForMetricInput{ + Namespace: aws.String(namespace), + MetricName: aws.String(metricName), + Period: aws.Int64(int64(period)), + Dimensions: qd, + ExtendedStatistic: aws.String(s), + } + resp, err := svc.DescribeAlarmsForMetric(params) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:DescribeAlarmsForMetric") + } + for _, alarm := range resp.MetricAlarms { + alarmNames = append(alarmNames, alarm.AlarmName) + } + } + } + + startTime, err := queryContext.TimeRange.ParseFrom() + if err != nil { + return nil, err + } + + endTime, err := queryContext.TimeRange.ParseTo() + if err != nil { + return nil, err + } + + annotations := make([]map[string]string, 0) + for _, alarmName := range alarmNames { + params := &cloudwatch.DescribeAlarmHistoryInput{ + AlarmName: alarmName, + StartDate: aws.Time(startTime), + EndDate: aws.Time(endTime), + } + resp, err := svc.DescribeAlarmHistory(params) + if err != nil { + return nil, errors.New("Failed to call cloudwatch:DescribeAlarmHistory") + } + for _, history := range resp.AlarmHistoryItems { + annotation := make(map[string]string) + annotation["time"] = history.Timestamp.UTC().Format(time.RFC3339) + annotation["title"] = *history.AlarmName + annotation["tags"] = *history.HistoryItemType + annotation["text"] = *history.HistorySummary + annotations = append(annotations, annotation) + } + } + + transformAnnotationToTable(annotations, queryResult) + result.Results[firstQuery.RefId] = queryResult + return result, err +} + +func transformAnnotationToTable(data []map[string]string, result *tsdb.QueryResult) { + table := &tsdb.Table{ + Columns: make([]tsdb.TableColumn, 4), + Rows: make([]tsdb.RowValues, 0), + } + table.Columns[0].Text = "time" + table.Columns[1].Text = "title" + table.Columns[2].Text = "tags" + table.Columns[3].Text = "text" + + for _, r := range data { + values := make([]interface{}, 4) + values[0] = r["time"] + values[1] = r["title"] + values[2] = r["tags"] + values[3] = r["text"] + table.Rows = append(table.Rows, values) + } + result.Tables = append(result.Tables, table) + result.Meta.Set("rowCount", len(data)) +} + +func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, metricName string, dimensions map[string]interface{}, statistics []string, extendedStatistics []string, period int64) []*string { + alarmNames := make([]*string, 0) + + for _, alarm := range alarms.MetricAlarms { + if namespace != "" && *alarm.Namespace != namespace { + continue + } + if metricName != "" && *alarm.MetricName != metricName { + continue + } + + match := true + for _, d := range alarm.Dimensions { + if _, ok := dimensions[*d.Name]; !ok { + match = false + } + } + if !match { + continue + } + if period != 0 && *alarm.Period != period { + continue + } + + if len(statistics) != 0 { + found := false + for _, s := range statistics { + if *alarm.Statistic == s { + found = true + } + } + if !found { + continue + } + } + + if len(extendedStatistics) != 0 { + found := false + for _, s := range extendedStatistics { + if *alarm.Statistic == s { + found = true + } + } + if !found { + continue + } + } + + alarmNames = append(alarmNames, alarm.AlarmName) + } + + return alarmNames +} diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 122cb868fac..01d8983e33a 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -60,6 +60,9 @@ func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSourc case "metricFindQuery": result, err = e.executeMetricFindQuery(ctx, queryContext) break + case "annotationQuery": + result, err = e.executeAnnotationQuery(ctx, queryContext) + break case "timeSeriesQuery": fallthrough default: diff --git a/public/app/plugins/datasource/cloudwatch/annotation_query.d.ts b/public/app/plugins/datasource/cloudwatch/annotation_query.d.ts deleted file mode 100644 index c3318b8e133..00000000000 --- a/public/app/plugins/datasource/cloudwatch/annotation_query.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare var test: any; -export default test; diff --git a/public/app/plugins/datasource/cloudwatch/annotation_query.js b/public/app/plugins/datasource/cloudwatch/annotation_query.js deleted file mode 100644 index 20d04314e83..00000000000 --- a/public/app/plugins/datasource/cloudwatch/annotation_query.js +++ /dev/null @@ -1,106 +0,0 @@ -define([ - 'lodash', -], -function (_) { - 'use strict'; - - function CloudWatchAnnotationQuery(datasource, annotation, $q, templateSrv) { - this.datasource = datasource; - this.annotation = annotation; - this.$q = $q; - this.templateSrv = templateSrv; - } - - CloudWatchAnnotationQuery.prototype.process = function(from, to) { - var self = this; - var usePrefixMatch = this.annotation.prefixMatching; - var region = this.templateSrv.replace(this.annotation.region); - var namespace = this.templateSrv.replace(this.annotation.namespace); - var metricName = this.templateSrv.replace(this.annotation.metricName); - var dimensions = this.datasource.convertDimensionFormat(this.annotation.dimensions); - var statistics = _.map(this.annotation.statistics, function(s) { return self.templateSrv.replace(s); }); - var defaultPeriod = usePrefixMatch ? '' : '300'; - var period = this.annotation.period || defaultPeriod; - period = parseInt(period, 10); - var actionPrefix = this.annotation.actionPrefix || ''; - var alarmNamePrefix = this.annotation.alarmNamePrefix || ''; - - var d = this.$q.defer(); - var allQueryPromise; - if (usePrefixMatch) { - allQueryPromise = [ - this.datasource.performDescribeAlarms(region, actionPrefix, alarmNamePrefix, [], '').then(function(alarms) { - alarms.MetricAlarms = self.filterAlarms(alarms, namespace, metricName, dimensions, statistics, period); - return alarms; - }) - ]; - } else { - if (!region || !namespace || !metricName || _.isEmpty(statistics)) { return this.$q.when([]); } - - allQueryPromise = _.map(statistics, function(statistic) { - return self.datasource.performDescribeAlarmsForMetric(region, namespace, metricName, dimensions, statistic, period); - }); - } - this.$q.all(allQueryPromise).then(function(alarms) { - var eventList = []; - - var start = self.datasource.convertToCloudWatchTime(from, false); - var end = self.datasource.convertToCloudWatchTime(to, true); - _.chain(alarms) - .map('MetricAlarms') - .flatten() - .each(function(alarm) { - if (!alarm) { - d.resolve(eventList); - return; - } - - self.datasource.performDescribeAlarmHistory(region, alarm.AlarmName, start, end).then(function(history) { - _.each(history.AlarmHistoryItems, function(h) { - var event = { - annotation: self.annotation, - time: Date.parse(h.Timestamp), - title: h.AlarmName, - tags: [h.HistoryItemType], - text: h.HistorySummary - }; - - eventList.push(event); - }); - - d.resolve(eventList); - }); - }) - .value(); - }); - - return d.promise; - }; - - CloudWatchAnnotationQuery.prototype.filterAlarms = function(alarms, namespace, metricName, dimensions, statistics, period) { - return _.filter(alarms.MetricAlarms, function(alarm) { - if (!_.isEmpty(namespace) && alarm.Namespace !== namespace) { - return false; - } - if (!_.isEmpty(metricName) && alarm.MetricName !== metricName) { - return false; - } - var sd = function(d) { - return d.Name; - }; - var isSameDimensions = JSON.stringify(_.sortBy(alarm.Dimensions, sd)) === JSON.stringify(_.sortBy(dimensions, sd)); - if (!_.isEmpty(dimensions) && !isSameDimensions) { - return false; - } - if (!_.isEmpty(statistics) && !_.includes(statistics, alarm.Statistic)) { - return false; - } - if (!_.isNaN(period) && alarm.Period !== period) { - return false; - } - return true; - }); - }; - - return CloudWatchAnnotationQuery; -}); diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 370f1f971d5..0da80c5f944 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -5,9 +5,8 @@ define([ 'app/core/utils/datemath', 'app/core/utils/kbn', 'app/features/templating/variable', - './annotation_query', ], -function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnotationQuery) { +function (angular, _, moment, dateMath, kbn, templatingVariable) { 'use strict'; /** @ngInject */ @@ -262,44 +261,52 @@ function (angular, _, moment, dateMath, kbn, templatingVariable, CloudWatchAnnot return $q.when([]); }; - this.performDescribeAlarms = function(region, actionPrefix, alarmNamePrefix, alarmNames, stateValue) { - return this.awsRequest({ - region: region, - action: 'DescribeAlarms', - parameters: { actionPrefix: actionPrefix, alarmNamePrefix: alarmNamePrefix, alarmNames: alarmNames, stateValue: stateValue } + this.annotationQuery = function (options) { + var annotation = options.annotation; + var defaultPeriod = annotation.prefixMatching ? '' : '300'; + var period = annotation.period || defaultPeriod; + period = parseInt(period, 10); + var dimensions = {}; + _.each(annotation.dimensions, function (value, key) { + dimensions[templateSrv.replace(key, options.scopedVars)] = templateSrv.replace(value, options.scopedVars); }); - }; + var parameters = { + prefixMatching: annotation.prefixMatching, + region: templateSrv.replace(annotation.region), + namespace: templateSrv.replace(annotation.namespace), + metricName: templateSrv.replace(annotation.metricName), + dimensions: dimensions, + statistics: _.map(annotation.statistics, function (s) { return templateSrv.replace(s); }), + period: period, + actionPrefix: annotation.actionPrefix || '', + alarmNamePrefix: annotation.alarmNamePrefix || '' + }; - this.performDescribeAlarmsForMetric = function(region, namespace, metricName, dimensions, statistic, period) { - var s = _.includes(self.standardStatistics, statistic) ? statistic : ''; - var es = _.includes(self.standardStatistics, statistic) ? '' : statistic; - return this.awsRequest({ - region: region, - action: 'DescribeAlarmsForMetric', - parameters: { - namespace: namespace, - metricName: metricName, - dimensions: dimensions, - statistic: s, - extendedStatistic: es, - period: period - } + return backendSrv.post('/api/tsdb/query', { + from: options.range.from, + to: options.range.to, + queries: [ + _.extend({ + refId: 'annotationQuery', + intervalMs: 1, // dummy + maxDataPoints: 1, // dummy + datasourceId: this.instanceSettings.id, + type: 'annotationQuery' + }, parameters) + ] + }).then(function (r) { + return _.map(r.results['annotationQuery'].tables[0].rows, function (v) { + return { + annotation: annotation, + time: Date.parse(v[0]), + title: v[1], + tags: [v[2]], + text: v[3] + }; + }); }); }; - this.performDescribeAlarmHistory = function(region, alarmName, startDate, endDate) { - return this.awsRequest({ - region: region, - action: 'DescribeAlarmHistory', - parameters: { alarmName: alarmName, startDate: startDate, endDate: endDate } - }); - }; - - this.annotationQuery = function(options) { - var annotationQuery = new CloudWatchAnnotationQuery(this, options.annotation, $q, templateSrv); - return annotationQuery.process(options.range.from, options.range.to); - }; - this.testDatasource = function() { /* use billing metrics for test */ var region = this.defaultRegion; diff --git a/public/app/plugins/datasource/cloudwatch/specs/annotation_query_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/annotation_query_specs.ts deleted file mode 100644 index 8e9bb5a0c9e..00000000000 --- a/public/app/plugins/datasource/cloudwatch/specs/annotation_query_specs.ts +++ /dev/null @@ -1,81 +0,0 @@ -import "../datasource"; -import {describe, beforeEach, it, expect, angularMocks} from 'test/lib/common'; -import moment from 'moment'; -import helpers from 'test/specs/helpers'; -import CloudWatchDatasource from "../datasource"; -import CloudWatchAnnotationQuery from '../annotation_query'; - -describe('CloudWatchAnnotationQuery', function() { - var ctx = new helpers.ServiceTestContext(); - var instanceSettings = { - jsonData: {defaultRegion: 'us-east-1', access: 'proxy'}, - }; - - beforeEach(angularMocks.module('grafana.core')); - beforeEach(angularMocks.module('grafana.services')); - beforeEach(angularMocks.module('grafana.controllers')); - beforeEach(ctx.providePhase(['templateSrv', 'backendSrv'])); - - beforeEach(angularMocks.inject(function($q, $rootScope, $httpBackend, $injector) { - ctx.$q = $q; - ctx.$httpBackend = $httpBackend; - ctx.$rootScope = $rootScope; - ctx.ds = $injector.instantiate(CloudWatchDatasource, {instanceSettings: instanceSettings}); - })); - - describe('When performing annotationQuery', function() { - var parameter = { - annotation: { - region: 'us-east-1', - namespace: 'AWS/EC2', - metricName: 'CPUUtilization', - dimensions: { - InstanceId: 'i-12345678' - }, - statistics: ['Average'], - period: 300 - }, - range: { - from: moment(1443438674760), - to: moment(1443460274760) - } - }; - var alarmResponse = { - MetricAlarms: [ - { - AlarmName: 'test_alarm_name' - } - ] - }; - var historyResponse = { - AlarmHistoryItems: [ - { - Timestamp: '2015-01-01T00:00:00.000Z', - HistoryItemType: 'StateUpdate', - AlarmName: 'test_alarm_name', - HistoryData: '{}', - HistorySummary: 'test_history_summary' - } - ] - }; - beforeEach(function() { - ctx.backendSrv.datasourceRequest = function(params) { - switch (params.data.action) { - case 'DescribeAlarmsForMetric': - return ctx.$q.when({data: alarmResponse}); - case 'DescribeAlarmHistory': - return ctx.$q.when({data: historyResponse}); - } - }; - }); - it('should return annotation list', function(done) { - var annotationQuery = new CloudWatchAnnotationQuery(ctx.ds, parameter.annotation, ctx.$q, ctx.templateSrv); - annotationQuery.process(parameter.range.from, parameter.range.to).then(function(result) { - expect(result[0].title).to.be('test_alarm_name'); - expect(result[0].text).to.be('test_history_summary'); - done(); - }); - ctx.$rootScope.$apply(); - }); - }); -}); From d98d8a404f4231f8b77cfe07fd221fb87d0af0bc Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 15:06:21 +0900 Subject: [PATCH 32/44] fix dimension convertion --- pkg/tsdb/cloudwatch/metric_find_query.go | 10 ++++----- .../datasource/cloudwatch/datasource.js | 21 ++++++------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index f09c7118137..e38e97e3d16 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -335,14 +335,14 @@ func (e *CloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param namespace := parameters.Get("namespace").MustString() metricName := parameters.Get("metricName").MustString() dimensionKey := parameters.Get("dimensionKey").MustString() - dimensionsJson := parameters.Get("dimensionKey").MustMap() + dimensionsJson := parameters.Get("dimensions").MustMap() var dimensions []*cloudwatch.DimensionFilter - for _, d := range dimensionsJson { - if dd, ok := d.(map[string]string); ok { + for k, v := range dimensionsJson { + if vv, ok := v.(string); ok { dimensions = append(dimensions, &cloudwatch.DimensionFilter{ - Name: aws.String(dd["Name"]), - Value: aws.String(dd["Value"]), + Name: aws.String(k), + Value: aws.String(vv), }) } } diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 0da80c5f944..9e655722c02 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -40,11 +40,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { item.region = templateSrv.replace(item.region, options.scopedVars); item.namespace = templateSrv.replace(item.namespace, options.scopedVars); item.metricName = templateSrv.replace(item.metricName, options.scopedVars); - var dimensions = {}; - _.each(item.dimensions, function (value, key) { - dimensions[templateSrv.replace(key, options.scopedVars)] = templateSrv.replace(value, options.scopedVars); - }); - item.dimensions = dimensions; + item.dimensions = self.convertDimensionFormat(item.dimensions, options.scopeVars); item.period = self.getPeriod(item, options); return _.extend({ @@ -266,16 +262,12 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { var defaultPeriod = annotation.prefixMatching ? '' : '300'; var period = annotation.period || defaultPeriod; period = parseInt(period, 10); - var dimensions = {}; - _.each(annotation.dimensions, function (value, key) { - dimensions[templateSrv.replace(key, options.scopedVars)] = templateSrv.replace(value, options.scopedVars); - }); var parameters = { prefixMatching: annotation.prefixMatching, region: templateSrv.replace(annotation.region), namespace: templateSrv.replace(annotation.namespace), metricName: templateSrv.replace(annotation.metricName), - dimensions: dimensions, + dimensions: this.convertDimensionFormat(annotation.dimensions, {}), statistics: _.map(annotation.statistics, function (s) { return templateSrv.replace(s); }), period: period, actionPrefix: annotation.actionPrefix || '', @@ -385,12 +377,11 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { }; this.convertDimensionFormat = function(dimensions, scopedVars) { - return _.map(dimensions, function(value, key) { - return { - Name: templateSrv.replace(key, scopedVars), - Value: templateSrv.replace(value, scopedVars) - }; + var convertedDimensions = {}; + _.each(dimensions, function (value, key) { + convertedDimensions[templateSrv.replace(key, scopedVars)] = templateSrv.replace(value, scopedVars); }); + return convertedDimensions; }; } From bf5268c0b4c7484a63376a49d60a9feddb29147f Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 15:25:49 +0900 Subject: [PATCH 33/44] fix time --- .../app/plugins/datasource/cloudwatch/datasource.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 9e655722c02..de369429b4f 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -60,8 +60,8 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { } var request = { - from: options.rangeRaw.from, - to: options.rangeRaw.to, + from: options.range.from.valueOf().toString(), + to: options.range.to.valueOf().toString(), queries: queries }; @@ -138,8 +138,8 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { this.doMetricQueryRequest = function (subtype, parameters) { var range = timeSrv.timeRange(); return backendSrv.post('/api/tsdb/query', { - from: range.from, - to: range.to, + from: range.from.valueOf().toString(), + to: range.to.valueOf().toString(), queries: [ _.extend({ refId: 'metricFindQuery', @@ -275,8 +275,8 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { }; return backendSrv.post('/api/tsdb/query', { - from: options.range.from, - to: options.range.to, + from: options.range.from.valueOf().toString(), + to: options.range.to.valueOf().toString(), queries: [ _.extend({ refId: 'annotationQuery', From e1fe15e0949164ad1379bad8678c4be1fdd889fb Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 15:45:52 +0900 Subject: [PATCH 34/44] fix annotation query --- pkg/tsdb/cloudwatch/annotation_query.go | 32 +++++++++++-------- .../datasource/cloudwatch/datasource.js | 4 ++- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go index e9680e9ff84..2283e12b64e 100644 --- a/pkg/tsdb/cloudwatch/annotation_query.go +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -20,16 +20,16 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: firstQuery.RefId} parameters := firstQuery.Model - usePrefixMatch := parameters.Get("prefixMatching").MustBool() + usePrefixMatch := parameters.Get("prefixMatching").MustBool(false) region := parameters.Get("region").MustString("") namespace := parameters.Get("namespace").MustString("") metricName := parameters.Get("metricName").MustString("") dimensions := parameters.Get("dimensions").MustMap() statistics := parameters.Get("statistics").MustStringArray() extendedStatistics := parameters.Get("extendedStatistics").MustStringArray() - period := int64(300) - if usePrefixMatch { - period = int64(parameters.Get("period").MustInt(0)) + period := int64(parameters.Get("period").MustInt(0)) + if period == 0 && !usePrefixMatch { + period = 300 } actionPrefix := parameters.Get("actionPrefix").MustString("") alarmNamePrefix := parameters.Get("alarmNamePrefix").MustString("") @@ -75,9 +75,9 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo params := &cloudwatch.DescribeAlarmsForMetricInput{ Namespace: aws.String(namespace), MetricName: aws.String(metricName), - Period: aws.Int64(int64(period)), Dimensions: qd, Statistic: aws.String(s), + Period: aws.Int64(int64(period)), } resp, err := svc.DescribeAlarmsForMetric(params) if err != nil { @@ -91,9 +91,9 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo params := &cloudwatch.DescribeAlarmsForMetricInput{ Namespace: aws.String(namespace), MetricName: aws.String(metricName), - Period: aws.Int64(int64(period)), Dimensions: qd, ExtendedStatistic: aws.String(s), + Period: aws.Int64(int64(period)), } resp, err := svc.DescribeAlarmsForMetric(params) if err != nil { @@ -109,7 +109,6 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo if err != nil { return nil, err } - endTime, err := queryContext.TimeRange.ParseTo() if err != nil { return nil, err @@ -175,17 +174,20 @@ func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, met } match := true - for _, d := range alarm.Dimensions { - if _, ok := dimensions[*d.Name]; !ok { - match = false + if len(dimensions) == 0 { + // all match + } else if len(alarm.Dimensions) != len(dimensions) { + match = false + } else { + for _, d := range alarm.Dimensions { + if _, ok := dimensions[*d.Name]; !ok { + match = false + } } } if !match { continue } - if period != 0 && *alarm.Period != period { - continue - } if len(statistics) != 0 { found := false @@ -211,6 +213,10 @@ func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, met } } + if period != 0 && *alarm.Period != period { + continue + } + alarmNames = append(alarmNames, alarm.AlarmName) } diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index de369429b4f..61222216b2a 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -259,6 +259,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { this.annotationQuery = function (options) { var annotation = options.annotation; + var statistics = _.map(annotation.statistics, function (s) { return templateSrv.replace(s); }); var defaultPeriod = annotation.prefixMatching ? '' : '300'; var period = annotation.period || defaultPeriod; period = parseInt(period, 10); @@ -268,7 +269,8 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { namespace: templateSrv.replace(annotation.namespace), metricName: templateSrv.replace(annotation.metricName), dimensions: this.convertDimensionFormat(annotation.dimensions, {}), - statistics: _.map(annotation.statistics, function (s) { return templateSrv.replace(s); }), + statistics: _.filter(statistics, function (s) { return _.includes(self.standardStatistics, s); }), + extendedStatistics: _.filter(statistics, function (s) { return !_.includes(self.standardStatistics, s); }), period: period, actionPrefix: annotation.actionPrefix || '', alarmNamePrefix: annotation.alarmNamePrefix || '' From 4f5f38f41b68c3fd2fb1955ed405d528e29f8a2f Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 17:59:34 +0900 Subject: [PATCH 35/44] remove old handler --- pkg/api/cloudwatch/cloudwatch.go | 169 ------------------------------- pkg/api/pluginproxy/ds_proxy.go | 6 -- 2 files changed, 175 deletions(-) diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/api/cloudwatch/cloudwatch.go index 7b6a189cfde..b6b0547a232 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/api/cloudwatch/cloudwatch.go @@ -1,10 +1,7 @@ package cloudwatch import ( - "encoding/json" - "errors" "fmt" - "io/ioutil" "os" "strings" "sync" @@ -16,16 +13,10 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/sts" - "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" ) -type actionHandler func(*cwRequest, *middleware.Context) - -var actionHandlers map[string]actionHandler - type cwRequest struct { Region string `json:"region"` Action string `json:"action"` @@ -69,14 +60,6 @@ func (req *cwRequest) GetDatasourceInfo() *DatasourceInfo { } } -func init() { - actionHandlers = map[string]actionHandler{ - "DescribeAlarms": handleDescribeAlarms, - "DescribeAlarmsForMetric": handleDescribeAlarmsForMetric, - "DescribeAlarmHistory": handleDescribeAlarmHistory, - } -} - type cache struct { credential *credentials.Credentials expiration *time.Time @@ -212,155 +195,3 @@ func getAwsConfig(req *cwRequest) (*aws.Config, error) { } return cfg, nil } - -func handleDescribeAlarms(req *cwRequest, c *middleware.Context) { - cfg, err := getAwsConfig(req) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - sess, err := session.NewSession(cfg) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - svc := cloudwatch.New(sess, cfg) - - reqParam := &struct { - Parameters struct { - ActionPrefix string `json:"actionPrefix"` - AlarmNamePrefix string `json:"alarmNamePrefix"` - AlarmNames []*string `json:"alarmNames"` - StateValue string `json:"stateValue"` - } `json:"parameters"` - }{} - json.Unmarshal(req.Body, reqParam) - - params := &cloudwatch.DescribeAlarmsInput{ - MaxRecords: aws.Int64(100), - } - if reqParam.Parameters.ActionPrefix != "" { - params.ActionPrefix = aws.String(reqParam.Parameters.ActionPrefix) - } - if reqParam.Parameters.AlarmNamePrefix != "" { - params.AlarmNamePrefix = aws.String(reqParam.Parameters.AlarmNamePrefix) - } - if len(reqParam.Parameters.AlarmNames) != 0 { - params.AlarmNames = reqParam.Parameters.AlarmNames - } - if reqParam.Parameters.StateValue != "" { - params.StateValue = aws.String(reqParam.Parameters.StateValue) - } - - resp, err := svc.DescribeAlarms(params) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - - c.JSON(200, resp) -} - -func handleDescribeAlarmsForMetric(req *cwRequest, c *middleware.Context) { - cfg, err := getAwsConfig(req) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - sess, err := session.NewSession(cfg) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - svc := cloudwatch.New(sess, cfg) - - reqParam := &struct { - Parameters struct { - Namespace string `json:"namespace"` - MetricName string `json:"metricName"` - Dimensions []*cloudwatch.Dimension `json:"dimensions"` - Statistic string `json:"statistic"` - ExtendedStatistic string `json:"extendedStatistic"` - Period int64 `json:"period"` - } `json:"parameters"` - }{} - json.Unmarshal(req.Body, reqParam) - - params := &cloudwatch.DescribeAlarmsForMetricInput{ - Namespace: aws.String(reqParam.Parameters.Namespace), - MetricName: aws.String(reqParam.Parameters.MetricName), - Period: aws.Int64(reqParam.Parameters.Period), - } - if len(reqParam.Parameters.Dimensions) != 0 { - params.Dimensions = reqParam.Parameters.Dimensions - } - if reqParam.Parameters.Statistic != "" { - params.Statistic = aws.String(reqParam.Parameters.Statistic) - } - if reqParam.Parameters.ExtendedStatistic != "" { - params.ExtendedStatistic = aws.String(reqParam.Parameters.ExtendedStatistic) - } - - resp, err := svc.DescribeAlarmsForMetric(params) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - - c.JSON(200, resp) -} - -func handleDescribeAlarmHistory(req *cwRequest, c *middleware.Context) { - cfg, err := getAwsConfig(req) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - sess, err := session.NewSession(cfg) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - svc := cloudwatch.New(sess, cfg) - - reqParam := &struct { - Parameters struct { - AlarmName string `json:"alarmName"` - HistoryItemType string `json:"historyItemType"` - StartDate int64 `json:"startDate"` - EndDate int64 `json:"endDate"` - } `json:"parameters"` - }{} - json.Unmarshal(req.Body, reqParam) - - params := &cloudwatch.DescribeAlarmHistoryInput{ - AlarmName: aws.String(reqParam.Parameters.AlarmName), - StartDate: aws.Time(time.Unix(reqParam.Parameters.StartDate, 0)), - EndDate: aws.Time(time.Unix(reqParam.Parameters.EndDate, 0)), - } - if reqParam.Parameters.HistoryItemType != "" { - params.HistoryItemType = aws.String(reqParam.Parameters.HistoryItemType) - } - - resp, err := svc.DescribeAlarmHistory(params) - if err != nil { - c.JsonApiErr(500, "Unable to call AWS API", err) - return - } - - c.JSON(200, resp) -} - -func HandleRequest(c *middleware.Context, ds *m.DataSource) { - var req cwRequest - req.Body, _ = ioutil.ReadAll(c.Req.Request.Body) - req.DataSource = ds - json.Unmarshal(req.Body, &req) - - if handler, found := actionHandlers[req.Action]; !found { - c.JsonApiErr(500, "Unexpected AWS Action", errors.New(req.Action)) - return - } else { - handler(&req, c) - } -} diff --git a/pkg/api/pluginproxy/ds_proxy.go b/pkg/api/pluginproxy/ds_proxy.go index 10fafe21fef..faac8c03c62 100644 --- a/pkg/api/pluginproxy/ds_proxy.go +++ b/pkg/api/pluginproxy/ds_proxy.go @@ -17,7 +17,6 @@ import ( "github.com/opentracing/opentracing-go" - "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" @@ -63,11 +62,6 @@ func NewDataSourceProxy(ds *m.DataSource, plugin *plugins.DataSourcePlugin, ctx } func (proxy *DataSourceProxy) HandleRequest() { - if proxy.ds.Type == m.DS_CLOUDWATCH { - cloudwatch.HandleRequest(proxy.ctx, proxy.ds) - return - } - if err := proxy.validateRequest(); err != nil { proxy.ctx.JsonApiErr(403, err.Error(), nil) return From fe9fca381c2bc75e5ece3deec778aca93244d619 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 18:01:07 +0900 Subject: [PATCH 36/44] move cloudwatch crendential related code --- pkg/tsdb/cloudwatch/cloudwatch.go | 5 ++-- .../cloudwatch/credentials.go} | 6 ++--- .../cloudwatch/credentials_test.go} | 0 pkg/tsdb/cloudwatch/metric_find_query.go | 26 +++++-------------- pkg/tsdb/cloudwatch/metric_find_query_test.go | 9 +++---- 5 files changed, 15 insertions(+), 31 deletions(-) rename pkg/{api/cloudwatch/cloudwatch.go => tsdb/cloudwatch/credentials.go} (97%) rename pkg/{api/cloudwatch/cloudwatch_test.go => tsdb/cloudwatch/credentials_test.go} (100%) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 01d8983e33a..17bf8ccd08a 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -17,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" - cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/metrics" @@ -128,7 +127,7 @@ func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, e } } - datasourceInfo := &cwapi.DatasourceInfo{ + datasourceInfo := &DatasourceInfo{ Region: region, Profile: e.DataSource.Database, AssumeRoleArn: assumeRoleArn, @@ -136,7 +135,7 @@ func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, e SecretKey: secretKey, } - credentials, err := cwapi.GetCredentials(datasourceInfo) + credentials, err := GetCredentials(datasourceInfo) if err != nil { return nil, err } diff --git a/pkg/api/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/credentials.go similarity index 97% rename from pkg/api/cloudwatch/cloudwatch.go rename to pkg/tsdb/cloudwatch/credentials.go index b6b0547a232..8921eaa9315 100644 --- a/pkg/api/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -183,14 +183,14 @@ func ec2RoleProvider(sess *session.Session) credentials.Provider { return &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute} } -func getAwsConfig(req *cwRequest) (*aws.Config, error) { - creds, err := GetCredentials(req.GetDatasourceInfo()) +func getAwsConfig(dsInfo *DatasourceInfo) (*aws.Config, error) { + creds, err := GetCredentials(dsInfo) if err != nil { return nil, err } cfg := &aws.Config{ - Region: aws.String(req.Region), + Region: aws.String(dsInfo.Region), Credentials: creds, } return cfg, nil diff --git a/pkg/api/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/credentials_test.go similarity index 100% rename from pkg/api/cloudwatch/cloudwatch_test.go rename to pkg/tsdb/cloudwatch/credentials_test.go diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index e38e97e3d16..12ca4ed2d78 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -14,7 +14,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/ec2" - cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/tsdb" @@ -211,7 +210,7 @@ func transformToTable(data []suggestData, result *tsdb.QueryResult) { result.Meta.Set("rowCount", len(data)) } -func (e *CloudWatchExecutor) getDsInfo(region string) *cwapi.DatasourceInfo { +func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo { assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() accessKey := "" secretKey := "" @@ -224,7 +223,7 @@ func (e *CloudWatchExecutor) getDsInfo(region string) *cwapi.DatasourceInfo { } } - datasourceInfo := &cwapi.DatasourceInfo{ + datasourceInfo := &DatasourceInfo{ Region: region, Profile: e.DataSource.Database, AssumeRoleArn: assumeRoleArn, @@ -460,19 +459,6 @@ func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, return result, nil } -func getAwsConfig(dsInfo *cwapi.DatasourceInfo) (*aws.Config, error) { - creds, err := cwapi.GetCredentials(dsInfo) - if err != nil { - return nil, err - } - - cfg := &aws.Config{ - Region: aws.String(dsInfo.Region), - Credentials: creds, - } - return cfg, nil -} - func (e *CloudWatchExecutor) cloudwatchListMetrics(region string, namespace string, metricName string, dimensions []*cloudwatch.DimensionFilter) (*cloudwatch.ListMetricsOutput, error) { dsInfo := e.getDsInfo(region) cfg, err := getAwsConfig(dsInfo) @@ -541,8 +527,8 @@ func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2. return &resp, nil } -func getAllMetrics(cwData *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { - creds, err := cwapi.GetCredentials(cwData) +func getAllMetrics(cwData *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + creds, err := GetCredentials(cwData) if err != nil { return cloudwatch.ListMetricsOutput{}, err } @@ -579,7 +565,7 @@ func getAllMetrics(cwData *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, var metricsCacheLock sync.Mutex -func getMetricsForCustomMetrics(dsInfo *cwapi.DatasourceInfo, getAllMetrics func(*cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +func getMetricsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { metricsCacheLock.Lock() defer metricsCacheLock.Unlock() @@ -616,7 +602,7 @@ func getMetricsForCustomMetrics(dsInfo *cwapi.DatasourceInfo, getAllMetrics func var dimensionsCacheLock sync.Mutex -func getDimensionsForCustomMetrics(dsInfo *cwapi.DatasourceInfo, getAllMetrics func(*cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +func getDimensionsForCustomMetrics(dsInfo *DatasourceInfo, getAllMetrics func(*DatasourceInfo) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { dimensionsCacheLock.Lock() defer dimensionsCacheLock.Unlock() diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index 2b5dcaec247..238e815fac1 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -5,20 +5,19 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" - cwapi "github.com/grafana/grafana/pkg/api/cloudwatch" . "github.com/smartystreets/goconvey/convey" ) func TestCloudWatchMetrics(t *testing.T) { Convey("When calling getMetricsForCustomMetrics", t, func() { - dsInfo := &cwapi.DatasourceInfo{ + dsInfo := &DatasourceInfo{ Region: "us-east-1", Namespace: "Foo", Profile: "default", AssumeRoleArn: "", } - f := func(dsInfo *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + f := func(dsInfo *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { return cloudwatch.ListMetricsOutput{ Metrics: []*cloudwatch.Metric{ { @@ -40,13 +39,13 @@ func TestCloudWatchMetrics(t *testing.T) { }) Convey("When calling getDimensionsForCustomMetrics", t, func() { - dsInfo := &cwapi.DatasourceInfo{ + dsInfo := &DatasourceInfo{ Region: "us-east-1", Namespace: "Foo", Profile: "default", AssumeRoleArn: "", } - f := func(dsInfo *cwapi.DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { + f := func(dsInfo *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) { return cloudwatch.ListMetricsOutput{ Metrics: []*cloudwatch.Metric{ { From 59cdd4d8d21cb0aff220f6be348fbdebabab0e84 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 18:08:46 +0900 Subject: [PATCH 37/44] remove obsolete code --- pkg/tsdb/cloudwatch/credentials.go | 33 ------------------------------ 1 file changed, 33 deletions(-) diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 8921eaa9315..882d2e3320a 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -14,16 +14,8 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sts" - m "github.com/grafana/grafana/pkg/models" ) -type cwRequest struct { - Region string `json:"region"` - Action string `json:"action"` - Body []byte `json:"-"` - DataSource *m.DataSource -} - type DatasourceInfo struct { Profile string Region string @@ -35,31 +27,6 @@ type DatasourceInfo struct { SecretKey string } -func (req *cwRequest) GetDatasourceInfo() *DatasourceInfo { - authType := req.DataSource.JsonData.Get("authType").MustString() - assumeRoleArn := req.DataSource.JsonData.Get("assumeRoleArn").MustString() - accessKey := "" - secretKey := "" - - for key, value := range req.DataSource.SecureJsonData.Decrypt() { - if key == "accessKey" { - accessKey = value - } - if key == "secretKey" { - secretKey = value - } - } - - return &DatasourceInfo{ - AuthType: authType, - AssumeRoleArn: assumeRoleArn, - Region: req.Region, - Profile: req.DataSource.Database, - AccessKey: accessKey, - SecretKey: secretKey, - } -} - type cache struct { credential *credentials.Credentials expiration *time.Time From a21f53cf82d0c6ddd41513540e441d01893f5840 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 26 Sep 2017 18:30:40 +0900 Subject: [PATCH 38/44] refactor cloudwatch code --- pkg/tsdb/cloudwatch/annotation_query.go | 11 +---- pkg/tsdb/cloudwatch/cloudwatch.go | 53 +++++------------------ pkg/tsdb/cloudwatch/credentials.go | 54 ++++++++++++++++++------ pkg/tsdb/cloudwatch/metric_find_query.go | 36 ++-------------- 4 files changed, 58 insertions(+), 96 deletions(-) diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go index 2283e12b64e..3736c19b100 100644 --- a/pkg/tsdb/cloudwatch/annotation_query.go +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -6,7 +6,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" @@ -34,16 +33,10 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo actionPrefix := parameters.Get("actionPrefix").MustString("") alarmNamePrefix := parameters.Get("alarmNamePrefix").MustString("") - dsInfo := e.getDsInfo(region) - cfg, err := getAwsConfig(dsInfo) + svc, err := e.getClient(region) if err != nil { - return nil, errors.New("Failed to call cloudwatch:ListMetrics") + return nil, err } - sess, err := session.NewSession(cfg) - if err != nil { - return nil, errors.New("Failed to call cloudwatch:ListMetrics") - } - svc := cloudwatch.New(sess, cfg) var alarmNames []*string if usePrefixMatch { diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 17bf8ccd08a..6f4cad3dec6 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -15,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" @@ -26,6 +25,17 @@ type CloudWatchExecutor struct { *models.DataSource } +type DatasourceInfo struct { + Profile string + Region string + AuthType string + AssumeRoleArn string + Namespace string + + AccessKey string + SecretKey string +} + func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { return &CloudWatchExecutor{}, nil } @@ -113,47 +123,6 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo return result, nil } -func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, error) { - assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() - - accessKey := "" - secretKey := "" - for key, value := range e.DataSource.SecureJsonData.Decrypt() { - if key == "accessKey" { - accessKey = value - } - if key == "secretKey" { - secretKey = value - } - } - - datasourceInfo := &DatasourceInfo{ - Region: region, - Profile: e.DataSource.Database, - AssumeRoleArn: assumeRoleArn, - AccessKey: accessKey, - SecretKey: secretKey, - } - - credentials, err := GetCredentials(datasourceInfo) - if err != nil { - return nil, err - } - - cfg := &aws.Config{ - Region: aws.String(region), - Credentials: credentials, - } - - sess, err := session.NewSession(cfg) - if err != nil { - return nil, err - } - - client := cloudwatch.New(sess, cfg) - return client, nil -} - func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) { query, err := parseQuery(parameters) if err != nil { diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 882d2e3320a..81e0262a3d4 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -13,20 +13,10 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/sts" ) -type DatasourceInfo struct { - Profile string - Region string - AuthType string - AssumeRoleArn string - Namespace string - - AccessKey string - SecretKey string -} - type cache struct { credential *credentials.Credentials expiration *time.Time @@ -150,7 +140,31 @@ func ec2RoleProvider(sess *session.Session) credentials.Provider { return &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute} } -func getAwsConfig(dsInfo *DatasourceInfo) (*aws.Config, error) { +func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo { + assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() + accessKey := "" + secretKey := "" + for key, value := range e.DataSource.SecureJsonData.Decrypt() { + if key == "accessKey" { + accessKey = value + } + if key == "secretKey" { + secretKey = value + } + } + + datasourceInfo := &DatasourceInfo{ + Region: region, + Profile: e.DataSource.Database, + AssumeRoleArn: assumeRoleArn, + AccessKey: accessKey, + SecretKey: secretKey, + } + + return datasourceInfo +} + +func (e *CloudWatchExecutor) getAwsConfig(dsInfo *DatasourceInfo) (*aws.Config, error) { creds, err := GetCredentials(dsInfo) if err != nil { return nil, err @@ -162,3 +176,19 @@ func getAwsConfig(dsInfo *DatasourceInfo) (*aws.Config, error) { } return cfg, nil } + +func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, error) { + datasourceInfo := e.getDsInfo(region) + cfg, err := e.getAwsConfig(datasourceInfo) + if err != nil { + return nil, err + } + + sess, err := session.NewSession(cfg) + if err != nil { + return nil, err + } + + client := cloudwatch.New(sess, cfg) + return client, nil +} diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 12ca4ed2d78..bc5525965d8 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -210,30 +210,6 @@ func transformToTable(data []suggestData, result *tsdb.QueryResult) { result.Meta.Set("rowCount", len(data)) } -func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo { - assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() - accessKey := "" - secretKey := "" - for key, value := range e.DataSource.SecureJsonData.Decrypt() { - if key == "accessKey" { - accessKey = value - } - if key == "secretKey" { - secretKey = value - } - } - - datasourceInfo := &DatasourceInfo{ - Region: region, - Profile: e.DataSource.Database, - AssumeRoleArn: assumeRoleArn, - AccessKey: accessKey, - SecretKey: secretKey, - } - - return datasourceInfo -} - // Whenever this list is updated, frontend list should also be updated. // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { @@ -460,16 +436,10 @@ func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, } func (e *CloudWatchExecutor) cloudwatchListMetrics(region string, namespace string, metricName string, dimensions []*cloudwatch.DimensionFilter) (*cloudwatch.ListMetricsOutput, error) { - dsInfo := e.getDsInfo(region) - cfg, err := getAwsConfig(dsInfo) + svc, err := e.getClient(region) if err != nil { - return nil, errors.New("Failed to call cloudwatch:ListMetrics") + return nil, err } - sess, err := session.NewSession(cfg) - if err != nil { - return nil, errors.New("Failed to call cloudwatch:ListMetrics") - } - svc := cloudwatch.New(sess, cfg) params := &cloudwatch.ListMetricsInput{ Namespace: aws.String(namespace), @@ -496,7 +466,7 @@ func (e *CloudWatchExecutor) cloudwatchListMetrics(region string, namespace stri func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.Filter, instanceIds []*string) (*ec2.DescribeInstancesOutput, error) { dsInfo := e.getDsInfo(region) - cfg, err := getAwsConfig(dsInfo) + cfg, err := e.getAwsConfig(dsInfo) if err != nil { return nil, errors.New("Failed to call ec2:DescribeInstances") } From f3a2dc7c5f0593cf3b1c99f458a196052a8ce05e Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 27 Sep 2017 00:00:38 +0900 Subject: [PATCH 39/44] improve cloudwatch tsdb --- pkg/tsdb/cloudwatch/annotation_query.go | 7 +-- pkg/tsdb/cloudwatch/metric_find_query.go | 56 ++++++++++++------------ 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go index 3736c19b100..a1c22d1542f 100644 --- a/pkg/tsdb/cloudwatch/annotation_query.go +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -110,9 +110,10 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo annotations := make([]map[string]string, 0) for _, alarmName := range alarmNames { params := &cloudwatch.DescribeAlarmHistoryInput{ - AlarmName: alarmName, - StartDate: aws.Time(startTime), - EndDate: aws.Time(endTime), + AlarmName: alarmName, + StartDate: aws.Time(startTime), + EndDate: aws.Time(endTime), + MaxRecords: aws.Int64(100), } resp, err := svc.DescribeAlarmHistory(params) if err != nil { diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index bc5525965d8..3f4f7bea9ef 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -392,40 +392,42 @@ func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, result := make([]suggestData, 0) dupCheck := make(map[string]bool) - for _, instance := range instances.Reservations[0].Instances { - tags := make(map[string]string) - for _, tag := range instance.Tags { - tags[*tag.Key] = *tag.Value - } + for _, reservation := range instances.Reservations { + for _, instance := range reservation.Instances { + tags := make(map[string]string) + for _, tag := range instance.Tags { + tags[*tag.Key] = *tag.Value + } - var data string - if strings.Index(attributeName, "Tags.") == 0 { - tagName := attributeName[5:] - data = tags[tagName] - } else { - attributePath := strings.Split(attributeName, ".") - v := reflect.ValueOf(instance) - for _, key := range attributePath { - if v.Kind() == reflect.Ptr { - v = v.Elem() + var data string + if strings.Index(attributeName, "Tags.") == 0 { + tagName := attributeName[5:] + data = tags[tagName] + } else { + attributePath := strings.Split(attributeName, ".") + v := reflect.ValueOf(instance) + for _, key := range attributePath { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Struct { + return nil, errors.New("invalid attribute path") + } + v = v.FieldByName(key) } - if v.Kind() != reflect.Struct { + if attr, ok := v.Interface().(*string); ok { + data = *attr + } else { return nil, errors.New("invalid attribute path") } - v = v.FieldByName(key) } - if attr, ok := v.Interface().(*string); ok { - data = *attr - } else { - return nil, errors.New("invalid attribute path") - } - } - if _, exists := dupCheck[data]; exists { - continue + if _, exists := dupCheck[data]; exists { + continue + } + dupCheck[data] = true + result = append(result, suggestData{Text: data, Value: data}) } - dupCheck[data] = true - result = append(result, suggestData{Text: data, Value: data}) } sort.Slice(result, func(i, j int) bool { From 6c375ed2cb605b6af9246a7f826d4ccf78619750 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 27 Sep 2017 00:18:05 +0900 Subject: [PATCH 40/44] fix assume role --- pkg/tsdb/cloudwatch/credentials.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 81e0262a3d4..784f3b729ac 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -141,6 +141,7 @@ func ec2RoleProvider(sess *session.Session) credentials.Provider { } func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo { + authType := e.DataSource.JsonData.Get("authType").MustString() assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() accessKey := "" secretKey := "" @@ -156,6 +157,7 @@ func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo { datasourceInfo := &DatasourceInfo{ Region: region, Profile: e.DataSource.Database, + AuthType: authType, AssumeRoleArn: assumeRoleArn, AccessKey: accessKey, SecretKey: secretKey, From 468e8c13ee481128bc46b22a88e25c67d125f3e2 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 27 Sep 2017 13:18:30 +0900 Subject: [PATCH 41/44] move extend statistics handling code to backend --- pkg/tsdb/cloudwatch/annotation_query.go | 6 ++++-- pkg/tsdb/cloudwatch/cloudwatch.go | 14 +++++++------- .../plugins/datasource/cloudwatch/datasource.js | 3 +-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go index a1c22d1542f..287f4e770ef 100644 --- a/pkg/tsdb/cloudwatch/annotation_query.go +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -24,8 +24,10 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo namespace := parameters.Get("namespace").MustString("") metricName := parameters.Get("metricName").MustString("") dimensions := parameters.Get("dimensions").MustMap() - statistics := parameters.Get("statistics").MustStringArray() - extendedStatistics := parameters.Get("extendedStatistics").MustStringArray() + statistics, extendedStatistics, err := parseStatistics(parameters) + if err != nil { + return nil, err + } period := int64(parameters.Get("period").MustInt(0)) if period == 0 && !usePrefixMatch { period = 300 diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 6f4cad3dec6..764aaceb65c 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -194,16 +194,16 @@ func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) { return result, nil } -func parseStatistics(model *simplejson.Json) ([]*string, []*string, error) { - var statistics []*string - var extendedStatistics []*string +func parseStatistics(model *simplejson.Json) ([]string, []string, error) { + var statistics []string + var extendedStatistics []string for _, s := range model.Get("statistics").MustArray() { if ss, ok := s.(string); ok { if _, isStandard := standardStatistics[ss]; isStandard { - statistics = append(statistics, &ss) + statistics = append(statistics, ss) } else { - extendedStatistics = append(extendedStatistics, &ss) + extendedStatistics = append(extendedStatistics, ss) } } else { return nil, nil, errors.New("failed to parse") @@ -269,8 +269,8 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { Namespace: namespace, MetricName: metricName, Dimensions: dimensions, - Statistics: statistics, - ExtendedStatistics: extendedStatistics, + Statistics: aws.StringSlice(statistics), + ExtendedStatistics: aws.StringSlice(extendedStatistics), Period: period, Alias: alias, }, nil diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 61222216b2a..192713b4bd4 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -269,8 +269,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { namespace: templateSrv.replace(annotation.namespace), metricName: templateSrv.replace(annotation.metricName), dimensions: this.convertDimensionFormat(annotation.dimensions, {}), - statistics: _.filter(statistics, function (s) { return _.includes(self.standardStatistics, s); }), - extendedStatistics: _.filter(statistics, function (s) { return !_.includes(self.standardStatistics, s); }), + statistics: statistics, period: period, actionPrefix: annotation.actionPrefix || '', alarmNamePrefix: annotation.alarmNamePrefix || '' From 5e88177f2849c8d626584bbaeb44af8990c7d9c6 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Thu, 28 Sep 2017 02:31:09 +0900 Subject: [PATCH 42/44] add debug log --- pkg/tsdb/cloudwatch/cloudwatch.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 764aaceb65c..081edcccbe5 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb" "github.com/aws/aws-sdk-go/aws" @@ -159,6 +160,10 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl params.ExtendedStatistics = query.ExtendedStatistics } + if setting.Env == setting.DEV { + plog.Debug("CloudWatch query", "raw query", params) + } + resp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second)) if err != nil { return nil, err From 8d6513a56490ccf61dcf59149f866667d38aa2c0 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Thu, 28 Sep 2017 03:00:17 +0900 Subject: [PATCH 43/44] fix cloudwatch alert bug --- pkg/tsdb/cloudwatch/cloudwatch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 081edcccbe5..266b71ec14e 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -94,7 +94,7 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo currentlyExecuting := 0 for i, model := range queryContext.Queries { queryType := model.Model.Get("type").MustString() - if queryType != "timeSeriesQuery" { + if queryType != "timeSeriesQuery" && queryType != "" { continue } currentlyExecuting++ From 5850ae51b180becf5a50d8a070921721fb40a8a3 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Fri, 29 Sep 2017 13:24:18 +0900 Subject: [PATCH 44/44] fix, add targetContainsTemplate() --- public/app/plugins/datasource/cloudwatch/datasource.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index 192713b4bd4..f6ea196e764 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -300,6 +300,15 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { }); }; + this.targetContainsTemplate = function(target) { + return templateSrv.variableExists(target.region) || + templateSrv.variableExists(target.namespace) || + templateSrv.variableExists(target.metricName) || + _.find(target.dimensions, function(v, k) { + return templateSrv.variableExists(k) || templateSrv.variableExists(v); + }); + }; + this.testDatasource = function() { /* use billing metrics for test */ var region = this.defaultRegion;