diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 8ed16980dee..1a522ed1a48 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -9,11 +9,17 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" @@ -45,75 +51,95 @@ var plog = log.New("tsdb.cloudwatch") var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) func init() { - tsdb.RegisterTsdbQueryEndpoint("cloudwatch", newcloudWatchExecutor) + tsdb.RegisterTsdbQueryEndpoint("cloudwatch", func(ds *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + return newExecutor(), nil + }) } -func newcloudWatchExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - e := &cloudWatchExecutor{ - DataSource: datasource, +func newExecutor() *cloudWatchExecutor { + return &cloudWatchExecutor{ + logsClientsByRegion: map[string]cloudwatchlogsiface.CloudWatchLogsAPI{}, } - - dsInfo := e.getDSInfo(defaultRegion) - defaultLogsClient, err := retrieveLogsClient(dsInfo) - if err != nil { - return nil, err - } - e.logsClientsByRegion = map[string]*cloudwatchlogs.CloudWatchLogs{ - dsInfo.Region: defaultLogsClient, - defaultRegion: defaultLogsClient, - } - - return e, nil } // cloudWatchExecutor executes CloudWatch requests. type cloudWatchExecutor struct { *models.DataSource - ec2Svc ec2iface.EC2API - rgtaSvc resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI - logsClientsByRegion map[string](*cloudwatchlogs.CloudWatchLogs) - mux sync.Mutex + ec2Client ec2iface.EC2API + rgtaClient resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI + logsClientsByRegion map[string]cloudwatchlogsiface.CloudWatchLogsAPI + mtx sync.Mutex } -func (e *cloudWatchExecutor) getCWClient(region string) (*cloudwatch.CloudWatch, error) { - datasourceInfo := e.getDSInfo(region) - cfg, err := getAwsConfig(datasourceInfo) +func (e *cloudWatchExecutor) newSession(region string) (*session.Session, error) { + dsInfo := e.getDSInfo(region) + creds, err := getCredentials(dsInfo) if err != nil { return nil, err } - sess, err := newSession(cfg) + cfg := &aws.Config{ + Region: aws.String(dsInfo.Region), + Credentials: creds, + } + return newSession(cfg) +} + +func (e *cloudWatchExecutor) getCWClient(region string) (cloudwatchiface.CloudWatchAPI, error) { + sess, err := e.newSession(region) if err != nil { return nil, err } - - client := cloudwatch.New(sess, cfg) - - client.Handlers.Send.PushFront(func(r *request.Request) { - r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) - }) - - return client, nil + return newCWClient(sess), nil } -func (e *cloudWatchExecutor) getCWLogsClient(region string) (*cloudwatchlogs.CloudWatchLogs, error) { - e.mux.Lock() - defer e.mux.Unlock() +func (e *cloudWatchExecutor) getCWLogsClient(region string) (cloudwatchlogsiface.CloudWatchLogsAPI, error) { + e.mtx.Lock() + defer e.mtx.Unlock() if logsClient, ok := e.logsClientsByRegion[region]; ok { return logsClient, nil } - dsInfo := e.getDSInfo(region) - newLogsClient, err := retrieveLogsClient(dsInfo) + sess, err := e.newSession(region) if err != nil { return nil, err } - e.logsClientsByRegion[region] = newLogsClient + logsClient := newCWLogsClient(sess) + e.logsClientsByRegion[region] = logsClient - return newLogsClient, nil + return logsClient, nil +} + +func (e *cloudWatchExecutor) getEC2Client(region string) (ec2iface.EC2API, error) { + if e.ec2Client != nil { + return e.ec2Client, nil + } + + sess, err := e.newSession(region) + if err != nil { + return nil, err + } + e.ec2Client = newEC2Client(sess) + + return e.ec2Client, nil +} + +func (e *cloudWatchExecutor) getRGTAClient(region string) (resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI, + error) { + if e.rgtaClient != nil { + return e.rgtaClient, nil + } + + sess, err := e.newSession(region) + if err != nil { + return nil, err + } + e.rgtaClient = newRGTAClient(sess) + + return e.rgtaClient, nil } func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, @@ -279,26 +305,44 @@ func (e *cloudWatchExecutor) getDSInfo(region string) *datasourceInfo { } } -func retrieveLogsClient(dsInfo *datasourceInfo) (*cloudwatchlogs.CloudWatchLogs, error) { - cfg, err := getAwsConfig(dsInfo) - if err != nil { - return nil, err - } - - sess, err := newSession(cfg) - if err != nil { - return nil, err - } - - client := cloudwatchlogs.New(sess, cfg) +func isTerminated(queryStatus string) bool { + return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout" +} +// CloudWatch client factory. +// +// Stubbable by tests. +var newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { + client := cloudwatch.New(sess) client.Handlers.Send.PushFront(func(r *request.Request) { r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) }) - return client, nil + return client } -func isTerminated(queryStatus string) bool { - return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout" +// CloudWatch logs client factory. +// +// Stubbable by tests. +var newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + client := cloudwatchlogs.New(sess) + client.Handlers.Send.PushFront(func(r *request.Request) { + r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) + }) + + return client +} + +// EC2 client factory. +// +// Stubbable by tests. +var newEC2Client = func(provider client.ConfigProvider) ec2iface.EC2API { + return ec2.New(provider) +} + +// RGTA client factory. +// +// Stubbable by tests. +var newRGTAClient = func(provider client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI { + return resourcegroupstaggingapi.New(provider) } diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 2f6cb5a17fd..8689983dc7c 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -174,17 +174,3 @@ func ecsCredProvider(sess *session.Session, uri string) credentials.Provider { func ec2RoleProvider(sess client.ConfigProvider) credentials.Provider { return &ec2rolecreds.EC2RoleProvider{Client: newEC2Metadata(sess), ExpiryWindow: 5 * time.Minute} } - -func getAwsConfig(dsInfo *datasourceInfo) (*aws.Config, error) { - creds, err := getCredentials(dsInfo) - if err != nil { - return nil, err - } - - cfg := &aws.Config{ - Region: aws.String(dsInfo.Region), - Credentials: creds, - } - - return cfg, nil -} diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go index 4cf0b16bf14..755ecf3903f 100644 --- a/pkg/tsdb/cloudwatch/log_actions_test.go +++ b/pkg/tsdb/cloudwatch/log_actions_test.go @@ -7,7 +7,9 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" @@ -16,84 +18,351 @@ import ( "github.com/stretchr/testify/require" ) -//*** -// LogActions Tests -//*** +func TestQuery_DescribeLogGroups(t *testing.T) { + origNewCWLogsClient := newCWLogsClient + t.Cleanup(func() { + newCWLogsClient = origNewCWLogsClient + }) -func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsEmpty(t *testing.T) { - executor := &cloudWatchExecutor{} + var cli fakeCWLogsClient - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), + newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + return cli + } + + t.Run("Empty log group name prefix", func(t *testing.T) { + cli = fakeCWLogsClient{ + logGroups: cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{ + { + LogGroupName: aws.String("group_a"), + }, + { + LogGroupName: aws.String("group_b"), + }, + { + LogGroupName: aws.String("group_c"), + }, + }, + }, + } + + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "DescribeLogGroups", + "limit": 50, + }), + }, + }, + }) + require.NoError(t, err) + require.NotNil(t, resp) + + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Dataframes: tsdb.NewDecodedDataFrames(data.Frames{ + &data.Frame{ + Name: "logGroups", + Fields: []*data.Field{ + data.NewField("logGroupName", nil, []*string{ + aws.String("group_a"), aws.String("group_b"), aws.String("group_c"), + }), + }, + Meta: &data.FrameMeta{ + PreferredVisualization: "logs", + }, + }, + }), + }, + }, + }, resp) + }) + + t.Run("Non-empty log group name prefix", func(t *testing.T) { + cli = fakeCWLogsClient{ + logGroups: cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{ + { + LogGroupName: aws.String("group_a"), + }, + { + LogGroupName: aws.String("group_b"), + }, + { + LogGroupName: aws.String("group_c"), + }, + }, + }, + } + + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "DescribeLogGroups", + "logGroupNamePrefix": "g", + }), + }, + }, + }) + require.NoError(t, err) + require.NotNil(t, resp) + + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Dataframes: tsdb.NewDecodedDataFrames(data.Frames{ + &data.Frame{ + Name: "logGroups", + Fields: []*data.Field{ + data.NewField("logGroupName", nil, []*string{ + aws.String("group_a"), aws.String("group_b"), aws.String("group_c"), + }), + }, + Meta: &data.FrameMeta{ + PreferredVisualization: "logs", + }, + }, + }), + }, + }, + }, resp) + }) +} + +func TestQuery_GetLogGroupFields(t *testing.T) { + origNewCWLogsClient := newCWLogsClient + t.Cleanup(func() { + newCWLogsClient = origNewCWLogsClient + }) + + var cli fakeCWLogsClient + + newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + return cli + } + + cli = fakeCWLogsClient{ + logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{ + LogGroupFields: []*cloudwatchlogs.LogGroupField{ + { + Name: aws.String("field_a"), + Percent: aws.Int64(100), + }, + { + Name: aws.String("field_b"), + Percent: aws.Int64(30), + }, + { + Name: aws.String("field_c"), + Percent: aws.Int64(55), + }, + }, }, } - params := simplejson.NewFromAny(map[string]interface{}{ - "limit": 50, + const refID = "A" + + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + RefId: refID, + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "GetLogGroupFields", + "logGroupName": "group_a", + "limit": 50, + }), + }, + }, }) + require.NoError(t, err) + require.NotNil(t, resp) - frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params) - - expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")}) - expectedFrame := data.NewFrame("logGroups", expectedField) - - assert.Equal(t, nil, err) - assert.Equal(t, expectedFrame, frame) -} - -func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) { - executor := &cloudWatchExecutor{} - - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), + expFrame := &data.Frame{ + Name: refID, + Fields: []*data.Field{ + data.NewField("name", nil, []*string{ + aws.String("field_a"), aws.String("field_b"), aws.String("field_c"), + }), + data.NewField("percent", nil, []*int64{ + aws.Int64(100), aws.Int64(30), aws.Int64(55), + }), + }, + Meta: &data.FrameMeta{ + PreferredVisualization: "logs", }, } - - params := simplejson.NewFromAny(map[string]interface{}{ - "logGroupNamePrefix": "g", - }) - - frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params) - - expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")}) - expectedFrame := data.NewFrame("logGroups", expectedField) - assert.Equal(t, nil, err) - assert.Equal(t, expectedFrame, frame) + expFrame.RefID = refID + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + refID: { + Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}), + RefId: refID, + }, + }, + }, resp) } -func TestHandleGetLogGroupFields_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) { - executor := &cloudWatchExecutor{} +func TestQuery_StartQuery(t *testing.T) { + origNewCWLogsClient := newCWLogsClient + t.Cleanup(func() { + newCWLogsClient = origNewCWLogsClient + }) - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), - }, + var cli fakeCWLogsClient + + newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + return cli } - params := simplejson.NewFromAny(map[string]interface{}{ - "logGroupName": "group_a", - "limit": 50, + t.Run("invalid time range", func(t *testing.T) { + cli = fakeCWLogsClient{ + logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{ + LogGroupFields: []*cloudwatchlogs.LogGroupField{ + { + Name: aws.String("field_a"), + Percent: aws.Int64(100), + }, + { + Name: aws.String("field_b"), + Percent: aws.Int64(30), + }, + { + Name: aws.String("field_c"), + Percent: aws.Int64(55), + }, + }, + }, + } + + timeRange := &tsdb.TimeRange{ + From: "1584873443000", + To: "1584700643000", + } + + executor := newExecutor() + _, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + TimeRange: timeRange, + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "StartQuery", + "limit": 50, + "region": "default", + "queryString": "fields @message", + }), + }, + }, + }) + require.Error(t, err) + + assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err) }) - frame, err := executor.handleGetLogGroupFields(context.Background(), logsClient, params, "A") + t.Run("valid time range", func(t *testing.T) { + const refID = "A" + cli = fakeCWLogsClient{ + logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{ + LogGroupFields: []*cloudwatchlogs.LogGroupField{ + { + Name: aws.String("field_a"), + Percent: aws.Int64(100), + }, + { + Name: aws.String("field_b"), + Percent: aws.Int64(30), + }, + { + Name: aws.String("field_c"), + Percent: aws.Int64(55), + }, + }, + }, + } - expectedNameField := data.NewField("name", nil, []*string{aws.String("field_a"), aws.String("field_b"), aws.String("field_c")}) - expectedPercentField := data.NewField("percent", nil, []*int64{aws.Int64(100), aws.Int64(30), aws.Int64(55)}) - expectedFrame := data.NewFrame("A", expectedNameField, expectedPercentField) - expectedFrame.RefID = "A" + timeRange := &tsdb.TimeRange{ + From: "1584700643000", + To: "1584873443000", + } - assert.Equal(t, nil, err) - assert.Equal(t, expectedFrame, frame) + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + TimeRange: timeRange, + Queries: []*tsdb.Query{ + { + RefId: refID, + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "StartQuery", + "limit": 50, + "region": "default", + "queryString": "fields @message", + }), + }, + }, + }) + require.NoError(t, err) + + expFrame := data.NewFrame( + refID, + data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"}), + ) + expFrame.RefID = refID + expFrame.Meta = &data.FrameMeta{ + Custom: map[string]interface{}{ + "Region": "default", + }, + PreferredVisualization: "logs", + } + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + refID: { + Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}), + RefId: refID, + }, + }, + }, resp) + }) } -func TestExecuteStartQuery(t *testing.T) { - executor := &cloudWatchExecutor{} +func TestQuery_StopQuery(t *testing.T) { + origNewCWLogsClient := newCWLogsClient + t.Cleanup(func() { + newCWLogsClient = origNewCWLogsClient + }) - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), + var cli fakeCWLogsClient + + newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + return cli + } + + cli = fakeCWLogsClient{ + logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{ + LogGroupFields: []*cloudwatchlogs.LogGroupField{ + { + Name: aws.String("field_a"), + Percent: aws.Int64(100), + }, + { + Name: aws.String("field_b"), + Percent: aws.Int64(30), + }, + { + Name: aws.String("field_c"), + Percent: aws.Int64(55), + }, + }, }, } @@ -102,109 +371,122 @@ func TestExecuteStartQuery(t *testing.T) { To: "1584700643000", } - params := simplejson.NewFromAny(map[string]interface{}{ - "region": "default", - "limit": 50, - "queryString": "fields @message", + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + TimeRange: timeRange, + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "StopQuery", + "queryId": "abcd-efgh-ijkl-mnop", + }), + }, + }, }) + require.NoError(t, err) - response, err := executor.executeStartQuery(context.Background(), logsClient, params, timeRange) - - var expectedResponse *cloudwatchlogs.StartQueryOutput = nil - - assert.Equal(t, expectedResponse, response) - assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err) + expFrame := &data.Frame{ + Name: "StopQueryResponse", + Fields: []*data.Field{ + data.NewField("success", nil, []bool{true}), + }, + Meta: &data.FrameMeta{ + PreferredVisualization: "logs", + }, + } + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}), + }, + }, + }, resp) } -func TestHandleStartQuery(t *testing.T) { - executor := &cloudWatchExecutor{} - - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), - }, - } - - timeRange := &tsdb.TimeRange{ - From: "1584700643000", - To: "1584873443000", - } - - params := simplejson.NewFromAny(map[string]interface{}{ - "region": "default", - "limit": 50, - "queryString": "fields @message", +func TestQuery_GetQueryResults(t *testing.T) { + origNewCWLogsClient := newCWLogsClient + t.Cleanup(func() { + newCWLogsClient = origNewCWLogsClient }) - frame, err := executor.handleStartQuery(context.Background(), logsClient, params, timeRange, "A") + var cli fakeCWLogsClient - expectedField := data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"}) - expectedFrame := data.NewFrame("A", expectedField) - expectedFrame.RefID = "A" - expectedFrame.Meta = &data.FrameMeta{ - Custom: map[string]interface{}{ - "Region": "default", + newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + return cli + } + + const refID = "A" + cli = fakeCWLogsClient{ + queryResults: cloudwatchlogs.GetQueryResultsOutput{ + Results: [][]*cloudwatchlogs.ResultField{ + { + { + Field: aws.String("@timestamp"), + Value: aws.String("2020-03-20 10:37:23.000"), + }, + { + Field: aws.String("field_b"), + Value: aws.String("b_1"), + }, + { + Field: aws.String("@ptr"), + Value: aws.String("abcdefg"), + }, + }, + { + { + Field: aws.String("@timestamp"), + Value: aws.String("2020-03-20 10:40:43.000"), + }, + { + Field: aws.String("field_b"), + Value: aws.String("b_2"), + }, + { + Field: aws.String("@ptr"), + Value: aws.String("hijklmnop"), + }, + }, + }, + Statistics: &cloudwatchlogs.QueryStatistics{ + BytesScanned: aws.Float64(512), + RecordsMatched: aws.Float64(256), + RecordsScanned: aws.Float64(1024), + }, + Status: aws.String("Complete"), }, } - assert.Equal(t, nil, err) - assert.Equal(t, expectedFrame, frame) -} - -func TestHandleStopQuery(t *testing.T) { - executor := &cloudWatchExecutor{} - - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + RefId: refID, + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "logAction", + "subtype": "GetQueryResults", + "queryId": "abcd-efgh-ijkl-mnop", + }), + }, }, - } - - params := simplejson.NewFromAny(map[string]interface{}{ - "queryId": "abcd-efgh-ijkl-mnop", }) - - frame, err := executor.handleStopQuery(context.Background(), logsClient, params) - - expectedField := data.NewField("success", nil, []bool{true}) - expectedFrame := data.NewFrame("StopQueryResponse", expectedField) - - assert.Equal(t, nil, err) - assert.Equal(t, expectedFrame, frame) -} - -func TestHandleGetQueryResults(t *testing.T) { - executor := &cloudWatchExecutor{} - - logsClient := &FakeLogsClient{ - Config: aws.Config{ - Region: aws.String("default"), - }, - } - - params := simplejson.NewFromAny(map[string]interface{}{ - "queryId": "abcd-efgh-ijkl-mnop", - }) - - frame, err := executor.handleGetQueryResults(context.Background(), logsClient, params, "A") require.NoError(t, err) - timeA, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000") - require.NoError(t, err) - timeB, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000") - require.NoError(t, err) - expectedTimeField := data.NewField("@timestamp", nil, []*time.Time{ - aws.Time(timeA), aws.Time(timeB), - }) - expectedTimeField.SetConfig(&data.FieldConfig{DisplayName: "Time"}) - expectedFieldB := data.NewField("field_b", nil, []*string{ + time1, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000") + require.NoError(t, err) + time2, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000") + require.NoError(t, err) + expField1 := data.NewField("@timestamp", nil, []*time.Time{ + aws.Time(time1), aws.Time(time2), + }) + expField1.SetConfig(&data.FieldConfig{DisplayName: "Time"}) + expField2 := data.NewField("field_b", nil, []*string{ aws.String("b_1"), aws.String("b_2"), }) - - expectedFrame := data.NewFrame("A", expectedTimeField, expectedFieldB) - expectedFrame.RefID = "A" - - expectedFrame.Meta = &data.FrameMeta{ + expFrame := data.NewFrame(refID, expField1, expField2) + expFrame.RefID = refID + expFrame.Meta = &data.FrameMeta{ Custom: map[string]interface{}{ "Status": "Complete", "Statistics": cloudwatchlogs.QueryStatistics{ @@ -213,9 +495,15 @@ func TestHandleGetQueryResults(t *testing.T) { RecordsScanned: aws.Float64(1024), }, }, + PreferredVisualization: "logs", } - assert.Equal(t, nil, err) - assert.ElementsMatch(t, expectedFrame.Fields, frame.Fields) - assert.Equal(t, expectedFrame.Meta, frame.Meta) + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + refID: { + RefId: refID, + Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}), + }, + }, + }, resp) } diff --git a/pkg/tsdb/cloudwatch/logs_client_mock.go b/pkg/tsdb/cloudwatch/logs_client_mock.go deleted file mode 100644 index 1a4a67f209c..00000000000 --- a/pkg/tsdb/cloudwatch/logs_client_mock.go +++ /dev/null @@ -1,106 +0,0 @@ -package cloudwatch - -import ( - "context" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" -) - -type FakeLogsClient struct { - cloudwatchlogsiface.CloudWatchLogsAPI - Config aws.Config -} - -func (f FakeLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { - return &cloudwatchlogs.DescribeLogGroupsOutput{ - LogGroups: []*cloudwatchlogs.LogGroup{ - { - LogGroupName: aws.String("group_a"), - }, - { - LogGroupName: aws.String("group_b"), - }, - { - LogGroupName: aws.String("group_c"), - }, - }, - }, nil -} - -func (f FakeLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) { - return &cloudwatchlogs.GetLogGroupFieldsOutput{ - LogGroupFields: []*cloudwatchlogs.LogGroupField{ - { - Name: aws.String("field_a"), - Percent: aws.Int64(100), - }, - { - Name: aws.String("field_b"), - Percent: aws.Int64(30), - }, - { - Name: aws.String("field_c"), - Percent: aws.Int64(55), - }, - }, - }, nil -} - -func (f FakeLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) { - return &cloudwatchlogs.StartQueryOutput{ - QueryId: aws.String("abcd-efgh-ijkl-mnop"), - }, nil -} - -func (f FakeLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) { - return &cloudwatchlogs.StopQueryOutput{ - Success: aws.Bool(true), - }, nil -} - -func (f FakeLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) { - return &cloudwatchlogs.GetQueryResultsOutput{ - Results: [][]*cloudwatchlogs.ResultField{ - { - { - Field: aws.String("@timestamp"), - Value: aws.String("2020-03-20 10:37:23.000"), - }, - { - Field: aws.String("field_b"), - Value: aws.String("b_1"), - }, - { - Field: aws.String("@ptr"), - Value: aws.String("abcdefg"), - }, - }, - - { - { - Field: aws.String("@timestamp"), - Value: aws.String("2020-03-20 10:40:43.000"), - }, - { - Field: aws.String("field_b"), - Value: aws.String("b_2"), - }, - { - Field: aws.String("@ptr"), - Value: aws.String("hijklmnop"), - }, - }, - }, - - Statistics: &cloudwatchlogs.QueryStatistics{ - BytesScanned: aws.Float64(512), - RecordsMatched: aws.Float64(256), - RecordsScanned: aws.Float64(1024), - }, - - Status: aws.String("Complete"), - }, nil -} diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index d892966c02f..39e0ff9ecc3 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -12,7 +12,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awsutil" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" @@ -310,8 +309,7 @@ func parseMultiSelectValue(input string) []string { // Whenever this list is updated, the frontend list should also be updated. // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { - const region = "default" - dsInfo := e.getDSInfo(region) + dsInfo := e.getDSInfo(defaultRegion) profile := dsInfo.Profile if cache, ok := regionCache.Load(profile); ok { if cache2, ok2 := cache.([]suggestData); ok2 { @@ -319,12 +317,12 @@ func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s } } - err := e.ensureClientSession("default") + client, err := e.getEC2Client(defaultRegion) if err != nil { return nil, err } regions := knownRegions - r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{}) + r, err := client.DescribeRegions(&ec2.DescribeRegionsInput{}) if err != nil { // ignore error for backward compatibility plog.Error("Failed to get regions", "error", err) @@ -389,7 +387,7 @@ func (e *cloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s dsInfo := e.getDSInfo(region) dsInfo.Namespace = namespace - if namespaceMetrics, err = e.getMetricsForCustomMetrics(region, e.getAllMetrics); err != nil { + if namespaceMetrics, err = e.getMetricsForCustomMetrics(region); err != nil { return nil, errors.New("Unable to call AWS API") } } @@ -418,7 +416,7 @@ func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters dsInfo := e.getDSInfo(region) dsInfo.Namespace = namespace - if dimensionValues, err = e.getDimensionsForCustomMetrics(region, e.getAllMetrics); err != nil { + if dimensionValues, err = e.getDimensionsForCustomMetrics(region); err != nil { return nil, errors.New("Unable to call AWS API") } } @@ -483,31 +481,10 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param return result, nil } -func (e *cloudWatchExecutor) ensureClientSession(region string) error { - if e.ec2Svc == nil { - dsInfo := e.getDSInfo(region) - cfg, err := getAwsConfig(dsInfo) - if err != nil { - return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err) - } - sess, err := session.NewSession(cfg) - if err != nil { - return fmt.Errorf("Failed to call ec2:NewSession, %w", err) - } - e.ec2Svc = ec2.New(sess, cfg) - } - return nil -} - func (e *cloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() instanceId := parameters.Get("instanceId").MustString() - err := e.ensureClientSession(region) - if err != nil { - return nil, err - } - instanceIds := aws.StringSlice(parseMultiSelectValue(instanceId)) instances, err := e.ec2DescribeInstances(region, nil, instanceIds) if err != nil { @@ -547,11 +524,6 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, } } - err := e.ensureClientSession(region) - if err != nil { - return nil, err - } - instances, err := e.ec2DescribeInstances(region, filters, nil) if err != nil { return nil, err @@ -610,32 +582,11 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, return result, nil } -func (e *cloudWatchExecutor) ensureRGTAClientSession(region string) error { - if e.rgtaSvc == nil { - dsInfo := e.getDSInfo(region) - cfg, err := getAwsConfig(dsInfo) - if err != nil { - return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err) - } - sess, err := session.NewSession(cfg) - if err != nil { - return fmt.Errorf("Failed to call ec2:NewSession, %w", err) - } - e.rgtaSvc = resourcegroupstaggingapi.New(sess, cfg) - } - return nil -} - func (e *cloudWatchExecutor) handleGetResourceArns(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { region := parameters.Get("region").MustString() resourceType := parameters.Get("resourceType").MustString() filterJson := parameters.Get("tags").MustMap() - err := e.ensureRGTAClientSession(region) - if err != nil { - return nil, err - } - var filters []*resourcegroupstaggingapi.TagFilter for k, v := range filterJson { if vv, ok := v.([]interface{}); ok { @@ -706,8 +657,13 @@ func (e *cloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2. InstanceIds: instanceIds, } + client, err := e.getEC2Client(region) + if err != nil { + return nil, err + } + var resp ec2.DescribeInstancesOutput - if err := e.ec2Svc.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { + if err := client.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { resp.Reservations = append(resp.Reservations, page.Reservations...) return !lastPage }); err != nil { @@ -724,8 +680,13 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [ TagFilters: filters, } + client, err := e.getRGTAClient(region) + if err != nil { + return nil, err + } + var resp resourcegroupstaggingapi.GetResourcesOutput - if err := e.rgtaSvc.GetResourcesPages(params, + if err := client.GetResourcesPages(params, func(page *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { resp.ResourceTagMappingList = append(resp.ResourceTagMappingList, page.ResourceTagMappingList...) return !lastPage @@ -737,27 +698,18 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [ } func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetricsOutput, error) { - dsInfo := e.getDSInfo(region) - creds, err := getCredentials(dsInfo) + client, err := e.getCWClient(region) if err != nil { return cloudwatch.ListMetricsOutput{}, err } - cfg := &aws.Config{ - Region: aws.String(dsInfo.Region), - Credentials: creds, - } - sess, err := session.NewSession(cfg) - if err != nil { - return cloudwatch.ListMetricsOutput{}, err - } - svc := cloudwatch.New(sess, cfg) + dsInfo := e.getDSInfo(region) params := &cloudwatch.ListMetricsInput{ Namespace: aws.String(dsInfo.Namespace), } var resp cloudwatch.ListMetricsOutput - err = svc.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { + err = client.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { metrics.MAwsCloudWatchListMetrics.Inc() metrics, err := awsutil.ValuesAtPath(page, "Metrics") if err != nil { @@ -769,12 +721,13 @@ func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetric } return !lastPage }) + return resp, err } var metricsCacheLock sync.Mutex -func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string) ([]string, error) { metricsCacheLock.Lock() defer metricsCacheLock.Unlock() @@ -794,7 +747,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil } - result, err := getAllMetrics(region) + result, err := e.getAllMetrics(region) if err != nil { return []string{}, err } @@ -813,7 +766,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet var dimensionsCacheLock sync.Mutex -func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { +func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string) ([]string, error) { dimensionsCacheLock.Lock() defer dimensionsCacheLock.Unlock() @@ -833,7 +786,7 @@ func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAll if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil } - result, err := getAllMetrics(region) + result, err := e.getAllMetrics(region) if err != nil { return []string{}, err } diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index b2c085d83c2..0edb7933940 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -5,241 +5,236 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" - "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type mockedEc2 struct { - ec2iface.EC2API - Resp ec2.DescribeInstancesOutput - RespRegions ec2.DescribeRegionsOutput -} - -type mockedRGTA struct { - resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI - Resp resourcegroupstaggingapi.GetResourcesOutput -} - -func (m mockedEc2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error { - fn(&m.Resp, true) - return nil -} -func (m mockedEc2) DescribeRegions(in *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) { - return &m.RespRegions, nil -} - -func (m mockedRGTA) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput, fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error { - fn(&m.Resp, true) - return nil -} - -func TestCloudWatchMetrics(t *testing.T) { - t.Run("When calling getMetricsForCustomMetrics", func(t *testing.T) { - const region = "us-east-1" - e := &cloudWatchExecutor{ - DataSource: &models.DataSource{ - Database: "default", - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "Region": region, - }), - }, - } - f := func(region string) (cloudwatch.ListMetricsOutput, error) { - return cloudwatch.ListMetricsOutput{ - Metrics: []*cloudwatch.Metric{ - { - MetricName: aws.String("Test_MetricName"), - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("Test_DimensionName"), - }, - }, - }, - }, - }, nil - } - metrics, err := e.getMetricsForCustomMetrics(region, f) - require.NoError(t, err) - - assert.Contains(t, metrics, "Test_MetricName") +func TestQuery_Metrics(t *testing.T) { + origNewCWClient := newCWClient + t.Cleanup(func() { + newCWClient = origNewCWClient }) - t.Run("When calling getDimensionsForCustomMetrics", func(t *testing.T) { - const region = "us-east-1" - e := &cloudWatchExecutor{ - DataSource: &models.DataSource{ - Database: "default", - JsonData: simplejson.NewFromAny(map[string]interface{}{ - "Region": region, - }), - }, - } - f := func(region string) (cloudwatch.ListMetricsOutput, error) { - return cloudwatch.ListMetricsOutput{ - Metrics: []*cloudwatch.Metric{ - { - MetricName: aws.String("Test_MetricName"), - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("Test_DimensionName"), - }, - }, - }, - }, - }, nil - } - dimensionKeys, err := e.getDimensionsForCustomMetrics(region, f) - require.NoError(t, err) + var client fakeCWClient - assert.Contains(t, dimensionKeys, "Test_DimensionName") - }) + newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { + return client + } - t.Run("When calling handleGetRegions", func(t *testing.T) { - executor := &cloudWatchExecutor{ - ec2Svc: mockedEc2{RespRegions: ec2.DescribeRegionsOutput{ - Regions: []*ec2.Region{ - { - RegionName: aws.String("ap-northeast-2"), - }, - }, - }}, - } - jsonData := simplejson.New() - jsonData.Set("defaultRegion", "default") - executor.DataSource = &models.DataSource{ - JsonData: jsonData, - SecureJsonData: securejsondata.SecureJsonData{}, - } - - result, err := executor.handleGetRegions(context.Background(), simplejson.New(), &tsdb.TsdbQuery{}) - require.NoError(t, err) - - assert.Equal(t, "af-south-1", result[0].Text) - assert.Equal(t, "ap-east-1", result[1].Text) - assert.Equal(t, "ap-northeast-1", result[2].Text) - assert.Equal(t, "ap-northeast-2", result[3].Text) - }) - - t.Run("When calling handleGetEc2InstanceAttribute", func(t *testing.T) { - executor := &cloudWatchExecutor{ - ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - { - Instances: []*ec2.Instance{ - { - InstanceId: aws.String("i-12345678"), - Tags: []*ec2.Tag{ - { - Key: aws.String("Environment"), - Value: aws.String("production"), - }, - }, - }, - }, - }, - }, - }}, - } - - json := simplejson.New() - json.Set("region", "us-east-1") - json.Set("attributeName", "InstanceId") - filters := make(map[string]interface{}) - filters["tag:Environment"] = []string{"production"} - json.Set("filters", filters) - result, err := executor.handleGetEc2InstanceAttribute(context.Background(), json, &tsdb.TsdbQuery{}) - require.NoError(t, err) - - assert.Equal(t, "i-12345678", result[0].Text) - }) - - t.Run("When calling handleGetEbsVolumeIds", func(t *testing.T) { - executor := &cloudWatchExecutor{ - ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{ - Reservations: []*ec2.Reservation{ - { - Instances: []*ec2.Instance{ - { - InstanceId: aws.String("i-1"), - BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-1")}}, - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-2")}}, - }, - }, - { - InstanceId: aws.String("i-2"), - BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-1")}}, - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-2")}}, - }, - }, - }, - }, - { - Instances: []*ec2.Instance{ - { - InstanceId: aws.String("i-3"), - BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-1")}}, - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-2")}}, - }, - }, - { - InstanceId: aws.String("i-4"), - BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-1")}}, - {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-2")}}, - }, - }, - }, - }, - }, - }}, - } - - json := simplejson.New() - json.Set("region", "us-east-1") - json.Set("instanceId", "{i-1, i-2, i-3, i-4}") - result, err := executor.handleGetEbsVolumeIds(context.Background(), json, &tsdb.TsdbQuery{}) - require.NoError(t, err) - - require.Len(t, result, 8) - assert.Equal(t, "vol-1-1", result[0].Text) - assert.Equal(t, "vol-1-2", result[1].Text) - assert.Equal(t, "vol-2-1", result[2].Text) - assert.Equal(t, "vol-2-2", result[3].Text) - assert.Equal(t, "vol-3-1", result[4].Text) - assert.Equal(t, "vol-3-2", result[5].Text) - assert.Equal(t, "vol-4-1", result[6].Text) - assert.Equal(t, "vol-4-2", result[7].Text) - }) - - t.Run("When calling handleGetResourceArns", func(t *testing.T) { - executor := &cloudWatchExecutor{ - rgtaSvc: mockedRGTA{ - Resp: resourcegroupstaggingapi.GetResourcesOutput{ - ResourceTagMappingList: []*resourcegroupstaggingapi.ResourceTagMapping{ + t.Run("Custom metrics", func(t *testing.T) { + client = fakeCWClient{ + metrics: []*cloudwatch.Metric{ + { + MetricName: aws.String("Test_MetricName"), + Dimensions: []*cloudwatch.Dimension{ { - ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"), - Tags: []*resourcegroupstaggingapi.Tag{ + Name: aws.String("Test_DimensionName"), + }, + }, + }, + }, + } + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "metricFindQuery", + "subtype": "metrics", + "region": "us-east-1", + "namespace": "custom", + }), + }, + }, + }) + require.NoError(t, err) + + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Meta: simplejson.NewFromAny(map[string]interface{}{ + "rowCount": 1, + }), + Tables: []*tsdb.Table{ + { + Columns: []tsdb.TableColumn{ { - Key: aws.String("Environment"), - Value: aws.String("production"), + Text: "text", + }, + { + Text: "value", + }, + }, + Rows: []tsdb.RowValues{ + { + "Test_MetricName", + "Test_MetricName", }, }, }, + }, + }, + }, + }, resp) + }) + + t.Run("Dimension keys for custom metrics", func(t *testing.T) { + client = fakeCWClient{ + metrics: []*cloudwatch.Metric{ + { + MetricName: aws.String("Test_MetricName"), + Dimensions: []*cloudwatch.Dimension{ { - ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321"), - Tags: []*resourcegroupstaggingapi.Tag{ + Name: aws.String("Test_DimensionName"), + }, + }, + }, + }, + } + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "metricFindQuery", + "subtype": "dimension_keys", + "region": "us-east-1", + "namespace": "custom", + }), + }, + }, + }) + require.NoError(t, err) + + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Meta: simplejson.NewFromAny(map[string]interface{}{ + "rowCount": 1, + }), + Tables: []*tsdb.Table{ + { + Columns: []tsdb.TableColumn{ + { + Text: "text", + }, + { + Text: "value", + }, + }, + Rows: []tsdb.RowValues{ + { + "Test_DimensionName", + "Test_DimensionName", + }, + }, + }, + }, + }, + }, + }, resp) + }) +} + +func TestQuery_Regions(t *testing.T) { + origNewEC2Client := newEC2Client + t.Cleanup(func() { + newEC2Client = origNewEC2Client + }) + + var cli fakeEC2Client + + newEC2Client = func(client.ConfigProvider) ec2iface.EC2API { + return cli + } + + t.Run("An extra region", func(t *testing.T) { + const regionName = "xtra-region" + cli = fakeEC2Client{ + regions: []string{regionName}, + } + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "metricFindQuery", + "subtype": "regions", + "region": "us-east-1", + "namespace": "custom", + }), + }, + }, + }) + require.NoError(t, err) + + rows := []tsdb.RowValues{} + for _, region := range knownRegions { + rows = append(rows, []interface{}{ + region, + region, + }) + } + rows = append(rows, []interface{}{ + regionName, + regionName, + }) + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Meta: simplejson.NewFromAny(map[string]interface{}{ + "rowCount": len(knownRegions) + 1, + }), + Tables: []*tsdb.Table{ + { + Columns: []tsdb.TableColumn{ + { + Text: "text", + }, + { + Text: "value", + }, + }, + Rows: rows, + }, + }, + }, + }, + }, resp) + }) +} + +func TestQuery_InstanceAttributes(t *testing.T) { + origNewEC2Client := newEC2Client + t.Cleanup(func() { + newEC2Client = origNewEC2Client + }) + + var cli fakeEC2Client + + newEC2Client = func(client.ConfigProvider) ec2iface.EC2API { + return cli + } + + t.Run("Get instance ID", func(t *testing.T) { + const instanceID = "i-12345678" + cli = fakeEC2Client{ + reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + InstanceId: aws.String(instanceID), + Tags: []*ec2.Tag{ { Key: aws.String("Environment"), Value: aws.String("production"), @@ -250,33 +245,257 @@ func TestCloudWatchMetrics(t *testing.T) { }, }, } - - json := simplejson.New() - json.Set("region", "us-east-1") - json.Set("resourceType", "ec2:instance") - tags := make(map[string]interface{}) - tags["Environment"] = []string{"production"} - json.Set("tags", tags) - result, err := executor.handleGetResourceArns(context.Background(), json, &tsdb.TsdbQuery{}) + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "metricFindQuery", + "subtype": "ec2_instance_attribute", + "region": "us-east-1", + "attributeName": "InstanceId", + "filters": map[string]interface{}{ + "tag:Environment": []string{"production"}, + }, + }), + }, + }, + }) require.NoError(t, err) - assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Text) - assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Value) - assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Text) - assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Value) + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Meta: simplejson.NewFromAny(map[string]interface{}{ + "rowCount": 1, + }), + Tables: []*tsdb.Table{ + { + Columns: []tsdb.TableColumn{ + { + Text: "text", + }, + { + Text: "value", + }, + }, + Rows: []tsdb.RowValues{ + { + instanceID, + instanceID, + }, + }, + }, + }, + }, + }, + }, resp) }) } -func TestParseMultiSelectValue(t *testing.T) { - values := parseMultiSelectValue(" i-someInstance ") - assert.Equal(t, []string{"i-someInstance"}, values) +func TestQuery_EBSVolumeIDs(t *testing.T) { + origNewEC2Client := newEC2Client + t.Cleanup(func() { + newEC2Client = origNewEC2Client + }) - values = parseMultiSelectValue("{i-05}") - assert.Equal(t, []string{"i-05"}, values) + var cli fakeEC2Client - values = parseMultiSelectValue(" {i-01, i-03, i-04} ") - assert.Equal(t, []string{"i-01", "i-03", "i-04"}, values) + newEC2Client = func(client.ConfigProvider) ec2iface.EC2API { + return cli + } - values = parseMultiSelectValue("i-{01}") - assert.Equal(t, []string{"i-{01}"}, values) + t.Run("", func(t *testing.T) { + const instanceIDs = "{i-1, i-2, i-3}" + + cli = fakeEC2Client{ + reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + InstanceId: aws.String("i-1"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-1")}}, + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-2")}}, + }, + }, + { + InstanceId: aws.String("i-2"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-1")}}, + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-2")}}, + }, + }, + }, + }, + { + Instances: []*ec2.Instance{ + { + InstanceId: aws.String("i-3"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-1")}}, + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-2")}}, + }, + }, + { + InstanceId: aws.String("i-4"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-1")}}, + {Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-2")}}, + }, + }, + }, + }, + }, + } + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "metricFindQuery", + "subtype": "ebs_volume_ids", + "region": "us-east-1", + "instanceId": instanceIDs, + }), + }, + }, + }) + require.NoError(t, err) + + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Meta: simplejson.NewFromAny(map[string]interface{}{ + "rowCount": 6, + }), + Tables: []*tsdb.Table{ + { + Columns: []tsdb.TableColumn{ + { + Text: "text", + }, + { + Text: "value", + }, + }, + Rows: []tsdb.RowValues{ + { + "vol-1-1", + "vol-1-1", + }, + { + "vol-1-2", + "vol-1-2", + }, + { + "vol-2-1", + "vol-2-1", + }, + { + "vol-2-2", + "vol-2-2", + }, + { + "vol-3-1", + "vol-3-1", + }, + { + "vol-3-2", + "vol-3-2", + }, + }, + }, + }, + }, + }, + }, resp) + }) +} + +func TestQuery_ResourceARNs(t *testing.T) { + origNewRGTAClient := newRGTAClient + t.Cleanup(func() { + newRGTAClient = origNewRGTAClient + }) + + var cli fakeRGTAClient + + newRGTAClient = func(client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI { + return cli + } + + t.Run("", func(t *testing.T) { + cli = fakeRGTAClient{ + tagMapping: []*resourcegroupstaggingapi.ResourceTagMapping{ + { + ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"), + Tags: []*resourcegroupstaggingapi.Tag{ + { + Key: aws.String("Environment"), + Value: aws.String("production"), + }, + }, + }, + { + ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321"), + Tags: []*resourcegroupstaggingapi.Tag{ + { + Key: aws.String("Environment"), + Value: aws.String("production"), + }, + }, + }, + }, + } + executor := newExecutor() + resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "type": "metricFindQuery", + "subtype": "resource_arns", + "region": "us-east-1", + "resourceType": "ec2:instance", + "tags": map[string]interface{}{ + "Environment": []string{"production"}, + }, + }), + }, + }, + }) + require.NoError(t, err) + + assert.Equal(t, &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{ + "": { + Meta: simplejson.NewFromAny(map[string]interface{}{ + "rowCount": 2, + }), + Tables: []*tsdb.Table{ + { + Columns: []tsdb.TableColumn{ + { + Text: "text", + }, + { + Text: "value", + }, + }, + Rows: []tsdb.RowValues{ + { + "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", + "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", + }, + { + "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", + "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", + }, + }, + }, + }, + }, + }, + }, resp) + }) } diff --git a/pkg/tsdb/cloudwatch/query_transformer_test.go b/pkg/tsdb/cloudwatch/query_transformer_test.go index 71794dfabee..6e8ca0bed20 100644 --- a/pkg/tsdb/cloudwatch/query_transformer_test.go +++ b/pkg/tsdb/cloudwatch/query_transformer_test.go @@ -10,7 +10,7 @@ import ( func TestQueryTransformer(t *testing.T) { Convey("TestQueryTransformer", t, func() { Convey("when transforming queries", func() { - executor := &cloudWatchExecutor{} + executor := newExecutor() Convey("one cloudwatchQuery is generated when its request query has one stat", func() { requestQueries := []*requestQuery{ { diff --git a/pkg/tsdb/cloudwatch/test_utils.go b/pkg/tsdb/cloudwatch/test_utils.go new file mode 100644 index 00000000000..9f81e0b0207 --- /dev/null +++ b/pkg/tsdb/cloudwatch/test_utils.go @@ -0,0 +1,133 @@ +package cloudwatch + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" + "github.com/grafana/grafana/pkg/components/securejsondata" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" +) + +func fakeDataSource() *models.DataSource { + jsonData := simplejson.New() + jsonData.Set("defaultRegion", "default") + return &models.DataSource{ + Id: 1, + Database: "default", + JsonData: jsonData, + SecureJsonData: securejsondata.SecureJsonData{}, + } +} + +type fakeCWLogsClient struct { + cloudwatchlogsiface.CloudWatchLogsAPI + logGroups cloudwatchlogs.DescribeLogGroupsOutput + logGroupFields cloudwatchlogs.GetLogGroupFieldsOutput + queryResults cloudwatchlogs.GetQueryResultsOutput +} + +func (m fakeCWLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) { + return &m.queryResults, nil +} + +func (m fakeCWLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) { + return &cloudwatchlogs.StartQueryOutput{ + QueryId: aws.String("abcd-efgh-ijkl-mnop"), + }, nil +} + +func (m fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) { + return &cloudwatchlogs.StopQueryOutput{ + Success: aws.Bool(true), + }, nil +} + +func (m fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { + return &m.logGroups, nil +} + +func (m fakeCWLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) { + return &m.logGroupFields, nil +} + +type fakeCWClient struct { + cloudwatchiface.CloudWatchAPI + + metrics []*cloudwatch.Metric +} + +func (c fakeCWClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error { + fn(&cloudwatch.ListMetricsOutput{ + Metrics: c.metrics, + }, true) + return nil +} + +type fakeEC2Client struct { + ec2iface.EC2API + + regions []string + reservations []*ec2.Reservation +} + +func (c fakeEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) { + regions := []*ec2.Region{} + for _, region := range c.regions { + regions = append(regions, &ec2.Region{ + RegionName: aws.String(region), + }) + } + return &ec2.DescribeRegionsOutput{ + Regions: regions, + }, nil +} + +func (c fakeEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput, + fn func(*ec2.DescribeInstancesOutput, bool) bool) error { + reservations := []*ec2.Reservation{} + for _, r := range c.reservations { + instances := []*ec2.Instance{} + for _, inst := range r.Instances { + if len(in.InstanceIds) == 0 { + instances = append(instances, inst) + continue + } + + for _, id := range in.InstanceIds { + if *inst.InstanceId == *id { + instances = append(instances, inst) + } + } + } + reservation := &ec2.Reservation{Instances: instances} + reservations = append(reservations, reservation) + } + fn(&ec2.DescribeInstancesOutput{ + Reservations: reservations, + }, true) + return nil +} + +type fakeRGTAClient struct { + resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI + + tagMapping []*resourcegroupstaggingapi.ResourceTagMapping +} + +func (c fakeRGTAClient) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput, + fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error { + fn(&resourcegroupstaggingapi.GetResourcesOutput{ + ResourceTagMappingList: c.tagMapping, + }, true) + return nil +} diff --git a/pkg/tsdb/cloudwatch/time_series_query_test.go b/pkg/tsdb/cloudwatch/time_series_query_test.go index 4af80a8804c..df087d5568e 100644 --- a/pkg/tsdb/cloudwatch/time_series_query_test.go +++ b/pkg/tsdb/cloudwatch/time_series_query_test.go @@ -9,7 +9,7 @@ import ( ) func TestTimeSeriesQuery(t *testing.T) { - executor := &cloudWatchExecutor{} + executor := newExecutor() t.Run("End time before start time should result in error", func(t *testing.T) { _, err := executor.executeTimeSeriesQuery(context.TODO(), &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")})