public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cloudwatch: Add support for template variables in new log group picker (#61243)

Browse Source
This commit is contained in:
Erik Sundell
2023-01-13 14:03:08 -05:00
committed by GitHub
parent b34377a0ef
commit 432cdbade4
25 changed files with 605 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudwatch/cloudwatch_test.go
+2 -157
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
@@ -257,152 +256,6 @@ func Test_executeLogAlertQuery(t *testing.T) {
})
}
func TestQuery_ResourceRequest_DescribeAllLogGroups(t *testing.T) {
origNewCWLogsClient := NewCWLogsClient
t.Cleanup(func() {
NewCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return &cli
}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return DataSource{Settings: models.CloudWatchSettings{}}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
sender := &mockedCallResourceResponseSenderForOauth{}
t.Run("multiple batches", func(t *testing.T) {
token := "foo"
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
},
NextToken: &token,
},
{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_x"),
},
{
LogGroupName: aws.String("group_y"),
},
{
LogGroupName: aws.String("group_z"),
},
},
},
},
}
req := &backend.CallResourceRequest{
Method: "GET",
Path: "/all-log-groups?limit=50",
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
ID: 0,
},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
suggestDataResponse := []suggestData{}
err = json.Unmarshal(sent.Body, &suggestDataResponse)
require.Nil(t, err)
assert.Equal(t, stringsToSuggestData([]string{
"group_a", "group_b", "group_c", "group_x", "group_y", "group_z",
}), suggestDataResponse)
})
t.Run("Should call api with LogGroupNamePrefix if passed in resource call", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{}},
},
}
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
req := &backend.CallResourceRequest{
Method: "GET",
Path: "/all-log-groups?logGroupNamePrefix=test",
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
ID: 0,
},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
assert.Equal(t, []*cloudwatchlogs.DescribeLogGroupsInput{
{
Limit: aws.Int64(defaultLogGroupLimit),
LogGroupNamePrefix: aws.String("test"),
},
}, cli.calls.describeLogGroups)
})
t.Run("Should call api without LogGroupNamePrefix when an empty string is passed in resource call", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{}},
},
}
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
req := &backend.CallResourceRequest{
Method: "GET",
Path: "/all-log-groups?logGroupNamePrefix=",
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
ID: 0,
},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
assert.Equal(t, []*cloudwatchlogs.DescribeLogGroupsInput{
{
Limit: aws.Int64(50),
},
}, cli.calls.describeLogGroups)
})
}
func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *testing.T) {
sender := &mockedCallResourceResponseSenderForOauth{}
origNewMetricsAPI := NewMetricsAPI
@@ -425,7 +278,7 @@ func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *te
return DataSource{Settings: models.CloudWatchSettings{}}, nil
})
t.Run("maps log group api response to resource response of describe-log-groups", func(t *testing.T) {
t.Run("maps log group api response to resource response of log-groups", func(t *testing.T) {
logsApi = mocks.LogsAPI{}
logsApi.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
@@ -434,7 +287,7 @@ func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *te
}, nil)
req := &backend.CallResourceRequest{
Method: "GET",
Path: `/describe-log-groups?logGroupPattern=some-pattern&accountId=some-account-id`,
Path: `/log-groups?logGroupPattern=some-pattern&accountId=some-account-id`,
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0},
PluginID: "cloudwatch",
@@ -464,11 +317,3 @@ func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *te
})
})
}
func stringsToSuggestData(values []string) []suggestData {
suggestDataArray := make([]suggestData, 0)
for _, v := range values {
suggestDataArray = append(suggestDataArray, suggestData{Text: v, Value: v, Label: v})
}
return suggestDataArray
}
pkg/tsdb/cloudwatch/metric_find_query.go
-42
View File
@@ -12,7 +12,6 @@ import (
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -288,44 +287,3 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(pluginCtx backend.Plugin
return &resp, nil
}
func (e *cloudWatchExecutor) handleGetAllLogGroups(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
var nextToken *string
logGroupNamePrefix := parameters.Get("logGroupNamePrefix")
var err error
logsClient, err := e.getCWLogsClient(pluginCtx, parameters.Get("region"))
if err != nil {
return nil, err
}
var response *cloudwatchlogs.DescribeLogGroupsOutput
result := make([]suggestData, 0)
for {
input := &cloudwatchlogs.DescribeLogGroupsInput{
Limit: aws.Int64(defaultLogGroupLimit),
NextToken: nextToken,
}
if len(logGroupNamePrefix) > 0 {
input.LogGroupNamePrefix = aws.String(logGroupNamePrefix)
}
response, err = logsClient.DescribeLogGroups(input)
if err != nil || response == nil {
return nil, err
}
for _, logGroup := range response.LogGroups {
logGroupName := *logGroup.LogGroupName
result = append(result, suggestData{Text: logGroupName, Value: logGroupName, Label: logGroupName})
}
if response.NextToken == nil {
break
}
nextToken = response.NextToken
}
return result, nil
}
pkg/tsdb/cloudwatch/models/resources/log_groups_resource_request.go
+2
View File
@@ -12,6 +12,7 @@ type LogGroupsRequest struct {
ResourceRequest
Limit int64
LogGroupNamePrefix, LogGroupNamePattern *string
ListAllLogGroups bool
}
func (r LogGroupsRequest) IsTargetingAllAccounts() bool {
@@ -33,6 +34,7 @@ func ParseLogGroupsRequest(parameters url.Values) (LogGroupsRequest, error) {
},
LogGroupNamePrefix: logGroupNamePrefix,
LogGroupNamePattern: logGroupPattern,
ListAllLogGroups: parameters.Get("listAllLogGroups") == "true",
}, nil
}
pkg/tsdb/cloudwatch/resource_handler.go
+1 -2
View File
@@ -18,8 +18,7 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
mux.HandleFunc("/ebs-volume-ids", handleResourceReq(e.handleGetEbsVolumeIds))
mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute))
mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns))
mux.HandleFunc("/describe-log-groups", routes.ResourceRequestMiddleware(routes.LogGroupsHandler, logger, e.getRequestContext))
mux.HandleFunc("/all-log-groups", handleResourceReq(e.handleGetAllLogGroups))
mux.HandleFunc("/log-groups", routes.ResourceRequestMiddleware(routes.LogGroupsHandler, logger, e.getRequestContext))
mux.HandleFunc("/metrics", routes.ResourceRequestMiddleware(routes.MetricsHandler, logger, e.getRequestContext))
mux.HandleFunc("/dimension-values", routes.ResourceRequestMiddleware(routes.DimensionValuesHandler, logger, e.getRequestContext))
mux.HandleFunc("/dimension-keys", routes.ResourceRequestMiddleware(routes.DimensionKeysHandler, logger, e.getRequestContext))
pkg/tsdb/cloudwatch/services/log_groups.go
+21 -13
View File
@@ -33,20 +33,28 @@ func (s *LogGroupsService) GetLogGroups(req resources.LogGroupsRequest) ([]resou
input.AccountIdentifiers = []*string{req.AccountId}
}
}
response, err := s.logGroupsAPI.DescribeLogGroups(input)
if err != nil || response == nil {
return nil, err
}
result := []resources.ResourceResponse[resources.LogGroup]{}
var result []resources.ResourceResponse[resources.LogGroup]
for _, logGroup := range response.LogGroups {
result = append(result, resources.ResourceResponse[resources.LogGroup]{
Value: resources.LogGroup{
Arn: *logGroup.Arn,
Name: *logGroup.LogGroupName,
},
AccountId: utils.Pointer(getAccountId(*logGroup.Arn)),
})
for {
response, err := s.logGroupsAPI.DescribeLogGroups(input)
if err != nil || response == nil {
return nil, err
}
for _, logGroup := range response.LogGroups {
result = append(result, resources.ResourceResponse[resources.LogGroup]{
Value: resources.LogGroup{
Arn: *logGroup.Arn,
Name: *logGroup.LogGroupName,
},
AccountId: utils.Pointer(getAccountId(*logGroup.Arn)),
})
}
if !req.ListAllLogGroups || response.NextToken == nil {
break
}
input.NextToken = response.NextToken
}
return result, nil
pkg/tsdb/cloudwatch/services/log_groups_test.go
+88
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
@@ -44,6 +45,17 @@ func TestGetLogGroups(t *testing.T) {
}, resp)
})
t.Run("Should return an empty error if api doesn't return any data", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil)
service := NewLogGroupsService(mockLogsAPI, false)
resp, err := service.GetLogGroups(resources.LogGroupsRequest{})
assert.NoError(t, err)
assert.Equal(t, []resources.ResourceResponse[resources.LogGroup]{}, resp)
})
t.Run("Should only use LogGroupNamePrefix even if LogGroupNamePattern passed in resource call", func(t *testing.T) {
// TODO: use LogGroupNamePattern when we have accounted for its behavior, still a little unexpected at the moment
mockLogsAPI := &mocks.LogsAPI{}
@@ -86,6 +98,82 @@ func TestGetLogGroups(t *testing.T) {
assert.Error(t, err)
assert.Equal(t, "some error", err.Error())
})
t.Run("Should only call the api once in case ListAllLogGroups is set to false", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
req := resources.LogGroupsRequest{
Limit: 2,
LogGroupNamePrefix: utils.Pointer("test"),
ListAllLogGroups: false,
}
mockLogsAPI.On("DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{
Limit: aws.Int64(req.Limit),
LogGroupNamePrefix: req.LogGroupNamePrefix,
}).Return(&cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{Arn: utils.Pointer("arn:aws:logs:us-east-1:111:log-group:group_a"), LogGroupName: utils.Pointer("group_a")},
},
NextToken: aws.String("next_token"),
}, nil)
service := NewLogGroupsService(mockLogsAPI, false)
resp, err := service.GetLogGroups(req)
assert.NoError(t, err)
mockLogsAPI.AssertNumberOfCalls(t, "DescribeLogGroups", 1)
assert.Equal(t, []resources.ResourceResponse[resources.LogGroup]{
{
AccountId: utils.Pointer("111"),
Value: resources.LogGroup{Arn: "arn:aws:logs:us-east-1:111:log-group:group_a", Name: "group_a"},
},
}, resp)
})
t.Run("Should keep on calling the api until NextToken is empty in case ListAllLogGroups is set to true", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
req := resources.LogGroupsRequest{
Limit: 2,
LogGroupNamePrefix: utils.Pointer("test"),
ListAllLogGroups: true,
}
// first call
mockLogsAPI.On("DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{
Limit: aws.Int64(req.Limit),
LogGroupNamePrefix: req.LogGroupNamePrefix,
}).Return(&cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{Arn: utils.Pointer("arn:aws:logs:us-east-1:111:log-group:group_a"), LogGroupName: utils.Pointer("group_a")},
},
NextToken: utils.Pointer("token"),
}, nil)
// second call
mockLogsAPI.On("DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{
Limit: aws.Int64(req.Limit),
LogGroupNamePrefix: req.LogGroupNamePrefix,
NextToken: utils.Pointer("token"),
}).Return(&cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{Arn: utils.Pointer("arn:aws:logs:us-east-1:222:log-group:group_b"), LogGroupName: utils.Pointer("group_b")},
},
}, nil)
service := NewLogGroupsService(mockLogsAPI, false)
resp, err := service.GetLogGroups(req)
assert.NoError(t, err)
mockLogsAPI.AssertNumberOfCalls(t, "DescribeLogGroups", 2)
assert.Equal(t, []resources.ResourceResponse[resources.LogGroup]{
{
AccountId: utils.Pointer("111"),
Value: resources.LogGroup{Arn: "arn:aws:logs:us-east-1:111:log-group:group_a", Name: "group_a"},
},
{
AccountId: utils.Pointer("222"),
Value: resources.LogGroup{Arn: "arn:aws:logs:us-east-1:222:log-group:group_b", Name: "group_b"},
},
}, resp)
})
}
func TestGetLogGroupsCrossAccountQuerying(t *testing.T) {