public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move describe log groups to resource api (#55485)

Browse Source
This commit is contained in:
Erik Sundell
2022-09-21 10:55:54 +02:00
committed by GitHub
parent 79a5e3e802
commit 28ebdf1641
21 changed files with 370 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudwatch/cloudwatch.go
+6 -11
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"time"
@@ -82,7 +83,6 @@ const (
var plog = log.New("tsdb.cloudwatch")
var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
var baseLimit = int64(1)
func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, features featuremgmt.FeatureToggles) *CloudWatchService {
plog.Debug("initing")
@@ -203,17 +203,12 @@ func (e *cloudWatchExecutor) checkHealthMetrics(pluginCtx backend.PluginContext)
return err
}
func (e *cloudWatchExecutor) checkHealthLogs(ctx context.Context, pluginCtx backend.PluginContext) error {
logsClient, err := e.getCWLogsClient(pluginCtx, defaultRegion)
if err != nil {
return err
func (e *cloudWatchExecutor) checkHealthLogs(pluginCtx backend.PluginContext) error {
parameters := url.Values{
"limit": []string{"1"},
}
parameters := LogQueryJson{
Limit: &baseLimit,
}
_, err = e.handleDescribeLogGroups(ctx, logsClient, parameters)
_, err := e.handleGetLogGroups(pluginCtx, parameters)
return err
}
@@ -228,7 +223,7 @@ func (e *cloudWatchExecutor) CheckHealth(ctx context.Context, req *backend.Check
metricsTest = fmt.Sprintf("CloudWatch metrics query failed: %s", err.Error())
}
err = e.checkHealthLogs(ctx, req.PluginContext)
err = e.checkHealthLogs(req.PluginContext)
if err != nil {
status = backend.HealthStatusError
logsTest = fmt.Sprintf("CloudWatch logs query failed: %s", err.Error())
pkg/tsdb/cloudwatch/cloudwatch_test.go
+139 -3
View File
@@ -4,11 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
awsrequest "github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
@@ -124,8 +124,7 @@ func Test_CheckHealth(t *testing.T) {
t.Run("successfully queries metrics, fails during logs query", func(t *testing.T) {
client = fakeCheckHealthClient{
describeLogGroupsWithContext: func(ctx aws.Context, input *cloudwatchlogs.DescribeLogGroupsInput,
options ...awsrequest.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
describeLogGroups: func(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
return nil, fmt.Errorf("some logs query error")
}}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
@@ -248,3 +247,140 @@ func Test_executeLogAlertQuery(t *testing.T) {
assert.Equal(t, []string{"instance manager's region"}, sess.calledRegions)
})
}
func TestQuery_ResourceRequest_DescribeLogGroups(t *testing.T) {
origNewCWLogsClient := NewCWLogsClient
t.Cleanup(func() {
NewCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return &cli
}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
sender := &mockedCallResourceResponseSenderForOauth{}
t.Run("Should map log groups to SuggestData response", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
}},
},
}
req := &backend.CallResourceRequest{
Method: "GET",
Path: "/log-groups",
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
ID: 0,
},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
suggestDataResponse := []suggestData{}
err = json.Unmarshal(sent.Body, &suggestDataResponse)
require.Nil(t, err)
assert.Equal(t, []suggestData{
{Text: "group_a", Value: "group_a", Label: "group_a"}, {Text: "group_b", Value: "group_b", Label: "group_b"}, {Text: "group_c", Value: "group_c", Label: "group_c"}}, suggestDataResponse)
})
t.Run("Should call api with LogGroupNamePrefix if passed in resource call", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{}},
},
}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
req := &backend.CallResourceRequest{
Method: "GET",
Path: "/log-groups?logGroupNamePrefix=test",
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
ID: 0,
},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
assert.Equal(t, []*cloudwatchlogs.DescribeLogGroupsInput{
{
Limit: aws.Int64(defaultLogGroupLimit),
LogGroupNamePrefix: aws.String("test"),
},
}, cli.calls.describeLogGroups)
})
t.Run("Should call api without LogGroupNamePrefix if not passed in resource call", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{}},
},
}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
req := &backend.CallResourceRequest{
Method: "GET",
Path: "/log-groups?limit=100",
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
ID: 0,
},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
assert.Equal(t, []*cloudwatchlogs.DescribeLogGroupsInput{
{
Limit: aws.Int64(100),
},
}, cli.calls.describeLogGroups)
})
}
pkg/tsdb/cloudwatch/log_actions.go
-36
View File
@@ -126,8 +126,6 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model LogQuer
var data *data.Frame = nil
switch model.SubType {
case "DescribeLogGroups":
data, err = e.handleDescribeLogGroups(ctx, logsClient, model)
case "DescribeAllLogGroups":
data, err = e.handleDescribeAllLogGroups(ctx, logsClient, model)
case "GetLogGroupFields":
@@ -203,40 +201,6 @@ func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient
return data.NewFrame("logEvents", timestampField, messageField), nil
}
func (e *cloudWatchExecutor) handleDescribeLogGroups(ctx context.Context,
logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) {
logGroupLimit := defaultLogGroupLimit
if parameters.Limit != nil && *parameters.Limit != 0 {
logGroupLimit = *parameters.Limit
}
var response *cloudwatchlogs.DescribeLogGroupsOutput = nil
var err error
if len(parameters.LogGroupNamePrefix) == 0 {
response, err = logsClient.DescribeLogGroupsWithContext(ctx, &cloudwatchlogs.DescribeLogGroupsInput{
Limit: aws.Int64(logGroupLimit),
})
} else {
response, err = logsClient.DescribeLogGroupsWithContext(ctx, &cloudwatchlogs.DescribeLogGroupsInput{
Limit: aws.Int64(logGroupLimit),
LogGroupNamePrefix: aws.String(parameters.LogGroupNamePrefix),
})
}
if err != nil || response == nil {
return nil, err
}
logGroupNames := make([]*string, 0)
for _, logGroup := range response.LogGroups {
logGroupNames = append(logGroupNames, logGroup.LogGroupName)
}
groupNamesField := data.NewField("logGroupName", nil, logGroupNames)
frame := data.NewFrame("logGroups", groupNamesField)
return frame, nil
}
func (e *cloudWatchExecutor) handleDescribeAllLogGroups(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) {
var namePrefix, nextToken *string
if len(parameters.LogGroupNamePrefix) != 0 {
pkg/tsdb/cloudwatch/log_actions_test.go
-127
View File
@@ -106,133 +106,6 @@ func TestQuery_GetLogEvents(t *testing.T) {
}
}
func TestQuery_DescribeLogGroups(t *testing.T) {
origNewCWLogsClient := NewCWLogsClient
t.Cleanup(func() {
NewCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return &cli
}
t.Run("Empty log group name prefix", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
}},
},
}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "logAction",
"subtype": "DescribeLogGroups",
"limit": 50
}`),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": backend.DataResponse{
Frames: data.Frames{
&data.Frame{
Name: "logGroups",
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
},
},
},
},
}, resp)
})
t.Run("Non-empty log group name prefix", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
{LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
}},
},
}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "logAction",
"subtype": "DescribeLogGroups",
"limit": 50,
"region": "default",
"logGroupNamePrefix": "g"
}`),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": backend.DataResponse{
Frames: data.Frames{
&data.Frame{
Name: "logGroups",
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
},
},
},
},
}, resp)
})
}
func TestQuery_DescribeAllLogGroups(t *testing.T) {
origNewCWLogsClient := NewCWLogsClient
t.Cleanup(func() {
pkg/tsdb/cloudwatch/metric_find_query.go
+37
View File
@@ -7,6 +7,7 @@ import (
"net/url"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -14,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -649,6 +651,41 @@ func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region, namespace str
return customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache, nil
}
func (e *cloudWatchExecutor) handleGetLogGroups(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
limit := parameters.Get("limit")
logGroupNamePrefix := parameters.Get("logGroupNamePrefix")
logsClient, err := e.getCWLogsClient(pluginCtx, region)
if err != nil {
return nil, err
}
logGroupLimit := defaultLogGroupLimit
intLimit, err := strconv.ParseInt(limit, 10, 64)
if err == nil && intLimit > 0 {
logGroupLimit = intLimit
}
var response *cloudwatchlogs.DescribeLogGroupsOutput = nil
input := &cloudwatchlogs.DescribeLogGroupsInput{Limit: aws.Int64(logGroupLimit)}
if len(logGroupNamePrefix) > 0 {
input.LogGroupNamePrefix = aws.String(logGroupNamePrefix)
}
response, err = logsClient.DescribeLogGroups(input)
if err != nil || response == nil {
return nil, err
}
result := make([]suggestData, 0)
for _, logGroup := range response.LogGroups {
logGroupName := *logGroup.LogGroupName
result = append(result, suggestData{Text: logGroupName, Value: logGroupName, Label: logGroupName})
}
return result, nil
}
func isDuplicate(nameList []string, target string) bool {
for _, name := range nameList {
if name == target {
pkg/tsdb/cloudwatch/resource_handler.go
+1
View File
@@ -21,6 +21,7 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
mux.HandleFunc("/ebs-volume-ids", handleResourceReq(e.handleGetEbsVolumeIds))
mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute))
mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns))
mux.HandleFunc("/log-groups", handleResourceReq(e.handleGetLogGroups))
return mux
}
pkg/tsdb/cloudwatch/utils_test.go
+23 -6
View File
@@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana-aws-sdk/pkg/awsds"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/setting"
)
@@ -33,6 +34,7 @@ type fakeCWLogsClient struct {
type logsQueryCalls struct {
startQueryWithContext []*cloudwatchlogs.StartQueryInput
getEventsWithContext []*cloudwatchlogs.GetLogEventsInput
describeLogGroups []*cloudwatchlogs.DescribeLogGroupsInput
}
func (m *fakeCWLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
@@ -53,6 +55,13 @@ func (m *fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *clou
}, nil
}
func (m *fakeCWLogsClient) DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
m.calls.describeLogGroups = append(m.calls.describeLogGroups, input)
output := &m.logGroups[m.logGroupsIndex]
m.logGroupsIndex++
return output, nil
}
func (m *fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
output := &m.logGroups[m.logGroupsIndex]
m.logGroupsIndex++
@@ -192,9 +201,8 @@ type fakeCheckHealthClient struct {
cloudwatchiface.CloudWatchAPI
cloudwatchlogsiface.CloudWatchLogsAPI
listMetricsPages func(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error
describeLogGroupsWithContext func(ctx aws.Context, input *cloudwatchlogs.DescribeLogGroupsInput,
options ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error)
listMetricsPages func(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error
describeLogGroups func(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error)
}
func (c fakeCheckHealthClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error {
@@ -204,9 +212,9 @@ func (c fakeCheckHealthClient) ListMetricsPages(input *cloudwatch.ListMetricsInp
return nil
}
func (c fakeCheckHealthClient) DescribeLogGroupsWithContext(ctx aws.Context, input *cloudwatchlogs.DescribeLogGroupsInput, options ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
if c.describeLogGroupsWithContext != nil {
return c.describeLogGroupsWithContext(ctx, input, options...)
func (c fakeCheckHealthClient) DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
if c.describeLogGroups != nil {
return c.describeLogGroups(input)
}
return nil, nil
}
@@ -247,3 +255,12 @@ func (s *fakeSessionCache) GetSession(c awsds.SessionConfig) (*session.Session,
Config: &aws.Config{},
}, nil
}
type mockedCallResourceResponseSenderForOauth struct {
Response *backend.CallResourceResponse
}
func (s *mockedCallResourceResponseSenderForOauth) Send(resp *backend.CallResourceResponse) error {
s.Response = resp
return nil
}