diff --git a/.betterer.results b/.betterer.results index 29264a986a6..37786d5e4ff 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5155,8 +5155,7 @@ exports[`better eslint`] = { "public/app/plugins/datasource/cloudwatch/language_provider.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], - [0, 0, 0, "Unexpected any. Specify a different type.", "2"], - [0, 0, 0, "Do not use any type assertions.", "3"] + [0, 0, 0, "Unexpected any. Specify a different type.", "2"] ], "public/app/plugins/datasource/cloudwatch/memoizedDebounce.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], diff --git a/pkg/tsdb/cloudwatch/cloudwatch_integration_test.go b/pkg/tsdb/cloudwatch/cloudwatch_integration_test.go index 27c8827f4ac..d0d24677b39 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_integration_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_integration_test.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" @@ -18,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -27,7 +29,12 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { origNewOAMAPI := NewOAMAPI origNewLogsAPI := NewLogsAPI NewOAMAPI = func(sess *session.Session) models.OAMAPIProvider { return nil } - NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { return nil } + + var logApi mocks.LogsAPI + NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { + return &logApi + } + t.Cleanup(func() { NewOAMAPI = origNewOAMAPI NewMetricsAPI = origNewMetricsAPI @@ -198,4 +205,38 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { require.Nil(t, err) assert.Equal(t, []resources.ResourceResponse[resources.Metric]{{Value: resources.Metric{Name: "Test_MetricName1", Namespace: "AWS/EC2"}}, {Value: resources.Metric{Name: "Test_MetricName2", Namespace: "AWS/EC2"}}, {Value: resources.Metric{Name: "Test_MetricName3", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName10", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName4", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName5", Namespace: "AWS/Redshift"}}}, res) }) + + t.Run("Should handle log group fields request", func(t *testing.T) { + logApi = mocks.LogsAPI{} + logApi.On("GetLogGroupFields", mock.Anything).Return(&cloudwatchlogs.GetLogGroupFieldsOutput{ + LogGroupFields: []*cloudwatchlogs.LogGroupField{ + { + Name: aws.String("field1"), + Percent: aws.Int64(50), + }, + { + Name: aws.String("field2"), + Percent: aws.Int64(50), + }, + }, + }, nil) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + + req := &backend.CallResourceRequest{ + Method: "GET", + Path: `/log-group-fields?region=us-east-2&logGroupName=test`, + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0}, + PluginID: "cloudwatch", + }, + } + err := executor.CallResource(context.Background(), req, sender) + + require.NoError(t, err) + sent := sender.Response + require.NotNil(t, sent) + require.Equal(t, http.StatusOK, sent.Status) + require.Nil(t, err) + assert.JSONEq(t, `[{"value":{"name":"field1","percent":50}},{"value":{"name":"field2","percent":50}}]`, string(sent.Body)) + }) } diff --git a/pkg/tsdb/cloudwatch/log_actions.go b/pkg/tsdb/cloudwatch/log_actions.go index 6649c237c98..8866f95e265 100644 --- a/pkg/tsdb/cloudwatch/log_actions.go +++ b/pkg/tsdb/cloudwatch/log_actions.go @@ -112,8 +112,6 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, logger log.Lo var data *data.Frame = nil switch logsQuery.SubType { - case "GetLogGroupFields": - data, err = e.handleGetLogGroupFields(ctx, logsClient, logsQuery, query.RefID) case "StartQuery": data, err = e.handleStartQuery(ctx, logger, logsClient, logsQuery, query.TimeRange, query.RefID) case "StopQuery": @@ -321,37 +319,6 @@ func (e *cloudWatchExecutor) handleGetQueryResults(ctx context.Context, logsClie return dataFrame, nil } -func (e *cloudWatchExecutor) handleGetLogGroupFields(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, - logsQuery models.LogsQuery, refID string) (*data.Frame, error) { - queryInput := &cloudwatchlogs.GetLogGroupFieldsInput{ - LogGroupName: aws.String(logsQuery.LogGroupName), - Time: aws.Int64(logsQuery.Time), - } - - getLogGroupFieldsOutput, err := logsClient.GetLogGroupFieldsWithContext(ctx, queryInput) - if err != nil { - return nil, err - } - - fieldNames := make([]*string, 0) - fieldPercentages := make([]*int64, 0) - - for _, logGroupField := range getLogGroupFieldsOutput.LogGroupFields { - fieldNames = append(fieldNames, logGroupField.Name) - fieldPercentages = append(fieldPercentages, logGroupField.Percent) - } - - dataFrame := data.NewFrame( - refID, - data.NewField("name", nil, fieldNames), - data.NewField("percent", nil, fieldPercentages), - ) - - dataFrame.RefID = refID - - return dataFrame, nil -} - func groupResponseFrame(frame *data.Frame, statsGroups []string) (data.Frames, error) { var dataFrames data.Frames diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go index 900e088ef52..51573eb205c 100644 --- a/pkg/tsdb/cloudwatch/log_actions_test.go +++ b/pkg/tsdb/cloudwatch/log_actions_test.go @@ -107,83 +107,6 @@ func TestQuery_GetLogEvents(t *testing.T) { } } -func TestQuery_GetLogGroupFields(t *testing.T) { - origNewCWLogsClient := NewCWLogsClient - t.Cleanup(func() { - NewCWLogsClient = origNewCWLogsClient - }) - - var cli fakeCWLogsClient - - NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { - return &cli - } - - cli = fakeCWLogsClient{ - logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{ - LogGroupFields: []*cloudwatchlogs.LogGroupField{ - { - Name: aws.String("field_a"), - Percent: aws.Int64(100), - }, - { - Name: aws.String("field_b"), - Percent: aws.Int64(30), - }, - { - Name: aws.String("field_c"), - Percent: aws.Int64(55), - }, - }, - }, - } - - const refID = "A" - - im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { - return DataSource{Settings: models.CloudWatchSettings{}}, nil - }) - - executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) - resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ - PluginContext: backend.PluginContext{ - DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, - }, - Queries: []backend.DataQuery{ - { - RefID: refID, - JSON: json.RawMessage(`{ - "type": "logAction", - "subtype": "GetLogGroupFields", - "logGroupName": "group_a", - "limit": 50 - }`), - }, - }, - }) - require.NoError(t, err) - require.NotNil(t, resp) - - expFrame := &data.Frame{ - Name: refID, - Fields: []*data.Field{ - data.NewField("name", nil, []*string{ - aws.String("field_a"), aws.String("field_b"), aws.String("field_c"), - }), - data.NewField("percent", nil, []*int64{ - aws.Int64(100), aws.Int64(30), aws.Int64(55), - }), - }, - } - expFrame.RefID = refID - assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{ - refID: backend.DataResponse{ - Frames: data.Frames{expFrame}, - }, - }, - }, resp) -} - func TestQuery_StartQuery(t *testing.T) { origNewCWLogsClient := NewCWLogsClient t.Cleanup(func() { diff --git a/pkg/tsdb/cloudwatch/mocks/logs.go b/pkg/tsdb/cloudwatch/mocks/logs.go index c6210eeceac..f080a961f42 100644 --- a/pkg/tsdb/cloudwatch/mocks/logs.go +++ b/pkg/tsdb/cloudwatch/mocks/logs.go @@ -16,6 +16,12 @@ func (l *LogsAPI) DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput return args.Get(0).(*cloudwatchlogs.DescribeLogGroupsOutput), args.Error(1) } +func (l *LogsAPI) GetLogGroupFields(input *cloudwatchlogs.GetLogGroupFieldsInput) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) { + args := l.Called(input) + + return args.Get(0).(*cloudwatchlogs.GetLogGroupFieldsOutput), args.Error(1) +} + type LogsService struct { mock.Mock } @@ -26,6 +32,12 @@ func (l *LogsService) GetLogGroups(request resources.LogGroupsRequest) ([]resour return args.Get(0).([]resources.ResourceResponse[resources.LogGroup]), args.Error(1) } +func (l *LogsService) GetLogGroupFields(request resources.LogGroupFieldsRequest) ([]resources.ResourceResponse[resources.LogGroupField], error) { + args := l.Called(request) + + return args.Get(0).([]resources.ResourceResponse[resources.LogGroupField]), args.Error(1) +} + type MockFeatures struct { mock.Mock } diff --git a/pkg/tsdb/cloudwatch/models/api.go b/pkg/tsdb/cloudwatch/models/api.go index ad833c87462..85b47e50b13 100644 --- a/pkg/tsdb/cloudwatch/models/api.go +++ b/pkg/tsdb/cloudwatch/models/api.go @@ -32,6 +32,7 @@ type ListMetricsProvider interface { type LogGroupsProvider interface { GetLogGroups(request resources.LogGroupsRequest) ([]resources.ResourceResponse[resources.LogGroup], error) + GetLogGroupFields(request resources.LogGroupFieldsRequest) ([]resources.ResourceResponse[resources.LogGroupField], error) } type AccountsProvider interface { @@ -50,6 +51,7 @@ type CloudWatchMetricsAPIProvider interface { type CloudWatchLogsAPIProvider interface { DescribeLogGroups(*cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) + GetLogGroupFields(*cloudwatchlogs.GetLogGroupFieldsInput) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) } type OAMAPIProvider interface { diff --git a/pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request.go b/pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request.go new file mode 100644 index 00000000000..8d2e8b5ad08 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request.go @@ -0,0 +1,31 @@ +package resources + +import ( + "fmt" + "net/url" +) + +type LogGroupFieldsRequest struct { + ResourceRequest + LogGroupName string + LogGroupARN string +} + +func ParseLogGroupFieldsRequest(parameters url.Values) (LogGroupFieldsRequest, error) { + resourceRequest, err := getResourceRequest(parameters) + if err != nil { + return LogGroupFieldsRequest{}, err + } + + request := LogGroupFieldsRequest{ + ResourceRequest: *resourceRequest, + LogGroupName: parameters.Get("logGroupName"), + LogGroupARN: parameters.Get("logGroupArn"), + } + + if request.LogGroupName == "" && request.LogGroupARN == "" { + return LogGroupFieldsRequest{}, fmt.Errorf("you need to specify either logGroupName or logGroupArn") + } + + return request, nil +} diff --git a/pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request_test.go b/pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request_test.go new file mode 100644 index 00000000000..8e1e9422c6f --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request_test.go @@ -0,0 +1,32 @@ +package resources + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLogGroupFieldsRequest(t *testing.T) { + t.Run("Should parse valid parameters", func(t *testing.T) { + request, err := ParseLogGroupFieldsRequest(map[string][]string{ + "region": {"us-east-1"}, + "logGroupName": {"my-log-group"}, + "logGroupArn": {"arn:aws:logs:us-east-1:123456789012:log-group:my-log-group"}}, + ) + require.NoError(t, err) + assert.Equal(t, "us-east-1", request.Region) + assert.Equal(t, "my-log-group", request.LogGroupName) + assert.Equal(t, "arn:aws:logs:us-east-1:123456789012:log-group:my-log-group", request.LogGroupARN) + }) + + t.Run("Should return an error if arn and name is missing ", func(t *testing.T) { + request, err := ParseLogGroupFieldsRequest(map[string][]string{ + "region": {"us-east-1"}, + }, + ) + require.Empty(t, request) + require.Error(t, fmt.Errorf("you need to specify either logGroupName or logGroupArn"), err) + }) +} diff --git a/pkg/tsdb/cloudwatch/models/resources/types.go b/pkg/tsdb/cloudwatch/models/resources/types.go index 728d74c5af6..8dd76bd7ea9 100644 --- a/pkg/tsdb/cloudwatch/models/resources/types.go +++ b/pkg/tsdb/cloudwatch/models/resources/types.go @@ -33,3 +33,8 @@ type LogGroup struct { Arn string `json:"arn"` Name string `json:"name"` } + +type LogGroupField struct { + Percent int64 `json:"percent"` + Name string `json:"name"` +} diff --git a/pkg/tsdb/cloudwatch/resource_handler.go b/pkg/tsdb/cloudwatch/resource_handler.go index 11cd293a891..2960c7bc77d 100644 --- a/pkg/tsdb/cloudwatch/resource_handler.go +++ b/pkg/tsdb/cloudwatch/resource_handler.go @@ -25,6 +25,7 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux { mux.HandleFunc("/dimension-keys", routes.ResourceRequestMiddleware(routes.DimensionKeysHandler, logger, e.getRequestContext)) mux.HandleFunc("/accounts", routes.ResourceRequestMiddleware(routes.AccountsHandler, logger, e.getRequestContext)) mux.HandleFunc("/namespaces", routes.ResourceRequestMiddleware(routes.NamespacesHandler, logger, e.getRequestContext)) + mux.HandleFunc("/log-group-fields", routes.ResourceRequestMiddleware(routes.LogGroupFieldsHandler, logger, e.getRequestContext)) return mux } diff --git a/pkg/tsdb/cloudwatch/routes/log_group_fields.go b/pkg/tsdb/cloudwatch/routes/log_group_fields.go new file mode 100644 index 00000000000..f09806ba6f7 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/log_group_fields.go @@ -0,0 +1,35 @@ +package routes + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" +) + +func LogGroupFieldsHandler(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) { + request, err := resources.ParseLogGroupFieldsRequest(parameters) + if err != nil { + return nil, models.NewHttpError("error in LogGroupFieldsHandler", http.StatusBadRequest, err) + } + + service, err := newLogGroupsService(pluginCtx, reqCtxFactory, request.Region) + if err != nil { + return nil, models.NewHttpError("newLogGroupsService error", http.StatusInternalServerError, err) + } + + logGroupFields, err := service.GetLogGroupFields(request) + if err != nil { + return nil, models.NewHttpError("GetLogGroupFields error", http.StatusInternalServerError, err) + } + + logGroupsResponse, err := json.Marshal(logGroupFields) + if err != nil { + return nil, models.NewHttpError("LogGroupFieldsHandler json error", http.StatusInternalServerError, err) + } + + return logGroupsResponse, nil +} diff --git a/pkg/tsdb/cloudwatch/routes/log_group_fields_test.go b/pkg/tsdb/cloudwatch/routes/log_group_fields_test.go new file mode 100644 index 00000000000..7f2350f84f0 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/log_group_fields_test.go @@ -0,0 +1,77 @@ +package routes + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestLogGroupFieldsRoute(t *testing.T) { + mockFeatures := mocks.MockFeatures{} + reqCtxFunc := func(pluginCtx backend.PluginContext, region string) (reqCtx models.RequestContext, err error) { + return models.RequestContext{Features: &mockFeatures}, err + } + t.Run("returns 400 if an invalid LogGroupFieldsRequest is used", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", `/log-group-fields?region=us-east-2`, nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupFieldsHandler, logger, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Equal(t, `{"Message":"error in LogGroupFieldsHandler: you need to specify either logGroupName or logGroupArn","Error":"you need to specify either logGroupName or logGroupArn","StatusCode":400}`, rr.Body.String()) + }) + + t.Run("returns 500 if GetLogGroupFields method fails", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroupFields", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroupField]{}, fmt.Errorf("error from api")) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-group-fields?region=us-east-2&logGroupName=test", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupFieldsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.Equal(t, `{"Message":"GetLogGroupFields error: error from api","Error":"error from api","StatusCode":500}`, rr.Body.String()) + }) + + t.Run("returns valid json response if everything is ok", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroupFields", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroupField]{ + { + AccountId: new(string), + Value: resources.LogGroupField{ + Name: "field1", + Percent: 50, + }, + }, + { + AccountId: new(string), + Value: resources.LogGroupField{ + Name: "field2", + Percent: 50, + }, + }, + }, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-group-fields?region=us-east-2&logGroupName=test", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupFieldsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + assert.JSONEq(t, `[{"accountId":"","value":{"name":"field1","percent":50}},{"accountId":"","value":{"name":"field2","percent":50}}]`, rr.Body.String()) + }) +} diff --git a/pkg/tsdb/cloudwatch/routes/log_groups_test.go b/pkg/tsdb/cloudwatch/routes/log_groups_test.go index 4902679c93a..ed395810ef2 100644 --- a/pkg/tsdb/cloudwatch/routes/log_groups_test.go +++ b/pkg/tsdb/cloudwatch/routes/log_groups_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/mock" ) -func Test_log_groups_route(t *testing.T) { +func TestLogGroupsRoute(t *testing.T) { origLogGroupsService := newLogGroupsService t.Cleanup(func() { newLogGroupsService = origLogGroupsService diff --git a/pkg/tsdb/cloudwatch/services/logs.go b/pkg/tsdb/cloudwatch/services/log_groups.go similarity index 62% rename from pkg/tsdb/cloudwatch/services/logs.go rename to pkg/tsdb/cloudwatch/services/log_groups.go index 64658bc51df..56ef087ad86 100644 --- a/pkg/tsdb/cloudwatch/services/logs.go +++ b/pkg/tsdb/cloudwatch/services/log_groups.go @@ -51,3 +51,31 @@ func (s *LogGroupsService) GetLogGroups(req resources.LogGroupsRequest) ([]resou return result, nil } + +func (s *LogGroupsService) GetLogGroupFields(request resources.LogGroupFieldsRequest) ([]resources.ResourceResponse[resources.LogGroupField], error) { + input := &cloudwatchlogs.GetLogGroupFieldsInput{ + LogGroupName: aws.String(request.LogGroupName), + } + // we should use LogGroupIdentifier instead of LogGroupName, but currently the api doesn't accept LogGroupIdentifier. need to check if it's a bug or not. + // if request.LogGroupARN != "" { + // input.LogGroupIdentifier = aws.String(strings.TrimSuffix(request.LogGroupARN, ":*")) + // input.LogGroupName = nil + // } + + getLogGroupFieldsOutput, err := s.logGroupsAPI.GetLogGroupFields(input) + if err != nil { + return nil, err + } + + result := make([]resources.ResourceResponse[resources.LogGroupField], 0) + for _, logGroupField := range getLogGroupFieldsOutput.LogGroupFields { + result = append(result, resources.ResourceResponse[resources.LogGroupField]{ + Value: resources.LogGroupField{ + Name: *logGroupField.Name, + Percent: *logGroupField.Percent, + }, + }) + } + + return result, nil +} diff --git a/pkg/tsdb/cloudwatch/services/logs_test.go b/pkg/tsdb/cloudwatch/services/log_groups_test.go similarity index 69% rename from pkg/tsdb/cloudwatch/services/logs_test.go rename to pkg/tsdb/cloudwatch/services/log_groups_test.go index a1b81a8d094..5bead8d9845 100644 --- a/pkg/tsdb/cloudwatch/services/logs_test.go +++ b/pkg/tsdb/cloudwatch/services/log_groups_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/mock" ) -func Test_GetLogGroups(t *testing.T) { +func TestGetLogGroups(t *testing.T) { t.Run("Should map log groups response", func(t *testing.T) { mockLogsAPI := &mocks.LogsAPI{} mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return( @@ -88,7 +88,7 @@ func Test_GetLogGroups(t *testing.T) { }) } -func Test_GetLogGroups_crossAccountQuerying(t *testing.T) { +func TestGetLogGroupsCrossAccountQuerying(t *testing.T) { t.Run("Should not includeLinkedAccounts or accountId if isCrossAccountEnabled is set to false", func(t *testing.T) { mockLogsAPI := &mocks.LogsAPI{} mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) @@ -198,3 +198,108 @@ func Test_GetLogGroups_crossAccountQuerying(t *testing.T) { }) }) } + +func TestGetLogGroupFields(t *testing.T) { + t.Run("Should map log group fields response", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return( + &cloudwatchlogs.GetLogGroupFieldsOutput{ + LogGroupFields: []*cloudwatchlogs.LogGroupField{ + { + Name: utils.Pointer("field1"), + Percent: utils.Pointer(int64(10)), + }, { + Name: utils.Pointer("field2"), + Percent: utils.Pointer(int64(10)), + }, { + Name: utils.Pointer("field3"), + Percent: utils.Pointer(int64(10)), + }, + }, + }, nil) + + service := NewLogGroupsService(mockLogsAPI, false) + resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{}) + + assert.NoError(t, err) + assert.Equal(t, []resources.ResourceResponse[resources.LogGroupField]{ + { + Value: resources.LogGroupField{ + Name: "field1", + Percent: 10, + }, + }, + { + Value: resources.LogGroupField{ + Name: "field2", + Percent: 10, + }, + }, + { + Value: resources.LogGroupField{ + Name: "field3", + Percent: 10, + }, + }, + }, resp) + }) + + // uncomment this test if when it's possible to pass only LogGroupIdentifier to the api + // t.Run("Should only set LogGroupIdentifier as api input in case both LogGroupName and LogGroupARN are specified", func(t *testing.T) { + // mockLogsAPI := &mocks.LogsAPI{} + // mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return( + // &cloudwatchlogs.GetLogGroupFieldsOutput{}, nil) + + // service := NewLogGroupsService(mockLogsAPI, false) + // resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{ + // LogGroupName: "logGroupName", + // LogGroupARN: "logGroupARN", + // }) + + // mockLogsAPI.AssertCalled(t, "GetLogGroupFields", &cloudwatchlogs.GetLogGroupFieldsInput{ + // LogGroupIdentifier: utils.Pointer("logGroupARN"), + // LogGroupName: nil, + // }) + // assert.NotNil(t, resp) + // assert.NoError(t, err) + // }) + + // remove this test once the above test is uncommented + t.Run("Should only set LogGroupName as api input in case both LogGroupName and LogGroupARN are specified", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return( + &cloudwatchlogs.GetLogGroupFieldsOutput{}, nil) + + service := NewLogGroupsService(mockLogsAPI, false) + resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{ + LogGroupName: "logGroupName", + LogGroupARN: "logGroupARN", + }) + + mockLogsAPI.AssertCalled(t, "GetLogGroupFields", &cloudwatchlogs.GetLogGroupFieldsInput{ + LogGroupIdentifier: nil, + LogGroupName: utils.Pointer("logGroupName"), + }) + assert.NotNil(t, resp) + assert.NoError(t, err) + }) + + t.Run("Should only set LogGroupName as api input in case only LogGroupName is specified", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return( + &cloudwatchlogs.GetLogGroupFieldsOutput{}, nil) + + service := NewLogGroupsService(mockLogsAPI, false) + resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{ + LogGroupName: "logGroupName", + LogGroupARN: "", + }) + + mockLogsAPI.AssertCalled(t, "GetLogGroupFields", &cloudwatchlogs.GetLogGroupFieldsInput{ + LogGroupIdentifier: nil, + LogGroupName: utils.Pointer("logGroupName"), + }) + assert.NotNil(t, resp) + assert.NoError(t, err) + }) +} diff --git a/pkg/tsdb/cloudwatch/test_utils.go b/pkg/tsdb/cloudwatch/test_utils.go index 0b3a07039c0..d04e3163eac 100644 --- a/pkg/tsdb/cloudwatch/test_utils.go +++ b/pkg/tsdb/cloudwatch/test_utils.go @@ -184,6 +184,10 @@ func (c fakeCheckHealthClient) DescribeLogGroups(input *cloudwatchlogs.DescribeL return nil, nil } +func (c fakeCheckHealthClient) GetLogGroupFields(input *cloudwatchlogs.GetLogGroupFieldsInput) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) { + return nil, nil +} + func newTestConfig() *setting.Cfg { return &setting.Cfg{AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true, AWSListMetricsPageLimit: 1000} } diff --git a/public/app/plugins/datasource/cloudwatch/api.ts b/public/app/plugins/datasource/cloudwatch/api.ts index 7b7c9578e38..7b7e4b97037 100644 --- a/public/app/plugins/datasource/cloudwatch/api.ts +++ b/public/app/plugins/datasource/cloudwatch/api.ts @@ -17,6 +17,8 @@ import { Account, ResourceRequest, ResourceResponse, + GetLogGroupFieldsRequest, + LogGroupField, } from './types'; export interface SelectableResourceValue extends SelectableValue { @@ -70,6 +72,18 @@ export class CloudWatchAPI extends CloudWatchRequest { }); } + async getLogGroupFields({ + region, + arn, + logGroupName, + }: GetLogGroupFieldsRequest): Promise>> { + return this.memoizedGetRequest>>('log-group-fields', { + region: this.templateSrv.replace(this.getActualRegion(region)), + logGroupName: this.templateSrv.replace(logGroupName, {}), + logGroupArn: this.templateSrv.replace(arn), + }); + } + async describeAllLogGroups(params: DescribeLogGroupsRequest) { return this.memoizedGetRequest('all-log-groups', { ...params, diff --git a/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx b/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx index 407afce7100..42f71d3522a 100644 --- a/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx @@ -61,7 +61,8 @@ export const CloudWatchLogsQueryField = (props: CloudWatchLogsQueryFieldProps) = }; const onTypeahead = async (typeahead: TypeaheadInput): Promise => { - const { logGroupNames } = query; + const { datasource, query } = props; + const { logGroups } = query; if (!datasource.languageProvider) { return { suggestions: [] }; @@ -76,7 +77,7 @@ export const CloudWatchLogsQueryField = (props: CloudWatchLogsQueryFieldProps) = { history, absoluteRange, - logGroupNames, + logGroups: logGroups, region: query.region, } ); diff --git a/public/app/plugins/datasource/cloudwatch/language_provider.test.ts b/public/app/plugins/datasource/cloudwatch/language_provider.test.ts index 9b93950c0ef..23ddde3b747 100644 --- a/public/app/plugins/datasource/cloudwatch/language_provider.test.ts +++ b/public/app/plugins/datasource/cloudwatch/language_provider.test.ts @@ -15,7 +15,7 @@ import { STRING_FUNCTIONS, FIELD_AND_FILTER_FUNCTIONS, } from './syntax'; -import { GetLogGroupFieldsResponse } from './types'; +import { LogGroupField, ResourceResponse } from './types'; const fields = ['field1', '@message']; @@ -109,9 +109,9 @@ async function runSuggestionTest(query: string, expectedItems: string[][]) { function makeDatasource(): CloudWatchDatasource { return { - logsQueryRunner: { - getLogGroupFields(): Promise { - return Promise.resolve({ logGroupFields: [{ name: 'field1' }, { name: '@message' }] }); + api: { + getLogGroupFields(): Promise>> { + return Promise.resolve([{ value: { name: 'field1' } }, { value: { name: '@message' } }]); }, }, } as any; @@ -131,7 +131,7 @@ function getProvideCompletionItems(query: string): Promise { { value, } as any, - { logGroupNames: ['logGroup1'], region: 'custom' } + { logGroups: [{ name: 'logGroup1', arn: 'logGroup1' }], region: 'custom' } ); } diff --git a/public/app/plugins/datasource/cloudwatch/language_provider.ts b/public/app/plugins/datasource/cloudwatch/language_provider.ts index 1a863f1a4e5..1588fb39cf1 100644 --- a/public/app/plugins/datasource/cloudwatch/language_provider.ts +++ b/public/app/plugins/datasource/cloudwatch/language_provider.ts @@ -1,4 +1,3 @@ -import { sortedUniq } from 'lodash'; import Prism, { Grammar } from 'prismjs'; import { lastValueFrom } from 'rxjs'; @@ -16,14 +15,14 @@ import syntax, { QUERY_COMMANDS, STRING_FUNCTIONS, } from './syntax'; -import { CloudWatchQuery, TSDBResponse } from './types'; +import { CloudWatchQuery, LogGroup, TSDBResponse } from './types'; export type CloudWatchHistoryItem = HistoryItem; type TypeaheadContext = { history?: CloudWatchHistoryItem[]; absoluteRange?: AbsoluteTimeRange; - logGroupNames?: string[]; + logGroups?: LogGroup[]; region: string; }; @@ -106,7 +105,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider { } if (isInsideFunctionParenthesis(curToken)) { - return await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default'); + return await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default'); } if (isAfterKeyword('by', curToken)) { @@ -127,44 +126,20 @@ export class CloudWatchLanguageProvider extends LanguageProvider { }; } - private fetchedFieldsCache: - | { - time: number; - logGroups: string[]; - fields: string[]; - } - | undefined; - - private fetchFields = async (logGroups: string[], region: string): Promise => { - if ( - this.fetchedFieldsCache && - Date.now() - this.fetchedFieldsCache.time < 30 * 1000 && - sortedUniq(this.fetchedFieldsCache.logGroups).join('|') === sortedUniq(logGroups).join('|') - ) { - return this.fetchedFieldsCache.fields; - } - + private fetchFields = async (logGroups: LogGroup[], region: string): Promise => { const results = await Promise.all( - logGroups.map((logGroup) => this.datasource.logsQueryRunner.getLogGroupFields({ logGroupName: logGroup, region })) + logGroups.map((logGroup) => + this.datasource.api + .getLogGroupFields({ logGroupName: logGroup.name, arn: logGroup.arn, region }) + .then((fields) => fields.filter((f) => f).map((f) => f.value.name ?? '')) + ) ); - const fields = [ - ...new Set( - results.reduce((acc: string[], cur) => acc.concat(cur.logGroupFields?.map((f) => f.name) as string[]), []) - ).values(), - ]; - - this.fetchedFieldsCache = { - time: Date.now(), - logGroups, - fields, - }; - - return fields; + return results.flat(); }; private handleKeyword = async (context?: TypeaheadContext): Promise => { - const suggs = await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default'); + const suggs = await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default'); const functionSuggestions: CompletionItemGroup[] = [ { searchFunctionType: SearchFunctionType.Prefix, @@ -192,7 +167,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider { if (queryCommand === 'parse') { if (currentTokenIsFirstArg) { - return await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default'); + return await this.getFieldCompletionItems(context?.logGroups ?? [], context?.region || 'default'); } } @@ -210,7 +185,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider { if (['display', 'fields'].includes(queryCommand)) { const typeaheadOutput = await this.getFieldCompletionItems( - context?.logGroupNames ?? [], + context?.logGroups ?? [], context?.region || 'default' ); typeaheadOutput.suggestions.push(...this.getFieldAndFilterFunctionCompletionItems().suggestions); @@ -229,7 +204,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider { } if (queryCommand === 'filter' && currentTokenIsFirstArg) { - const sugg = await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default'); + const sugg = await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default'); const boolFuncs = this.getBoolFuncCompletionItems(); sugg.suggestions.push(...boolFuncs.suggestions); return sugg; @@ -243,7 +218,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider { context?: TypeaheadContext ): Promise { if (isFirstArgument) { - return await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default'); + return await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default'); } else if (isTokenType(prevNonWhitespaceToken(curToken), 'field-name')) { // suggest sort options return { @@ -266,10 +241,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider { } private handleComparison = async (context?: TypeaheadContext) => { - const fieldsSuggestions = await this.getFieldCompletionItems( - context?.logGroupNames ?? [], - context?.region || 'default' - ); + const fieldsSuggestions = await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default'); const comparisonSuggestions = this.getComparisonCompletionItems(); fieldsSuggestions.suggestions.push(...comparisonSuggestions.suggestions); return fieldsSuggestions; @@ -321,9 +293,15 @@ export class CloudWatchLanguageProvider extends LanguageProvider { }; }; - private getFieldCompletionItems = async (logGroups: string[], region: string): Promise => { - const fields = await this.fetchFields(logGroups, region); + private getFieldCompletionItems = async ( + logGroups: LogGroup[] | undefined, + region: string + ): Promise => { + if (!logGroups) { + return { suggestions: [] }; + } + const fields = await this.fetchFields(logGroups, region); return { suggestions: [ { diff --git a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts index 80c8e3d59e7..686687214a4 100644 --- a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts +++ b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts @@ -1,6 +1,6 @@ import { interval, lastValueFrom, of } from 'rxjs'; -import { dataFrameToJSON, DataQueryErrorType, FieldType, LogLevel, LogRowModel, MutableDataFrame } from '@grafana/data'; +import { DataQueryErrorType, FieldType, LogLevel, LogRowModel, MutableDataFrame } from '@grafana/data'; import { genMockFrames, setupMockedLogsQueryRunner } from '../__mocks__/LogsQueryRunner'; import { validLogsQuery } from '../__mocks__/queries'; @@ -51,34 +51,6 @@ describe('CloudWatchLogsQueryRunner', () => { }); }); - describe('getLogGroupFields', () => { - it('passes region correctly', async () => { - const { runner, fetchMock } = setupMockedLogsQueryRunner(); - fetchMock.mockReturnValueOnce( - of({ - data: { - results: { - A: { - frames: [ - dataFrameToJSON( - new MutableDataFrame({ - fields: [ - { name: 'key', values: [] }, - { name: 'val', values: [] }, - ], - }) - ), - ], - }, - }, - }, - }) - ); - await runner.getLogGroupFields({ region: 'us-west-1', logGroupName: 'test' }); - expect(fetchMock.mock.calls[0][0].data.queries[0].region).toBe('us-west-1'); - }); - }); - describe('logs query', () => { beforeEach(() => { jest.spyOn(rxjsUtils, 'increasingInterval').mockImplementation(() => interval(100)); diff --git a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts index 15d129cd036..e20eb202f5d 100644 --- a/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts +++ b/public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts @@ -42,7 +42,6 @@ import { DescribeLogGroupsRequest, GetLogEventsRequest, GetLogGroupFieldsRequest, - GetLogGroupFieldsResponse, LogAction, StartQueryRequest, } from '../types'; @@ -415,18 +414,6 @@ export class CloudWatchLogsQueryRunner extends CloudWatchRequest { }; }; - async getLogGroupFields(params: GetLogGroupFieldsRequest): Promise { - const dataFrames = await lastValueFrom(this.makeLogActionRequest('GetLogGroupFields', [params])); - - const fieldNames = dataFrames[0].fields[0].values.toArray(); - const fieldPercentages = dataFrames[0].fields[1].values.toArray(); - const getLogGroupFieldsResponse = { - logGroupFields: fieldNames.map((val, i) => ({ name: val, percent: fieldPercentages[i] })) ?? [], - }; - - return getLogGroupFieldsResponse; - } - private filterQuery(query: CloudWatchLogsQuery) { const hasMissingLegacyLogGroupNames = !query.logGroupNames?.length; const hasMissingLogGroups = !query.logGroups?.length; diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index 76f05b0df4e..acfc480cdbe 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -267,36 +267,17 @@ export interface TSDBTimeSeries { } export type TSDBTimePoint = [number, number]; -export interface GetLogGroupFieldsRequest { - /** - * The name of the log group to search. - */ - logGroupName: string; - /** - * The time to set as the center of the query. If you specify time, the 8 minutes before and 8 minutes after this time are searched. If you omit time, the past 15 minutes are queried. The time value is specified as epoch time, the number of seconds since January 1, 1970, 00:00:00 UTC. - */ - time?: number; - region: string; -} - export interface LogGroupField { /** * The name of a log field. */ - name?: string; + name: string; /** * The percentage of log events queried that contained the field. */ percent?: number; } -export interface GetLogGroupFieldsResponse { - /** - * The array of fields found in the query. Each object in the array contains the name of the field, along with the percentage of time it appeared in the log events that were queried. - */ - logGroupFields?: LogGroupField[]; -} - export interface StartQueryRequest { /** * The log group on which to perform the query. A StartQuery operation must include a logGroupNames or a logGroupName parameter, but not both. @@ -423,6 +404,17 @@ export interface ResourceRequest { accountId?: string; } +export interface GetLogGroupFieldsRequest extends ResourceRequest { + /** + * The log group identifier + */ + arn?: string; + /** + * The name of the log group to search. + */ + logGroupName: string; +} + export interface GetDimensionKeysRequest extends ResourceRequest { metricName?: string; namespace?: string;