diff --git a/pkg/tsdb/cloudwatch/clients/metrics.go b/pkg/tsdb/cloudwatch/clients/metrics.go new file mode 100644 index 00000000000..d205c968cee --- /dev/null +++ b/pkg/tsdb/cloudwatch/clients/metrics.go @@ -0,0 +1,36 @@ +package clients + +import ( + "github.com/aws/aws-sdk-go/aws/awsutil" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/infra/metrics" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" +) + +type metricsClient struct { + models.CloudWatchMetricsAPIProvider + config *setting.Cfg +} + +func NewMetricsClient(api models.CloudWatchMetricsAPIProvider, config *setting.Cfg) *metricsClient { + return &metricsClient{CloudWatchMetricsAPIProvider: api, config: config} +} + +func (l *metricsClient) ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) { + var cloudWatchMetrics []*cloudwatch.Metric + pageNum := 0 + err := l.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { + pageNum++ + metrics.MAwsCloudWatchListMetrics.Inc() + metrics, err := awsutil.ValuesAtPath(page, "Metrics") + if err == nil { + for _, metric := range metrics { + cloudWatchMetrics = append(cloudWatchMetrics, metric.(*cloudwatch.Metric)) + } + } + return !lastPage && pageNum < l.config.AWSListMetricsPageLimit + }) + + return cloudWatchMetrics, err +} diff --git a/pkg/tsdb/cloudwatch/clients/metrics_test.go b/pkg/tsdb/cloudwatch/clients/metrics_test.go new file mode 100644 index 00000000000..91636775ba5 --- /dev/null +++ b/pkg/tsdb/cloudwatch/clients/metrics_test.go @@ -0,0 +1,49 @@ +package clients + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetricsClient(t *testing.T) { + metrics := []*cloudwatch.Metric{ + {MetricName: aws.String("Test_MetricName1")}, + {MetricName: aws.String("Test_MetricName2")}, + {MetricName: aws.String("Test_MetricName3")}, + {MetricName: aws.String("Test_MetricName4")}, + {MetricName: aws.String("Test_MetricName5")}, + {MetricName: aws.String("Test_MetricName6")}, + {MetricName: aws.String("Test_MetricName7")}, + {MetricName: aws.String("Test_MetricName8")}, + {MetricName: aws.String("Test_MetricName9")}, + {MetricName: aws.String("Test_MetricName10")}, + } + + t.Run("List Metrics and page limit is reached", func(t *testing.T) { + pageLimit := 3 + fakeApi := &mocks.FakeMetricsAPI{Metrics: metrics, MetricsPerPage: 2} + client := NewMetricsClient(fakeApi, &setting.Cfg{AWSListMetricsPageLimit: pageLimit}) + response, err := client.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{}) + require.NoError(t, err) + + expectedMetrics := fakeApi.MetricsPerPage * pageLimit + assert.Equal(t, expectedMetrics, len(response)) + }) + + t.Run("List Metrics and page limit is not reached", func(t *testing.T) { + pageLimit := 2 + fakeApi := &mocks.FakeMetricsAPI{Metrics: metrics} + client := NewMetricsClient(fakeApi, &setting.Cfg{AWSListMetricsPageLimit: pageLimit}) + + response, err := client.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{}) + require.NoError(t, err) + + assert.Equal(t, len(metrics), len(response)) + }) +} diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 2171bf48e72..409e994f6ab 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -29,7 +29,9 @@ import ( "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/clients" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/cwlog" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" ) type datasourceInfo struct { @@ -104,14 +106,34 @@ type SessionCache interface { } func newExecutor(im instancemgmt.InstanceManager, cfg *setting.Cfg, sessions SessionCache, features featuremgmt.FeatureToggles) *cloudWatchExecutor { - cwe := &cloudWatchExecutor{ + e := &cloudWatchExecutor{ im: im, cfg: cfg, sessions: sessions, features: features, } - cwe.resourceHandler = httpadapter.New(cwe.newResourceMux()) - return cwe + + e.resourceHandler = httpadapter.New(e.newResourceMux()) + return e +} + +func (e *cloudWatchExecutor) getClients(pluginCtx backend.PluginContext, region string) (models.Clients, error) { + r := region + if region == defaultRegion { + dsInfo, err := e.getDSInfo(pluginCtx) + if err != nil { + return models.Clients{}, err + } + r = dsInfo.region + } + + sess, err := e.newSession(pluginCtx, r) + if err != nil { + return models.Clients{}, err + } + return models.Clients{ + MetricsClientProvider: clients.NewMetricsClient(NewMetricsAPI(sess), e.cfg), + }, nil } func NewInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc { @@ -440,6 +462,13 @@ func isTerminated(queryStatus string) bool { return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout" } +// NewMetricsAPI is a CloudWatch metrics api factory. +// +// Stubbable by tests. +var NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvider { + return cloudwatch.New(sess) +} + // NewCWClient is a CloudWatch client factory. // // Stubbable by tests. diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go index c428435649e..7276740c504 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -21,6 +21,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -463,6 +466,105 @@ func TestQuery_ResourceRequest_DescribeLogGroups(t *testing.T) { }) } +func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { + sender := &mockedCallResourceResponseSenderForOauth{} + origNewMetricsAPI := NewMetricsAPI + t.Cleanup(func() { + NewMetricsAPI = origNewMetricsAPI + }) + var api mocks.FakeMetricsAPI + NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvider { + return &api + } + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { + return datasourceInfo{}, nil + }) + + t.Run("Should handle dimension key filter query and return keys from the api", func(t *testing.T) { + pageLimit := 3 + api = mocks.FakeMetricsAPI{Metrics: []*cloudwatch.Metric{ + {MetricName: aws.String("Test_MetricName1"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}, {Name: aws.String("Test_DimensionName2")}}}, + {MetricName: aws.String("Test_MetricName2"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + {MetricName: aws.String("Test_MetricName3"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName2")}}}, + {MetricName: aws.String("Test_MetricName10"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName4")}, {Name: aws.String("Test_DimensionName5")}}}, + {MetricName: aws.String("Test_MetricName4"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName2")}}}, + {MetricName: aws.String("Test_MetricName5"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + {MetricName: aws.String("Test_MetricName6"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + {MetricName: aws.String("Test_MetricName7"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName4")}}}, + {MetricName: aws.String("Test_MetricName8"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName4")}}}, + {MetricName: aws.String("Test_MetricName9"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + }, MetricsPerPage: 2} + executor := newExecutor(im, &setting.Cfg{AWSListMetricsPageLimit: pageLimit}, &fakeSessionCache{}, featuremgmt.WithFeatures()) + + req := &backend.CallResourceRequest{ + Method: "GET", + Path: `/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization&dimensionFilters={"NodeID":["Shared"],"stage":["QueryCommit"]}`, + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0}, + PluginID: "cloudwatch", + }, + } + err := executor.CallResource(context.Background(), req, sender) + + require.NoError(t, err) + sent := sender.Response + require.NotNil(t, sent) + require.Equal(t, http.StatusOK, sent.Status) + res := []string{} + err = json.Unmarshal(sent.Body, &res) + require.Nil(t, err) + assert.Equal(t, []string{"Test_DimensionName1", "Test_DimensionName2", "Test_DimensionName4", "Test_DimensionName5"}, res) + }) + + t.Run("Should handle standard dimension key query and return hard coded keys", func(t *testing.T) { + api = mocks.FakeMetricsAPI{} + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + + req := &backend.CallResourceRequest{ + Method: "GET", + Path: `/dimension-keys?region=us-east-2&namespace=AWS/CloudSearch&metricName=CPUUtilization`, + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0}, + PluginID: "cloudwatch", + }, + } + err := executor.CallResource(context.Background(), req, sender) + + require.NoError(t, err) + sent := sender.Response + require.NotNil(t, sent) + require.Equal(t, http.StatusOK, sent.Status) + res := []string{} + err = json.Unmarshal(sent.Body, &res) + require.Nil(t, err) + assert.Equal(t, []string{"ClientId", "DomainName"}, res) + }) + + t.Run("Should handle custom namespace dimension key query and return hard coded keys", func(t *testing.T) { + api = mocks.FakeMetricsAPI{} + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + + req := &backend.CallResourceRequest{ + Method: "GET", + Path: `/dimension-keys?region=us-east-2&namespace=AWS/CloudSearch&metricName=CPUUtilization`, + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0}, + PluginID: "cloudwatch", + }, + } + err := executor.CallResource(context.Background(), req, sender) + + require.NoError(t, err) + sent := sender.Response + require.NotNil(t, sent) + require.Equal(t, http.StatusOK, sent.Status) + res := []string{} + err = json.Unmarshal(sent.Body, &res) + require.Nil(t, err) + assert.Equal(t, []string{"ClientId", "DomainName"}, res) + }) +} + func stringsToSuggestData(values []string) []suggestData { suggestDataArray := make([]suggestData, 0) for _, v := range values { diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index 521dac628c2..ff686ad2c04 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -36,8 +36,6 @@ type customMetricsCache struct { } var customMetricsMetricsMap = make(map[string]map[string]map[string]*customMetricsCache) -var customMetricsDimensionsMap = make(map[string]map[string]map[string]*customMetricsCache) - var regionCache sync.Map func parseMultiSelectValue(input string) []string { @@ -168,104 +166,6 @@ func (e *cloudWatchExecutor) handleGetAllMetrics(pluginCtx backend.PluginContext return result, nil } -// handleGetDimensionKeys returns a slice of suggestData structs with dimension keys. -// If a dimension filters parameter is specified, a new api call to list metrics will be issued to load dimension keys for the given filter. -// If no dimension filter is specified, dimension keys will be retrieved from the hard coded map in this file. -func (e *cloudWatchExecutor) handleGetDimensionKeys(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) { - region := parameters.Get("region") - namespace := parameters.Get("namespace") - metricName := parameters.Get("metricName") - dimensionFilterJson := parameters.Get("dimensionFilters") - - dimensionFilters := map[string]interface{}{} - if dimensionFilterJson != "" { - err := json.Unmarshal([]byte(dimensionFilterJson), &dimensionFilters) - if err != nil { - return nil, fmt.Errorf("error unmarshaling dimensionFilters: %v", err) - } - } - - var dimensionValues []string - if !isCustomMetrics(namespace) { - if len(dimensionFilters) != 0 { - var dimensions []*cloudwatch.DimensionFilter - addDimension := func(key string, value string) { - filter := &cloudwatch.DimensionFilter{ - Name: aws.String(key), - } - // if value is not specified or a wildcard is used, simply don't use the value field - if value != "" && value != "*" { - filter.Value = aws.String(value) - } - dimensions = append(dimensions, filter) - } - for k, v := range dimensionFilters { - // due to legacy, value can be a string, a string slice or nil - if vv, ok := v.(string); ok { - addDimension(k, vv) - } else if vv, ok := v.([]interface{}); ok { - for _, v := range vv { - addDimension(k, v.(string)) - } - } else if v == nil { - addDimension(k, "") - } - } - - input := &cloudwatch.ListMetricsInput{ - Namespace: aws.String(namespace), - Dimensions: dimensions, - } - - if metricName != "" { - input.MetricName = aws.String(metricName) - } - - metrics, err := e.listMetrics(pluginCtx, - region, input) - - if err != nil { - return nil, fmt.Errorf("%v: %w", "unable to call AWS API", err) - } - - dupCheck := make(map[string]bool) - for _, metric := range metrics { - for _, dim := range metric.Dimensions { - if _, exists := dupCheck[*dim.Name]; exists { - continue - } - - // keys in the dimension filter should not be included - if _, ok := dimensionFilters[*dim.Name]; ok { - continue - } - - dupCheck[*dim.Name] = true - dimensionValues = append(dimensionValues, *dim.Name) - } - } - } else { - var exists bool - if dimensionValues, exists = constants.NamespaceDimensionKeysMap[namespace]; !exists { - return nil, fmt.Errorf("unable to find dimension %q", namespace) - } - } - } else { - var err error - if dimensionValues, err = e.getDimensionsForCustomMetrics(region, namespace, pluginCtx); err != nil { - return nil, fmt.Errorf("%v: %w", "unable to call AWS API", err) - } - } - sort.Strings(dimensionValues) - - result := make([]suggestData, 0) - for _, name := range dimensionValues { - result = append(result, suggestData{Text: name, Value: name, Label: name}) - } - - return result, nil -} - // handleGetDimensionValues returns a slice of suggestData structs with dimension values. // A call to the list metrics api is issued to retrieve the dimension values. All parameters are used as input args to the list metrics call. func (e *cloudWatchExecutor) handleGetDimensionValues(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) { @@ -608,51 +508,6 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region, namespace string return customMetricsMetricsMap[dsInfo.profile][dsInfo.region][namespace].Cache, nil } -var dimensionsCacheLock sync.Mutex - -func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region, namespace string, pluginCtx backend.PluginContext) ([]string, error) { - dimensionsCacheLock.Lock() - defer dimensionsCacheLock.Unlock() - - dsInfo, err := e.getDSInfo(pluginCtx) - if err != nil { - return nil, err - } - - if _, ok := customMetricsDimensionsMap[dsInfo.profile]; !ok { - customMetricsDimensionsMap[dsInfo.profile] = make(map[string]map[string]*customMetricsCache) - } - if _, ok := customMetricsDimensionsMap[dsInfo.profile][dsInfo.region]; !ok { - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region] = make(map[string]*customMetricsCache) - } - if _, ok := customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace]; !ok { - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace] = &customMetricsCache{} - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache = make([]string, 0) - } - - if customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Expire.After(time.Now()) { - return customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache, nil - } - metrics, err := e.listMetrics(pluginCtx, region, &cloudwatch.ListMetricsInput{Namespace: aws.String(namespace)}) - if err != nil { - return []string{}, err - } - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache = make([]string, 0) - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Expire = time.Now().Add(5 * time.Minute) - - for _, metric := range metrics { - for _, dimension := range metric.Dimensions { - if isDuplicate(customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache, *dimension.Name) { - continue - } - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache = append( - customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache, *dimension.Name) - } - } - - return customMetricsDimensionsMap[dsInfo.profile][dsInfo.region][namespace].Cache, nil -} - func (e *cloudWatchExecutor) handleGetLogGroups(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) { region := parameters.Get("region") limit := parameters.Get("limit") diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index c4930131984..c618ebed9a5 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -19,8 +19,8 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -31,14 +31,14 @@ func TestQuery_Metrics(t *testing.T) { NewCWClient = origNewCWClient }) - var cwClient fakeCWClient + var cwClient mocks.FakeMetricsAPI NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { return &cwClient } t.Run("Custom metrics", func(t *testing.T) { - cwClient = fakeCWClient{ + cwClient = mocks.FakeMetricsAPI{ Metrics: []*cloudwatch.Metric{ { MetricName: aws.String("Test_MetricName"), @@ -71,41 +71,6 @@ func TestQuery_Metrics(t *testing.T) { } assert.Equal(t, expResponse, resp) }) - - t.Run("Dimension keys for custom metrics", func(t *testing.T) { - cwClient = fakeCWClient{ - Metrics: []*cloudwatch.Metric{ - { - MetricName: aws.String("Test_MetricName"), - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("Test_DimensionName"), - }, - }, - }, - }, - } - - im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { - return datasourceInfo{}, nil - }) - - executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) - resp, err := executor.handleGetDimensionKeys( - backend.PluginContext{ - DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, - }, url.Values{ - "region": []string{"us-east-1"}, - "namespace": []string{"custom"}, - }, - ) - require.NoError(t, err) - - expResponse := []suggestData{ - {Text: "Test_DimensionName", Value: "Test_DimensionName", Label: "Test_DimensionName"}, - } - assert.Equal(t, expResponse, resp) - }) } func TestQuery_Regions(t *testing.T) { @@ -395,184 +360,3 @@ func TestQuery_GetAllMetrics(t *testing.T) { assert.Equal(t, metricCount, len(resp)) }) } - -func TestQuery_GetDimensionKeys(t *testing.T) { - origNewCWClient := NewCWClient - t.Cleanup(func() { - NewCWClient = origNewCWClient - }) - - var client fakeCWClient - - NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { - return &client - } - - metrics := []*cloudwatch.Metric{ - {MetricName: aws.String("Test_MetricName1"), Dimensions: []*cloudwatch.Dimension{ - {Name: aws.String("Dimension1"), Value: aws.String("Dimension1")}, - {Name: aws.String("Dimension2"), Value: aws.String("Dimension2")}, - }}, - {MetricName: aws.String("Test_MetricName2"), Dimensions: []*cloudwatch.Dimension{ - {Name: aws.String("Dimension2"), Value: aws.String("Dimension2")}, - {Name: aws.String("Dimension3"), Value: aws.String("Dimension3")}, - }}, - } - - t.Run("should fetch dimension keys from list metrics api and return unique dimensions when a dimension filter is specified", func(t *testing.T) { - client = fakeCWClient{Metrics: metrics, MetricsPerPage: 2} - im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { - return datasourceInfo{}, nil - }) - - executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) - resp, err := executor.handleGetDimensionKeys( - backend.PluginContext{ - DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, - }, - url.Values{ - "region": []string{"us-east-1"}, - "namespace": []string{"AWS/EC2"}, - "dimensionFilters": []string{`{ - "InstanceId": "", - "AutoscalingGroup": [] - }`}, - }, - ) - require.NoError(t, err) - - expValues := []string{"Dimension1", "Dimension2", "Dimension3"} - expResponse := []suggestData{} - for _, val := range expValues { - expResponse = append(expResponse, suggestData{val, val, val}) - } - - assert.Equal(t, expResponse, resp) - }) - - t.Run("should return hard coded metrics when no dimension filter is specified", func(t *testing.T) { - im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { - return datasourceInfo{}, nil - }) - - executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) - resp, err := executor.handleGetDimensionKeys( - backend.PluginContext{ - DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, - }, - url.Values{ - "region": []string{"us-east-1"}, - "namespace": []string{"AWS/EC2"}, - "dimensionFilters": []string{`{}`}, - }, - ) - require.NoError(t, err) - - expValues := constants.NamespaceDimensionKeysMap["AWS/EC2"] - expResponse := []suggestData{} - for _, val := range expValues { - expResponse = append(expResponse, suggestData{val, val, val}) - } - - assert.Equal(t, expResponse, resp) - }) -} -func Test_isCustomMetrics(t *testing.T) { - constants.NamespaceMetricsMap = map[string][]string{ - "AWS/EC2": {"ExampleMetric"}, - } - - type args struct { - namespace string - } - - tests := []struct { - name string - args args - want bool - }{ - {name: "A custom metric should return true", - want: true, - args: args{ - namespace: "Custom/MyApp", - }, - }, - {name: "An AWS metric not included in this package should return true", - want: true, - args: args{ - namespace: "AWS/MyApp", - }, - }, - {name: "An AWS metric included in this package should return false", - want: false, - args: args{ - namespace: "AWS/EC2", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isCustomMetrics(tt.args.namespace); got != tt.want { - t.Errorf("isCustomMetrics() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestQuery_ListMetricsPagination(t *testing.T) { - origNewCWClient := NewCWClient - t.Cleanup(func() { - NewCWClient = origNewCWClient - }) - - var client fakeCWClient - - NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { - return &client - } - - metrics := []*cloudwatch.Metric{ - {MetricName: aws.String("Test_MetricName1")}, - {MetricName: aws.String("Test_MetricName2")}, - {MetricName: aws.String("Test_MetricName3")}, - {MetricName: aws.String("Test_MetricName4")}, - {MetricName: aws.String("Test_MetricName5")}, - {MetricName: aws.String("Test_MetricName6")}, - {MetricName: aws.String("Test_MetricName7")}, - {MetricName: aws.String("Test_MetricName8")}, - {MetricName: aws.String("Test_MetricName9")}, - {MetricName: aws.String("Test_MetricName10")}, - } - - t.Run("List Metrics and page limit is reached", func(t *testing.T) { - client = fakeCWClient{Metrics: metrics, MetricsPerPage: 2} - im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { - return datasourceInfo{}, nil - }) - executor := newExecutor(im, &setting.Cfg{AWSListMetricsPageLimit: 3, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}, &fakeSessionCache{}, - featuremgmt.WithFeatures()) - response, err := executor.listMetrics(backend.PluginContext{ - DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, - }, "default", &cloudwatch.ListMetricsInput{}) - require.NoError(t, err) - - expectedMetrics := client.MetricsPerPage * executor.cfg.AWSListMetricsPageLimit - assert.Equal(t, expectedMetrics, len(response)) - }) - - t.Run("List Metrics and page limit is not reached", func(t *testing.T) { - client = fakeCWClient{Metrics: metrics, MetricsPerPage: 2} - im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { - return datasourceInfo{}, nil - }) - executor := newExecutor(im, &setting.Cfg{AWSListMetricsPageLimit: 1000, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}, &fakeSessionCache{}, - featuremgmt.WithFeatures()) - response, err := executor.listMetrics(backend.PluginContext{ - DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, - }, "default", &cloudwatch.ListMetricsInput{}) - require.NoError(t, err) - - assert.Equal(t, len(metrics), len(response)) - }) -} diff --git a/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go b/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go new file mode 100644 index 00000000000..0e928b93292 --- /dev/null +++ b/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go @@ -0,0 +1,58 @@ +package mocks + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" +) + +type FakeMetricsAPI struct { + cloudwatchiface.CloudWatchAPI + cloudwatch.GetMetricDataOutput + + Metrics []*cloudwatch.Metric + MetricsPerPage int + + CallsGetMetricDataWithContext []*cloudwatch.GetMetricDataInput +} + +func (c *FakeMetricsAPI) GetMetricDataWithContext(ctx aws.Context, input *cloudwatch.GetMetricDataInput, opts ...request.Option) (*cloudwatch.GetMetricDataOutput, error) { + c.CallsGetMetricDataWithContext = append(c.CallsGetMetricDataWithContext, input) + + return &c.GetMetricDataOutput, nil +} + +func (c *FakeMetricsAPI) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error { + if c.MetricsPerPage == 0 { + c.MetricsPerPage = 1000 + } + chunks := chunkSlice(c.Metrics, c.MetricsPerPage) + + for i, metrics := range chunks { + response := fn(&cloudwatch.ListMetricsOutput{ + Metrics: metrics, + }, i+1 == len(chunks)) + if !response { + break + } + } + return nil +} + +func chunkSlice(slice []*cloudwatch.Metric, chunkSize int) [][]*cloudwatch.Metric { + var chunks [][]*cloudwatch.Metric + for { + if len(slice) == 0 { + break + } + if len(slice) < chunkSize { + chunkSize = len(slice) + } + + chunks = append(chunks, slice[0:chunkSize]) + slice = slice[chunkSize:] + } + + return chunks +} diff --git a/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go b/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go new file mode 100644 index 00000000000..cf18ce6db01 --- /dev/null +++ b/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go @@ -0,0 +1,28 @@ +package mocks + +import ( + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/stretchr/testify/mock" +) + +type ListMetricsServiceMock struct { + mock.Mock +} + +func (a *ListMetricsServiceMock) GetDimensionKeysByDimensionFilter(*models.DimensionKeysRequest) ([]string, error) { + args := a.Called() + + return args.Get(0).([]string), args.Error(1) +} + +func (a *ListMetricsServiceMock) GetDimensionKeysByNamespace(string) ([]string, error) { + args := a.Called() + + return args.Get(0).([]string), args.Error(1) +} + +func (a *ListMetricsServiceMock) GetHardCodedDimensionKeysByNamespace(string) ([]string, error) { + args := a.Called() + + return args.Get(0).([]string), args.Error(1) +} diff --git a/pkg/tsdb/cloudwatch/mocks/metrics_client.go b/pkg/tsdb/cloudwatch/mocks/metrics_client.go new file mode 100644 index 00000000000..3b0d818f7f4 --- /dev/null +++ b/pkg/tsdb/cloudwatch/mocks/metrics_client.go @@ -0,0 +1,15 @@ +package mocks + +import ( + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/stretchr/testify/mock" +) + +type FakeMetricsClient struct { + mock.Mock +} + +func (m *FakeMetricsClient) ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) { + args := m.Called(params) + return args.Get(0).([]*cloudwatch.Metric), args.Error(1) +} diff --git a/pkg/tsdb/cloudwatch/models/dimension_keys_request.go b/pkg/tsdb/cloudwatch/models/dimension_keys_request.go new file mode 100644 index 00000000000..3f4342f71d1 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/dimension_keys_request.go @@ -0,0 +1,88 @@ +package models + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" +) + +type DimensionKeysRequestType uint32 + +const ( + StandardDimensionKeysRequest DimensionKeysRequestType = iota + FilterDimensionKeysRequest + CustomMetricDimensionKeysRequest +) + +type Dimension struct { + Name string + Value string +} + +type DimensionKeysRequest struct { + Region string `json:"region"` + Namespace string `json:"namespace"` + MetricName string `json:"metricName"` + DimensionFilter []*Dimension +} + +func (q *DimensionKeysRequest) Type() DimensionKeysRequestType { + if _, exist := constants.NamespaceMetricsMap[q.Namespace]; !exist { + return CustomMetricDimensionKeysRequest + } + + if len(q.DimensionFilter) > 0 { + return FilterDimensionKeysRequest + } + + return StandardDimensionKeysRequest +} + +func GetDimensionKeysRequest(parameters url.Values) (*DimensionKeysRequest, error) { + req := &DimensionKeysRequest{ + Region: parameters.Get("region"), + Namespace: parameters.Get("namespace"), + MetricName: parameters.Get("metricName"), + DimensionFilter: []*Dimension{}, + } + + if req.Region == "" { + return nil, fmt.Errorf("region is required") + } + + dimensionFilters := map[string]interface{}{} + dimensionFilterJson := []byte(parameters.Get("dimensionFilters")) + if len(dimensionFilterJson) > 0 { + err := json.Unmarshal(dimensionFilterJson, &dimensionFilters) + if err != nil { + return nil, fmt.Errorf("error unmarshaling dimensionFilters: %v", err) + } + } + addDimension := func(key string, value string) { + d := &Dimension{ + Name: key, + } + // if value is not specified or a wildcard is used, simply don't use the value field + if value != "" && value != "*" { + d.Value = value + } + req.DimensionFilter = append(req.DimensionFilter, d) + } + + for k, v := range dimensionFilters { + // due to legacy, value can be a string, a string slice or nil + if vv, ok := v.(string); ok { + addDimension(k, vv) + } else if vv, ok := v.([]interface{}); ok { + for _, v := range vv { + addDimension(k, v.(string)) + } + } else if v == nil { + addDimension(k, "") + } + } + + return req, nil +} diff --git a/pkg/tsdb/cloudwatch/models/dimension_keys_request_test.go b/pkg/tsdb/cloudwatch/models/dimension_keys_request_test.go new file mode 100644 index 00000000000..0c35a1e7337 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/dimension_keys_request_test.go @@ -0,0 +1,72 @@ +package models + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDimensionKeyQuery(t *testing.T) { + t.Run("Should parse parameters without dimension filter", func(t *testing.T) { + req, err := GetDimensionKeysRequest(map[string][]string{ + "region": {"us-east-1"}, + "namespace": {"AWS/EC2"}, + "metricName": {"CPUUtilization"}}, + ) + require.NoError(t, err) + assert.Equal(t, "us-east-1", req.Region) + assert.Equal(t, "AWS/EC2", req.Namespace) + assert.Equal(t, "CPUUtilization", req.MetricName) + }) + + t.Run("Should parse parameters with single valued dimension filter", func(t *testing.T) { + req, err := GetDimensionKeysRequest(map[string][]string{ + "region": {"us-east-1"}, + "namespace": {"AWS/EC2"}, + "metricName": {"CPUUtilization"}, + "dimensionFilters": {"{\"InstanceId\": \"i-1234567890abcdef0\"}"}, + }) + require.NoError(t, err) + assert.Equal(t, "us-east-1", req.Region) + assert.Equal(t, "AWS/EC2", req.Namespace) + assert.Equal(t, "CPUUtilization", req.MetricName) + assert.Equal(t, 1, len(req.DimensionFilter)) + assert.Equal(t, "InstanceId", req.DimensionFilter[0].Name) + assert.Equal(t, "i-1234567890abcdef0", req.DimensionFilter[0].Value) + }) + + t.Run("Should parse parameters with multi-valued dimension filter", func(t *testing.T) { + req, err := GetDimensionKeysRequest(map[string][]string{ + "region": {"us-east-1"}, + "namespace": {"AWS/EC2"}, + "metricName": {"CPUUtilization"}, + "dimensionFilters": {"{\"InstanceId\": [\"i-1234567890abcdef0\", \"i-1234567890abcdef1\"]}"}, + }) + require.NoError(t, err) + assert.Equal(t, "us-east-1", req.Region) + assert.Equal(t, "AWS/EC2", req.Namespace) + assert.Equal(t, "CPUUtilization", req.MetricName) + assert.Equal(t, 2, len(req.DimensionFilter)) + assert.Equal(t, "InstanceId", req.DimensionFilter[0].Name) + assert.Equal(t, "i-1234567890abcdef0", req.DimensionFilter[0].Value) + assert.Equal(t, "InstanceId", req.DimensionFilter[1].Name) + assert.Equal(t, "i-1234567890abcdef1", req.DimensionFilter[1].Value) + }) + + t.Run("Should parse parameters with wildcard dimension filter", func(t *testing.T) { + req, err := GetDimensionKeysRequest(map[string][]string{ + "region": {"us-east-1"}, + "namespace": {"AWS/EC2"}, + "metricName": {"CPUUtilization"}, + "dimensionFilters": {"{\"InstanceId\": [\"*\"]}"}, + }) + require.NoError(t, err) + assert.Equal(t, "us-east-1", req.Region) + assert.Equal(t, "AWS/EC2", req.Namespace) + assert.Equal(t, "CPUUtilization", req.MetricName) + assert.Equal(t, 1, len(req.DimensionFilter)) + assert.Equal(t, "InstanceId", req.DimensionFilter[0].Name) + assert.Equal(t, "", req.DimensionFilter[0].Value) + }) +} diff --git a/pkg/tsdb/cloudwatch/models/http_error.go b/pkg/tsdb/cloudwatch/models/http_error.go new file mode 100644 index 00000000000..850eb2eaf59 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/http_error.go @@ -0,0 +1,22 @@ +package models + +import "fmt" + +type HttpError struct { + Message string + Error string + StatusCode int +} + +func NewHttpError(message string, statusCode int, err error) *HttpError { + httpError := &HttpError{ + Message: message, + StatusCode: statusCode, + } + if err != nil { + httpError.Error = err.Error() + httpError.Message = fmt.Sprintf("%s: %s", message, err) + } + + return httpError +} diff --git a/pkg/tsdb/cloudwatch/models/metric_types.go b/pkg/tsdb/cloudwatch/models/metric_types.go new file mode 100644 index 00000000000..40af418e09e --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/metric_types.go @@ -0,0 +1,19 @@ +package models + +import ( + "github.com/aws/aws-sdk-go/service/cloudwatch" +) + +type ListMetricsProvider interface { + GetDimensionKeysByDimensionFilter(*DimensionKeysRequest) ([]string, error) + GetHardCodedDimensionKeysByNamespace(string) ([]string, error) + GetDimensionKeysByNamespace(string) ([]string, error) +} + +type MetricsClientProvider interface { + ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) +} + +type CloudWatchMetricsAPIProvider interface { + ListMetricsPages(*cloudwatch.ListMetricsInput, func(*cloudwatch.ListMetricsOutput, bool) bool) error +} diff --git a/pkg/tsdb/cloudwatch/models/types.go b/pkg/tsdb/cloudwatch/models/types.go index a3835e078dc..9d098f9341e 100644 --- a/pkg/tsdb/cloudwatch/models/types.go +++ b/pkg/tsdb/cloudwatch/models/types.go @@ -1,5 +1,19 @@ package models +import ( + "net/url" + + "github.com/grafana/grafana-plugin-sdk-go/backend" +) + +type Clients struct { + MetricsClientProvider MetricsClientProvider +} + +type ClientsFactoryFunc func(pluginCtx backend.PluginContext, region string) (clients Clients, err error) + +type RouteHandlerFunc func(pluginCtx backend.PluginContext, clientFactory ClientsFactoryFunc, parameters url.Values) ([]byte, *HttpError) + type cloudWatchLink struct { View string `json:"view"` Stacked bool `json:"stacked"` diff --git a/pkg/tsdb/cloudwatch/resource_handler.go b/pkg/tsdb/cloudwatch/resource_handler.go index 805d40bdc66..0db3c5531b5 100644 --- a/pkg/tsdb/cloudwatch/resource_handler.go +++ b/pkg/tsdb/cloudwatch/resource_handler.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/cwlog" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/routes" ) func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux { @@ -17,13 +18,13 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux { mux.HandleFunc("/namespaces", handleResourceReq(e.handleGetNamespaces)) mux.HandleFunc("/metrics", handleResourceReq(e.handleGetMetrics)) mux.HandleFunc("/all-metrics", handleResourceReq(e.handleGetAllMetrics)) - mux.HandleFunc("/dimension-keys", handleResourceReq(e.handleGetDimensionKeys)) mux.HandleFunc("/dimension-values", handleResourceReq(e.handleGetDimensionValues)) mux.HandleFunc("/ebs-volume-ids", handleResourceReq(e.handleGetEbsVolumeIds)) mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute)) mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns)) mux.HandleFunc("/log-groups", handleResourceReq(e.handleGetLogGroups)) mux.HandleFunc("/all-log-groups", handleResourceReq(e.handleGetAllLogGroups)) + mux.HandleFunc("/dimension-keys", routes.ResourceRequestMiddleware(routes.DimensionKeysHandler, e.getClients)) return mux } diff --git a/pkg/tsdb/cloudwatch/routes/dimension_keys.go b/pkg/tsdb/cloudwatch/routes/dimension_keys.go new file mode 100644 index 00000000000..8a2eaacbb66 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/dimension_keys.go @@ -0,0 +1,55 @@ +package routes + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" +) + +func DimensionKeysHandler(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) { + dimensionKeysRequest, err := models.GetDimensionKeysRequest(parameters) + if err != nil { + return nil, models.NewHttpError("error in DimensionKeyHandler", http.StatusBadRequest, err) + } + + service, err := newListMetricsService(pluginCtx, clientFactory, dimensionKeysRequest.Region) + if err != nil { + return nil, models.NewHttpError("error in DimensionKeyHandler", http.StatusInternalServerError, err) + } + + dimensionKeys := []string{} + switch dimensionKeysRequest.Type() { + case models.StandardDimensionKeysRequest: + dimensionKeys, err = service.GetHardCodedDimensionKeysByNamespace(dimensionKeysRequest.Namespace) + case models.FilterDimensionKeysRequest: + dimensionKeys, err = service.GetDimensionKeysByDimensionFilter(dimensionKeysRequest) + case models.CustomMetricDimensionKeysRequest: + dimensionKeys, err = service.GetDimensionKeysByNamespace(dimensionKeysRequest.Namespace) + } + if err != nil { + return nil, models.NewHttpError("error in DimensionKeyHandler", http.StatusInternalServerError, err) + } + + dimensionKeysResponse, err := json.Marshal(dimensionKeys) + if err != nil { + return nil, models.NewHttpError("error in DimensionKeyHandler", http.StatusInternalServerError, err) + } + + return dimensionKeysResponse, nil +} + +// newListMetricsService is an list metrics service factory. +// +// Stubbable by tests. +var newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + metricClient, err := clientFactory(pluginCtx, region) + if err != nil { + return nil, err + } + + return services.NewListMetricsService(metricClient.MetricsClientProvider), nil +} diff --git a/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go b/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go new file mode 100644 index 00000000000..d47b19c940e --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go @@ -0,0 +1,81 @@ +package routes + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/stretchr/testify/assert" +) + +func Test_DimensionKeys_Route(t *testing.T) { + t.Run("rejects POST method", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/dimension-keys?region=us-east-1", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusMethodNotAllowed, rr.Code) + }) + + t.Run("requires region query value", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/dimension-keys", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + }) + + tests := []struct { + url string + methodName string + requestType string + }{ + { + url: "/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization", + methodName: "GetHardCodedDimensionKeysByNamespace", + requestType: "StandardDimensionKeysRequest"}, + { + url: `/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization&dimensionFilters={"NodeID":["Shared"],"stage":["QueryCommit"]}`, + methodName: "GetDimensionKeysByDimensionFilter", + requestType: "FilterDimensionKeysRequest"}, + { + url: `/dimension-keys?region=us-east-2&namespace=customNamespace&metricName=CPUUtilization`, + methodName: "GetDimensionKeysByNamespace", + requestType: "CustomMetricDimensionKeysRequest"}, + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("calls %s when a StandardDimensionKeysRequest is passed", tc.requestType), func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On(tc.methodName).Return([]string{}, nil) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", tc.url, nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + mockListMetricsService.AssertNumberOfCalls(t, tc.methodName, 1) + }) + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("return 500 if %s returns an error", tc.requestType), func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On(tc.methodName).Return([]string{}, fmt.Errorf("some error")) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", tc.url, nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.Equal(t, `{"Message":"error in DimensionKeyHandler: some error","Error":"some error","StatusCode":500}`, rr.Body.String()) + }) + } +} diff --git a/pkg/tsdb/cloudwatch/routes/http_helpers.go b/pkg/tsdb/cloudwatch/routes/http_helpers.go new file mode 100644 index 00000000000..1694413bb65 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/http_helpers.go @@ -0,0 +1,22 @@ +package routes + +import ( + "encoding/json" + "net/http" + + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" +) + +func respondWithError(rw http.ResponseWriter, httpError *models.HttpError) { + response, err := json.Marshal(httpError) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + rw.Header().Set("Content-Type", "application/json") + rw.WriteHeader(httpError.StatusCode) + _, err = rw.Write(response) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + } +} diff --git a/pkg/tsdb/cloudwatch/routes/middleware.go b/pkg/tsdb/cloudwatch/routes/middleware.go new file mode 100644 index 00000000000..c730921fe57 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/middleware.go @@ -0,0 +1,34 @@ +package routes + +import ( + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/cwlog" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" +) + +func ResourceRequestMiddleware(handleFunc models.RouteHandlerFunc, clientFactory models.ClientsFactoryFunc) func(rw http.ResponseWriter, req *http.Request) { + return func(rw http.ResponseWriter, req *http.Request) { + if req.Method != "GET" { + respondWithError(rw, models.NewHttpError("Invalid method", http.StatusMethodNotAllowed, nil)) + return + } + + ctx := req.Context() + pluginContext := httpadapter.PluginConfigFromContext(ctx) + json, httpError := handleFunc(pluginContext, clientFactory, req.URL.Query()) + if httpError != nil { + cwlog.Error("error handling resource request", "error", httpError.Message) + respondWithError(rw, httpError) + return + } + + rw.Header().Set("Content-Type", "application/json") + _, err := rw.Write(json) + if err != nil { + cwlog.Error("error handling resource request", "error", err) + respondWithError(rw, models.NewHttpError("error writing response in resource request middleware", http.StatusInternalServerError, err)) + } + } +} diff --git a/pkg/tsdb/cloudwatch/services/list_metrics.go b/pkg/tsdb/cloudwatch/services/list_metrics.go new file mode 100644 index 00000000000..672683d4542 --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/list_metrics.go @@ -0,0 +1,102 @@ +package services + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" +) + +type ListMetricsService struct { + models.MetricsClientProvider +} + +func NewListMetricsService(metricsClient models.MetricsClientProvider) models.ListMetricsProvider { + return &ListMetricsService{metricsClient} +} + +func (*ListMetricsService) GetHardCodedDimensionKeysByNamespace(namespace string) ([]string, error) { + var dimensionKeys []string + exists := false + if dimensionKeys, exists = constants.NamespaceDimensionKeysMap[namespace]; !exists { + return nil, fmt.Errorf("unable to find dimensions for namespace '%q'", namespace) + } + return dimensionKeys, nil +} + +func (l *ListMetricsService) GetDimensionKeysByDimensionFilter(r *models.DimensionKeysRequest) ([]string, error) { + input := &cloudwatch.ListMetricsInput{} + if r.Namespace != "" { + input.Namespace = aws.String(r.Namespace) + } + if r.MetricName != "" { + input.MetricName = aws.String(r.MetricName) + } + for _, dimension := range r.DimensionFilter { + df := &cloudwatch.DimensionFilter{ + Name: aws.String(dimension.Name), + } + if dimension.Value != "" { + df.Value = aws.String(dimension.Value) + } + input.Dimensions = append(input.Dimensions, df) + } + + metrics, err := l.ListMetricsWithPageLimit(input) + if err != nil { + return nil, fmt.Errorf("%v: %w", "unable to call AWS API", err) + } + + var dimensionKeys []string + // remove duplicates + dupCheck := make(map[string]struct{}) + for _, metric := range metrics { + for _, dim := range metric.Dimensions { + if _, exists := dupCheck[*dim.Name]; exists { + continue + } + + // keys in the dimension filter should not be included + dimensionFilterExist := false + for _, d := range r.DimensionFilter { + if d.Name == *dim.Name { + dimensionFilterExist = true + break + } + } + + if dimensionFilterExist { + continue + } + + dupCheck[*dim.Name] = struct{}{} + dimensionKeys = append(dimensionKeys, *dim.Name) + } + } + + return dimensionKeys, nil +} + +func (l *ListMetricsService) GetDimensionKeysByNamespace(namespace string) ([]string, error) { + metrics, err := l.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{Namespace: aws.String(namespace)}) + if err != nil { + return []string{}, err + } + + var dimensionKeys []string + dupCheck := make(map[string]struct{}) + for _, metric := range metrics { + for _, dim := range metric.Dimensions { + if _, exists := dupCheck[*dim.Name]; exists { + continue + } + + dupCheck[*dim.Name] = struct{}{} + dimensionKeys = append(dimensionKeys, *dim.Name) + } + } + + return dimensionKeys, nil +} diff --git a/pkg/tsdb/cloudwatch/services/list_metrics_test.go b/pkg/tsdb/cloudwatch/services/list_metrics_test.go new file mode 100644 index 00000000000..73a92788b82 --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/list_metrics_test.go @@ -0,0 +1,92 @@ +package services + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +var metricResponse = []*cloudwatch.Metric{ + { + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.Dimension{ + {Name: aws.String("InstanceId"), Value: aws.String("i-1234567890abcdef0")}, + {Name: aws.String("InstanceType"), Value: aws.String("t2.micro")}, + }, + }, + { + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.Dimension{ + {Name: aws.String("InstanceId"), Value: aws.String("i-5234567890abcdef0")}, + {Name: aws.String("InstanceType"), Value: aws.String("t2.micro")}, + {Name: aws.String("AutoScalingGroupName"), Value: aws.String("my-asg")}, + }, + }, + { + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.Dimension{ + {Name: aws.String("InstanceId"), Value: aws.String("i-64234567890abcdef0")}, + {Name: aws.String("InstanceType"), Value: aws.String("t3.micro")}, + {Name: aws.String("AutoScalingGroupName"), Value: aws.String("my-asg2")}, + }, + }, +} + +func TestListMetricsService_GetHardCodedDimensionKeysByNamespace(t *testing.T) { + t.Run("Should return an error in case namespace doesnt exist in map", func(t *testing.T) { + listMetricsService := NewListMetricsService(&mocks.FakeMetricsClient{}) + resp, err := listMetricsService.GetHardCodedDimensionKeysByNamespace("unknownNamespace") + require.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, err.Error(), "unable to find dimensions for namespace '\"unknownNamespace\"'") + }) + + t.Run("Should return keys if namespace exist", func(t *testing.T) { + listMetricsService := NewListMetricsService(&mocks.FakeMetricsClient{}) + resp, err := listMetricsService.GetHardCodedDimensionKeysByNamespace("AWS/EC2") + require.NoError(t, err) + assert.Equal(t, []string{"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"}, resp) + }) +} + +func TestListMetricsService_GetDimensionKeysByDimensionFilter(t *testing.T) { + t.Run("Should filter out duplicates and keys matching dimension filter keys", func(t *testing.T) { + fakeMetricsClient := &mocks.FakeMetricsClient{} + fakeMetricsClient.On("ListMetricsWithPageLimit", mock.Anything).Return(metricResponse, nil) + listMetricsService := NewListMetricsService(fakeMetricsClient) + + resp, err := listMetricsService.GetDimensionKeysByDimensionFilter(&models.DimensionKeysRequest{ + Region: "us-east-1", + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + DimensionFilter: []*models.Dimension{ + {Name: "InstanceId", Value: ""}, + }, + }) + + require.NoError(t, err) + assert.Equal(t, []string{"InstanceType", "AutoScalingGroupName"}, resp) + }) +} + +func TestListMetricsService_GetDimensionKeysByNamespace(t *testing.T) { + t.Run("Should filter out duplicates and keys matching dimension filter keys", func(t *testing.T) { + fakeMetricsClient := &mocks.FakeMetricsClient{} + fakeMetricsClient.On("ListMetricsWithPageLimit", mock.Anything).Return(metricResponse, nil) + listMetricsService := NewListMetricsService(fakeMetricsClient) + + resp, err := listMetricsService.GetDimensionKeysByNamespace("AWS/EC2") + + require.NoError(t, err) + assert.Equal(t, []string{"InstanceId", "InstanceType", "AutoScalingGroupName"}, resp) + }) +} diff --git a/pkg/tsdb/cloudwatch/time_series_query_test.go b/pkg/tsdb/cloudwatch/time_series_query_test.go index 08403336362..610f96f1f1f 100644 --- a/pkg/tsdb/cloudwatch/time_series_query_test.go +++ b/pkg/tsdb/cloudwatch/time_series_query_test.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" "github.com/stretchr/testify/assert" @@ -29,15 +30,14 @@ func TestTimeSeriesQuery(t *testing.T) { t.Cleanup(func() { NewCWClient = origNewCWClient }) - - var cwClient fakeCWClient + var api mocks.FakeMetricsAPI NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { - return &cwClient + return &api } t.Run("Custom metrics", func(t *testing.T) { - cwClient = fakeCWClient{ + api = mocks.FakeMetricsAPI{ CloudWatchAPI: nil, GetMetricDataOutput: cloudwatch.GetMetricDataOutput{ NextToken: nil, @@ -209,9 +209,11 @@ func Test_QueryData_timeSeriesQuery_GetMetricDataWithContext(t *testing.T) { t.Cleanup(func() { NewCWClient = origNewCWClient }) - var cwClient fakeCWClient + + var api mocks.FakeMetricsAPI + NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { - return &cwClient + return &api } im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { @@ -219,7 +221,7 @@ func Test_QueryData_timeSeriesQuery_GetMetricDataWithContext(t *testing.T) { }) t.Run("passes query label as GetMetricData label when dynamic labels feature toggle is enabled", func(t *testing.T) { - cwClient = fakeCWClient{} + api = mocks.FakeMetricsAPI{} executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchDynamicLabels)) query := newTestQuery(t, queryParameters{ Label: aws.String("${PROP('Period')} some words ${PROP('Dim.InstanceId')}"), @@ -240,11 +242,11 @@ func Test_QueryData_timeSeriesQuery_GetMetricDataWithContext(t *testing.T) { }) assert.NoError(t, err) - require.Len(t, cwClient.callsGetMetricDataWithContext, 1) - require.Len(t, cwClient.callsGetMetricDataWithContext[0].MetricDataQueries, 1) - require.NotNil(t, cwClient.callsGetMetricDataWithContext[0].MetricDataQueries[0].Label) + require.Len(t, api.CallsGetMetricDataWithContext, 1) + require.Len(t, api.CallsGetMetricDataWithContext[0].MetricDataQueries, 1) + require.NotNil(t, api.CallsGetMetricDataWithContext[0].MetricDataQueries[0].Label) - assert.Equal(t, "${PROP('Period')} some words ${PROP('Dim.InstanceId')}", *cwClient.callsGetMetricDataWithContext[0].MetricDataQueries[0].Label) + assert.Equal(t, "${PROP('Period')} some words ${PROP('Dim.InstanceId')}", *api.CallsGetMetricDataWithContext[0].MetricDataQueries[0].Label) }) testCases := map[string]struct { @@ -266,7 +268,7 @@ func Test_QueryData_timeSeriesQuery_GetMetricDataWithContext(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - cwClient = fakeCWClient{} + api = mocks.FakeMetricsAPI{} executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, tc.feature) _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ @@ -284,10 +286,10 @@ func Test_QueryData_timeSeriesQuery_GetMetricDataWithContext(t *testing.T) { }) assert.NoError(t, err) - require.Len(t, cwClient.callsGetMetricDataWithContext, 1) - require.Len(t, cwClient.callsGetMetricDataWithContext[0].MetricDataQueries, 1) + require.Len(t, api.CallsGetMetricDataWithContext, 1) + require.Len(t, api.CallsGetMetricDataWithContext[0].MetricDataQueries, 1) - assert.Nil(t, cwClient.callsGetMetricDataWithContext[0].MetricDataQueries[0].Label) + assert.Nil(t, api.CallsGetMetricDataWithContext[0].MetricDataQueries[0].Label) }) } } @@ -297,12 +299,14 @@ func Test_QueryData_response_data_frame_names(t *testing.T) { t.Cleanup(func() { NewCWClient = origNewCWClient }) - var cwClient fakeCWClient + var api mocks.FakeMetricsAPI + NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { - return &cwClient + return &api } + labelFromGetMetricData := "some label" - cwClient = fakeCWClient{ + api = mocks.FakeMetricsAPI{ GetMetricDataOutput: cloudwatch.GetMetricDataOutput{ MetricDataResults: []*cloudwatch.MetricDataResult{ {StatusCode: aws.String("Complete"), Id: aws.String(queryId), Label: aws.String(labelFromGetMetricData), diff --git a/pkg/tsdb/cloudwatch/utils_test.go b/pkg/tsdb/cloudwatch/utils_test.go index 73094be8d10..989b8764cff 100644 --- a/pkg/tsdb/cloudwatch/utils_test.go +++ b/pkg/tsdb/cloudwatch/utils_test.go @@ -80,39 +80,6 @@ func (m *fakeCWLogsClient) GetLogEventsWithContext(ctx context.Context, input *c }, nil } -type fakeCWClient struct { - cloudwatchiface.CloudWatchAPI - cloudwatch.GetMetricDataOutput - - Metrics []*cloudwatch.Metric - MetricsPerPage int - - callsGetMetricDataWithContext []*cloudwatch.GetMetricDataInput -} - -func (c *fakeCWClient) GetMetricDataWithContext(ctx aws.Context, input *cloudwatch.GetMetricDataInput, opts ...request.Option) (*cloudwatch.GetMetricDataOutput, error) { - c.callsGetMetricDataWithContext = append(c.callsGetMetricDataWithContext, input) - - return &c.GetMetricDataOutput, nil -} - -func (c *fakeCWClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error { - if c.MetricsPerPage == 0 { - c.MetricsPerPage = 1000 - } - chunks := chunkSlice(c.Metrics, c.MetricsPerPage) - - for i, metrics := range chunks { - response := fn(&cloudwatch.ListMetricsOutput{ - Metrics: metrics, - }, i+1 == len(chunks)) - if !response { - break - } - } - return nil -} - type fakeCWAnnotationsClient struct { cloudwatchiface.CloudWatchAPI calls annontationsQueryCalls @@ -219,23 +186,6 @@ func (c fakeCheckHealthClient) DescribeLogGroups(input *cloudwatchlogs.DescribeL return nil, nil } -func chunkSlice(slice []*cloudwatch.Metric, chunkSize int) [][]*cloudwatch.Metric { - var chunks [][]*cloudwatch.Metric - for { - if len(slice) == 0 { - break - } - if len(slice) < chunkSize { - chunkSize = len(slice) - } - - chunks = append(chunks, slice[0:chunkSize]) - slice = slice[chunkSize:] - } - - return chunks -} - func newTestConfig() *setting.Cfg { return &setting.Cfg{AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true, AWSListMetricsPageLimit: 1000} } diff --git a/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts b/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts index 4cf7c5ad931..734c61690c7 100644 --- a/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts +++ b/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts @@ -81,6 +81,7 @@ export function setupMockedDataSource({ datasource.api.describeLogGroups = jest.fn().mockResolvedValue([]); datasource.api.getNamespaces = jest.fn().mockResolvedValue([]); datasource.api.getRegions = jest.fn().mockResolvedValue([]); + datasource.api.getDimensionKeys = jest.fn().mockResolvedValue([]); datasource.logsQueryRunner.defaultLogGroups = []; const fetchMock = jest.fn().mockReturnValue(of({})); setBackendSrv({ diff --git a/public/app/plugins/datasource/cloudwatch/api.ts b/public/app/plugins/datasource/cloudwatch/api.ts index 615bfcfb6aa..bb2d6700eb1 100644 --- a/public/app/plugins/datasource/cloudwatch/api.ts +++ b/public/app/plugins/datasource/cloudwatch/api.ts @@ -1,11 +1,11 @@ import { memoize } from 'lodash'; -import { DataSourceInstanceSettings, SelectableValue } from '@grafana/data'; +import { DataSourceInstanceSettings, SelectableValue, toOption } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { CloudWatchRequest } from './query-runner/CloudWatchRequest'; -import { CloudWatchJsonData, DescribeLogGroupsRequest, Dimensions, MultiFilters } from './types'; +import { CloudWatchJsonData, DescribeLogGroupsRequest, GetDimensionKeysRequest, MultiFilters } from './types'; export interface SelectableResourceValue extends SelectableValue { text: string; @@ -69,22 +69,18 @@ export class CloudWatchAPI extends CloudWatchRequest { return values.map((v) => ({ metricName: v.value, namespace: v.text })); } - async getDimensionKeys( - namespace: string | undefined, - region: string, - dimensionFilters: Dimensions = {}, - metricName = '' - ) { - if (!namespace) { - return []; - } - - return this.memoizedGetRequest('dimension-keys', { + async getDimensionKeys({ + region, + namespace = '', + dimensionFilters = {}, + metricName = '', + }: GetDimensionKeysRequest): Promise>> { + return this.memoizedGetRequest('dimension-keys', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), dimensionFilters: JSON.stringify(this.convertDimensionFormat(dimensionFilters, {})), metricName, - }); + }).then((dimensionKeys) => dimensionKeys.map(toOption)); } async getDimensionValues( diff --git a/public/app/plugins/datasource/cloudwatch/cloudwatch-sql/completion/CompletionItemProvider.ts b/public/app/plugins/datasource/cloudwatch/cloudwatch-sql/completion/CompletionItemProvider.ts index 80948f92068..5cc03c47279 100644 --- a/public/app/plugins/datasource/cloudwatch/cloudwatch-sql/completion/CompletionItemProvider.ts +++ b/public/app/plugins/datasource/cloudwatch/cloudwatch-sql/completion/CompletionItemProvider.ts @@ -163,7 +163,7 @@ export class SQLCompletionItemProvider extends CompletionItemProvider { const metricNameToken = getMetricNameToken(currentToken); const namespaceToken = getNamespaceToken(currentToken); if (namespaceToken?.value) { - let dimensionFilter = {}; + let dimensionFilters = {}; let labelKeyTokens; if (statementPosition === StatementPosition.SchemaFuncExtraArgument) { labelKeyTokens = namespaceToken?.getNextUntil(this.tokenTypes.Parenthesis, [ @@ -176,15 +176,15 @@ export class SQLCompletionItemProvider extends CompletionItemProvider { this.tokenTypes.Whitespace, ]); } - dimensionFilter = (labelKeyTokens || []).reduce((acc, curr) => { + dimensionFilters = (labelKeyTokens || []).reduce((acc, curr) => { return { ...acc, [curr.value]: null }; }, {}); - const keys = await this.api.getDimensionKeys( - this.templateSrv.replace(namespaceToken.value.replace(/\"/g, '')), - this.templateSrv.replace(this.region), - dimensionFilter, - metricNameToken?.value ?? '' - ); + const keys = await this.api.getDimensionKeys({ + namespace: this.templateSrv.replace(namespaceToken.value.replace(/\"/g, '')), + region: this.templateSrv.replace(this.region), + metricName: metricNameToken?.value, + dimensionFilters, + }); keys.map((m) => { const key = /[\s\.-]/.test(m.value ?? '') ? `"${m.value}"` : m.value; key && addSuggestion(key); diff --git a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx index 3a7934091b2..b619282aa99 100644 --- a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx @@ -30,7 +30,7 @@ export function MetricStatEditor({ const { region, namespace, metricName, dimensions } = metricStat; const namespaces = useNamespaces(datasource); const metrics = useMetrics(datasource, region, namespace); - const dimensionKeys = useDimensionKeys(datasource, region, namespace, metricName, dimensions ?? {}); + const dimensionKeys = useDimensionKeys(datasource, { region, namespace, metricName, dimensionFilters: dimensions }); const onMetricStatChange = (metricStat: MetricStat) => { onChange(metricStat); diff --git a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderEditor.test.tsx b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderEditor.test.tsx index cc6b563b61a..30148482919 100644 --- a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderEditor.test.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderEditor.test.tsx @@ -95,12 +95,12 @@ describe('Cloudwatch SQLBuilderEditor', () => { render(); await waitFor(() => - expect(datasource.api.getDimensionKeys).toHaveBeenCalledWith( - 'AWS/EC2', - query.region, - { InstanceId: null }, - undefined - ) + expect(datasource.api.getDimensionKeys).toHaveBeenCalledWith({ + namespace: 'AWS/EC2', + region: query.region, + dimensionFilters: { InstanceId: null }, + metricName: undefined, + }) ); expect(screen.getByText('AWS/EC2')).toBeInTheDocument(); expect(screen.getByLabelText('With schema')).toBeChecked(); diff --git a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx index 722dead533a..b6b3ecabdeb 100644 --- a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx @@ -49,7 +49,12 @@ const SQLBuilderSelectRow: React.FC = ({ datasource, q const namespaceOptions = useNamespaces(datasource); const metricOptions = useMetrics(datasource, query.region, namespace); const existingFilters = useMemo(() => stringArrayToDimensions(schemaLabels ?? []), [schemaLabels]); - const unusedDimensionKeys = useDimensionKeys(datasource, query.region, namespace, metricName, existingFilters); + const unusedDimensionKeys = useDimensionKeys(datasource, { + region: query.region, + namespace, + metricName, + dimensionFilters: existingFilters, + }); const dimensionKeys = useMemo( () => (schemaLabels?.length ? [...unusedDimensionKeys, ...schemaLabels.map(toOption)] : unusedDimensionKeys), [unusedDimensionKeys, schemaLabels] diff --git a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLFilter.tsx b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLFilter.tsx index 8e0ed8c2075..376151e92eb 100644 --- a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLFilter.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLFilter.tsx @@ -101,7 +101,7 @@ const FilterItem: React.FC = (props) => { const namespace = getNamespaceFromExpression(sql.from); const metricName = getMetricNameFromExpression(sql.select); - const dimensionKeys = useDimensionKeys(datasource, query.region, namespace, metricName); + const dimensionKeys = useDimensionKeys(datasource, { region: query.region, namespace, metricName }); const loadDimensionValues = async () => { if (!filter.property?.name) { diff --git a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.test.tsx b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.test.tsx index c3eb94fa6c1..2492bcb2b11 100644 --- a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.test.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.test.tsx @@ -22,6 +22,8 @@ const makeSQLQuery = (sql?: SQLExpression): CloudWatchMetricsQuery => ({ sql: sql, }); +datasource.api.getDimensionKeys = jest.fn().mockResolvedValue([]); + describe('Cloudwatch SQLGroupBy', () => { const baseProps = { query: makeSQLQuery(), diff --git a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.tsx b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.tsx index 657a4034e8f..93df84f74aa 100644 --- a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLGroupBy.tsx @@ -30,7 +30,7 @@ const SQLGroupBy: React.FC = ({ query, datasource, onQueryChang const namespace = getNamespaceFromExpression(sql.from); const metricName = getMetricNameFromExpression(sql.select); - const baseOptions = useDimensionKeys(datasource, query.region, namespace, metricName); + const baseOptions = useDimensionKeys(datasource, { region: query.region, namespace, metricName }); const options = useMemo( // Exclude options we've already selected () => baseOptions.filter((option) => !groupBysFromQuery.some((v) => v.property.name === option.value)), diff --git a/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.test.tsx b/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.test.tsx index fcf388f3f97..1e4720c58cb 100644 --- a/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.test.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.test.tsx @@ -1,10 +1,10 @@ -import { fireEvent, render, screen, waitFor, within } from '@testing-library/react'; +import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; import { select } from 'react-select-event'; import { setupMockedDataSource } from '../../__mocks__/CloudWatchDataSource'; -import { Dimensions, VariableQueryType } from '../../types'; +import { GetDimensionKeysRequest, VariableQueryType } from '../../types'; import { VariableQueryEditor, Props } from './VariableQueryEditor'; @@ -39,7 +39,7 @@ ds.datasource.api.getMetrics = jest.fn().mockResolvedValue([ ]); ds.datasource.api.getDimensionKeys = jest .fn() - .mockImplementation((_namespace: string, region: string, dimensionFilters?: Dimensions) => { + .mockImplementation(({ namespace: region, dimensionFilters }: GetDimensionKeysRequest) => { if (!!dimensionFilters) { return Promise.resolve([ { label: 's4', value: 's4' }, @@ -128,8 +128,9 @@ describe('VariableEditor', () => { dimensionKey: 's4', dimensionFilters: { s4: 'foo' }, }; - render(); - + await act(async () => { + render(); + }); const filterItem = screen.getByTestId('cloudwatch-dimensions-filter-item'); expect(filterItem).toBeInTheDocument(); expect(within(filterItem).getByText('s4')).toBeInTheDocument(); @@ -141,7 +142,12 @@ describe('VariableEditor', () => { await select(keySelect, 'v4', { container: document.body, }); - expect(ds.datasource.api.getDimensionKeys).toHaveBeenCalledWith('z2', 'a1', {}, ''); + expect(ds.datasource.api.getDimensionKeys).toHaveBeenCalledWith({ + namespace: 'z2', + region: 'a1', + metricName: 'i3', + dimensionFilters: undefined, + }); expect(onChange).toHaveBeenCalledWith({ ...defaultQuery, queryType: VariableQueryType.DimensionValues, @@ -225,7 +231,7 @@ describe('VariableEditor', () => { }); expect(ds.datasource.api.getMetrics).toHaveBeenCalledWith('z2', 'b1'); - expect(ds.datasource.api.getDimensionKeys).toHaveBeenCalledWith('z2', 'b1'); + expect(ds.datasource.api.getDimensionKeys).toHaveBeenCalledWith({ namespace: 'z2', region: 'b1' }); expect(props.onChange).toHaveBeenCalledWith({ ...defaultQuery, refId: 'CloudWatchVariableQueryEditor-VariableQuery', diff --git a/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.tsx index 6bca0d4bc31..59e500e34fe 100644 --- a/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/VariableQueryEditor/VariableQueryEditor.tsx @@ -35,8 +35,8 @@ export const VariableQueryEditor = ({ query, datasource, onChange }: Props) => { const [regions, regionIsLoading] = useRegions(datasource); const namespaces = useNamespaces(datasource); const metrics = useMetrics(datasource, region, namespace); - const dimensionKeys = useDimensionKeys(datasource, region, namespace, metricName); - const keysForDimensionFilter = useDimensionKeys(datasource, region, namespace, metricName, dimensionFilters ?? {}); + const dimensionKeys = useDimensionKeys(datasource, { region, namespace, metricName }); + const keysForDimensionFilter = useDimensionKeys(datasource, { region, namespace, metricName, dimensionFilters }); const onRegionChange = async (region: string) => { const validatedQuery = await sanitizeQuery({ @@ -72,7 +72,7 @@ export const VariableQueryEditor = ({ query, datasource, onChange }: Props) => { }); } if (dimensionKey) { - await datasource.api.getDimensionKeys(namespace, region).then((result: Array>) => { + await datasource.api.getDimensionKeys({ namespace, region }).then((result: Array>) => { if (!result.find((key) => key.value === dimensionKey)) { dimensionKey = ''; dimensionFilters = {}; diff --git a/public/app/plugins/datasource/cloudwatch/hooks.ts b/public/app/plugins/datasource/cloudwatch/hooks.ts index afad87e256f..d2b72fb2a06 100644 --- a/public/app/plugins/datasource/cloudwatch/hooks.ts +++ b/public/app/plugins/datasource/cloudwatch/hooks.ts @@ -4,7 +4,7 @@ import { useDeepCompareEffect } from 'react-use'; import { SelectableValue, toOption } from '@grafana/data'; import { CloudWatchDatasource } from './datasource'; -import { Dimensions } from './types'; +import { GetDimensionKeysRequest } from './types'; import { appendTemplateVariables } from './utils/utils'; export const useRegions = (datasource: CloudWatchDatasource): [Array>, boolean] => { @@ -52,21 +52,18 @@ export const useMetrics = (datasource: CloudWatchDatasource, region: string, nam export const useDimensionKeys = ( datasource: CloudWatchDatasource, - region: string, - namespace: string | undefined, - metricName: string | undefined, - dimensionFilter?: Dimensions + { namespace, region, dimensionFilters, metricName }: GetDimensionKeysRequest ) => { const [dimensionKeys, setDimensionKeys] = useState>>([]); // doing deep comparison to avoid making new api calls to list metrics unless dimension filter object props changes useDeepCompareEffect(() => { datasource.api - .getDimensionKeys(namespace, region, dimensionFilter, metricName) + .getDimensionKeys({ namespace, region, dimensionFilters, metricName }) .then((result: Array>) => { setDimensionKeys(appendTemplateVariables(datasource, result)); }); - }, [datasource, region, namespace, metricName, dimensionFilter]); + }, [datasource, region, namespace, metricName, dimensionFilters]); return dimensionKeys; }; diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index b64111dc7f5..b84678baf0e 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -450,3 +450,13 @@ export interface LegacyAnnotationQuery extends MetricStat, DataQuery { }; type: string; } + +export interface ResourceRequest { + region: string; +} + +export interface GetDimensionKeysRequest extends ResourceRequest { + metricName?: string; + namespace?: string; + dimensionFilters?: Dimensions; +} diff --git a/public/app/plugins/datasource/cloudwatch/variables.ts b/public/app/plugins/datasource/cloudwatch/variables.ts index a33e3ad8145..eff700a70e3 100644 --- a/public/app/plugins/datasource/cloudwatch/variables.ts +++ b/public/app/plugins/datasource/cloudwatch/variables.ts @@ -93,7 +93,7 @@ export class CloudWatchVariableSupport extends CustomVariableSupport ({ text: s.label, value: s.value,