Cloudwatch: ListMetrics API page limit (#31788) (#31851)

* Cloudwatch: ListMetrics API page limit (#31788)

* add list metrics api page limit

* Update docs/sources/datasources/cloudwatch.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
(cherry picked from commit d512c5a1b4)

* update go sum

* remove spaces

* revert go.sum

conf/defaults.ini

@@ -515,6 +515,9 @@ allowed_auth_providers = default,keys,credentials
 # If true, assume role will be enabled for all AWS authentication providers that are specified in aws_auth_providers
 assume_role_enabled = true
 
+# Specify max no of pages to be returned by the ListMetricPages API
+list_metrics_page_limit = 500
+
 #################################### SMTP / Emailing #####################
 [smtp]
 enabled = false

docs/sources/administration/configuration.md

@@ -788,6 +788,10 @@ Set to `false` to disable AWS authentication from using an assumed role with tem
 
 If this option is disabled, the **Assume Role** and the **External Id** field are removed from the AWS data source configuration page. If the plugin is configured using provisioning, it is possible to use an assumed role as long as `assume_role_enabled` is set to `true`.
 
+### list_metrics_page_limit
+
+Use the [List Metrics API](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html) option to load metrics for custom namespaces in the CloudWatch data source. By default, the page limit is 500.
+
 <hr />
 
 ## [smtp]

docs/sources/datasources/cloudwatch.md

@@ -387,6 +387,10 @@ Specify which authentication providers are allowed for the CloudWatch data sourc
 
 Allows you to disable `assume role (ARN)` in the CloudWatch data source. By default, assume role (ARN) is enabled for OSS Grafana.
 
+### list_metrics_page_limit
+
+When a custom namespace is specified in the query editor, the [List Metrics API](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html) is used to populate the _Metrics_ field and the _Dimension_ fields. The API is paginated and returns up to 500 results per page. The CloudWatch data source also limits the number of pages to 500. However, you can change this limit using the `list_metrics_page_limit` variable in the [grafana configuration file](https://grafana.com/docs/grafana/latest/administration/configuration/#aws).
+
 ## Configure the data source with provisioning
 
 It's now possible to configure data sources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for data sources on the [provisioning docs page]({{< relref "../administration/provisioning/#datasources" >}})

pkg/setting/setting.go

@@ -276,6 +276,7 @@ type Cfg struct {
 	// AWS Plugin Auth
 	AWSAllowedAuthProviders []string
 	AWSAssumeRoleEnabled    bool
+	AWSListMetricsPageLimit int
 
 	// Auth proxy settings
 	AuthProxyEnabled          bool
@@ -942,6 +943,7 @@ func (cfg *Cfg) readAWSConfig() {
 			cfg.AWSAllowedAuthProviders = append(cfg.AWSAllowedAuthProviders, authProvider)
 		}
 	}
+	cfg.AWSListMetricsPageLimit = awsPluginSec.Key("list_metrics_page_limit").MustInt(500)
 }
 
 func (cfg *Cfg) readSessionConfig() {

pkg/tsdb/cloudwatch/metric_find_query.go

@@ -461,14 +461,21 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param
 		}
 	}
 
-	metrics, err := e.cloudwatchListMetrics(region, namespace, metricName, dimensions)
+	params := &cloudwatch.ListMetricsInput{
+		Namespace:  aws.String(namespace),
+		Dimensions: dimensions,
+	}
+	if metricName != "" {
+		params.MetricName = aws.String(metricName)
+	}
+	metrics, err := e.listMetrics(region, params)
 	if err != nil {
 		return nil, err
 	}
 
 	result := make([]suggestData, 0)
 	dupCheck := make(map[string]bool)
-	for _, metric := range metrics.Metrics {
+	for _, metric := range metrics {
 		for _, dim := range metric.Dimensions {
 			if *dim.Name == dimensionKey {
 				if _, exists := dupCheck[*dim.Value]; exists {
@@ -630,36 +637,29 @@ func (e *cloudWatchExecutor) handleGetResourceArns(ctx context.Context, paramete
 	return result, nil
 }
 
-func (e *cloudWatchExecutor) cloudwatchListMetrics(region string, namespace string, metricName string,
-	dimensions []*cloudwatch.DimensionFilter) (*cloudwatch.ListMetricsOutput, error) {
-	svc, err := e.getCWClient(region)
+func (e *cloudWatchExecutor) listMetrics(region string, params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) {
+	client, err := e.getCWClient(region)
 	if err != nil {
 		return nil, err
 	}
 
-	params := &cloudwatch.ListMetricsInput{
-		Namespace:  aws.String(namespace),
-		Dimensions: dimensions,
-	}
+	plog.Debug("Listing metrics pages")
+	cloudWatchMetrics := []*cloudwatch.Metric{}
 
-	if metricName != "" {
-		params.MetricName = aws.String(metricName)
-	}
-
-	var resp cloudwatch.ListMetricsOutput
-	if err := svc.ListMetricsPages(params,
-		func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
-			metrics.MAwsCloudWatchListMetrics.Inc()
-			metrics, _ := awsutil.ValuesAtPath(page, "Metrics")
+	pageNum := 0
+	err = client.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
+		pageNum++
+		metrics.MAwsCloudWatchListMetrics.Inc()
+		metrics, err := awsutil.ValuesAtPath(page, "Metrics")
+		if err == nil {
 			for _, metric := range metrics {
-				resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric))
+				cloudWatchMetrics = append(cloudWatchMetrics, metric.(*cloudwatch.Metric))
 			}
-			return !lastPage
-		}); err != nil {
-		return nil, fmt.Errorf("failed to call cloudwatch:ListMetrics: %w", err)
-	}
+		}
+		return !lastPage && pageNum < e.cfg.AWSListMetricsPageLimit
+	})
 
-	return &resp, nil
+	return cloudWatchMetrics, err
 }
 
 func (e *cloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.Filter, instanceIds []*string) (*ec2.DescribeInstancesOutput, error) {
@@ -708,34 +708,6 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
 	return &resp, nil
 }
 
-func (e *cloudWatchExecutor) getAllMetrics(region, namespace string) (cloudwatch.ListMetricsOutput, error) {
-	client, err := e.getCWClient(region)
-	if err != nil {
-		return cloudwatch.ListMetricsOutput{}, err
-	}
-
-	params := &cloudwatch.ListMetricsInput{
-		Namespace: aws.String(namespace),
-	}
-
-	plog.Debug("Listing metrics pages")
-	var resp cloudwatch.ListMetricsOutput
-	err = client.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
-		metrics.MAwsCloudWatchListMetrics.Inc()
-		metrics, err := awsutil.ValuesAtPath(page, "Metrics")
-		if err != nil {
-			return !lastPage
-		}
-
-		for _, metric := range metrics {
-			resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric))
-		}
-		return !lastPage
-	})
-
-	return resp, err
-}
-
 var metricsCacheLock sync.Mutex
 
 func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region, namespace string) ([]string, error) {
@@ -759,7 +731,10 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region, namespace string
 	if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][namespace].Expire.After(time.Now()) {
 		return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][namespace].Cache, nil
 	}
-	result, err := e.getAllMetrics(region, namespace)
+	metrics, err := e.listMetrics(region, &cloudwatch.ListMetricsInput{
+		Namespace: aws.String(namespace),
+	})
+
 	if err != nil {
 		return []string{}, err
 	}
@@ -767,7 +742,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region, namespace string
 	customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][namespace].Cache = make([]string, 0)
 	customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][namespace].Expire = time.Now().Add(5 * time.Minute)
 
-	for _, metric := range result.Metrics {
+	for _, metric := range metrics {
 		if isDuplicate(customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][namespace].Cache, *metric.MetricName) {
 			continue
 		}
@@ -800,14 +775,15 @@ func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region, namespace str
 	if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][namespace].Expire.After(time.Now()) {
 		return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][namespace].Cache, nil
 	}
-	result, err := e.getAllMetrics(region, namespace)
+
+	metrics, err := e.listMetrics(region, &cloudwatch.ListMetricsInput{Namespace: aws.String(namespace)})
 	if err != nil {
 		return []string{}, err
 	}
 	customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][namespace].Cache = make([]string, 0)
 	customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][namespace].Expire = time.Now().Add(5 * time.Minute)
 
-	for _, metric := range result.Metrics {
+	for _, metric := range metrics {
 		for _, dimension := range metric.Dimensions {
 			if isDuplicate(customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][namespace].Cache, *dimension.Name) {
 				continue

pkg/tsdb/cloudwatch/metric_find_query_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -499,3 +500,50 @@ func TestQuery_ResourceARNs(t *testing.T) {
 		}, resp)
 	})
 }
+
+func TestQuery_ListMetricsPagination(t *testing.T) {
+	origNewCWClient := NewCWClient
+	t.Cleanup(func() {
+		NewCWClient = origNewCWClient
+	})
+
+	var client FakeCWClient
+
+	NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
+		return client
+	}
+
+	metrics := []*cloudwatch.Metric{
+		{MetricName: aws.String("Test_MetricName1")},
+		{MetricName: aws.String("Test_MetricName2")},
+		{MetricName: aws.String("Test_MetricName3")},
+		{MetricName: aws.String("Test_MetricName4")},
+		{MetricName: aws.String("Test_MetricName5")},
+		{MetricName: aws.String("Test_MetricName6")},
+		{MetricName: aws.String("Test_MetricName7")},
+		{MetricName: aws.String("Test_MetricName8")},
+		{MetricName: aws.String("Test_MetricName9")},
+		{MetricName: aws.String("Test_MetricName10")},
+	}
+
+	t.Run("List Metrics and page limit is reached", func(t *testing.T) {
+		client = FakeCWClient{Metrics: metrics, MetricsPerPage: 2}
+		executor := newExecutor(nil, &setting.Cfg{AWSListMetricsPageLimit: 3, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true})
+		executor.DataSource = fakeDataSource()
+		response, err := executor.listMetrics("default", &cloudwatch.ListMetricsInput{})
+		require.NoError(t, err)
+
+		expectedMetrics := client.MetricsPerPage * executor.cfg.AWSListMetricsPageLimit
+		assert.Equal(t, expectedMetrics, len(response))
+	})
+
+	t.Run("List Metrics and page limit is not reached", func(t *testing.T) {
+		client = FakeCWClient{Metrics: metrics, MetricsPerPage: 2}
+		executor := newExecutor(nil, &setting.Cfg{AWSListMetricsPageLimit: 1000, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true})
+		executor.DataSource = fakeDataSource()
+		response, err := executor.listMetrics("default", &cloudwatch.ListMetricsInput{})
+		require.NoError(t, err)
+
+		assert.Equal(t, len(metrics), len(response))
+	})
+}

pkg/tsdb/cloudwatch/test_utils.go

@@ -79,12 +79,24 @@ type FakeCWClient struct {
 	cloudwatchiface.CloudWatchAPI
 
 	Metrics []*cloudwatch.Metric
+
+	MetricsPerPage int
 }
 
 func (c FakeCWClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error {
-	fn(&cloudwatch.ListMetricsOutput{
-		Metrics: c.Metrics,
-	}, true)
+	if c.MetricsPerPage == 0 {
+		c.MetricsPerPage = 1000
+	}
+	chunks := chunkSlice(c.Metrics, c.MetricsPerPage)
+
+	for i, metrics := range chunks {
+		response := fn(&cloudwatch.ListMetricsOutput{
+			Metrics: metrics,
+		}, i+1 == len(chunks))
+		if !response {
+			break
+		}
+	}
 	return nil
 }
 
@@ -147,6 +159,23 @@ func (c fakeRGTAClient) GetResourcesPages(in *resourcegroupstaggingapi.GetResour
 	return nil
 }
 
+func chunkSlice(slice []*cloudwatch.Metric, chunkSize int) [][]*cloudwatch.Metric {
+	var chunks [][]*cloudwatch.Metric
+	for {
+		if len(slice) == 0 {
+			break
+		}
+		if len(slice) < chunkSize {
+			chunkSize = len(slice)
+		}
+
+		chunks = append(chunks, slice[0:chunkSize])
+		slice = slice[chunkSize:]
+	}
+
+	return chunks
+}
+
 func newTestConfig() *setting.Cfg {
-	return &setting.Cfg{AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}
+	return &setting.Cfg{AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true, AWSListMetricsPageLimit: 1000}
 }