Cloudwatch: Enable dimension filtering when loading dimension values (#41566)

* fix dimension filter * refactor tests * add comments * fix typo
2021-11-11 16:48:35 +01:00
parent 0f36152127
commit eb40723bcb
6 changed files with 418 additions and 54 deletions
@@ -276,6 +276,8 @@ func (e *cloudWatchExecutor) executeMetricFindQuery(ctx context.Context, model *
 		data, err = e.handleGetNamespaces(ctx, model, pluginCtx)
 	case "metrics":
 		data, err = e.handleGetMetrics(ctx, model, pluginCtx)
+	case "all_metrics":
+		data, err = e.handleGetAllMetrics(ctx, model, pluginCtx)
 	case "dimension_keys":
 		data, err = e.handleGetDimensions(ctx, model, pluginCtx)
 	case "dimension_values":
@@ -434,15 +436,90 @@ func (e *cloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s
 	return result, nil
 }

+// handleGetAllMetrics returns a slice of suggestData structs with metric and its namespace
+func (e *cloudWatchExecutor) handleGetAllMetrics(ctx context.Context, parameters *simplejson.Json, pluginCtx backend.PluginContext) ([]suggestData, error) {
+	result := make([]suggestData, 0)
+	for namespace, metrics := range metricsMap {
+		for _, metric := range metrics {
+			result = append(result, suggestData{Text: namespace, Value: metric})
+		}
+	}
+
+	return result, nil
+}
+
+// handleGetDimensions returns a slice of suggestData structs with dimension keys.
+// If a dimension filters parameter is specified, a new api call to list metrics will be issued to load dimension keys for the given filter.
+// If no dimension filter is specified, dimension keys will be retrieved from the hard coded map in this file.
 func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters *simplejson.Json, pluginCtx backend.PluginContext) ([]suggestData, error) {
 	region := parameters.Get("region").MustString()
 	namespace := parameters.Get("namespace").MustString()
+	metricName := parameters.Get("metricName").MustString("")
+	dimensionFilters := parameters.Get("dimensionFilters").MustMap()

 	var dimensionValues []string
 	if !isCustomMetrics(namespace) {
-		var exists bool
-		if dimensionValues, exists = dimensionsMap[namespace]; !exists {
-			return nil, fmt.Errorf("unable to find dimension %q", namespace)
+		if len(dimensionFilters) != 0 {
+			var dimensions []*cloudwatch.DimensionFilter
+			addDimension := func(key string, value string) {
+				filter := &cloudwatch.DimensionFilter{
+					Name: aws.String(key),
+				}
+				// if value is not specified or a wildcard is used, simply don't use the value field
+				if value != "" && value != "*" {
+					filter.Value = aws.String(value)
+				}
+				dimensions = append(dimensions, filter)
+			}
+			for k, v := range dimensionFilters {
+				// due to legacy, value can be a string, a string slice or nil
+				if vv, ok := v.(string); ok {
+					addDimension(k, vv)
+				} else if vv, ok := v.([]interface{}); ok {
+					for _, v := range vv {
+						addDimension(k, v.(string))
+					}
+				} else if v == nil {
+					addDimension(k, "")
+				}
+			}
+
+			input := &cloudwatch.ListMetricsInput{
+				Namespace:  aws.String(namespace),
+				Dimensions: dimensions,
+			}
+
+			if metricName != "" {
+				input.MetricName = aws.String(metricName)
+			}
+
+			metrics, err := e.listMetrics(region, input, pluginCtx)
+
+			if err != nil {
+				return nil, errutil.Wrap("unable to call AWS API", err)
+			}
+
+			dupCheck := make(map[string]bool)
+			for _, metric := range metrics {
+				for _, dim := range metric.Dimensions {
+					if _, exists := dupCheck[*dim.Name]; exists {
+						continue
+					}
+
+					// keys in the dimension filter should not be included
+					if _, ok := dimensionFilters[*dim.Name]; ok {
+						continue
+					}
+
+					dupCheck[*dim.Name] = true
+					dimensionValues = append(dimensionValues, *dim.Name)
+				}
+			}
+		} else {
+			var exists bool
+			if dimensionValues, exists = dimensionsMap[namespace]; !exists {
+				return nil, fmt.Errorf("unable to find dimension %q", namespace)
+			}
 		}
 	} else {
 		var err error
@@ -460,6 +537,8 @@ func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters
 	return result, nil
 }

+// handleGetDimensionValues returns a slice of suggestData structs with dimension values.
+// A call to the list metrics api is issued to retrieve the dimension values. All parameters are used as input args to the list metrics call.
 func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, parameters *simplejson.Json, pluginCtx backend.PluginContext) ([]suggestData, error) {
 	region := parameters.Get("region").MustString()
 	namespace := parameters.Get("namespace").MustString()
@@ -468,19 +547,26 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param
 	dimensionsJson := parameters.Get("dimensions").MustMap()

 	var dimensions []*cloudwatch.DimensionFilter
+	addDimension := func(key string, value string) {
+		filter := &cloudwatch.DimensionFilter{
+			Name: aws.String(key),
+		}
+		// if value is not specified or a wildcard is used, simply don't use the value field
+		if value != "" && value != "*" {
+			filter.Value = aws.String(value)
+		}
+		dimensions = append(dimensions, filter)
+	}
 	for k, v := range dimensionsJson {
+		// due to legacy, value can be a string, a string slice or nil
 		if vv, ok := v.(string); ok {
-			dimensions = append(dimensions, &cloudwatch.DimensionFilter{
-				Name:  aws.String(k),
-				Value: aws.String(vv),
-			})
+			addDimension(k, vv)
 		} else if vv, ok := v.([]interface{}); ok {
 			for _, v := range vv {
-				dimensions = append(dimensions, &cloudwatch.DimensionFilter{
-					Name:  aws.String(k),
-					Value: aws.String(v.(string)),
-				})
+				addDimension(k, v.(string))
 			}
+		} else if v == nil {
+			addDimension(k, "")
 		}
 	}

@@ -465,6 +465,154 @@ func TestQuery_ResourceARNs(t *testing.T) {
 	})
 }

+func TestQuery_GetAllMetrics(t *testing.T) {
+	t.Run("all metrics in all namespaces are being returned", func(t *testing.T) {
+		im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+			return datasourceInfo{}, nil
+		})
+
+		executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{})
+		resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
+			},
+			Queries: []backend.DataQuery{
+				{
+					JSON: json.RawMessage(`{
+						"type":      "metricFindQuery",
+						"subtype":   "all_metrics",
+						"region":    "us-east-1"
+					}`),
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		metricCount := 0
+		for _, metrics := range metricsMap {
+			metricCount += len(metrics)
+		}
+
+		assert.Equal(t, metricCount, resp.Responses[""].Frames[0].Fields[1].Len())
+	})
+}
+
+func TestQuery_GetDimensionKeys(t *testing.T) {
+	origNewCWClient := NewCWClient
+	t.Cleanup(func() {
+		NewCWClient = origNewCWClient
+	})
+
+	var client FakeCWClient
+
+	NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
+		return client
+	}
+
+	metrics := []*cloudwatch.Metric{
+		{MetricName: aws.String("Test_MetricName1"), Dimensions: []*cloudwatch.Dimension{
+			{Name: aws.String("Dimension1"), Value: aws.String("Dimension1")},
+			{Name: aws.String("Dimension2"), Value: aws.String("Dimension2")},
+		}},
+		{MetricName: aws.String("Test_MetricName2"), Dimensions: []*cloudwatch.Dimension{
+			{Name: aws.String("Dimension2"), Value: aws.String("Dimension2")},
+			{Name: aws.String("Dimension3"), Value: aws.String("Dimension3")},
+		}},
+	}
+
+	t.Run("should fetch dimension keys from list metrics api and return unique dimensions when a dimension filter is specified", func(t *testing.T) {
+		client = FakeCWClient{Metrics: metrics, MetricsPerPage: 2}
+		im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+			return datasourceInfo{}, nil
+		})
+
+		executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{})
+		resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
+			},
+			Queries: []backend.DataQuery{
+				{
+					JSON: json.RawMessage(`{
+						"type":      "metricFindQuery",
+						"subtype":   "dimension_keys",
+						"region":    "us-east-1",
+						"namespace": "AWS/EC2",
+						"dimensionFilters": {
+							"InstanceId": "",
+							"AutoscalingGroup": []
+						}
+					}`),
+				},
+			},
+		})
+
+		require.NoError(t, err)
+
+		expValues := []string{"Dimension1", "Dimension2", "Dimension3"}
+		expFrame := data.NewFrame(
+			"",
+			data.NewField("text", nil, expValues),
+			data.NewField("value", nil, expValues),
+		)
+		expFrame.Meta = &data.FrameMeta{
+			Custom: map[string]interface{}{
+				"rowCount": len(expValues),
+			},
+		}
+
+		assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
+			"": {
+				Frames: data.Frames{expFrame},
+			},
+		},
+		}, resp)
+	})
+
+	t.Run("should return hard coded metrics when no dimension filter is specified", func(t *testing.T) {
+		im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+			return datasourceInfo{}, nil
+		})
+
+		executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{})
+		resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
+			},
+			Queries: []backend.DataQuery{
+				{
+					JSON: json.RawMessage(`{
+						"type":      "metricFindQuery",
+						"subtype":   "dimension_keys",
+						"region":    "us-east-1",
+						"namespace": "AWS/EC2",
+						"dimensionFilters": {}
+					}`),
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		expValues := dimensionsMap["AWS/EC2"]
+		expFrame := data.NewFrame(
+			"",
+			data.NewField("text", nil, expValues),
+			data.NewField("value", nil, expValues),
+		)
+		expFrame.Meta = &data.FrameMeta{
+			Custom: map[string]interface{}{
+				"rowCount": len(expValues),
+			},
+		}
+
+		assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
+			"": {
+				Frames: data.Frames{expFrame},
+			},
+		},
+		}, resp)
+	})
+}
 func Test_isCustomMetrics(t *testing.T) {
 	metricsMap = map[string][]string{
 		"AWS/EC2": {"ExampleMetric"},