CloudWatch: Add support for AWS Metric Insights (#42487)

* add support for code editor and builder

* refactor cloudwatch migration

* Add tooltip to editor field (#56)

* add tooltip

* add old tooltips

* Bug bash feedback fixes (#58)

* make ASC the default option

* update sql preview whenever sql changes

* don't allow queries without aggregation

* set default value for aggregation

* use new input field

* cleanup

* pr feedback

* prevent unnecessary rerenders

* use frame error instead of main error

* remove not used snapshot

* Use dimension filter in schema picker  (#63)

* use dimension key filter in group by and schema labels

* add dimension filter also to code editor

* add tests

* fix build error

* fix strict error

* remove debug code

* fix annotation editor (#64)

* fix annotation editor

* fix broken test

* revert annotation backend change

* PR feedback (#67)

* pr feedback

* removed dimension filter from group by

* add spacing between common fields and rest

* do not generate deep link for metric queries (#70)

* update docs (#69)

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>

* fix lint problem caused by merge conflict

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
This commit is contained in:
Erik Sundell
2021-11-30 10:53:31 +01:00
committed by GitHub
co-authored by Erik Sundell achatterjee-grafana
parent 2a50c029b2
commit bab78a9e64
86 changed files with 6487 additions and 1010 deletions
+44 -17
View File
@@ -9,34 +9,57 @@ import (
)
type cloudWatchQuery struct {
RefId string
Region string
Id string
Namespace string
MetricName string
Statistic string
Expression string
ReturnData bool
Dimensions map[string][]string
Period int
Alias string
MatchExact bool
UsedExpression string
RefId string
Region string
Id string
Namespace string
MetricName string
Statistic string
Expression string
SqlExpression string
ReturnData bool
Dimensions map[string][]string
Period int
Alias string
MatchExact bool
UsedExpression string
MetricQueryType metricQueryType
MetricEditorMode metricEditorMode
}
func (q *cloudWatchQuery) getGMDAPIMode() gmdApiMode {
if q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeBuilder {
if q.isInferredSearchExpression() {
return GMDApiModeInferredSearchExpression
}
return GMDApiModeMetricStat
} else if q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeRaw {
return GMDApiModeMathExpression
} else if q.MetricQueryType == MetricQueryTypeQuery {
return GMDApiModeSQLExpression
}
plog.Warn("Could not resolve CloudWatch metric query type. Falling back to metric stat.", "query", q)
return GMDApiModeMetricStat
}
func (q *cloudWatchQuery) isMathExpression() bool {
return q.Expression != "" && !q.isUserDefinedSearchExpression()
return q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeRaw && !q.isUserDefinedSearchExpression()
}
func (q *cloudWatchQuery) isSearchExpression() bool {
return q.isUserDefinedSearchExpression() || q.isInferredSearchExpression()
return q.MetricQueryType == MetricQueryTypeSearch && (q.isUserDefinedSearchExpression() || q.isInferredSearchExpression())
}
func (q *cloudWatchQuery) isUserDefinedSearchExpression() bool {
return strings.Contains(q.Expression, "SEARCH(")
return q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeRaw && strings.Contains(q.Expression, "SEARCH(")
}
func (q *cloudWatchQuery) isInferredSearchExpression() bool {
if q.MetricQueryType != MetricQueryTypeSearch || q.MetricEditorMode != MetricEditorModeBuilder {
return false
}
if len(q.Dimensions) == 0 {
return !q.MatchExact
}
@@ -58,6 +81,10 @@ func (q *cloudWatchQuery) isInferredSearchExpression() bool {
}
func (q *cloudWatchQuery) isMultiValuedDimensionExpression() bool {
if q.MetricQueryType != MetricQueryTypeSearch || q.MetricEditorMode != MetricEditorModeBuilder {
return false
}
for _, values := range q.Dimensions {
for _, v := range values {
if v == "*" {
@@ -74,7 +101,7 @@ func (q *cloudWatchQuery) isMultiValuedDimensionExpression() bool {
}
func (q *cloudWatchQuery) buildDeepLink(startTime time.Time, endTime time.Time) (string, error) {
if q.isMathExpression() {
if q.isMathExpression() || q.MetricQueryType == MetricQueryTypeQuery {
return "", nil
}
+25 -7
View File
@@ -2,11 +2,36 @@ package cloudwatch
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCloudWatchQuery(t *testing.T) {
t.Run("Deeplink is not generated for MetricQueryTypeQuery", func(t *testing.T) {
startTime := time.Now()
endTime := startTime.Add(2 * time.Hour)
query := &cloudWatchQuery{
RefId: "A",
Region: "us-east-1",
Expression: "",
Statistic: "Average",
Period: 300,
Id: "id1",
MatchExact: true,
Dimensions: map[string][]string{
"InstanceId": {"i-12345678"},
},
MetricQueryType: MetricQueryTypeQuery,
MetricEditorMode: MetricEditorModeBuilder,
}
deepLink, err := query.buildDeepLink(startTime, endTime)
require.NoError(t, err)
assert.Empty(t, deepLink)
})
t.Run("SEARCH(someexpression) was specified in the query editor", func(t *testing.T) {
query := &cloudWatchQuery{
RefId: "A",
@@ -107,14 +132,12 @@ func TestCloudWatchQuery(t *testing.T) {
query.MatchExact = false
assert.True(t, query.isSearchExpression(), "Expected a search expression")
assert.False(t, query.isMathExpression(), "Expected not math expression")
assert.False(t, query.isMetricStat(), "Expected not metric stat")
})
t.Run("Match exact is true", func(t *testing.T) {
query.MatchExact = true
assert.False(t, query.isSearchExpression(), "Exxpected not search expression")
assert.False(t, query.isMathExpression(), "Expected not math expression")
assert.True(t, query.isMetricStat(), "Expected a metric stat")
})
})
@@ -134,10 +157,5 @@ func TestCloudWatchQuery(t *testing.T) {
assert.True(t, query.isSearchExpression(), "Expected search expression")
assert.False(t, query.isMathExpression(), "Expected not math expression")
assert.False(t, query.isMetricStat(), "Expected not metric stat")
})
}
func (q *cloudWatchQuery) isMetricStat() bool {
return !q.isSearchExpression() && !q.isMathExpression()
}
@@ -16,30 +16,32 @@ func (e *cloudWatchExecutor) buildMetricDataQuery(query *cloudWatchQuery) (*clou
ReturnData: aws.Bool(query.ReturnData),
}
if query.Expression != "" {
mdq.Expression = aws.String(query.Expression)
switch query.getGMDAPIMode() {
case GMDApiModeMathExpression:
mdq.Period = aws.Int64(int64(query.Period))
} else {
if query.isSearchExpression() {
mdq.Expression = aws.String(buildSearchExpression(query, query.Statistic))
} else {
mdq.MetricStat = &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
Namespace: aws.String(query.Namespace),
MetricName: aws.String(query.MetricName),
Dimensions: make([]*cloudwatch.Dimension, 0),
},
Period: aws.Int64(int64(query.Period)),
}
for key, values := range query.Dimensions {
mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions,
&cloudwatch.Dimension{
Name: aws.String(key),
Value: aws.String(values[0]),
})
}
mdq.MetricStat.Stat = aws.String(query.Statistic)
mdq.Expression = aws.String(query.Expression)
case GMDApiModeSQLExpression:
mdq.Period = aws.Int64(int64(query.Period))
mdq.Expression = aws.String(query.SqlExpression)
case GMDApiModeInferredSearchExpression:
mdq.Expression = aws.String(buildSearchExpression(query, query.Statistic))
case GMDApiModeMetricStat:
mdq.MetricStat = &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
Namespace: aws.String(query.Namespace),
MetricName: aws.String(query.MetricName),
Dimensions: make([]*cloudwatch.Dimension, 0),
},
Period: aws.Int64(int64(query.Period)),
}
for key, values := range query.Dimensions {
mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions,
&cloudwatch.Dimension{
Name: aws.String(key),
Value: aws.String(values[0]),
})
}
mdq.MetricStat.Stat = aws.String(query.Statistic)
}
if mdq.Expression != nil {
@@ -7,21 +7,63 @@ import (
"github.com/stretchr/testify/require"
)
func TestMetricDataQueryBuilder_buildSearchExpression(t *testing.T) {
func TestMetricDataQueryBuilder(t *testing.T) {
t.Run("buildMetricDataQuery", func(t *testing.T) {
t.Run("should use metric stat", func(t *testing.T) {
executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{})
query := getBaseQuery()
query.MetricEditorMode = MetricEditorModeBuilder
query.MetricQueryType = MetricQueryTypeSearch
mdq, err := executor.buildMetricDataQuery(query)
require.NoError(t, err)
require.Empty(t, mdq.Expression)
assert.Equal(t, query.MetricName, *mdq.MetricStat.Metric.MetricName)
assert.Equal(t, query.Namespace, *mdq.MetricStat.Metric.Namespace)
})
t.Run("should use custom built expression", func(t *testing.T) {
executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{})
query := getBaseQuery()
query.MetricEditorMode = MetricEditorModeBuilder
query.MetricQueryType = MetricQueryTypeSearch
query.MatchExact = false
mdq, err := executor.buildMetricDataQuery(query)
require.NoError(t, err)
require.Nil(t, mdq.MetricStat)
assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"="lb1"', '', 300))`, *mdq.Expression)
})
t.Run("should use sql expression", func(t *testing.T) {
executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{})
query := getBaseQuery()
query.MetricEditorMode = MetricEditorModeRaw
query.MetricQueryType = MetricQueryTypeQuery
query.SqlExpression = `SELECT SUM(CPUUTilization) FROM "AWS/EC2"`
mdq, err := executor.buildMetricDataQuery(query)
require.NoError(t, err)
require.Nil(t, mdq.MetricStat)
assert.Equal(t, query.SqlExpression, *mdq.Expression)
})
t.Run("should use user defined math expression", func(t *testing.T) {
executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{})
query := getBaseQuery()
query.MetricEditorMode = MetricEditorModeRaw
query.MetricQueryType = MetricQueryTypeSearch
query.Expression = `SUM(x+y)`
mdq, err := executor.buildMetricDataQuery(query)
require.NoError(t, err)
require.Nil(t, mdq.MetricStat)
assert.Equal(t, query.Expression, *mdq.Expression)
})
t.Run("should set period in user defined expression", func(t *testing.T) {
executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{})
query := &cloudWatchQuery{
Namespace: "AWS/EC2",
MetricName: "CPUUtilization",
Dimensions: map[string][]string{
"LoadBalancer": {"lb1"},
},
Period: 300,
Expression: "SUM([a,b])",
MatchExact: true,
}
query := getBaseQuery()
query.MetricEditorMode = MetricEditorModeRaw
query.MetricQueryType = MetricQueryTypeSearch
query.MatchExact = false
query.Expression = `SUM([a,b])`
mdq, err := executor.buildMetricDataQuery(query)
require.NoError(t, err)
require.Nil(t, mdq.MetricStat)
@@ -235,3 +277,17 @@ func TestMetricDataQueryBuilder_buildSearchExpression(t *testing.T) {
assert.Contains(t, res, `lb4\"\"`, "Expected escape double quotes")
})
}
func getBaseQuery() *cloudWatchQuery {
query := &cloudWatchQuery{
Namespace: "AWS/EC2",
MetricName: "CPUUtilization",
Dimensions: map[string][]string{
"LoadBalancer": {"lb1"},
},
Period: 300,
Expression: "",
MatchExact: true,
}
return query
}
+27 -13
View File
@@ -143,6 +143,7 @@ func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time
id = fmt.Sprintf("query%s", refId)
}
expression := model.Get("expression").MustString("")
sqlExpression := model.Get("sqlExpression").MustString("")
alias := model.Get("alias").MustString()
returnData := !model.Get("hide").MustBool(false)
queryType := model.Get("type").MustString()
@@ -154,21 +155,34 @@ func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time
}
matchExact := model.Get("matchExact").MustBool(true)
metricQueryType := metricQueryType(model.Get("metricQueryType").MustInt(0))
var metricEditorModeValue metricEditorMode
memv, err := model.Get("metricEditorMode").Int()
if err != nil && len(expression) > 0 {
// this should only ever happen if this is an alerting query that has not yet been migrated in the frontend
metricEditorModeValue = MetricEditorModeRaw
} else {
metricEditorModeValue = metricEditorMode(memv)
}
return &cloudWatchQuery{
RefId: refId,
Region: region,
Id: id,
Namespace: namespace,
MetricName: metricName,
Statistic: statistic,
Expression: expression,
ReturnData: returnData,
Dimensions: dimensions,
Period: period,
Alias: alias,
MatchExact: matchExact,
UsedExpression: "",
RefId: refId,
Region: region,
Id: id,
Namespace: namespace,
MetricName: metricName,
Statistic: statistic,
Expression: expression,
ReturnData: returnData,
Dimensions: dimensions,
Period: period,
Alias: alias,
MatchExact: matchExact,
UsedExpression: "",
MetricQueryType: metricQueryType,
MetricEditorMode: metricEditorModeValue,
SqlExpression: sqlExpression,
}, nil
}
@@ -272,4 +272,54 @@ func TestRequestParser(t *testing.T) {
assert.Equal(t, 21600, res.Period)
})
})
t.Run("Metric query type, metric editor mode and query api mode", func(t *testing.T) {
timeRange := legacydata.NewDataTimeRange("now-1h", "now-2h")
from, err := timeRange.ParseFrom()
require.NoError(t, err)
to, err := timeRange.ParseTo()
require.NoError(t, err)
t.Run("when metric query type and metric editor mode is not specified", func(t *testing.T) {
t.Run("it should be metric search builder", func(t *testing.T) {
query := getBaseJsonQuery()
res, err := parseRequestQuery(query, "ref1", from, to)
require.NoError(t, err)
assert.Equal(t, MetricQueryTypeSearch, res.MetricQueryType)
assert.Equal(t, MetricEditorModeBuilder, res.MetricEditorMode)
assert.Equal(t, GMDApiModeMetricStat, res.getGMDAPIMode())
})
t.Run("and an expression is specified it should be metric search builder", func(t *testing.T) {
query := getBaseJsonQuery()
query.Set("expression", "SUM(a)")
res, err := parseRequestQuery(query, "ref1", from, to)
require.NoError(t, err)
assert.Equal(t, MetricQueryTypeSearch, res.MetricQueryType)
assert.Equal(t, MetricEditorModeRaw, res.MetricEditorMode)
assert.Equal(t, GMDApiModeMathExpression, res.getGMDAPIMode())
})
})
t.Run("and an expression is specified it should be metric search builder", func(t *testing.T) {
query := getBaseJsonQuery()
query.Set("expression", "SUM(a)")
res, err := parseRequestQuery(query, "ref1", from, to)
require.NoError(t, err)
assert.Equal(t, MetricQueryTypeSearch, res.MetricQueryType)
assert.Equal(t, MetricEditorModeRaw, res.MetricEditorMode)
assert.Equal(t, GMDApiModeMathExpression, res.getGMDAPIMode())
})
})
}
func getBaseJsonQuery() *simplejson.Json {
return simplejson.NewFromAny(map[string]interface{}{
"refId": "ref1",
"region": "us-east-1",
"namespace": "ec2",
"metricName": "CPUUtilization",
"statistic": "Average",
"period": "900",
})
}
+15 -7
View File
@@ -229,19 +229,27 @@ func formatAlias(query *cloudWatchQuery, stat string, dimensions map[string]stri
if len(query.Alias) == 0 && query.isInferredSearchExpression() && !query.isMultiValuedDimensionExpression() {
return label
}
if len(query.Alias) == 0 && query.MetricQueryType == MetricQueryTypeQuery {
return label
}
// common fields
data := map[string]string{
"region": region,
"namespace": namespace,
"metric": metricName,
"stat": stat,
"period": period,
"region": region,
"period": period,
}
if len(label) != 0 {
data["label"] = label
}
for k, v := range dimensions {
data[k] = v
// since the SQL query string is not (yet) parsed, we don't know what namespace, metric, statistic and labels it's using at this point
if query.MetricQueryType != MetricQueryTypeQuery {
data["namespace"] = namespace
data["metric"] = metricName
data["stat"] = stat
for k, v := range dimensions {
data[k] = v
}
}
result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
+76 -18
View File
@@ -3,6 +3,7 @@ package cloudwatch
import (
"encoding/json"
"io/ioutil"
"strings"
"testing"
"time"
@@ -104,9 +105,11 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb1", "lb2"},
"TargetGroup": {"tg"},
},
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
MetricQueryType: MetricQueryTypeSearch,
MetricEditorMode: MetricEditorModeBuilder,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
@@ -166,9 +169,11 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb1", "lb2"},
"TargetGroup": {"tg"},
},
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
MetricQueryType: MetricQueryTypeSearch,
MetricEditorMode: MetricEditorModeBuilder,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
@@ -229,9 +234,11 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"*"},
"TargetGroup": {"tg"},
},
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
MetricQueryType: MetricQueryTypeSearch,
MetricEditorMode: MetricEditorModeBuilder,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
@@ -266,9 +273,11 @@ func TestCloudWatchResponseParser(t *testing.T) {
Dimensions: map[string][]string{
"LoadBalancer": {"lb1", "lb2"},
},
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
MetricQueryType: MetricQueryTypeSearch,
MetricEditorMode: MetricEditorModeBuilder,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
@@ -307,9 +316,11 @@ func TestCloudWatchResponseParser(t *testing.T) {
"InstanceType": {"micro"},
"Resource": {"res"},
},
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded {{InstanceType}} - {{Resource}}",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded {{InstanceType}} - {{Resource}}",
MetricQueryType: MetricQueryTypeSearch,
MetricEditorMode: MetricEditorModeBuilder,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
@@ -319,6 +330,51 @@ func TestCloudWatchResponseParser(t *testing.T) {
assert.Equal(t, "lb2 Expanded micro - res", frames[1].Name)
})
t.Run("Should only expand certain fields when using SQL queries", func(t *testing.T) {
timestamp := time.Unix(0, 0)
response := &queryRowResponse{
Labels: []string{"lb3"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
},
Values: []*float64{aws.Float64(23)},
StatusCode: aws.String("Complete"),
},
},
}
query := &cloudWatchQuery{
RefId: "refId1",
Region: "us-east-1",
Namespace: "AWS/ApplicationELB",
MetricName: "TargetResponseTime",
Dimensions: map[string][]string{
"LoadBalancer": {"lb1"},
"InstanceType": {"micro"},
"Resource": {"res"},
},
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} {{InstanceType}} {{metric}} {{namespace}} {{stat}} {{region}} {{period}}",
MetricQueryType: MetricQueryTypeQuery,
MetricEditorMode: MetricEditorModeRaw,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
assert.False(t, strings.Contains(frames[0].Name, "AWS/ApplicationELB"))
assert.False(t, strings.Contains(frames[0].Name, "lb1"))
assert.False(t, strings.Contains(frames[0].Name, "micro"))
assert.False(t, strings.Contains(frames[0].Name, "AWS/ApplicationELB"))
assert.True(t, strings.Contains(frames[0].Name, "us-east-1"))
assert.True(t, strings.Contains(frames[0].Name, "60"))
})
t.Run("Parse cloudwatch response", func(t *testing.T) {
timestamp := time.Unix(0, 0)
response := &queryRowResponse{
@@ -351,9 +407,11 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb"},
"TargetGroup": {"tg"},
},
Statistic: "Average",
Period: 60,
Alias: "{{namespace}}_{{metric}}_{{stat}}",
Statistic: "Average",
Period: 60,
Alias: "{{namespace}}_{{metric}}_{{stat}}",
MetricQueryType: MetricQueryTypeSearch,
MetricEditorMode: MetricEditorModeBuilder,
}
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
-1
View File
@@ -91,7 +91,6 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, req *ba
resultChan <- &responseWrapper{
DataResponse: &dataResponse,
}
return nil, err
}
close(resultChan)
+23
View File
@@ -31,3 +31,26 @@ type metricStatMeta struct {
Stat string `json:"stat"`
Period int `json:"period"`
}
type metricQueryType uint32
const (
MetricQueryTypeSearch metricQueryType = iota
MetricQueryTypeQuery
)
type metricEditorMode uint32
const (
MetricEditorModeBuilder metricEditorMode = iota
MetricEditorModeRaw
)
type gmdApiMode uint32
const (
GMDApiModeMetricStat gmdApiMode = iota
GMDApiModeInferredSearchExpression
GMDApiModeMathExpression
GMDApiModeSQLExpression
)