public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cloudwatch: Migrate queries that use multiple stats to one query per stat (#36925)

Browse Source 
* migrate queries that use multiple stats - squash commits

* fix typo
This commit is contained in:
Erik Sundell
2021-09-08 16:06:43 +02:00
committed by GitHub
parent ae9343f8ae
commit 5e38b02f94
34 changed files with 2303 additions and 1492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudwatch/response_parser_test.go
+207 -157
View File
@@ -1,6 +1,8 @@
package cloudwatch
import (
"encoding/json"
"io/ioutil"
"testing"
"time"
@@ -10,40 +12,86 @@ import (
"github.com/stretchr/testify/require"
)
func loadGetMetricDataOutputsFromFile() ([]*cloudwatch.GetMetricDataOutput, error) {
var getMetricDataOutputs []*cloudwatch.GetMetricDataOutput
jsonBody, err := ioutil.ReadFile("./test-data/multiple-outputs.json")
if err != nil {
return getMetricDataOutputs, err
}
err = json.Unmarshal(jsonBody, &getMetricDataOutputs)
return getMetricDataOutputs, err
}
func TestCloudWatchResponseParser(t *testing.T) {
startTime := time.Now()
endTime := startTime.Add(2 * time.Hour)
t.Run("when aggregating response", func(t *testing.T) {
getMetricDataOutputs, err := loadGetMetricDataOutputsFromFile()
require.NoError(t, err)
aggregatedResponse := aggregateResponse(getMetricDataOutputs)
t.Run("response for id a", func(t *testing.T) {
idA := "a"
t.Run("should have two labels", func(t *testing.T) {
assert.Len(t, aggregatedResponse[idA].Labels, 2)
assert.Len(t, aggregatedResponse[idA].Metrics, 2)
})
t.Run("should have points for label1 taken from both getMetricDataOutputs", func(t *testing.T) {
assert.Len(t, aggregatedResponse[idA].Metrics["label1"].Values, 10)
})
t.Run("should have statuscode 'Complete'", func(t *testing.T) {
assert.Equal(t, "Complete", aggregatedResponse[idA].StatusCode)
})
t.Run("should have exceeded request limit", func(t *testing.T) {
assert.True(t, aggregatedResponse[idA].RequestExceededMaxLimit)
})
})
t.Run("response for id b", func(t *testing.T) {
idB := "b"
t.Run("should have statuscode is 'Partial'", func(t *testing.T) {
assert.Equal(t, "Partial", aggregatedResponse[idB].StatusCode)
})
t.Run("should have an arithmetic error and an error message", func(t *testing.T) {
assert.True(t, aggregatedResponse[idB].HasArithmeticError)
assert.Equal(t, "One or more data-points have been dropped due to non-numeric values (NaN, -Infinite, +Infinite)", aggregatedResponse[idB].ArithmeticErrorMessage)
})
})
})
t.Run("Expand dimension value using exact match", func(t *testing.T) {
timestamp := time.Unix(0, 0)
labels := []string{"lb1", "lb2"}
mdrs := map[string]*cloudwatch.MetricDataResult{
"lb1": {
Id: aws.String("id1"),
Label: aws.String("lb1"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
response := &queryRowResponse{
Labels: []string{"lb1", "lb2"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb1": {
Id: aws.String("id1"),
Label: aws.String("lb1"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
"lb2": {
Id: aws.String("id2"),
Label: aws.String("lb2"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
StatusCode: aws.String("Complete"),
},
"lb2": {
Id: aws.String("id2"),
Label: aws.String("lb2"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
}
@@ -56,15 +104,14 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb1", "lb2"},
"TargetGroup": {"tg"},
},
Stats: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
}
frames, partialData, err := parseMetricResults(mdrs, labels, query)
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
frame1 := frames[0]
assert.False(t, partialData)
assert.Equal(t, "lb1 Expanded", frame1.Name)
assert.Equal(t, "lb1", frame1.Fields[1].Labels["LoadBalancer"])
@@ -75,39 +122,40 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Expand dimension value using substring", func(t *testing.T) {
timestamp := time.Unix(0, 0)
labels := []string{"lb1 Sum", "lb2 Average"}
mdrs := map[string]*cloudwatch.MetricDataResult{
"lb1 Sum": {
Id: aws.String("id1"),
Label: aws.String("lb1 Sum"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
response := &queryRowResponse{
Labels: []string{"lb1 Sum", "lb2 Average"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb1 Sum": {
Id: aws.String("id1"),
Label: aws.String("lb1 Sum"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
"lb2 Average": {
Id: aws.String("id2"),
Label: aws.String("lb2 Average"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
StatusCode: aws.String("Complete"),
},
"lb2 Average": {
Id: aws.String("id2"),
Label: aws.String("lb2 Average"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
}
}}
query := &cloudWatchQuery{
RefId: "refId1",
@@ -118,15 +166,14 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb1", "lb2"},
"TargetGroup": {"tg"},
},
Stats: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
}
frames, partialData, err := parseMetricResults(mdrs, labels, query)
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
frame1 := frames[0]
assert.False(t, partialData)
assert.Equal(t, "lb1 Expanded", frame1.Name)
assert.Equal(t, "lb1", frame1.Fields[1].Labels["LoadBalancer"])
@@ -137,37 +184,39 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Expand dimension value using wildcard", func(t *testing.T) {
timestamp := time.Unix(0, 0)
labels := []string{"lb3", "lb4"}
mdrs := map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
response := &queryRowResponse{
Labels: []string{"lb3", "lb4"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
"lb4": {
Id: aws.String("lb4"),
Label: aws.String("lb4"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
StatusCode: aws.String("Complete"),
},
"lb4": {
Id: aws.String("lb4"),
Label: aws.String("lb4"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
}
@@ -180,35 +229,35 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"*"},
"TargetGroup": {"tg"},
},
Stats: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
}
frames, partialData, err := parseMetricResults(mdrs, labels, query)
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
assert.False(t, partialData)
assert.Equal(t, "lb3 Expanded", frames[0].Name)
assert.Equal(t, "lb4 Expanded", frames[1].Name)
})
t.Run("Expand dimension value when no values are returned and a multi-valued template variable is used", func(t *testing.T) {
timestamp := time.Unix(0, 0)
labels := []string{"lb3"}
mdrs := map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
response := &queryRowResponse{
Labels: []string{"lb3"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{},
StatusCode: aws.String("Complete"),
},
Values: []*float64{},
StatusCode: aws.String("Complete"),
},
}
query := &cloudWatchQuery{
RefId: "refId1",
Region: "us-east-1",
@@ -217,14 +266,13 @@ func TestCloudWatchResponseParser(t *testing.T) {
Dimensions: map[string][]string{
"LoadBalancer": {"lb1", "lb2"},
},
Stats: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded",
}
frames, partialData, err := parseMetricResults(mdrs, labels, query)
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
assert.False(t, partialData)
assert.Len(t, frames, 2)
assert.Equal(t, "lb1 Expanded", frames[0].Name)
assert.Equal(t, "lb2 Expanded", frames[1].Name)
@@ -232,18 +280,20 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Expand dimension value when no values are returned and a multi-valued template variable and two single-valued dimensions are used", func(t *testing.T) {
timestamp := time.Unix(0, 0)
labels := []string{"lb3"}
mdrs := map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
response := &queryRowResponse{
Labels: []string{"lb3"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb3": {
Id: aws.String("lb3"),
Label: aws.String("lb3"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{},
StatusCode: aws.String("Complete"),
},
Values: []*float64{},
StatusCode: aws.String("Complete"),
},
}
@@ -257,14 +307,13 @@ func TestCloudWatchResponseParser(t *testing.T) {
"InstanceType": {"micro"},
"Resource": {"res"},
},
Stats: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded {{InstanceType}} - {{Resource}}",
Statistic: "Average",
Period: 60,
Alias: "{{LoadBalancer}} Expanded {{InstanceType}} - {{Resource}}",
}
frames, partialData, err := parseMetricResults(mdrs, labels, query)
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
assert.False(t, partialData)
assert.Len(t, frames, 2)
assert.Equal(t, "lb1 Expanded micro - res", frames[0].Name)
assert.Equal(t, "lb2 Expanded micro - res", frames[1].Name)
@@ -272,22 +321,24 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Parse cloudwatch response", func(t *testing.T) {
timestamp := time.Unix(0, 0)
labels := []string{"lb"}
mdrs := map[string]*cloudwatch.MetricDataResult{
"lb": {
Id: aws.String("id1"),
Label: aws.String("lb"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
response := &queryRowResponse{
Labels: []string{"lb"},
Metrics: map[string]*cloudwatch.MetricDataResult{
"lb": {
Id: aws.String("id1"),
Label: aws.String("lb"),
Timestamps: []*time.Time{
aws.Time(timestamp),
aws.Time(timestamp.Add(60 * time.Second)),
aws.Time(timestamp.Add(180 * time.Second)),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
Values: []*float64{
aws.Float64(10),
aws.Float64(20),
aws.Float64(30),
},
StatusCode: aws.String("Complete"),
},
}
@@ -300,15 +351,14 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb"},
"TargetGroup": {"tg"},
},
Stats: "Average",
Period: 60,
Alias: "{{namespace}}_{{metric}}_{{stat}}",
Statistic: "Average",
Period: 60,
Alias: "{{namespace}}_{{metric}}_{{stat}}",
}
frames, partialData, err := parseMetricResults(mdrs, labels, query)
frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
frame := frames[0]
assert.False(t, partialData)
assert.Equal(t, "AWS/ApplicationELB_TargetResponseTime_Average", frame.Name)
assert.Equal(t, "Time", frame.Fields[0].Name)
assert.Equal(t, "lb", frame.Fields[1].Labels["LoadBalancer"])