public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Datasource/CloudWatch: Better handling of stats grouping (#24789)

Browse Source 
* Datasource/CloudWatch: Better handling of stats grouping
This commit is contained in:
kay delaney
2020-05-21 15:18:09 +01:00
committed by GitHub
parent 44ca05272a
commit e1f4287f70
15 changed files with 10840 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudwatch/cloudwatch.go
+30 -54
View File
@@ -4,14 +4,12 @@ import (
"context"
"fmt"
"regexp"
"strconv"
"sync"
"time"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
@@ -137,7 +135,7 @@ func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSourc
*/
queryParams := queryContext.Queries[0].Model
_, fromAlert := queryContext.Headers["FromAlert"]
isLogAlertQuery := fromAlert && queryParams.Get("mode").MustString("") == "Logs"
isLogAlertQuery := fromAlert && queryParams.Get("queryMode").MustString("") == "Logs"
if isLogAlertQuery {
return e.executeLogAlertQuery(ctx, queryContext)
@@ -192,11 +190,39 @@ func (e *CloudWatchExecutor) executeLogAlertQuery(ctx context.Context, queryCont
return nil, err
}
dataframe, err := queryResultsToDataframe(getQueryResultsOutput)
dataframe, err := logsResultsToDataframes(getQueryResultsOutput)
if err != nil {
return nil, err
}
statsGroups := queryParams.Get("statsGroups").MustStringArray()
if len(statsGroups) > 0 && len(dataframe.Fields) > 0 {
groupedFrames, err := groupResults(dataframe, statsGroups)
if err != nil {
return nil, err
}
encodedFrames := make([][]byte, 0)
for _, frame := range groupedFrames {
dataframeEnc, err := frame.MarshalArrow()
if err != nil {
return nil, err
}
encodedFrames = append(encodedFrames, dataframeEnc)
}
response := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult),
}
response.Results["A"] = &tsdb.QueryResult{
RefId: "A",
Dataframes: encodedFrames,
}
return response, nil
}
dataframeEnc, err := dataframe.MarshalArrow()
if err != nil {
return nil, err
@@ -213,56 +239,6 @@ func (e *CloudWatchExecutor) executeLogAlertQuery(ctx context.Context, queryCont
return response, nil
}
func queryResultsToDataframe(results *cloudwatchlogs.GetQueryResultsOutput) (*data.Frame, error) {
rowCount := len(results.Results)
fieldValues := make(map[string]interface{})
for i, row := range results.Results {
for _, resultField := range row {
// Strip @ptr field from results as it's not needed
if *resultField.Field == "@ptr" {
continue
}
if _, exists := fieldValues[*resultField.Field]; !exists {
if _, err := time.Parse(cloudWatchTSFormat, *resultField.Value); err == nil {
fieldValues[*resultField.Field] = make([]*time.Time, rowCount)
} else if _, err := strconv.ParseFloat(*resultField.Value, 64); err == nil {
fieldValues[*resultField.Field] = make([]*float64, rowCount)
} else {
continue
}
}
if timeField, ok := fieldValues[*resultField.Field].([]*time.Time); ok {
parsedTime, err := time.Parse(cloudWatchTSFormat, *resultField.Value)
if err != nil {
return nil, err
}
timeField[i] = &parsedTime
} else if numericField, ok := fieldValues[*resultField.Field].([]*float64); ok {
parsedFloat, err := strconv.ParseFloat(*resultField.Value, 64)
if err != nil {
return nil, err
}
numericField[i] = &parsedFloat
}
}
}
newFields := make([]*data.Field, 0)
for fieldName, vals := range fieldValues {
newFields = append(newFields, data.NewField(fieldName, nil, vals))
if fieldName == "@timestamp" {
newFields[len(newFields)-1].SetConfig(&data.FieldConfig{Title: "Time"})
}
}
frame := data.NewFrame("CloudWatchLogsResponse", newFields...)
return frame, nil
}
func isTerminated(queryStatus string) bool {
return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout"
}
pkg/tsdb/cloudwatch/log_actions.go
+4 -23
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"sort"
"strconv"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -33,12 +32,11 @@ func (e *CloudWatchExecutor) executeLogActions(ctx context.Context, queryContext
// the query response is in, there does not seem to be a way to tell
// by the response alone if/how the results should be grouped.
// Because of this, if the frontend sees that a "stats ... by ..." query is being made
// the "groupResults" parameter is sent along with the query to the backend so that we
// the "statsGroups" parameter is sent along with the query to the backend so that we
// can correctly group the CloudWatch logs response.
if query.Model.Get("groupResults").MustBool() && len(dataframe.Fields) > 0 {
groupingFields := findGroupingFields(dataframe.Fields)
groupedFrames, err := groupResults(dataframe, groupingFields)
statsGroups := query.Model.Get("statsGroups").MustStringArray()
if len(statsGroups) > 0 && len(dataframe.Fields) > 0 {
groupedFrames, err := groupResults(dataframe, statsGroups)
if err != nil {
return err
}
@@ -81,23 +79,6 @@ func (e *CloudWatchExecutor) executeLogActions(ctx context.Context, queryContext
return response, nil
}
func findGroupingFields(fields []*data.Field) []string {
groupingFields := make([]string, 0)
for _, field := range fields {
if field.Type().Numeric() || field.Type() == data.FieldTypeNullableTime || field.Type() == data.FieldTypeTime {
continue
}
if _, err := strconv.ParseFloat(*field.At(0).(*string), 64); err == nil {
continue
}
groupingFields = append(groupingFields, field.Name)
}
return groupingFields
}
func (e *CloudWatchExecutor) executeLogAction(ctx context.Context, queryContext *tsdb.TsdbQuery, query *tsdb.Query) (*data.Frame, error) {
parameters := query.Model
subType := query.Model.Get("subtype").MustString()
pkg/tsdb/cloudwatch/log_query.go
+12
View File
@@ -1,6 +1,8 @@
package cloudwatch
import (
"sort"
"strconv"
"time"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
@@ -47,6 +49,8 @@ func logsResultsToDataframes(response *cloudwatchlogs.GetQueryResultsOutput) (*d
// Check if field is time field
if _, err := time.Parse(cloudWatchTSFormat, *resultField.Value); err == nil {
fieldValues[*resultField.Field] = make([]*time.Time, rowCount)
} else if _, err := strconv.ParseFloat(*resultField.Value, 64); err == nil {
fieldValues[*resultField.Field] = make([]*float64, rowCount)
} else {
fieldValues[*resultField.Field] = make([]*string, rowCount)
}
@@ -59,6 +63,12 @@ func logsResultsToDataframes(response *cloudwatchlogs.GetQueryResultsOutput) (*d
}
timeField[i] = &parsedTime
} else if numericField, ok := fieldValues[*resultField.Field].([]*float64); ok {
parsedFloat, err := strconv.ParseFloat(*resultField.Value, 64)
if err != nil {
return nil, err
}
numericField[i] = &parsedFloat
} else {
fieldValues[*resultField.Field].([]*string)[i] = resultField.Value
}
@@ -90,6 +100,8 @@ func logsResultsToDataframes(response *cloudwatchlogs.GetQueryResultsOutput) (*d
},
}
// Results aren't guaranteed to come ordered by time (ascending), so we need to sort
sort.Sort(ByTime(*frame))
return frame, nil
}
pkg/tsdb/cloudwatch/sort_frame.go
+41
View File
@@ -0,0 +1,41 @@
package cloudwatch
import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
)
// ByTime implements sort.Interface for data.Frame based on the frame's time field
type ByTime data.Frame
func (a ByTime) Len() int {
if len(a.Fields) > 0 {
return a.Fields[0].Len()
}
return 0
}
func (a ByTime) Swap(i, j int) {
for _, field := range a.Fields {
temp := field.At(i)
field.Set(i, field.At(j))
field.Set(j, temp)
}
}
func (a ByTime) Less(i, j int) bool {
var timeField *data.Field = nil
for _, field := range a.Fields {
if field.Type() == data.FieldTypeNullableTime {
timeField = field
break
}
}
if timeField == nil {
return false
}
return (timeField.At(i).(*time.Time)).Before(*timeField.At(j).(*time.Time))
}
pkg/tsdb/cloudwatch/sort_frame_test.go
+57
View File
@@ -0,0 +1,57 @@
package cloudwatch
import (
"sort"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/assert"
)
func TestFrameSort(t *testing.T) {
timeA, _ := time.Parse("2006-01-02 15:04:05.000", "2020-03-02 17:04:05.000")
timeB, _ := time.Parse("2006-01-02 15:04:05.000", "2020-03-02 16:04:05.000")
timeC, _ := time.Parse("2006-01-02 15:04:05.000", "2020-03-02 15:04:05.000")
timeVals := []*time.Time{
&timeA, &timeB, &timeC,
}
timeField := data.NewField("@timestamp", nil, timeVals)
stringField := data.NewField("line", nil, []*string{
aws.String("test message 1"),
aws.String("test message 2"),
aws.String("test message 3"),
})
numberField := data.NewField("nums", nil, []*float64{
aws.Float64(20.0),
aws.Float64(50.0),
aws.Float64(17.0),
})
expectedDataframe := &data.Frame{
Name: "CloudWatchLogsResponse",
Fields: []*data.Field{
timeField,
stringField,
numberField,
},
}
sort.Sort(ByTime(*expectedDataframe))
for i := 1; i < timeField.Len(); i++ {
assert.True(t, timeField.At(i).(*time.Time).After(*(timeField.At(i - 1).(*time.Time))))
}
assert.Equal(t, *stringField.At(0).(*string), "test message 3")
assert.Equal(t, *stringField.At(1).(*string), "test message 2")
assert.Equal(t, *stringField.At(2).(*string), "test message 1")
assert.Equal(t, *numberField.At(0).(*float64), 17.0)
assert.Equal(t, *numberField.At(1).(*float64), 50.0)
assert.Equal(t, *numberField.At(2).(*float64), 20.0)
}