From 464e0cf540e13ab84d5a5f9f32c9a505c3f3fb65 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 2 Oct 2018 19:42:30 +0900 Subject: [PATCH 01/21] stackdriver heatmap support (cherry picked from commit 6770f2e9401547863048f58d88a52e3e9dbcb975) --- pkg/tsdb/stackdriver/stackdriver.go | 126 +++++++++++++----- pkg/tsdb/stackdriver/types.go | 40 +++++- .../datasource/stackdriver/constants.ts | 2 +- 3 files changed, 133 insertions(+), 35 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index 586e154cd5d..ebf468be877 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -300,29 +300,6 @@ func (e *StackdriverExecutor) parseResponse(queryRes *tsdb.QueryResult, data Sta for _, series := range data.TimeSeries { points := make([]tsdb.TimePoint, 0) - // reverse the order to be ascending - for i := len(series.Points) - 1; i >= 0; i-- { - point := series.Points[i] - value := point.Value.DoubleValue - - if series.ValueType == "INT64" { - parsedValue, err := strconv.ParseFloat(point.Value.IntValue, 64) - if err == nil { - value = parsedValue - } - } - - if series.ValueType == "BOOL" { - if point.Value.BoolValue { - value = 1 - } else { - value = 0 - } - } - - points = append(points, tsdb.NewTimePoint(null.FloatFrom(value), float64((point.Interval.EndTime).Unix())*1000)) - } - defaultMetricName := series.Metric.Type for key, value := range series.Metric.Labels { @@ -338,18 +315,87 @@ func (e *StackdriverExecutor) parseResponse(queryRes *tsdb.QueryResult, data Sta if !containsLabel(resourceLabels[key], value) { resourceLabels[key] = append(resourceLabels[key], value) } - if containsLabel(query.GroupBys, "resource.label."+key) { defaultMetricName += " " + value } } - metricName := formatLegendKeys(series.Metric.Type, defaultMetricName, series.Metric.Labels, series.Resource.Labels, query) + // reverse the order to be ascending + if series.ValueType != "DISTRIBUTION" { + for i := len(series.Points) - 1; i >= 0; i-- { + point := series.Points[i] + value := point.Value.DoubleValue - queryRes.Series = append(queryRes.Series, &tsdb.TimeSeries{ - Name: metricName, - Points: points, - }) + if series.ValueType == "INT64" { + parsedValue, err := strconv.ParseFloat(point.Value.IntValue, 64) + if err == nil { + value = parsedValue + } + } + + if series.ValueType == "BOOL" { + if point.Value.BoolValue { + value = 1 + } else { + value = 0 + } + } + + points = append(points, tsdb.NewTimePoint(null.FloatFrom(value), float64((point.Interval.EndTime).Unix())*1000)) + } + + metricName := formatLegendKeys(series.Metric.Type, defaultMetricName, series.Metric.Labels, series.Resource.Labels, make(map[string]string), query) + + queryRes.Series = append(queryRes.Series, &tsdb.TimeSeries{ + Name: metricName, + Points: points, + }) + } else { + buckets := make(map[int]*tsdb.TimeSeries) + + for i := len(series.Points) - 1; i >= 0; i-- { + point := series.Points[i] + if len(point.Value.DistributionValue.BucketCounts) == 0 { + continue + } + maxKey := 0 + for i := 0; i < len(point.Value.DistributionValue.BucketCounts); i++ { + value, err := strconv.ParseFloat(point.Value.DistributionValue.BucketCounts[i], 64) + if err != nil { + continue + } + if _, ok := buckets[i]; !ok { + // set lower bounds + // https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries#Distribution + bucketBound := calcBucketBound(point.Value.DistributionValue.BucketOptions, i) + additionalLabels := map[string]string{"bucket": bucketBound} + buckets[i] = &tsdb.TimeSeries{ + Name: formatLegendKeys(series.Metric.Type, defaultMetricName, series.Metric.Labels, series.Resource.Labels, additionalLabels, query), + Points: make([]tsdb.TimePoint, 0), + } + if maxKey < i { + maxKey = i + } + } + buckets[i].Points = append(buckets[i].Points, tsdb.NewTimePoint(null.FloatFrom(value), float64((point.Interval.EndTime).Unix())*1000)) + } + + // fill empty bucket + for i := 0; i < maxKey; i++ { + if _, ok := buckets[i]; !ok { + bucketBound := calcBucketBound(point.Value.DistributionValue.BucketOptions, i) + additionalLabels := map[string]string{"bucket": bucketBound} + buckets[i] = &tsdb.TimeSeries{ + Name: formatLegendKeys(series.Metric.Type, defaultMetricName, series.Metric.Labels, series.Resource.Labels, additionalLabels, query), + Points: make([]tsdb.TimePoint, 0), + } + } + } + } + for i := 0; i < len(buckets); i++ { + queryRes.Series = append(queryRes.Series, buckets[i]) + } + } } queryRes.Meta.Set("resourceLabels", resourceLabels) @@ -368,7 +414,7 @@ func containsLabel(labels []string, newLabel string) bool { return false } -func formatLegendKeys(metricType string, defaultMetricName string, metricLabels map[string]string, resourceLabels map[string]string, query *StackdriverQuery) string { +func formatLegendKeys(metricType string, defaultMetricName string, metricLabels map[string]string, resourceLabels map[string]string, additionalLabels map[string]string, query *StackdriverQuery) string { if query.AliasBy == "" { return defaultMetricName } @@ -400,6 +446,10 @@ func formatLegendKeys(metricType string, defaultMetricName string, metricLabels return []byte(val) } + if val, exists := additionalLabels[metaPartName]; exists { + return []byte(val) + } + return in }) @@ -425,6 +475,22 @@ func replaceWithMetricPart(metaPartName string, metricType string) []byte { return nil } +func calcBucketBound(bucketOptions StackdriverBucketOptions, n int) string { + bucketBound := "0" + if n == 0 { + return bucketBound + } + + if bucketOptions.LinearBuckets != nil { + bucketBound = strconv.FormatInt(bucketOptions.LinearBuckets.Offset+(bucketOptions.LinearBuckets.Width*int64(n-1)), 10) + } else if bucketOptions.ExponentialBuckets != nil { + bucketBound = strconv.FormatInt(int64(bucketOptions.ExponentialBuckets.Scale*math.Pow(bucketOptions.ExponentialBuckets.GrowthFactor, float64(n-1))), 10) + } else if bucketOptions.ExplicitBuckets != nil { + bucketBound = strconv.FormatInt(bucketOptions.ExplicitBuckets.Bounds[(n-1)], 10) + } + return bucketBound +} + func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { u, _ := url.Parse(dsInfo.Url) u.Path = path.Join(u.Path, "render") diff --git a/pkg/tsdb/stackdriver/types.go b/pkg/tsdb/stackdriver/types.go index c58ac2968f2..3821ce7ceda 100644 --- a/pkg/tsdb/stackdriver/types.go +++ b/pkg/tsdb/stackdriver/types.go @@ -14,6 +14,22 @@ type StackdriverQuery struct { AliasBy string } +type StackdriverBucketOptions struct { + LinearBuckets *struct { + NumFiniteBuckets int64 `json:"numFiniteBuckets"` + Width int64 `json:"width"` + Offset int64 `json:"offset"` + } `json:"linearBuckets"` + ExponentialBuckets *struct { + NumFiniteBuckets int64 `json:"numFiniteBuckets"` + GrowthFactor float64 `json:"growthFactor"` + Scale float64 `json:"scale"` + } `json:"exponentialBuckets"` + ExplicitBuckets *struct { + Bounds []int64 `json:"bounds"` + } `json:"explicitBuckets"` +} + // StackdriverResponse is the data returned from the external Google Stackdriver API type StackdriverResponse struct { TimeSeries []struct { @@ -33,10 +49,26 @@ type StackdriverResponse struct { EndTime time.Time `json:"endTime"` } `json:"interval"` Value struct { - DoubleValue float64 `json:"doubleValue"` - StringValue string `json:"stringValue"` - BoolValue bool `json:"boolValue"` - IntValue string `json:"int64Value"` + DoubleValue float64 `json:"doubleValue"` + StringValue string `json:"stringValue"` + BoolValue bool `json:"boolValue"` + IntValue string `json:"int64Value"` + DistributionValue struct { + Count string `json:"count"` + Mean float64 `json:"mean"` + SumOfSquaredDeviation float64 `json:"sumOfSquaredDeviation"` + Range struct { + Min int `json:"min"` + Max int `json:"max"` + } `json:"range"` + BucketOptions StackdriverBucketOptions `json:"bucketOptions"` + BucketCounts []string `json:"bucketCounts"` + Examplars []struct { + Value float64 `json:"value"` + Timestamp string `json:"timestamp"` + // attachments + } `json:"examplars"` + } `json:"distributionValue"` } `json:"value"` } `json:"points"` } `json:"timeSeries"` diff --git a/public/app/plugins/datasource/stackdriver/constants.ts b/public/app/plugins/datasource/stackdriver/constants.ts index 628e480c3db..b11f4a1bcb1 100644 --- a/public/app/plugins/datasource/stackdriver/constants.ts +++ b/public/app/plugins/datasource/stackdriver/constants.ts @@ -19,7 +19,7 @@ export const alignOptions = [ { text: 'delta', value: 'ALIGN_DELTA', - valueTypes: [ValueTypes.INT64, ValueTypes.DOUBLE, ValueTypes.MONEY], + valueTypes: [ValueTypes.INT64, ValueTypes.DOUBLE, ValueTypes.MONEY, ValueTypes.DISTRIBUTION], metricKinds: [MetricKind.CUMULATIVE, MetricKind.DELTA], }, { From 221341b3e81124a30fc617001d67a9635e0f4a4b Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 2 Oct 2018 22:45:54 +0900 Subject: [PATCH 02/21] add test (cherry picked from commit c2c0cdb49c92126c39150b5aaaef766eb404511f) --- pkg/tsdb/stackdriver/stackdriver_test.go | 42 +++++++ .../3-series-response-distribution.json | 112 ++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go index da4d6890207..1e8e9cea025 100644 --- a/pkg/tsdb/stackdriver/stackdriver_test.go +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math" + "strconv" "testing" "time" @@ -341,6 +343,46 @@ func TestStackdriver(t *testing.T) { }) }) }) + + Convey("when data from query is distribution", func() { + data, err := loadTestFile("./test-data/3-series-response-distribution.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 1) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"} + query := &StackdriverQuery{AliasBy: "{{bucket}}"} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + So(len(res.Series), ShouldEqual, 11) + for i := 0; i < 11; i++ { + if i == 0 { + So(res.Series[i].Name, ShouldEqual, "0") + } else { + So(res.Series[i].Name, ShouldEqual, strconv.FormatInt(int64(math.Pow(float64(2), float64(i-1))), 10)) + } + So(len(res.Series[i].Points), ShouldEqual, 3) + } + + Convey("timestamps should be in ascending order", func() { + So(res.Series[0].Points[0][1].Float64, ShouldEqual, 1536668940000) + So(res.Series[0].Points[1][1].Float64, ShouldEqual, 1536669000000) + So(res.Series[0].Points[2][1].Float64, ShouldEqual, 1536669060000) + }) + + Convey("value should be correct", func() { + So(res.Series[8].Points[0][0].Float64, ShouldEqual, 1) + So(res.Series[9].Points[0][0].Float64, ShouldEqual, 1) + So(res.Series[10].Points[0][0].Float64, ShouldEqual, 1) + So(res.Series[8].Points[1][0].Float64, ShouldEqual, 0) + So(res.Series[9].Points[1][0].Float64, ShouldEqual, 0) + So(res.Series[10].Points[1][0].Float64, ShouldEqual, 1) + So(res.Series[8].Points[2][0].Float64, ShouldEqual, 0) + So(res.Series[9].Points[2][0].Float64, ShouldEqual, 1) + So(res.Series[10].Points[2][0].Float64, ShouldEqual, 0) + }) + }) + }) }) } diff --git a/pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json b/pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json new file mode 100644 index 00000000000..8603f78eab4 --- /dev/null +++ b/pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json @@ -0,0 +1,112 @@ +{ + "timeSeries": [ + { + "metric": { + "type": "loadbalancing.googleapis.com\/https\/backend_latencies" + }, + "resource": { + "type": "https_lb_rule", + "labels": { + "project_id": "grafana-prod" + } + }, + "metricKind": "DELTA", + "valueType": "DISTRIBUTION", + "points": [ + { + "interval": { + "startTime": "2018-09-11T12:30:00Z", + "endTime": "2018-09-11T12:31:00Z" + }, + "value": { + "distributionValue": { + "count": "1", + "bucketOptions": { + "exponentialBuckets": { + "numFiniteBuckets": 10, + "growthFactor": 2, + "scale": 1 + } + }, + "bucketCounts": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0" + ] + } + } + }, + { + "interval": { + "startTime": "2018-09-11T12:29:00Z", + "endTime": "2018-09-11T12:30:00Z" + }, + "value": { + "distributionValue": { + "count": "1", + "bucketOptions": { + "exponentialBuckets": { + "numFiniteBuckets": 10, + "growthFactor": 2, + "scale": 1 + } + }, + "bucketCounts": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1" + ] + } + } + }, + { + "interval": { + "startTime": "2018-09-11T12:28:00Z", + "endTime": "2018-09-11T12:29:00Z" + }, + "value": { + "distributionValue": { + "count": "3", + "bucketOptions": { + "exponentialBuckets": { + "numFiniteBuckets": 10, + "growthFactor": 2, + "scale": 1 + } + }, + "bucketCounts": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "1", + "1" + ] + } + } + } + ] + } + ] +} From 56c32963d6820cd5c96dcfc2eac00c5c40639744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Sat, 6 Oct 2018 12:09:30 -0700 Subject: [PATCH 03/21] ux: minor update to look of stackdriver query help (cherry picked from commit 3fa83d2755ec44ad48cccd9c2bfc91d2280f3dd8) --- public/app/features/teams/TeamList.tsx | 2 +- .../__snapshots__/TeamList.test.tsx.snap | 5 +-- .../stackdriver/partials/query.editor.html | 36 ++++++++++++------- public/sass/_variables.dark.scss | 4 +-- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/public/app/features/teams/TeamList.tsx b/public/app/features/teams/TeamList.tsx index 985d73d9a52..7b153746f9f 100644 --- a/public/app/features/teams/TeamList.tsx +++ b/public/app/features/teams/TeamList.tsx @@ -103,7 +103,7 @@ export class TeamList extends PureComponent {
- New team + New team
diff --git a/public/app/features/teams/__snapshots__/TeamList.test.tsx.snap b/public/app/features/teams/__snapshots__/TeamList.test.tsx.snap index 45d0f78126e..7cf5951dba3 100644 --- a/public/app/features/teams/__snapshots__/TeamList.test.tsx.snap +++ b/public/app/features/teams/__snapshots__/TeamList.test.tsx.snap @@ -62,10 +62,7 @@ exports[`Render should render teams table 1`] = ` className="btn btn-success" href="org/teams/new" > - - New team + New team 
{{ctrl.lastQueryMeta.rawQueryString}}
-
-
Alias Patterns
Format the legend keys any way you want by using alias patterns.
+  

+    
Alias Patterns

 
-{{metric.name}} - {{metric.label.instance_name}}
+    Format the legend keys any way you want by using alias patterns.
 

 
-cpu/usage_time - server1-europe-west-1
+    Example: {{metric.name}} - {{metric.label.instance_name}}

+    Result:   cpu/usage_time - server1-europe-west-1


 
-
-{{metric.type}} = metric type e.g. compute.googleapis.com/instance/cpu/usage_time
-{{metric.name}} = name part of metric e.g. instance/cpu/usage_time
-{{metric.service}} = service part of metric e.g. compute
-
-{{metric.label.label_name}} = Metric label metadata e.g. metric.label.instance_name
-{{resource.label.label_name}} = Resource label metadata e.g. resource.label.zone
-
+ Patterns
+
    +
  • + {{metric.type}} = metric type e.g. compute.googleapis.com/instance/cpu/usage_time +
    • +
  • + {{metric.name}} = name part of metric e.g. instance/cpu/usage_time +
    • +
  • + {{metric.service}} = service part of metric e.g. compute +
    • +
  • + {{metric.label.label_name}} = Metric label metadata e.g. + metric.label.instance_name +
    • +
  • + {{resource.label.label_name}} = Resource label metadata e.g. resource.label.zone +
    • + 
{{ctrl.lastQueryError}}
diff --git a/public/sass/_variables.dark.scss b/public/sass/_variables.dark.scss index 01590ace585..ae52fbc5ab5 100644 --- a/public/sass/_variables.dark.scss +++ b/public/sass/_variables.dark.scss @@ -115,8 +115,8 @@ $tight-form-func-bg: #333334; $tight-form-func-highlight-bg: #444445; $modal-backdrop-bg: #353c42; -$code-tag-bg: $gray-1; -$code-tag-border: lighten($code-tag-bg, 2%); +$code-tag-bg: $dark-1; +$code-tag-border: $dark-4; // cards $card-background: linear-gradient(135deg, #2f2f32, #262628); From a4e148e300e29df4313afd43cde59e648f50a644 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 17:07:46 +0200 Subject: [PATCH 04/21] stackdriver: interpolate stackdriver filter wildcards when asterix is used in filter (cherry picked from commit 4d8f594d31ea3fa6d82f6fb249abf16059f8decd) --- pkg/tsdb/stackdriver/stackdriver.go | 56 ++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index ebf468be877..2885c07505a 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -15,6 +15,8 @@ import ( "strings" "time" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "golang.org/x/net/context/ctxhttp" "github.com/grafana/grafana/pkg/api/pluginproxy" @@ -159,6 +161,53 @@ func (e *StackdriverExecutor) buildQueries(tsdbQuery *tsdb.TsdbQuery) ([]*Stackd return stackdriverQueries, nil } +func reverse(s string) string { + chars := []rune(s) + for i, j := 0, len(chars)-1; i < j; i, j = i+1, j-1 { + chars[i], chars[j] = chars[j], chars[i] + } + return string(chars) +} + +func escapeDoubleBackslash(target string) string { + var re = regexp.MustCompile(`\\`) + return re.ReplaceAllString(target, `\\\\`) + // return strings.Replace(target, `\`, "", -1) +} + +func escapeIllegalCharacters(target string) string { + var re = regexp.MustCompile(`[-\/^$+?.()|[\]{}]`) + return string(re.ReplaceAllFunc([]byte(target), func(in []byte) []byte { + return []byte(strings.Replace(string(in), string(in), `\\`+string(in), 1)) + })) +} + +func replaceSingleAsterixCharacters(target string) string { + return strings.Replace(target, "*", ".*", -1) +} + +func interpolateFilterWildcards(value string) string { + if strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { + value = strings.Replace(value, "*", "", 1) + value = fmt.Sprintf(`has_substring("%s")`, value) + } else if strings.HasPrefix(value, "*") { + value = strings.Replace(value, "*", "", 1) + value = fmt.Sprintf(`ends_with("%s")`, value) + } else if strings.HasSuffix(value, "*") { + value = reverse(strings.Replace(reverse(value), "*", "", 1)) + value = fmt.Sprintf(`starts_with("%s")`, value) + } else if strings.Contains(value, "*") { + value = escapeIllegalCharacters(value) + value = replaceSingleAsterixCharacters(value) + value = strings.Replace(value, `"`, `\\"`, -1) + value = fmt.Sprintf(`monitoring.regex.full_match("^%s$")`, value) + } + + logger.Info("filter", "filter", value) + + return value +} + func buildFilterString(metricType string, filterParts []interface{}) string { filterString := "" for i, part := range filterParts { @@ -166,7 +215,11 @@ func buildFilterString(metricType string, filterParts []interface{}) string { if part == "AND" { filterString += " " } else if mod == 2 { - filterString += fmt.Sprintf(`"%s"`, part) + if strings.Contains(part.(string), "*") { + filterString += interpolateFilterWildcards(part.(string)) + } else { + filterString += fmt.Sprintf(`"%s"`, part) + } } else { filterString += part.(string) } @@ -231,6 +284,7 @@ func (e *StackdriverExecutor) executeQuery(ctx context.Context, query *Stackdriv } req.URL.RawQuery = query.Params.Encode() + logger.Info("req.URL.RawQuery", "req.URL.RawQuery", req.URL.RawQuery) queryResult.Meta.Set("rawQuery", req.URL.RawQuery) alignmentPeriod, ok := req.URL.Query()["aggregation.alignmentPeriod"] From 897cf51e7597c8eeed0a91c5d20822f515c5846e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 17:11:05 +0200 Subject: [PATCH 05/21] stackdriver: remove not necessary helper functions (cherry picked from commit 2e665fba0f6c8a9b83f58e10922a9538d1ede966) --- pkg/tsdb/stackdriver/stackdriver.go | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index 2885c07505a..8e575db9063 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -169,23 +169,6 @@ func reverse(s string) string { return string(chars) } -func escapeDoubleBackslash(target string) string { - var re = regexp.MustCompile(`\\`) - return re.ReplaceAllString(target, `\\\\`) - // return strings.Replace(target, `\`, "", -1) -} - -func escapeIllegalCharacters(target string) string { - var re = regexp.MustCompile(`[-\/^$+?.()|[\]{}]`) - return string(re.ReplaceAllFunc([]byte(target), func(in []byte) []byte { - return []byte(strings.Replace(string(in), string(in), `\\`+string(in), 1)) - })) -} - -func replaceSingleAsterixCharacters(target string) string { - return strings.Replace(target, "*", ".*", -1) -} - func interpolateFilterWildcards(value string) string { if strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { value = strings.Replace(value, "*", "", 1) @@ -197,8 +180,11 @@ func interpolateFilterWildcards(value string) string { value = reverse(strings.Replace(reverse(value), "*", "", 1)) value = fmt.Sprintf(`starts_with("%s")`, value) } else if strings.Contains(value, "*") { - value = escapeIllegalCharacters(value) - value = replaceSingleAsterixCharacters(value) + re := regexp.MustCompile(`[-\/^$+?.()|[\]{}]`) + value = string(re.ReplaceAllFunc([]byte(value), func(in []byte) []byte { + return []byte(strings.Replace(string(in), string(in), `\\`+string(in), 1)) + })) + value = strings.Replace(value, "*", ".*", -1) value = strings.Replace(value, `"`, `\\"`, -1) value = fmt.Sprintf(`monitoring.regex.full_match("^%s$")`, value) } From dee26f3d2fab5c60f4ffb4ddf671f3af9de31391 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 17:29:51 +0200 Subject: [PATCH 06/21] stackdriver: fix broken substring. also adds tests (cherry picked from commit 68332c595171a1ba83dc9193411d2ac0d3c69490) --- pkg/tsdb/stackdriver/stackdriver.go | 7 +++++-- pkg/tsdb/stackdriver/stackdriver_test.go | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index 8e575db9063..e4c5134a97d 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -170,8 +170,11 @@ func reverse(s string) string { } func interpolateFilterWildcards(value string) string { - if strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { - value = strings.Replace(value, "*", "", 1) + re := regexp.MustCompile("[*]") + matches := re.FindAllStringIndex(value, -1) + logger.Info("len", "len", len(matches)) + if len(matches) == 2 && strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { + value = strings.Replace(value, "*", "", -1) value = fmt.Sprintf(`has_substring("%s")`, value) } else if strings.HasPrefix(value, "*") { value = strings.Replace(value, "*", "", 1) diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go index 1e8e9cea025..8841493d13f 100644 --- a/pkg/tsdb/stackdriver/stackdriver_test.go +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -384,6 +384,19 @@ func TestStackdriver(t *testing.T) { }) }) + + Convey("when interpolating filter wildcards", func() { + Convey("and wildcard is used in the beginning and the end of the word", func() { + Convey("and theres no wildcard in the middle of the word", func() { + value := interpolateFilterWildcards("*-central1*") + So(value, ShouldEqual, `has_substring("-central1")`) + }) + Convey("and there is a wildcard in the middle of the word", func() { + value := interpolateFilterWildcards("*-cent*ral1*") + So(value, ShouldNotStartWith, `has_substring`) + }) + }) + }) }) } From 0ef06d467acf5f4075303d74e3c94ad22095d68b Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 17:52:26 +0200 Subject: [PATCH 07/21] stackdriver: add more tests (cherry picked from commit 035be6cbbe5354aa4f0c2b0db2f09b228e2effe7) --- pkg/tsdb/stackdriver/stackdriver.go | 12 +++---- pkg/tsdb/stackdriver/stackdriver_test.go | 44 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index e4c5134a97d..0eac85afde5 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -171,18 +171,18 @@ func reverse(s string) string { func interpolateFilterWildcards(value string) string { re := regexp.MustCompile("[*]") - matches := re.FindAllStringIndex(value, -1) - logger.Info("len", "len", len(matches)) - if len(matches) == 2 && strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { + matches := len(re.FindAllStringIndex(value, -1)) + logger.Info("len", "len", matches) + if matches == 2 && strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { value = strings.Replace(value, "*", "", -1) value = fmt.Sprintf(`has_substring("%s")`, value) - } else if strings.HasPrefix(value, "*") { + } else if matches == 1 && strings.HasPrefix(value, "*") { value = strings.Replace(value, "*", "", 1) value = fmt.Sprintf(`ends_with("%s")`, value) - } else if strings.HasSuffix(value, "*") { + } else if matches == 1 && strings.HasSuffix(value, "*") { value = reverse(strings.Replace(reverse(value), "*", "", 1)) value = fmt.Sprintf(`starts_with("%s")`, value) - } else if strings.Contains(value, "*") { + } else if matches == 1 { re := regexp.MustCompile(`[-\/^$+?.()|[\]{}]`) value = string(re.ReplaceAllFunc([]byte(value), func(in []byte) []byte { return []byte(strings.Replace(string(in), string(in), `\\`+string(in), 1)) diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go index 8841493d13f..fdf8afe5c89 100644 --- a/pkg/tsdb/stackdriver/stackdriver_test.go +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -396,6 +396,50 @@ func TestStackdriver(t *testing.T) { So(value, ShouldNotStartWith, `has_substring`) }) }) + + Convey("and wildcard is used in the beginning of the word", func() { + Convey("and there is not a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("*-central1") + So(value, ShouldEqual, `ends_with("-central1")`) + }) + Convey("and there is a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("*-cent*al1") + So(value, ShouldNotStartWith, `ends_with`) + }) + }) + + Convey("and wildcard is used at the end of the word", func() { + Convey("and there is not a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("us-central*") + So(value, ShouldEqual, `starts_with("us-central")`) + }) + Convey("and there is a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("*us-central*") + So(value, ShouldNotStartWith, `starts_with`) + }) + }) + + Convey("and wildcard is used in the middle of the word", func() { + Convey("and there is only one wildcard", func() { + value := interpolateFilterWildcards("us-ce*tral1-b") + So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tral1\\-b$")`) + }) + + Convey("and there is more than one wildcard", func() { + value := interpolateFilterWildcards("us-ce*tra*1-b") + So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tra.*1\\-b$")`) + }) + }) + + Convey("and wildcard is used in the middle of the word and in the beginning of the word", func() { + value := interpolateFilterWildcards("*s-ce*tral1-b") + So(value, ShouldEqual, `monitoring.regex.full_match("^.*s\\-ce.*tral1\\-b$")`) + }) + + Convey("and wildcard is used in the middle of the word and in the ending of the word", func() { + value := interpolateFilterWildcards("us-ce*tral1-*") + So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tral1\\-.*$")`) + }) }) }) } From 84094b505195819a056cbe07da14e68ffb420ec3 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 17:53:19 +0200 Subject: [PATCH 08/21] stackdriver: remove debug logging (cherry picked from commit 2a0d7a88039224627acee3291b70dbc5b1bd814c) --- pkg/tsdb/stackdriver/stackdriver.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index 0eac85afde5..e3d914b05e8 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -15,8 +15,6 @@ import ( "strings" "time" - "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" - "golang.org/x/net/context/ctxhttp" "github.com/grafana/grafana/pkg/api/pluginproxy" @@ -172,7 +170,6 @@ func reverse(s string) string { func interpolateFilterWildcards(value string) string { re := regexp.MustCompile("[*]") matches := len(re.FindAllStringIndex(value, -1)) - logger.Info("len", "len", matches) if matches == 2 && strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { value = strings.Replace(value, "*", "", -1) value = fmt.Sprintf(`has_substring("%s")`, value) @@ -192,8 +189,6 @@ func interpolateFilterWildcards(value string) string { value = fmt.Sprintf(`monitoring.regex.full_match("^%s$")`, value) } - logger.Info("filter", "filter", value) - return value } @@ -273,7 +268,6 @@ func (e *StackdriverExecutor) executeQuery(ctx context.Context, query *Stackdriv } req.URL.RawQuery = query.Params.Encode() - logger.Info("req.URL.RawQuery", "req.URL.RawQuery", req.URL.RawQuery) queryResult.Meta.Set("rawQuery", req.URL.RawQuery) alignmentPeriod, ok := req.URL.Query()["aggregation.alignmentPeriod"] From 93fb427310b00a5fff25b3e20294cc0d4dca8517 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 17:58:31 +0200 Subject: [PATCH 09/21] stackdriver: test that no interpolation is done when there are no wildcards (cherry picked from commit 5f7795aa1f525e34f5aba659175827887ede3a91) --- pkg/tsdb/stackdriver/stackdriver.go | 2 +- pkg/tsdb/stackdriver/stackdriver_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index e3d914b05e8..f38c45ebfe0 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -179,7 +179,7 @@ func interpolateFilterWildcards(value string) string { } else if matches == 1 && strings.HasSuffix(value, "*") { value = reverse(strings.Replace(reverse(value), "*", "", 1)) value = fmt.Sprintf(`starts_with("%s")`, value) - } else if matches == 1 { + } else if matches != 0 { re := regexp.MustCompile(`[-\/^$+?.()|[\]{}]`) value = string(re.ReplaceAllFunc([]byte(value), func(in []byte) []byte { return []byte(strings.Replace(string(in), string(in), `\\`+string(in), 1)) diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go index fdf8afe5c89..2a862c7c118 100644 --- a/pkg/tsdb/stackdriver/stackdriver_test.go +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -440,7 +440,13 @@ func TestStackdriver(t *testing.T) { value := interpolateFilterWildcards("us-ce*tral1-*") So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tral1\\-.*$")`) }) + + Convey("and no wildcard is used", func() { + value := interpolateFilterWildcards("us-central1-a}") + So(value, ShouldEqual, `us-central1-a}`) + }) }) + }) } From 322535a2b72a8c353c611c6e2f53204f9567aa3e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 2 Oct 2018 18:09:42 +0200 Subject: [PATCH 10/21] stackdriver: test build filter string (cherry picked from commit a3122a4b854672f210892f6f158f7a074dd1d8f5) --- pkg/tsdb/stackdriver/stackdriver_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go index 2a862c7c118..4685362aedc 100644 --- a/pkg/tsdb/stackdriver/stackdriver_test.go +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -447,6 +447,19 @@ func TestStackdriver(t *testing.T) { }) }) + Convey("when building filter string", func() { + Convey("and there are wildcards in a filter value", func() { + filterParts := []interface{}{"zone", "=", "*-central1*"} + value := buildFilterString("somemetrictype", filterParts) + So(value, ShouldEqual, `metric.type="somemetrictype" zone=has_substring("-central1")`) + }) + + Convey("and there are no wildcards in any filter value", func() { + filterParts := []interface{}{"zone", "=", "us-central1-a"} + value := buildFilterString("somemetrictype", filterParts) + So(value, ShouldEqual, `metric.type="somemetrictype" zone="us-central1-a"`) + }) + }) }) } From a109c53ceaf4e093209f1f29fbf0fcbb8d509133 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 10:52:18 +0200 Subject: [PATCH 11/21] stackdriver: always use regex full match for =~ and !=~operator (cherry picked from commit 46ca306c2f742223d3f6aa546f4805c8d30cb31f) --- .../features/datasources/stackdriver.md | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/sources/features/datasources/stackdriver.md b/docs/sources/features/datasources/stackdriver.md index 96f3ba3382e..6c493829e50 100644 --- a/docs/sources/features/datasources/stackdriver.md +++ b/docs/sources/features/datasources/stackdriver.md @@ -74,8 +74,12 @@ Click on the links above and click the `Enable` button: Choose a metric from the `Metric` dropdown. +### Filter + To add a filter, click the plus icon and choose a field to filter by and enter a filter value e.g. `instance_name = grafana-1` +It is also possible to add wildcards to the filter value field. E.g `us-*` to capture all values that starts with "us-", `*central-a` to capture all that ends with "central-a". `*-central-*` captures values that has the substring of -central-. + ### Aggregation The aggregation field lets you combine time series based on common statistics. Read more about this option [here](https://cloud.google.com/monitoring/charts/metrics-selector#aggregation-options). @@ -105,20 +109,20 @@ The Alias By field allows you to control the format of the legend keys. The defa #### Metric Type Patterns -Alias Pattern | Description | Example Result ------------------ | ---------------------------- | ------------- -`{{metric.type}}` | returns the full Metric Type | `compute.googleapis.com/instance/cpu/utilization` -`{{metric.name}}` | returns the metric name part | `instance/cpu/utilization` -`{{metric.service}}` | returns the service part | `compute` +| Alias Pattern | Description | Example Result | +| -------------------- | ---------------------------- | ------------------------------------------------- | +| `{{metric.type}}` | returns the full Metric Type | `compute.googleapis.com/instance/cpu/utilization` | +| `{{metric.name}}` | returns the metric name part | `instance/cpu/utilization` | +| `{{metric.service}}` | returns the service part | `compute` | #### Label Patterns In the Group By dropdown, you can see a list of metric and resource labels for a metric. These can be included in the legend key using alias patterns. -Alias Pattern Format | Description | Alias Pattern Example | Example Result ----------------------- | ---------------------------------- | ---------------------------- | ------------- -`{{metric.label.xxx}}` | returns the metric label value | `{{metric.label.instance_name}}` | `grafana-1-prod` -`{{resource.label.xxx}}` | returns the resource label value | `{{resource.label.zone}}` | `us-east1-b` +| Alias Pattern Format | Description | Alias Pattern Example | Example Result | +| ------------------------ | -------------------------------- | -------------------------------- | ---------------- | +| `{{metric.label.xxx}}` | returns the metric label value | `{{metric.label.instance_name}}` | `grafana-1-prod` | +| `{{resource.label.xxx}}` | returns the resource label value | `{{resource.label.zone}}` | `us-east1-b` | Example Alias By: `{{metric.type}} - {{metric.labels.instance_name}}` From 25f255f56047e5dc239440ac2cd0c5b226bb2638 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 11:08:14 +0200 Subject: [PATCH 12/21] stackdriver: add tests from regex matching (cherry picked from commit 7e6a5c0a7436e175383dbbe93e9f869d15c4ccbb) --- pkg/tsdb/stackdriver/stackdriver_test.go | 29 ++++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go index 4685362aedc..784bf4a7fbb 100644 --- a/pkg/tsdb/stackdriver/stackdriver_test.go +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -448,16 +448,31 @@ func TestStackdriver(t *testing.T) { }) Convey("when building filter string", func() { - Convey("and there are wildcards in a filter value", func() { - filterParts := []interface{}{"zone", "=", "*-central1*"} - value := buildFilterString("somemetrictype", filterParts) - So(value, ShouldEqual, `metric.type="somemetrictype" zone=has_substring("-central1")`) + Convey("and theres no regex operator", func() { + Convey("and there are wildcards in a filter value", func() { + filterParts := []interface{}{"zone", "=", "*-central1*"} + value := buildFilterString("somemetrictype", filterParts) + So(value, ShouldEqual, `metric.type="somemetrictype" zone=has_substring("-central1")`) + }) + + Convey("and there are no wildcards in any filter value", func() { + filterParts := []interface{}{"zone", "!=", "us-central1-a"} + value := buildFilterString("somemetrictype", filterParts) + So(value, ShouldEqual, `metric.type="somemetrictype" zone!="us-central1-a"`) + }) }) - Convey("and there are no wildcards in any filter value", func() { - filterParts := []interface{}{"zone", "=", "us-central1-a"} + Convey("and there is a regex operator", func() { + filterParts := []interface{}{"zone", "=~", "us-central1-a~"} value := buildFilterString("somemetrictype", filterParts) - So(value, ShouldEqual, `metric.type="somemetrictype" zone="us-central1-a"`) + Convey("it should remove the ~ character from the operator that belongs to the value", func() { + So(value, ShouldNotContainSubstring, `=~`) + So(value, ShouldContainSubstring, `zone=`) + }) + + Convey("it should insert monitoring.regex.full_match before filter value", func() { + So(value, ShouldContainSubstring, `zone=monitoring.regex.full_match("us-central1-a~")`) + }) }) }) }) From 0d0df00b8e42ba920bcbc1e538cc7bdf06c55ade Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 11:12:26 +0200 Subject: [PATCH 13/21] stackdriver: always use regex full match for =~ and !=~operator (cherry picked from commit 8d53799bcdd2f7ce434ef20e389df44464271828) --- pkg/tsdb/stackdriver/stackdriver.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go index f38c45ebfe0..96242dfdec4 100644 --- a/pkg/tsdb/stackdriver/stackdriver.go +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -199,7 +199,11 @@ func buildFilterString(metricType string, filterParts []interface{}) string { if part == "AND" { filterString += " " } else if mod == 2 { - if strings.Contains(part.(string), "*") { + operator := filterParts[i-1] + if operator == "=~" || operator == "!=~" { + filterString = reverse(strings.Replace(reverse(filterString), "~", "", 1)) + filterString += fmt.Sprintf(`monitoring.regex.full_match("%s")`, part) + } else if strings.Contains(part.(string), "*") { filterString += interpolateFilterWildcards(part.(string)) } else { filterString += fmt.Sprintf(`"%s"`, part) From b67e69bc52cade730d4f4f0d6ff60fee49abc13a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 8 Oct 2018 12:01:11 +0200 Subject: [PATCH 14/21] stackdriver: improve filter docs for wildcards and regular expressions (cherry picked from commit 11b9f9691cb181f7b3322ef24cd85b2d616e73dc) --- docs/sources/features/datasources/stackdriver.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/sources/features/datasources/stackdriver.md b/docs/sources/features/datasources/stackdriver.md index 6c493829e50..c525130aebb 100644 --- a/docs/sources/features/datasources/stackdriver.md +++ b/docs/sources/features/datasources/stackdriver.md @@ -76,9 +76,15 @@ Choose a metric from the `Metric` dropdown. ### Filter -To add a filter, click the plus icon and choose a field to filter by and enter a filter value e.g. `instance_name = grafana-1` +To add a filter, click the plus icon and choose a field to filter by and enter a filter value e.g. `instance_name = grafana-1`. You can remove the filter by clicking on the filter name and select `--remove filter--`. -It is also possible to add wildcards to the filter value field. E.g `us-*` to capture all values that starts with "us-", `*central-a` to capture all that ends with "central-a". `*-central-*` captures values that has the substring of -central-. +#### Simple wildcards + +When the operator is set to `=` or `!=` it is possible to add wildcards to the filter value field. E.g `us-*` will capture all values that starts with "us-" and `*central-a` will capture all values that ends with "central-a". `*-central-*` captures all values that has the substring of -central-. Simple wildcards are less expensive than regular expressions. + +#### Regular expressions + +When the operator is set to `=~` or `!=~` it is possible to add regular expressions to the filter value field. E.g `us-central[1-3]-[af]` would match all values that starts with "us-central", is followed by a number in the range of 1 to 3, a dash and then either an "a" or an "f". Leading and trailing slashes are not needed when creating regular expressions. ### Aggregation From 5250c84ca7aacdc0e5cf2e7743f435e323a734ea Mon Sep 17 00:00:00 2001 From: Daniel Lee Date: Mon, 8 Oct 2018 15:34:28 +0200 Subject: [PATCH 15/21] stackdriver metric name fix. Fixes #13562 Sets metric name even when the metric does not have a displayName field. Closes #13562. (cherry picked from commit 6fce178ec7a94d1b63a0a08fc57e2c45b11b70e2) --- .../plugins/datasource/stackdriver/datasource.ts | 12 +++++++++++- .../datasource/stackdriver/query_filter_ctrl.ts | 12 ++++-------- .../datasource/stackdriver/specs/datasource.test.ts | 13 +++++++++---- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/public/app/plugins/datasource/stackdriver/datasource.ts b/public/app/plugins/datasource/stackdriver/datasource.ts index 8ff81f3160a..7ea748e1082 100644 --- a/public/app/plugins/datasource/stackdriver/datasource.ts +++ b/public/app/plugins/datasource/stackdriver/datasource.ts @@ -241,7 +241,17 @@ export default class StackdriverDatasource { try { const metricsApiPath = `v3/projects/${projectId}/metricDescriptors`; const { data } = await this.doRequest(`${this.baseUrl}${metricsApiPath}`); - return data.metricDescriptors; + + const metrics = data.metricDescriptors.map(m => { + const [service] = m.type.split('/'); + const [serviceShortName] = service.split('.'); + m.service = service; + m.serviceShortName = serviceShortName; + m.displayName = m.displayName || m.type; + return m; + }); + + return metrics; } catch (error) { console.log(error); } diff --git a/public/app/plugins/datasource/stackdriver/query_filter_ctrl.ts b/public/app/plugins/datasource/stackdriver/query_filter_ctrl.ts index ac279eec0d5..786b2831e89 100644 --- a/public/app/plugins/datasource/stackdriver/query_filter_ctrl.ts +++ b/public/app/plugins/datasource/stackdriver/query_filter_ctrl.ts @@ -96,11 +96,9 @@ export class StackdriverFilterCtrl { getServicesList() { const defaultValue = { value: this.$scope.defaultServiceValue, text: this.$scope.defaultServiceValue }; const services = this.metricDescriptors.map(m => { - const [service] = m.type.split('/'); - const [serviceShortName] = service.split('.'); return { - value: service, - text: serviceShortName, + value: m.service, + text: m.serviceShortName, }; }); @@ -113,12 +111,10 @@ export class StackdriverFilterCtrl { getMetricsList() { const metrics = this.metricDescriptors.map(m => { - const [service] = m.type.split('/'); - const [serviceShortName] = service.split('.'); return { - service, + service: m.service, value: m.type, - serviceShortName, + serviceShortName: m.serviceShortName, text: m.displayName, title: m.description, }; diff --git a/public/app/plugins/datasource/stackdriver/specs/datasource.test.ts b/public/app/plugins/datasource/stackdriver/specs/datasource.test.ts index 80830fd4d68..3117be402a9 100644 --- a/public/app/plugins/datasource/stackdriver/specs/datasource.test.ts +++ b/public/app/plugins/datasource/stackdriver/specs/datasource.test.ts @@ -164,11 +164,11 @@ describe('StackdriverDataSource', () => { metricDescriptors: [ { displayName: 'test metric name 1', - type: 'test metric type 1', + type: 'compute.googleapis.com/instance/cpu/test-metric-type-1', + description: 'A description', }, { - displayName: 'test metric name 2', - type: 'test metric type 2', + type: 'logging.googleapis.com/user/logbased-metric-with-no-display-name', }, ], }, @@ -180,8 +180,13 @@ describe('StackdriverDataSource', () => { }); it('should return successfully', () => { expect(result.length).toBe(2); - expect(result[0].type).toBe('test metric type 1'); + expect(result[0].service).toBe('compute.googleapis.com'); + expect(result[0].serviceShortName).toBe('compute'); + expect(result[0].type).toBe('compute.googleapis.com/instance/cpu/test-metric-type-1'); expect(result[0].displayName).toBe('test metric name 1'); + expect(result[0].description).toBe('A description'); + expect(result[1].type).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name'); + expect(result[1].displayName).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name'); }); }); From cc57377f03b6a98ca605ac43c076be7a977d4708 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Tue, 9 Oct 2018 12:11:18 +0900 Subject: [PATCH 16/21] set unit for CloudWatch GetMetricStatistics result (cherry picked from commit 6ed1cbd5bb8bba9c80022710c9e71dd7c169db41) --- pkg/tsdb/cloudwatch/cloudwatch.go | 7 +++++ pkg/tsdb/cloudwatch/constants.go | 30 +++++++++++++++++++ .../datasource/cloudwatch/datasource.ts | 2 +- .../cloudwatch/specs/datasource.test.ts | 2 ++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 pkg/tsdb/cloudwatch/constants.go diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index be14c6f96ec..fab8b92ef66 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -362,6 +362,7 @@ func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, regi } queryRes.Series = append(queryRes.Series, &series) + queryRes.Meta = simplejson.New() queryResponses = append(queryResponses, queryRes) } @@ -565,6 +566,12 @@ func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatch } queryRes.Series = append(queryRes.Series, &series) + queryRes.Meta = simplejson.New() + if len(resp.Datapoints) > 0 && resp.Datapoints[0].Unit != nil { + if unit, ok := cloudwatchUnitMappings[*resp.Datapoints[0].Unit]; ok { + queryRes.Meta.Set("unit", unit) + } + } } return queryRes, nil diff --git a/pkg/tsdb/cloudwatch/constants.go b/pkg/tsdb/cloudwatch/constants.go new file mode 100644 index 00000000000..23817b1d133 --- /dev/null +++ b/pkg/tsdb/cloudwatch/constants.go @@ -0,0 +1,30 @@ +package cloudwatch + +var cloudwatchUnitMappings = map[string]string{ + "Seconds": "s", + "Microseconds": "µs", + "Milliseconds": "ms", + "Bytes": "bytes", + "Kilobytes": "kbytes", + "Megabytes": "mbytes", + "Gigabytes": "gbytes", + //"Terabytes": "", + "Bits": "bits", + //"Kilobits": "", + //"Megabits": "", + //"Gigabits": "", + //"Terabits": "", + "Percent": "percent", + //"Count": "", + "Bytes/Second": "Bps", + "Kilobytes/Second": "KBs", + "Megabytes/Second": "MBs", + "Gigabytes/Second": "GBs", + //"Terabytes/Second": "", + "Bits/Second": "bps", + "Kilobits/Second": "Kbits", + "Megabits/Second": "Mbits", + "Gigabits/Second": "Gbits", + //"Terabits/Second": "", + //"Count/Second": "", +} diff --git a/public/app/plugins/datasource/cloudwatch/datasource.ts b/public/app/plugins/datasource/cloudwatch/datasource.ts index e2b99d69df9..e096e44ac25 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.ts +++ b/public/app/plugins/datasource/cloudwatch/datasource.ts @@ -131,7 +131,7 @@ export default class CloudWatchDatasource { if (res.results) { _.forEach(res.results, queryRes => { _.forEach(queryRes.series, series => { - data.push({ target: series.name, datapoints: series.points }); + data.push({ target: series.name, datapoints: series.points, unit: queryRes.meta.unit || 'none' }); }); }); } diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts index 497c773687f..2825539f223 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts @@ -60,6 +60,7 @@ describe('CloudWatchDatasource', () => { A: { error: '', refId: 'A', + meta: {}, series: [ { name: 'CPUUtilization_Average', @@ -221,6 +222,7 @@ describe('CloudWatchDatasource', () => { A: { error: '', refId: 'A', + meta: {}, series: [ { name: 'TargetResponseTime_p90.00', From 55712d61f4517273222729c7e8e8273273ff07e9 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 10 Oct 2018 13:56:58 +0900 Subject: [PATCH 17/21] fix crach bug (cherry picked from commit 37e749f6dab874ae891dc43fb0bf3b86c7a41d18) --- pkg/tsdb/cloudwatch/cloudwatch.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index fab8b92ef66..53ae2eae727 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -129,10 +129,12 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" { return err } - result.Results[queryRes.RefId] = queryRes if err != nil { - result.Results[queryRes.RefId].Error = err + result.Results[query.RefId] = &tsdb.QueryResult{ + Error: err, + } } + result.Results[queryRes.RefId] = queryRes return nil }) } @@ -269,7 +271,7 @@ func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, regi for _, query := range queries { // 1 minutes resolution metrics is stored for 15 days, 15 * 24 * 60 = 21600 if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) { - return nil, errors.New("too long query period") + return queryResponses, errors.New("too long query period") } mdq := &cloudwatch.MetricDataQuery{ From 002da27e982c69ad4d13b8f82aa2a163831f0041 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 10 Oct 2018 14:07:08 +0900 Subject: [PATCH 18/21] add test for automatically unit set (cherry picked from commit f0fb8123ae2ff377d861ffbe3b1ecfde07eba5e7) --- pkg/tsdb/cloudwatch/cloudwatch_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go index 719edba08ba..32b8c910f2b 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -71,6 +71,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(30.0), "p90.00": aws.Float64(40.0), }, + Unit: aws.String("Seconds"), }, }, } @@ -103,6 +104,7 @@ func TestCloudWatch(t *testing.T) { So(queryRes.Series[1].Points[0][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) So(queryRes.Series[2].Points[0][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) So(queryRes.Series[3].Points[0][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) + So(queryRes.Meta.Get("unit").MustString(), ShouldEqual, "s") }) Convey("terminate gap of data points", func() { @@ -118,6 +120,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(30.0), "p90.00": aws.Float64(40.0), }, + Unit: aws.String("Seconds"), }, { Timestamp: aws.Time(timestamp.Add(60 * time.Second)), @@ -127,6 +130,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(40.0), "p90.00": aws.Float64(50.0), }, + Unit: aws.String("Seconds"), }, { Timestamp: aws.Time(timestamp.Add(180 * time.Second)), @@ -136,6 +140,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(50.0), "p90.00": aws.Float64(60.0), }, + Unit: aws.String("Seconds"), }, }, } From 04ba06ccadaab498c8af86e4672fcd8648b4a754 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 10 Oct 2018 09:38:42 +0200 Subject: [PATCH 19/21] cloudwatch: return early if execute query returns error This will stop a segfault from happening (cherry picked from commit 0612ce9b75c20ce385401a51b719d4561488dce9) --- pkg/tsdb/cloudwatch/cloudwatch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 53ae2eae727..61bbc04394a 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -133,6 +133,7 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo result.Results[query.RefId] = &tsdb.QueryResult{ Error: err, } + return nil } result.Results[queryRes.RefId] = queryRes return nil From 6611aefea462aaf8c24dd41285e17ae31dc6baed Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 10 Oct 2018 10:54:47 +0200 Subject: [PATCH 20/21] release 5.3.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a6db5bdc1e5..333cd361ac4 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "company": "Grafana Labs" }, "name": "grafana", - "version": "5.3.0-beta3", + "version": "5.3.0", "repository": { "type": "git", "url": "http://github.com/grafana/grafana.git" From 4e2607b8e75c469015a8fcdb77a6418777edc188 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanda Date: Wed, 10 Oct 2018 12:19:57 +0900 Subject: [PATCH 21/21] fix id validation (cherry picked from commit 6e32c9bb3ff20237d25c872de5152933368f6483) --- .../datasource/cloudwatch/partials/query.parameter.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html b/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html index 7da6e7d2a83..2a951bc9257 100644 --- a/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html +++ b/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html @@ -37,8 +37,7 @@ Id Id can include numbers, letters, and underscore, and must start with a lowercase letter. - +