public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stackdriver: Support for SLO queries (#22917)

Browse Source 
* wip: add slo support

* Export DataSourcePlugin

* wip: break out metric query editor into its own component

* wip: refactor frontend - keep SLO and Metric query in differnt objects

* wip - load services and slos

* Fix broken test

* Add interactive slo expression builder

* Change order of dropdowns

* Refactoring backend model. slo unit testing in progress

* Unit test migration and SLOs

* Cleanup SLO editor

* Simplify alias by component

* Support alias by for slos

* Support slos in variable queries

* Fix broken last query error

* Update Help section to include SLO aliases

* streamline datasource resource cache

* Break out api specific stuff in datasource to its own file

* Move get projects call to frontend

* Refactor api caching

* Unit test api service

* Fix lint go issue

* Fix typescript strict errors

* Fix test datasource

* Use budget fraction selector instead of budget

* Reset SLO when service is changed

* Handle error in case resource call returned no data

* Show real SLI display name

* Use unsafe prefix on will mount hook

* Store goal in query model since it will be used as soon as graph panel supports adding a threshold

* Add comment to describe why componentWillMount is used

* Interpolate sloid

* Break out SLO aggregation into its own func

* Also test group bys for metricquery test

* Remove not used type fields

* Remove annoying stackdriver prefix from error message

* Default view param to FULL

* Add part about SLO query builder in docs

* Use new images

* Fixes after feedback

* Add one more group by test

* Make stackdriver types internal

* Update docs/sources/features/datasources/stackdriver.md

Co-Authored-By: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Update docs/sources/features/datasources/stackdriver.md

Co-Authored-By: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Update docs/sources/features/datasources/stackdriver.md

Co-Authored-By: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Updates after PR feedback

* add test for when no alias by defined

* fix infinite loop when newVariables feature flag is on

onChange being called in componentDidUpdate produces an
infinite loop when using the new React template variable
implementation.

Also fixes a spelling mistake

* implements feedback for documentation changes

* more doc changes

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
Co-authored-by: Daniel Lee <dan.limerick@gmail.com>
This commit is contained in:
Erik Sundell
2020-03-27 12:01:16 +01:00
committed by GitHub
parent e19493ae24
commit a111cc0d5c
38 changed files with 1416 additions and 1127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/stackdriver/stackdriver_test.go
+148 -16
View File
@@ -19,7 +19,7 @@ func TestStackdriver(t *testing.T) {
Convey("Stackdriver", t, func() {
executor := &StackdriverExecutor{}
Convey("Parse queries from frontend and build Stackdriver API queries", func() {
Convey("Parse migrated queries from frontend and build Stackdriver API queries", func() {
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
tsdbQuery := &tsdb.TsdbQuery{
TimeRange: &tsdb.TimeRange{
@@ -208,6 +208,99 @@ func TestStackdriver(t *testing.T) {
})
Convey("Parse queries from frontend and build Stackdriver API queries", func() {
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
tsdbQuery := &tsdb.TsdbQuery{
TimeRange: &tsdb.TimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": metricQueryType,
"metricQuery": map[string]interface{}{
"metricType": "a/metric/type",
"view": "FULL",
"aliasBy": "testalias",
"type": "timeSeriesQuery",
"groupBys": []interface{}{"metric.label.group1", "metric.label.group2"},
},
}),
RefId: "A",
},
},
}
Convey("and query type is metrics", func() {
queries, err := executor.buildQueries(tsdbQuery)
So(err, ShouldBeNil)
So(len(queries), ShouldEqual, 1)
So(queries[0].RefID, ShouldEqual, "A")
So(queries[0].Target, ShouldEqual, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.groupByFields=metric.label.group1&aggregation.groupByFields=metric.label.group2&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL")
So(len(queries[0].Params), ShouldEqual, 8)
So(queries[0].Params["aggregation.groupByFields"][0], ShouldEqual, "metric.label.group1")
So(queries[0].Params["aggregation.groupByFields"][1], ShouldEqual, "metric.label.group2")
So(queries[0].Params["interval.startTime"][0], ShouldEqual, "2018-03-15T13:00:00Z")
So(queries[0].Params["interval.endTime"][0], ShouldEqual, "2018-03-15T13:34:00Z")
So(queries[0].Params["aggregation.perSeriesAligner"][0], ShouldEqual, "ALIGN_MEAN")
So(queries[0].Params["filter"][0], ShouldEqual, "metric.type=\"a/metric/type\"")
So(queries[0].Params["view"][0], ShouldEqual, "FULL")
So(queries[0].AliasBy, ShouldEqual, "testalias")
So(queries[0].GroupBys, ShouldResemble, []string{"metric.label.group1", "metric.label.group2"})
})
Convey("and query type is SLOs", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{
"queryType": sloQueryType,
"metricQuery": map[string]interface{}{},
"sloQuery": map[string]interface{}{
"projectName": "test-proj",
"alignmentPeriod": "stackdriver-auto",
"perSeriesAligner": "ALIGN_NEXT_OLDER",
"aliasBy": "",
"selectorName": "select_slo_health",
"serviceId": "test-service",
"sloId": "test-slo",
},
})
queries, err := executor.buildQueries(tsdbQuery)
So(err, ShouldBeNil)
So(len(queries), ShouldEqual, 1)
So(queries[0].RefID, ShouldEqual, "A")
So(queries[0].Params["interval.startTime"][0], ShouldEqual, "2018-03-15T13:00:00Z")
So(queries[0].Params["interval.endTime"][0], ShouldEqual, "2018-03-15T13:34:00Z")
So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+60s`)
So(queries[0].AliasBy, ShouldEqual, "")
So(queries[0].Params["aggregation.perSeriesAligner"][0], ShouldEqual, "ALIGN_MEAN")
So(queries[0].Target, ShouldEqual, `aggregation.alignmentPeriod=%2B60s&aggregation.perSeriesAligner=ALIGN_MEAN&filter=select_slo_health%28%22projects%2Ftest-proj%2Fservices%2Ftest-service%2FserviceLevelObjectives%2Ftest-slo%22%29&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z`)
So(len(queries[0].Params), ShouldEqual, 5)
Convey("and perSeriesAligner is inferred by SLO selector", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{
"queryType": sloQueryType,
"metricQuery": map[string]interface{}{},
"sloQuery": map[string]interface{}{
"projectName": "test-proj",
"alignmentPeriod": "stackdriver-auto",
"perSeriesAligner": "ALIGN_NEXT_OLDER",
"aliasBy": "",
"selectorName": "select_slo_compliance",
"serviceId": "test-service",
"sloId": "test-slo",
},
})
queries, err := executor.buildQueries(tsdbQuery)
So(err, ShouldBeNil)
So(queries[0].Params["aggregation.perSeriesAligner"][0], ShouldEqual, "ALIGN_NEXT_OLDER")
})
})
})
Convey("Parse stackdriver response in the time series format", func() {
Convey("when data from query aggregated to one time series", func() {
data, err := loadTestFile("./test-data/1-series-response-agg-one-metric.json")
@@ -215,7 +308,7 @@ func TestStackdriver(t *testing.T) {
So(len(data.TimeSeries), ShouldEqual, 1)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{}
query := &stackdriverQuery{}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -241,7 +334,7 @@ func TestStackdriver(t *testing.T) {
So(len(data.TimeSeries), ShouldEqual, 3)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{}
query := &stackdriverQuery{}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -283,7 +376,7 @@ func TestStackdriver(t *testing.T) {
So(len(data.TimeSeries), ShouldEqual, 3)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}
query := &stackdriverQuery{GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -304,7 +397,7 @@ func TestStackdriver(t *testing.T) {
Convey("and the alias pattern is for metric type, a metric label and a resource label", func() {
query := &StackdriverQuery{AliasBy: "{{metric.type}} - {{metric.label.instance_name}} - {{resource.label.zone}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}
query := &stackdriverQuery{AliasBy: "{{metric.type}} - {{metric.label.instance_name}} - {{resource.label.zone}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -318,7 +411,7 @@ func TestStackdriver(t *testing.T) {
Convey("and the alias pattern is for metric name", func() {
query := &StackdriverQuery{AliasBy: "metric {{metric.name}} service {{metric.service}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}
query := &stackdriverQuery{AliasBy: "metric {{metric.name}} service {{metric.service}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -337,7 +430,7 @@ func TestStackdriver(t *testing.T) {
So(len(data.TimeSeries), ShouldEqual, 1)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{AliasBy: "{{bucket}}"}
query := &stackdriverQuery{AliasBy: "{{bucket}}"}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -384,7 +477,7 @@ func TestStackdriver(t *testing.T) {
So(len(data.TimeSeries), ShouldEqual, 1)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{AliasBy: "{{bucket}}"}
query := &stackdriverQuery{AliasBy: "{{bucket}}"}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
@@ -424,7 +517,7 @@ func TestStackdriver(t *testing.T) {
So(len(data.TimeSeries), ShouldEqual, 3)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{AliasBy: "{{bucket}}"}
query := &stackdriverQuery{AliasBy: "{{bucket}}"}
err = executor.parseResponse(res, data, query)
labels := res.Meta.Get("labels").Interface().(map[string][]string)
So(err, ShouldBeNil)
@@ -463,7 +556,7 @@ func TestStackdriver(t *testing.T) {
Convey("and systemlabel contains key with array of string", func() {
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{AliasBy: "{{metadata.system_labels.test}}"}
query := &stackdriverQuery{AliasBy: "{{metadata.system_labels.test}}"}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
So(len(res.Series), ShouldEqual, 3)
@@ -475,7 +568,7 @@ func TestStackdriver(t *testing.T) {
Convey("and systemlabel contains key with array of string2", func() {
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &StackdriverQuery{AliasBy: "{{metadata.system_labels.test2}}"}
query := &stackdriverQuery{AliasBy: "{{metadata.system_labels.test2}}"}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
So(len(res.Series), ShouldEqual, 3)
@@ -483,6 +576,45 @@ func TestStackdriver(t *testing.T) {
So(res.Series[2].Name, ShouldEqual, "testvalue")
})
})
Convey("when data from query returns slo and alias by is defined", func() {
data, err := loadTestFile("./test-data/6-series-response-slo.json")
So(err, ShouldBeNil)
So(len(data.TimeSeries), ShouldEqual, 1)
Convey("and alias by is expanded", func() {
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &stackdriverQuery{
ProjectName: "test-proj",
Selector: "select_slo_compliance",
Service: "test-service",
Slo: "test-slo",
AliasBy: "{{project}} - {{service}} - {{slo}} - {{selector}}",
}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
So(res.Series[0].Name, ShouldEqual, "test-proj - test-service - test-slo - select_slo_compliance")
})
})
Convey("when data from query returns slo and alias by is not defined", func() {
data, err := loadTestFile("./test-data/6-series-response-slo.json")
So(err, ShouldBeNil)
So(len(data.TimeSeries), ShouldEqual, 1)
Convey("and alias by is expanded", func() {
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
query := &stackdriverQuery{
ProjectName: "test-proj",
Selector: "select_slo_compliance",
Service: "test-service",
Slo: "test-slo",
}
err = executor.parseResponse(res, data, query)
So(err, ShouldBeNil)
So(res.Series[0].Name, ShouldEqual, "select_slo_compliance(\"projects/test-proj/services/test-service/serviceLevelObjectives/test-slo\")")
})
})
})
Convey("when interpolating filter wildcards", func() {
@@ -550,20 +682,20 @@ func TestStackdriver(t *testing.T) {
Convey("when building filter string", func() {
Convey("and theres no regex operator", func() {
Convey("and there are wildcards in a filter value", func() {
filterParts := []interface{}{"zone", "=", "*-central1*"}
filterParts := []string{"zone", "=", "*-central1*"}
value := buildFilterString("somemetrictype", filterParts)
So(value, ShouldEqual, `metric.type="somemetrictype" zone=has_substring("-central1")`)
})
Convey("and there are no wildcards in any filter value", func() {
filterParts := []interface{}{"zone", "!=", "us-central1-a"}
filterParts := []string{"zone", "!=", "us-central1-a"}
value := buildFilterString("somemetrictype", filterParts)
So(value, ShouldEqual, `metric.type="somemetrictype" zone!="us-central1-a"`)
})
})
Convey("and there is a regex operator", func() {
filterParts := []interface{}{"zone", "=~", "us-central1-a~"}
filterParts := []string{"zone", "=~", "us-central1-a~"}
value := buildFilterString("somemetrictype", filterParts)
Convey("it should remove the ~ character from the operator that belongs to the value", func() {
So(value, ShouldNotContainSubstring, `=~`)
@@ -578,8 +710,8 @@ func TestStackdriver(t *testing.T) {
})
}
func loadTestFile(path string) (StackdriverResponse, error) {
var data StackdriverResponse
func loadTestFile(path string) (stackdriverResponse, error) {
var data stackdriverResponse
jsonBody, err := ioutil.ReadFile(path)
if err != nil {