public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GoogleCloudMonitoring: Adapt frontend to the new API format (#60173)

Browse Source 
* GoogleCloudMonitoring: Migrate queries to the new format

* Refactor Aligment and AligmentFunction components (#60235)

* Adapt CloudMonitoringDatasource and CloudMonitoringAnnotationSupport (#60177)

* Fix: avoid migration for new queries (#60375)

* Move preprocessor handling to the backend (#60383)

* Other fixes and new function (#60411)

* Adapt components to the new API (#60451)

* Split metrics query type in time series list and query (#60475)

* Clean up metricQuery references (#60478)

* More bug fixes (#60525)
This commit is contained in:
Andres Martinez Gotor
2022-12-20 12:47:49 +01:00
committed by GitHub
parent f6f140c412
commit 4d693863c0
38 changed files with 1007 additions and 526 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudmonitoring/cloudmonitoring.go
+43 -62
View File
@@ -56,7 +56,8 @@ const (
gceAuthentication = "gce"
jwtAuthentication = "jwt"
annotationQueryType = "annotation"
metricQueryType = "metrics"
timeSeriesListQueryType = "timeSeriesList"
timeSeriesQueryQueryType = "timeSeriesQuery"
sloQueryType = "slo"
crossSeriesReducerDefault = "REDUCE_NONE"
perSeriesAlignerDefault = "ALIGN_MEAN"
@@ -216,32 +217,6 @@ func migrateMetricTypeFilter(metricTypeFilter string, prevFilters interface{}) [
return metricTypeFilterArray
}
func migratePreprocessor(tsl *timeSeriesList, preprocessor string) {
// In case a preprocessor is defined, the preprocessor becomes the primary aggregation
// and the aggregation that is specified in the UI becomes the secondary aggregation
// Rules are specified in this issue: https://github.com/grafana/grafana/issues/30866
t := toPreprocessorType(preprocessor)
if t != PreprocessorTypeNone {
// Move aggregation to secondaryAggregation
tsl.SecondaryAlignmentPeriod = tsl.AlignmentPeriod
tsl.SecondaryCrossSeriesReducer = tsl.CrossSeriesReducer
tsl.SecondaryPerSeriesAligner = tsl.PerSeriesAligner
tsl.SecondaryGroupBys = tsl.GroupBys
// Set a default cross series reducer if grouped
if len(tsl.GroupBys) == 0 {
tsl.CrossSeriesReducer = crossSeriesReducerDefault
}
// Set aligner based on preprocessor type
aligner := "ALIGN_RATE"
if t == PreprocessorTypeDelta {
aligner = "ALIGN_DELTA"
}
tsl.PerSeriesAligner = aligner
}
}
func migrateRequest(req *backend.QueryDataRequest) error {
for i, q := range req.Queries {
var rawQuery map[string]interface{}
@@ -250,14 +225,17 @@ func migrateRequest(req *backend.QueryDataRequest) error {
return err
}
if rawQuery["metricQuery"] == nil {
if rawQuery["metricQuery"] == nil &&
rawQuery["timeSeriesQuery"] == nil &&
rawQuery["timeSeriesList"] == nil &&
rawQuery["sloQuery"] == nil {
// migrate legacy query
var mq timeSeriesList
err = json.Unmarshal(q.JSON, &mq)
if err != nil {
return err
}
q.QueryType = metricQueryType
q.QueryType = timeSeriesListQueryType
gq := grafanaQuery{
TimeSeriesList: &mq,
}
@@ -268,9 +246,6 @@ func migrateRequest(req *backend.QueryDataRequest) error {
// metricType should be a filter
gq.TimeSeriesList.Filters = migrateMetricTypeFilter(rawQuery["metricType"].(string), rawQuery["filters"])
}
if rawQuery["preprocessor"] != nil {
migratePreprocessor(gq.TimeSeriesList, rawQuery["preprocessor"].(string))
}
b, err := json.Marshal(gq)
if err != nil {
@@ -288,7 +263,7 @@ func migrateRequest(req *backend.QueryDataRequest) error {
}
// Metric query was divided between timeSeriesList and timeSeriesQuery API calls
if rawQuery["metricQuery"] != nil {
if rawQuery["metricQuery"] != nil && q.QueryType == "metrics" {
metricQuery := rawQuery["metricQuery"].(map[string]interface{})
if metricQuery["editorMode"] != nil && toString(metricQuery["editorMode"]) == "mql" {
@@ -297,6 +272,7 @@ func migrateRequest(req *backend.QueryDataRequest) error {
Query: toString(metricQuery["query"]),
GraphPeriod: toString(metricQuery["graphPeriod"]),
}
q.QueryType = timeSeriesQueryQueryType
} else {
tslb, err := json.Marshal(metricQuery)
if err != nil {
@@ -311,11 +287,10 @@ func migrateRequest(req *backend.QueryDataRequest) error {
// metricType should be a filter
tsl.Filters = migrateMetricTypeFilter(metricQuery["metricType"].(string), metricQuery["filters"])
}
if rawQuery["preprocessor"] != nil {
migratePreprocessor(tsl, rawQuery["preprocessor"].(string))
}
rawQuery["timeSeriesList"] = tsl
q.QueryType = timeSeriesListQueryType
}
// AliasBy is now a top level property
if metricQuery["aliasBy"] != nil {
rawQuery["aliasBy"] = metricQuery["aliasBy"]
}
@@ -323,12 +298,22 @@ func migrateRequest(req *backend.QueryDataRequest) error {
if err != nil {
return err
}
if q.QueryType == "" {
q.QueryType = metricQueryType
}
q.JSON = b
}
if rawQuery["sloQuery"] != nil && q.QueryType == sloQueryType {
sloQuery := rawQuery["sloQuery"].(map[string]interface{})
// AliasBy is now a top level property
if sloQuery["aliasBy"] != nil {
rawQuery["aliasBy"] = sloQuery["aliasBy"]
b, err := json.Marshal(rawQuery)
if err != nil {
return err
}
q.JSON = b
}
}
req.Queries[i] = q
}
@@ -408,29 +393,25 @@ func (s *Service) buildQueryExecutors(logger log.Logger, req *backend.QueryDataR
var queryInterface cloudMonitoringQueryExecutor
switch query.QueryType {
case metricQueryType, annotationQueryType:
if q.TimeSeriesQuery != nil {
queryInterface = &cloudMonitoringTimeSeriesQuery{
refID: query.RefID,
aliasBy: q.AliasBy,
parameters: q.TimeSeriesQuery,
IntervalMS: query.Interval.Milliseconds(),
timeRange: req.Queries[0].TimeRange,
}
} else if q.TimeSeriesList != nil {
cmtsf := &cloudMonitoringTimeSeriesList{
refID: query.RefID,
logger: logger,
aliasBy: q.AliasBy,
}
if q.TimeSeriesList.View == "" {
q.TimeSeriesList.View = "FULL"
}
cmtsf.parameters = q.TimeSeriesList
cmtsf.setParams(startTime, endTime, durationSeconds, query.Interval.Milliseconds())
queryInterface = cmtsf
} else {
return nil, fmt.Errorf("missing query info")
case timeSeriesListQueryType, annotationQueryType:
cmtsf := &cloudMonitoringTimeSeriesList{
refID: query.RefID,
logger: logger,
aliasBy: q.AliasBy,
}
if q.TimeSeriesList.View == "" {
q.TimeSeriesList.View = "FULL"
}
cmtsf.parameters = q.TimeSeriesList
cmtsf.setParams(startTime, endTime, durationSeconds, query.Interval.Milliseconds())
queryInterface = cmtsf
case timeSeriesQueryQueryType:
queryInterface = &cloudMonitoringTimeSeriesQuery{
refID: query.RefID,
aliasBy: q.AliasBy,
parameters: q.TimeSeriesQuery,
IntervalMS: query.Interval.Milliseconds(),
timeRange: req.Queries[0].TimeRange,
}
case sloQueryType:
cmslo := &cloudMonitoringSLO{
pkg/tsdb/cloudmonitoring/cloudmonitoring_test.go
+4 -4
View File
@@ -664,7 +664,7 @@ func TestCloudMonitoring(t *testing.T) {
"projectName": "test-proj",
"alignmentPeriod": "stackdriver-auto",
"perSeriesAligner": "ALIGN_NEXT_OLDER",
"aliasBy": "",
"aliasBy": "test-alias",
"selectorName": "select_slo_health",
"serviceId": "test-service",
"sloId": "test-slo"
@@ -683,7 +683,7 @@ func TestCloudMonitoring(t *testing.T) {
assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0])
assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0])
assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0])
assert.Equal(t, "", queries[0].aliasBy)
assert.Equal(t, "test-alias", queries[0].aliasBy)
assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0])
assert.Equal(t, `aggregation.alignmentPeriod=%2B60s&aggregation.perSeriesAligner=ALIGN_MEAN&filter=select_slo_health%28%22projects%2Ftest-proj%2Fservices%2Ftest-service%2FserviceLevelObjectives%2Ftest-slo%22%29&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z`, queries[0].params.Encode())
assert.Equal(t, 5, len(queries[0].params))
@@ -1103,7 +1103,7 @@ func baseTimeSeriesList() *backend.QueryDataRequest {
From: fromStart,
To: fromStart.Add(34 * time.Minute),
},
QueryType: "metrics",
QueryType: timeSeriesListQueryType,
JSON: json.RawMessage(`{
"timeSeriesList": {
"filters": ["metric.type=\"a/metric/type\""],
@@ -1127,7 +1127,7 @@ func baseTimeSeriesQuery() *backend.QueryDataRequest {
From: fromStart,
To: fromStart.Add(34 * time.Minute),
},
QueryType: "metrics",
QueryType: timeSeriesQueryQueryType,
JSON: json.RawMessage(`{
"queryType": "metrics",
"timeSeriesQuery": {
pkg/tsdb/cloudmonitoring/time_series_filter.go
+28
View File
@@ -147,6 +147,32 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesList) getFilter() string {
return strings.Trim(filterString, " ")
}
func (timeSeriesFilter *cloudMonitoringTimeSeriesList) setPreprocessor() {
// In case a preprocessor is defined, the preprocessor becomes the primary aggregation
// and the aggregation that is specified in the UI becomes the secondary aggregation
// Rules are specified in this issue: https://github.com/grafana/grafana/issues/30866
t := toPreprocessorType(timeSeriesFilter.parameters.Preprocessor)
if t != PreprocessorTypeNone {
// Move aggregation to secondaryAggregation
timeSeriesFilter.parameters.SecondaryAlignmentPeriod = timeSeriesFilter.parameters.AlignmentPeriod
timeSeriesFilter.parameters.SecondaryCrossSeriesReducer = timeSeriesFilter.parameters.CrossSeriesReducer
timeSeriesFilter.parameters.SecondaryPerSeriesAligner = timeSeriesFilter.parameters.PerSeriesAligner
timeSeriesFilter.parameters.SecondaryGroupBys = timeSeriesFilter.parameters.GroupBys
// Set a default cross series reducer if grouped
if len(timeSeriesFilter.parameters.GroupBys) == 0 {
timeSeriesFilter.parameters.CrossSeriesReducer = crossSeriesReducerDefault
}
// Set aligner based on preprocessor type
aligner := "ALIGN_RATE"
if t == PreprocessorTypeDelta {
aligner = "ALIGN_DELTA"
}
timeSeriesFilter.parameters.PerSeriesAligner = aligner
}
}
func (timeSeriesFilter *cloudMonitoringTimeSeriesList) setParams(startTime time.Time, endTime time.Time, durationSeconds int, intervalMs int64) {
params := url.Values{}
query := timeSeriesFilter.parameters
@@ -165,6 +191,8 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesList) setParams(startTime time.
query.PerSeriesAligner = perSeriesAlignerDefault
}
timeSeriesFilter.setPreprocessor()
alignmentPeriod := calculateAlignmentPeriod(query.AlignmentPeriod, intervalMs, durationSeconds)
params.Add("aggregation.alignmentPeriod", alignmentPeriod)
if query.CrossSeriesReducer != "" {
pkg/tsdb/cloudmonitoring/time_series_filter_test.go
+36
View File
@@ -18,6 +18,42 @@ import (
)
func TestTimeSeriesFilter(t *testing.T) {
t.Run("parses params", func(t *testing.T) {
query := &cloudMonitoringTimeSeriesList{parameters: &timeSeriesList{}}
query.setParams(time.Time{}, time.Time{}, 0, 0)
assert.Equal(t, "0001-01-01T00:00:00Z", query.params.Get("interval.startTime"))
assert.Equal(t, "0001-01-01T00:00:00Z", query.params.Get("interval.endTime"))
assert.Equal(t, "", query.params.Get("filter"))
assert.Equal(t, "", query.params.Get("view"))
assert.Equal(t, "+60s", query.params.Get("aggregation.alignmentPeriod"))
assert.Equal(t, "REDUCE_NONE", query.params.Get("aggregation.crossSeriesReducer"))
assert.Equal(t, "ALIGN_MEAN", query.params.Get("aggregation.perSeriesAligner"))
assert.Equal(t, "", query.params.Get("aggregation.groupByFields"))
assert.Equal(t, "", query.params.Get("secondaryAggregation.alignmentPeriod"))
assert.Equal(t, "", query.params.Get("secondaryAggregation.crossSeriesReducer"))
assert.Equal(t, "", query.params.Get("secondaryAggregation.perSeriesAligner"))
assert.Equal(t, "", query.params.Get("secondaryAggregation.groupByFields"))
})
t.Run("parses params with preprocessor", func(t *testing.T) {
query := &cloudMonitoringTimeSeriesList{parameters: &timeSeriesList{Preprocessor: "rate"}}
query.setParams(time.Time{}, time.Time{}, 0, 0)
assert.Equal(t, "0001-01-01T00:00:00Z", query.params.Get("interval.startTime"))
assert.Equal(t, "0001-01-01T00:00:00Z", query.params.Get("interval.endTime"))
assert.Equal(t, "", query.params.Get("filter"))
assert.Equal(t, "", query.params.Get("view"))
assert.Equal(t, "+60s", query.params.Get("aggregation.alignmentPeriod"))
assert.Equal(t, "REDUCE_NONE", query.params.Get("aggregation.crossSeriesReducer"))
assert.Equal(t, "ALIGN_RATE", query.params.Get("aggregation.perSeriesAligner"))
assert.Equal(t, "", query.params.Get("aggregation.groupByFields"))
assert.Equal(t, "", query.params.Get("secondaryAggregation.alignmentPeriod"))
assert.Equal(t, "REDUCE_NONE", query.params.Get("secondaryAggregation.crossSeriesReducer"))
assert.Equal(t, "ALIGN_MEAN", query.params.Get("secondaryAggregation.perSeriesAligner"))
assert.Equal(t, "", query.params.Get("secondaryAggregation.groupByFields"))
})
t.Run("when data from query aggregated to one time series", func(t *testing.T) {
data, err := loadTestFile("./test-data/1-series-response-agg-one-metric.json")
require.NoError(t, err)
pkg/tsdb/cloudmonitoring/types.go
+4
View File
@@ -50,6 +50,10 @@ type (
SecondaryCrossSeriesReducer string `json:"secondaryCrossSeriesReducer"`
SecondaryPerSeriesAligner string `json:"secondaryPerSeriesAligner"`
SecondaryGroupBys []string `json:"secondaryGroupBys"`
// Preprocessor is not part of the GCM API but added for simplicity
// It will overwrite AligmentPeriod, CrossSeriesReducer, PerSeriesAligner, GroupBys
// and its secondary counterparts
Preprocessor string `json:"preprocessor"`
}
// sloQuery is an internal convention but the API is the same as timeSeriesList