From 4952f6fc582348c726c5b618f3c424ed6d808dd2 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Fri, 10 Sep 2021 16:56:15 -0400 Subject: [PATCH] Prometheus: Add interpolation for built-in-time variables to backend (#39051) * Add grafana variable interpolation to backend * Remove newlines --- pkg/tsdb/intervalv2/intervalv2.go | 29 +-- pkg/tsdb/prometheus/prometheus.go | 90 +++++++--- pkg/tsdb/prometheus/prometheus_test.go | 235 +++++++++++++++++++++---- 3 files changed, 280 insertions(+), 74 deletions(-) diff --git a/pkg/tsdb/intervalv2/intervalv2.go b/pkg/tsdb/intervalv2/intervalv2.go index 886c521e379..f62139bdc29 100644 --- a/pkg/tsdb/intervalv2/intervalv2.go +++ b/pkg/tsdb/intervalv2/intervalv2.go @@ -82,31 +82,40 @@ func (ic *intervalCalculator) CalculateSafeInterval(timerange backend.TimeRange, // queryIntervalMS is a pre-calculated numeric representation of the query interval in milliseconds. func GetIntervalFrom(dsInterval, queryInterval string, queryIntervalMS int64, defaultInterval time.Duration) (time.Duration, error) { // Apparently we are setting default value of queryInterval to 0s now - if queryInterval == "0s" { - queryInterval = "" + interval := queryInterval + if interval == "0s" { + interval = "" } - - if queryInterval == "" { + if interval == "" { if queryIntervalMS != 0 { return time.Duration(queryIntervalMS) * time.Millisecond, nil } } - interval := queryInterval - if queryInterval == "" && dsInterval != "" { + if interval == "" && dsInterval != "" { interval = dsInterval } if interval == "" { return defaultInterval, nil } - interval = strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1) - isPureNum, err := regexp.MatchString(`^\d+$`, interval) + + parsedInterval, err := ParseIntervalStringToTimeDuration(interval) + if err != nil { + return time.Duration(0), err + } + + return parsedInterval, nil +} + +func ParseIntervalStringToTimeDuration(interval string) (time.Duration, error) { + formattedInterval := strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1) + isPureNum, err := regexp.MatchString(`^\d+$`, formattedInterval) if err != nil { return time.Duration(0), err } if isPureNum { - interval += "s" + formattedInterval += "s" } - parsedInterval, err := time.ParseDuration(interval) + parsedInterval, err := time.ParseDuration(formattedInterval) if err != nil { return time.Duration(0), err } diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 001ef65ce53..a8b00d92399 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -5,8 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "math" "net/http" "regexp" + "strconv" "strings" "time" @@ -15,7 +17,6 @@ import ( sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/plugins/backendplugin" @@ -41,6 +42,17 @@ type DatasourceInfo struct { TimeInterval string } +type QueryModel struct { + Expr string `json:"expr"` + LegendFormat string `json:"legendFormat"` + Interval string `json:"interval"` + IntervalMS int64 `json:"intervalMS"` + StepMode string `json:"stepMode"` + RangeQuery bool `json:"range"` + InstantQuery bool `json:"instant"` + IntervalFactor int64 `json:"intervalFactor"` +} + type Service struct { httpClientProvider httpclient.Provider intervalCalculator intervalv2.Calculator @@ -130,7 +142,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) Responses: backend.Responses{}, } - queries, err := s.parseQuery(req.Queries, dsInfo) + queries, err := s.parseQuery(req, dsInfo) if err != nil { return &result, err } @@ -225,48 +237,59 @@ func formatLegend(metric model.Metric, query *PrometheusQuery) string { return string(result) } -func (s *Service) parseQuery(queries []backend.DataQuery, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) { +func (s *Service) parseQuery(queryContext *backend.QueryDataRequest, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) { qs := []*PrometheusQuery{} - for _, queryModel := range queries { - jsonModel, err := simplejson.NewJson(queryModel.JSON) - if err != nil { - return nil, err - } - expr, err := jsonModel.Get("expr").String() + for _, query := range queryContext.Queries { + model := &QueryModel{} + err := json.Unmarshal(query.JSON, model) if err != nil { return nil, err } - format := jsonModel.Get("legendFormat").MustString("") - - start := queryModel.TimeRange.From - end := queryModel.TimeRange.To - queryInterval := jsonModel.Get("interval").MustString("") - - minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, 0, 15*time.Second) + //Calculate interval + queryInterval := model.Interval + //If we are using variable or interval/step, we will replace it with calculated interval + if queryInterval == "$__interval" || queryInterval == "$__interval_ms" { + queryInterval = "" + } + minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, model.IntervalMS, 15*time.Second) if err != nil { return nil, err } - calculatedInterval := s.intervalCalculator.Calculate(queries[0].TimeRange, minInterval) - - safeInterval := s.intervalCalculator.CalculateSafeInterval(queries[0].TimeRange, int64(safeRes)) + calculatedInterval := s.intervalCalculator.Calculate(query.TimeRange, minInterval) + safeInterval := s.intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeRes)) adjustedInterval := safeInterval.Value if calculatedInterval.Value > safeInterval.Value { adjustedInterval = calculatedInterval.Value } - intervalFactor := jsonModel.Get("intervalFactor").MustInt64(1) - step := time.Duration(int64(adjustedInterval) * intervalFactor) + intervalFactor := model.IntervalFactor + if intervalFactor == 0 { + intervalFactor = 1 + } + + interval := time.Duration(int64(adjustedInterval) * intervalFactor) + intervalMs := int64(interval / time.Millisecond) + rangeS := query.TimeRange.To.Unix() - query.TimeRange.From.Unix() + + // Interpolate variables in expr + expr := model.Expr + expr = strings.ReplaceAll(expr, "$__interval_ms", strconv.FormatInt(intervalMs, 10)) + expr = strings.ReplaceAll(expr, "$__interval", intervalv2.FormatDuration(interval)) + expr = strings.ReplaceAll(expr, "$__range_ms", strconv.FormatInt(rangeS*1000, 10)) + expr = strings.ReplaceAll(expr, "$__range_s", strconv.FormatInt(rangeS, 10)) + expr = strings.ReplaceAll(expr, "$__range", strconv.FormatInt(rangeS, 10)+"s") + expr = strings.ReplaceAll(expr, "$__rate_interval", intervalv2.FormatDuration(calculateRateInterval(interval, dsInfo.TimeInterval, s.intervalCalculator))) qs = append(qs, &PrometheusQuery{ Expr: expr, - Step: step, - LegendFormat: format, - Start: start, - End: end, - RefId: queryModel.RefID, + Step: interval, + LegendFormat: model.LegendFormat, + Start: query.TimeRange.From, + End: query.TimeRange.To, + RefId: query.RefID, }) } @@ -317,3 +340,18 @@ func ConvertAPIError(err error) error { } return err } + +func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration { + scrape := scrapeInterval + if scrape == "" { + scrape = "15s" + } + + scrapeIntervalDuration, err := intervalv2.ParseIntervalStringToTimeDuration(scrape) + if err != nil { + return time.Duration(0) + } + + rateInterval := time.Duration(int(math.Max(float64(interval+scrapeIntervalDuration), float64(4)*float64(scrapeIntervalDuration)))) + return rateInterval +} diff --git a/pkg/tsdb/prometheus/prometheus_test.go b/pkg/tsdb/prometheus/prometheus_test.go index e4604af19bd..3293c167252 100644 --- a/pkg/tsdb/prometheus/prometheus_test.go +++ b/pkg/tsdb/prometheus/prometheus_test.go @@ -13,11 +13,7 @@ import ( var now = time.Now() -func TestPrometheus(t *testing.T) { - service := Service{ - intervalCalculator: intervalv2.NewCalculator(), - } - +func TestPrometheus_formatLeged(t *testing.T) { t.Run("converting metric name", func(t *testing.T) { metric := map[p.LabelName]p.LabelValue{ p.LabelName("app"): p.LabelValue("backend"), @@ -44,89 +40,252 @@ func TestPrometheus(t *testing.T) { require.Equal(t, `http_request_total{app="backend", device="mobile"}`, formatLegend(metric, query)) }) +} + +func TestPrometheus_parseQuery(t *testing.T) { + service := Service{ + intervalCalculator: intervalv2.NewCalculator(), + } t.Run("parsing query model with step", func(t *testing.T) { - query := queryContext(`{ - "expr": "go_goroutines", - "format": "time_series", - "refId": "A" - }`) timeRange := backend.TimeRange{ From: now, To: now.Add(12 * time.Hour), } - query.TimeRange = timeRange - models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{}) + + query := queryContext(`{ + "expr": "go_goroutines", + "format": "time_series", + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) require.NoError(t, err) require.Equal(t, time.Second*30, models[0].Step) }) t.Run("parsing query model without step parameter", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(1 * time.Hour), + } + query := queryContext(`{ "expr": "go_goroutines", "format": "time_series", "intervalFactor": 1, "refId": "A" - }`) - models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{}) - require.NoError(t, err) - require.Equal(t, time.Minute*2, models[0].Step) + }`, timeRange) - timeRange := backend.TimeRange{ - From: now, - To: now.Add(1 * time.Hour), - } - query.TimeRange = timeRange - models, err = service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{}) + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) require.NoError(t, err) require.Equal(t, time.Second*15, models[0].Step) }) t.Run("parsing query model with high intervalFactor", func(t *testing.T) { - models, err := service.parseQuery([]backend.DataQuery{queryContext(`{ + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ "expr": "go_goroutines", "format": "time_series", "intervalFactor": 10, "refId": "A" - }`)}, &DatasourceInfo{}) + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) require.NoError(t, err) require.Equal(t, time.Minute*20, models[0].Step) }) t.Run("parsing query model with low intervalFactor", func(t *testing.T) { - models, err := service.parseQuery([]backend.DataQuery{queryContext(`{ + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ "expr": "go_goroutines", "format": "time_series", "intervalFactor": 1, "refId": "A" - }`)}, &DatasourceInfo{}) + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) require.NoError(t, err) require.Equal(t, time.Minute*2, models[0].Step) }) t.Run("parsing query model specified scrape-interval in the data source", func(t *testing.T) { - models, err := service.parseQuery([]backend.DataQuery{queryContext(`{ + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ "expr": "go_goroutines", "format": "time_series", "intervalFactor": 1, "refId": "A" - }`)}, &DatasourceInfo{ + }`, timeRange) + + dsInfo := &DatasourceInfo{ TimeInterval: "240s", - }) + } + models, err := service.parseQuery(query, dsInfo) require.NoError(t, err) require.Equal(t, time.Minute*4, models[0].Step) }) + + t.Run("parsing query model with $__interval variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__interval]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [2m]})", models[0].Expr) + }) + + t.Run("parsing query model with $__interval_ms variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__interval_ms]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [120000]})", models[0].Expr) + }) + + t.Run("parsing query model with $__interval_ms and $__interval variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__interval_ms]}) + rate(ALERTS{job=\"test\" [$__interval]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [120000]}) + rate(ALERTS{job=\"test\" [2m]})", models[0].Expr) + }) + + t.Run("parsing query model with $__range variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__range]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [172800s]})", models[0].Expr) + }) + + t.Run("parsing query model with $__range_s variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__range_s]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [172800]})", models[0].Expr) + }) + + t.Run("parsing query model with $__range_ms variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__range_ms]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [172800000]})", models[0].Expr) + }) + + t.Run("parsing query model with $__rate_interval variable", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(5 * time.Minute), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [1m]})", models[0].Expr) + }) } -func queryContext(json string) backend.DataQuery { - timeRange := backend.TimeRange{ - From: now, - To: now.Add(48 * time.Hour), - } - return backend.DataQuery{ - TimeRange: timeRange, - RefID: "A", - JSON: []byte(json), +func queryContext(json string, timeRange backend.TimeRange) *backend.QueryDataRequest { + return &backend.QueryDataRequest{ + Queries: []backend.DataQuery{ + { + JSON: []byte(json), + TimeRange: timeRange, + RefID: "A", + }, + }, } }