Prometheus: Implement Streaming JSON Parser (#48477)

use `prometheusStreamingJSONParser` feature toggle to enable
This commit is contained in:
Todd Treece
2022-05-13 20:28:54 +02:00
committed by GitHub
parent e528f2e430
commit 87e8521591
39 changed files with 2877 additions and 34 deletions
+230
View File
@@ -0,0 +1,230 @@
package models
import (
"encoding/json"
"math"
"strconv"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/intervalv2"
)
//Internal interval and range variables
const (
varInterval = "$__interval"
varIntervalMs = "$__interval_ms"
varRange = "$__range"
varRangeS = "$__range_s"
varRangeMs = "$__range_ms"
varRateInterval = "$__rate_interval"
)
//Internal interval and range variables with {} syntax
//Repetitive code, we should have functionality to unify these
const (
varIntervalAlt = "${__interval}"
varIntervalMsAlt = "${__interval_ms}"
varRangeAlt = "${__range}"
varRangeSAlt = "${__range_s}"
varRangeMsAlt = "${__range_ms}"
varRateIntervalAlt = "${__rate_interval}"
)
type TimeSeriesQueryType string
const (
RangeQueryType TimeSeriesQueryType = "range"
InstantQueryType TimeSeriesQueryType = "instant"
ExemplarQueryType TimeSeriesQueryType = "exemplar"
UnknownQueryType TimeSeriesQueryType = "unknown"
)
var safeResolution = 11000
type QueryModel struct {
Expr string `json:"expr"`
LegendFormat string `json:"legendFormat"`
Interval string `json:"interval"`
IntervalMS int64 `json:"intervalMS"`
StepMode string `json:"stepMode"`
RangeQuery bool `json:"range"`
InstantQuery bool `json:"instant"`
ExemplarQuery bool `json:"exemplar"`
IntervalFactor int64 `json:"intervalFactor"`
UtcOffsetSec int64 `json:"utcOffsetSec"`
}
type TimeRange struct {
Start time.Time
End time.Time
Step time.Duration
}
type Query struct {
Expr string
Step time.Duration
LegendFormat string
Start time.Time
End time.Time
RefId string
InstantQuery bool
RangeQuery bool
ExemplarQuery bool
UtcOffsetSec int64
}
func Parse(query backend.DataQuery, timeInterval string, intervalCalculator intervalv2.Calculator, fromAlert bool) (*Query, error) {
model := &QueryModel{}
if err := json.Unmarshal(query.JSON, model); err != nil {
return nil, err
}
//Final interval value
interval, err := calculatePrometheusInterval(model, timeInterval, query, intervalCalculator)
if err != nil {
return nil, err
}
// Interpolate variables in expr
timeRange := query.TimeRange.To.Sub(query.TimeRange.From)
expr := interpolateVariables(model, interval, timeRange, intervalCalculator, timeInterval)
rangeQuery := model.RangeQuery
if !model.InstantQuery && !model.RangeQuery {
// In older dashboards, we were not setting range query param and !range && !instant was run as range query
rangeQuery = true
}
// We never want to run exemplar query for alerting
exemplarQuery := model.ExemplarQuery
if fromAlert {
exemplarQuery = false
}
return &Query{
Expr: expr,
Step: interval,
LegendFormat: model.LegendFormat,
Start: query.TimeRange.From,
End: query.TimeRange.To,
RefId: query.RefID,
InstantQuery: model.InstantQuery,
RangeQuery: rangeQuery,
ExemplarQuery: exemplarQuery,
UtcOffsetSec: model.UtcOffsetSec,
}, nil
}
func (query *Query) Type() TimeSeriesQueryType {
if query.InstantQuery {
return InstantQueryType
}
if query.RangeQuery {
return RangeQueryType
}
if query.ExemplarQuery {
return ExemplarQueryType
}
return UnknownQueryType
}
func (query *Query) TimeRange() TimeRange {
return TimeRange{
Step: query.Step,
// Align query range to step. It rounds start and end down to a multiple of step.
Start: alignTimeRange(query.Start, query.Step, query.UtcOffsetSec),
End: alignTimeRange(query.End, query.Step, query.UtcOffsetSec),
}
}
func calculatePrometheusInterval(model *QueryModel, timeInterval string, query backend.DataQuery, intervalCalculator intervalv2.Calculator) (time.Duration, error) {
queryInterval := model.Interval
//If we are using variable for interval/step, we will replace it with calculated interval
if isVariableInterval(queryInterval) {
queryInterval = ""
}
minInterval, err := intervalv2.GetIntervalFrom(timeInterval, queryInterval, model.IntervalMS, 15*time.Second)
if err != nil {
return time.Duration(0), err
}
calculatedInterval := intervalCalculator.Calculate(query.TimeRange, minInterval, query.MaxDataPoints)
safeInterval := intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeResolution))
adjustedInterval := safeInterval.Value
if calculatedInterval.Value > safeInterval.Value {
adjustedInterval = calculatedInterval.Value
}
if model.Interval == varRateInterval || model.Interval == varRateIntervalAlt {
// Rate interval is final and is not affected by resolution
return calculateRateInterval(adjustedInterval, timeInterval, intervalCalculator), nil
} else {
intervalFactor := model.IntervalFactor
if intervalFactor == 0 {
intervalFactor = 1
}
return time.Duration(int64(adjustedInterval) * intervalFactor), nil
}
}
func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration {
scrape := scrapeInterval
if scrape == "" {
scrape = "15s"
}
scrapeIntervalDuration, err := intervalv2.ParseIntervalStringToTimeDuration(scrape)
if err != nil {
return time.Duration(0)
}
rateInterval := time.Duration(int(math.Max(float64(interval+scrapeIntervalDuration), float64(4)*float64(scrapeIntervalDuration))))
return rateInterval
}
func interpolateVariables(model *QueryModel, interval time.Duration, timeRange time.Duration, intervalCalculator intervalv2.Calculator, timeInterval string) string {
expr := model.Expr
rangeMs := timeRange.Milliseconds()
rangeSRounded := int64(math.Round(float64(rangeMs) / 1000.0))
var rateInterval time.Duration
if model.Interval == varRateInterval || model.Interval == varRateIntervalAlt {
rateInterval = interval
} else {
rateInterval = calculateRateInterval(interval, timeInterval, intervalCalculator)
}
expr = strings.ReplaceAll(expr, varIntervalMs, strconv.FormatInt(int64(interval/time.Millisecond), 10))
expr = strings.ReplaceAll(expr, varInterval, intervalv2.FormatDuration(interval))
expr = strings.ReplaceAll(expr, varRangeMs, strconv.FormatInt(rangeMs, 10))
expr = strings.ReplaceAll(expr, varRangeS, strconv.FormatInt(rangeSRounded, 10))
expr = strings.ReplaceAll(expr, varRange, strconv.FormatInt(rangeSRounded, 10)+"s")
expr = strings.ReplaceAll(expr, varRateInterval, rateInterval.String())
// Repetitive code, we should have functionality to unify these
expr = strings.ReplaceAll(expr, varIntervalMsAlt, strconv.FormatInt(int64(interval/time.Millisecond), 10))
expr = strings.ReplaceAll(expr, varIntervalAlt, intervalv2.FormatDuration(interval))
expr = strings.ReplaceAll(expr, varRangeMsAlt, strconv.FormatInt(rangeMs, 10))
expr = strings.ReplaceAll(expr, varRangeSAlt, strconv.FormatInt(rangeSRounded, 10))
expr = strings.ReplaceAll(expr, varRangeAlt, strconv.FormatInt(rangeSRounded, 10)+"s")
expr = strings.ReplaceAll(expr, varRateIntervalAlt, rateInterval.String())
return expr
}
func isVariableInterval(interval string) bool {
if interval == varInterval || interval == varIntervalMs || interval == varRateInterval {
return true
}
//Repetitive code, we should have functionality to unify these
if interval == varIntervalAlt || interval == varIntervalMsAlt || interval == varRateIntervalAlt {
return true
}
return false
}
func alignTimeRange(t time.Time, step time.Duration, offset int64) time.Time {
return time.Unix(int64(math.Floor((float64(t.Unix()+offset)/step.Seconds()))*step.Seconds()-float64(offset)), 0)
}
+451
View File
@@ -0,0 +1,451 @@
package models_test
import (
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/grafana/grafana/pkg/tsdb/prometheus/models"
"github.com/stretchr/testify/require"
)
var (
now = time.Now()
intervalCalculator = intervalv2.NewCalculator()
)
func TestPrometheus_timeSeriesQuery_parseTimeSeriesQuery(t *testing.T) {
t.Run("parsing query from unified alerting", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(12 * time.Hour),
}
queryJson := `{
"expr": "go_goroutines",
"refId": "A",
"exemplar": true
}`
q := backend.DataQuery{
JSON: []byte(queryJson),
TimeRange: timeRange,
RefID: "A",
}
res, err := models.Parse(q, "15s", intervalCalculator, true)
require.NoError(t, err)
require.Equal(t, false, res.ExemplarQuery)
})
t.Run("parsing query model with step", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(12 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, time.Second*30, res.Step)
})
t.Run("parsing query model without step parameter", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(1 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, time.Second*15, res.Step)
})
t.Run("parsing query model with high intervalFactor", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 10,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, time.Minute*20, res.Step)
})
t.Run("parsing query model with low intervalFactor", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, time.Minute*2, res.Step)
})
t.Run("parsing query model specified scrape-interval in the data source", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "240s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, time.Minute*4, res.Step)
})
t.Run("parsing query model with $__interval variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__interval]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [2m]})", res.Expr)
})
t.Run("parsing query model with ${__interval} variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [${__interval}]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [2m]})", res.Expr)
})
t.Run("parsing query model with $__interval_ms variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__interval_ms]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [120000]})", res.Expr)
})
t.Run("parsing query model with $__interval_ms and $__interval variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__interval_ms]}) + rate(ALERTS{job=\"test\" [$__interval]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [120000]}) + rate(ALERTS{job=\"test\" [2m]})", res.Expr)
})
t.Run("parsing query model with ${__interval_ms} and ${__interval} variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [${__interval_ms}]}) + rate(ALERTS{job=\"test\" [${__interval}]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [120000]}) + rate(ALERTS{job=\"test\" [2m]})", res.Expr)
})
t.Run("parsing query model with $__range variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800s]})", res.Expr)
})
t.Run("parsing query model with $__range_s variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_s]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800]})", res.Expr)
})
t.Run("parsing query model with ${__range_s} variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [${__range_s}s]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800s]})", res.Expr)
})
t.Run("parsing query model with $__range_s variable below 0.5s", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(40 * time.Millisecond),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_s]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [0]})", res.Expr)
})
t.Run("parsing query model with $__range_s variable between 1-0.5s", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(800 * time.Millisecond),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_s]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [1]})", res.Expr)
})
t.Run("parsing query model with $__range_ms variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_ms]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800000]})", res.Expr)
})
t.Run("parsing query model with $__range_ms variable below 1s", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(20 * time.Millisecond),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_ms]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [20]})", res.Expr)
})
t.Run("parsing query model with $__rate_interval variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(5 * time.Minute),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})",
"format": "time_series",
"intervalFactor": 1,
"interval": "5m",
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [5m15s]})", res.Expr)
})
t.Run("parsing query model with $__rate_interval variable in expr and interval", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(5 * time.Minute),
}
q := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})",
"format": "time_series",
"intervalFactor": 1,
"interval": "$__rate_interval",
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [1m0s]})", res.Expr)
require.Equal(t, 1*time.Minute, res.Step)
})
t.Run("parsing query model of range query", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A",
"range": true
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, true, res.RangeQuery)
})
t.Run("parsing query model of range and instant query", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A",
"range": true,
"instant": true
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, true, res.RangeQuery)
require.Equal(t, true, res.InstantQuery)
})
t.Run("parsing query model of with no query type", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
q := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
res, err := models.Parse(q, "15s", intervalCalculator, false)
require.NoError(t, err)
require.Equal(t, true, res.RangeQuery)
})
}
func queryContext(json string, timeRange backend.TimeRange) backend.DataQuery {
return backend.DataQuery{
JSON: []byte(json),
TimeRange: timeRange,
RefID: "A",
}
}