public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce TSDB service (#31520)

Browse Source 
* Introduce TSDB service

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.org>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Zoltán Bedi <zoltan.bedi@gmail.com>
This commit is contained in:
Arve Knudsen
2021-03-08 07:02:49 +01:00
committed by GitHub
co-authored by Erik Sundell Will Browne Torkel Ödegaard Will Browne Zoltán Bedi
parent c899bf3592
commit b79e61656a
203 changed files with 5269 additions and 4776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/prometheus/prometheus.go
+35 -30
View File
@@ -15,7 +15,8 @@ import (
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/interval"
"github.com/prometheus/client_golang/api"
apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/common/model"
@@ -23,6 +24,8 @@ import (
type PrometheusExecutor struct {
Transport http.RoundTripper
intervalCalculator interval.Calculator
}
type basicAuthTransport struct {
@@ -37,28 +40,25 @@ func (bat basicAuthTransport) RoundTrip(req *http.Request) (*http.Response, erro
return bat.Transport.RoundTrip(req)
}
func NewPrometheusExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
transport, err := dsInfo.GetHttpTransport()
if err != nil {
return nil, err
}
return &PrometheusExecutor{
Transport: transport,
Transport: transport,
intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
}, nil
}
var (
plog log.Logger
legendFormat *regexp.Regexp
intervalCalculator tsdb.IntervalCalculator
plog log.Logger
legendFormat *regexp.Regexp = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
)
func init() {
plog = log.New("tsdb.prometheus")
tsdb.RegisterTsdbQueryEndpoint("prometheus", NewPrometheusExecutor)
legendFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Second * 1})
}
func (e *PrometheusExecutor) getClient(dsInfo *models.DataSource) (apiv1.API, error) {
@@ -83,19 +83,20 @@ func (e *PrometheusExecutor) getClient(dsInfo *models.DataSource) (apiv1.API, er
return apiv1.NewAPI(client), nil
}
func (e *PrometheusExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
func (e *PrometheusExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
tsdbQuery plugins.DataQuery) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
client, err := e.getClient(dsInfo)
if err != nil {
return nil, err
return result, err
}
queries, err := parseQuery(dsInfo, tsdbQuery.Queries, tsdbQuery)
queries, err := e.parseQuery(dsInfo, tsdbQuery)
if err != nil {
return nil, err
return result, err
}
for _, query := range queries {
@@ -116,12 +117,12 @@ func (e *PrometheusExecutor) Query(ctx context.Context, dsInfo *models.DataSourc
value, _, err := client.QueryRange(ctx, query.Expr, timeRange)
if err != nil {
return nil, err
return result, err
}
queryResult, err := parseResponse(value, query)
if err != nil {
return nil, err
return result, err
}
result.Results[query.RefId] = queryResult
}
@@ -147,9 +148,10 @@ func formatLegend(metric model.Metric, query *PrometheusQuery) string {
return string(result)
}
func parseQuery(dsInfo *models.DataSource, queries []*tsdb.Query, queryContext *tsdb.TsdbQuery) ([]*PrometheusQuery, error) {
func (e *PrometheusExecutor) parseQuery(dsInfo *models.DataSource, query plugins.DataQuery) (
[]*PrometheusQuery, error) {
qs := []*PrometheusQuery{}
for _, queryModel := range queries {
for _, queryModel := range query.Queries {
expr, err := queryModel.Model.Get("expr").String()
if err != nil {
return nil, err
@@ -157,23 +159,23 @@ func parseQuery(dsInfo *models.DataSource, queries []*tsdb.Query, queryContext *
format := queryModel.Model.Get("legendFormat").MustString("")
start, err := queryContext.TimeRange.ParseFrom()
start, err := query.TimeRange.ParseFrom()
if err != nil {
return nil, err
}
end, err := queryContext.TimeRange.ParseTo()
end, err := query.TimeRange.ParseTo()
if err != nil {
return nil, err
}
dsInterval, err := tsdb.GetIntervalFrom(dsInfo, queryModel.Model, time.Second*15)
dsInterval, err := interval.GetIntervalFrom(dsInfo, queryModel.Model, time.Second*15)
if err != nil {
return nil, err
}
intervalFactor := queryModel.Model.Get("intervalFactor").MustInt64(1)
interval := intervalCalculator.Calculate(queryContext.TimeRange, dsInterval)
interval := e.intervalCalculator.Calculate(*query.TimeRange, dsInterval)
step := time.Duration(int64(interval.Value) * intervalFactor)
qs = append(qs, &PrometheusQuery{
@@ -182,15 +184,15 @@ func parseQuery(dsInfo *models.DataSource, queries []*tsdb.Query, queryContext *
LegendFormat: format,
Start: start,
End: end,
RefId: queryModel.RefId,
RefId: queryModel.RefID,
})
}
return qs, nil
}
func parseResponse(value model.Value, query *PrometheusQuery) (*tsdb.QueryResult, error) {
queryRes := tsdb.NewQueryResult()
func parseResponse(value model.Value, query *PrometheusQuery) (plugins.DataQueryResult, error) {
var queryRes plugins.DataQueryResult
data, ok := value.(model.Matrix)
if !ok {
@@ -198,10 +200,10 @@ func parseResponse(value model.Value, query *PrometheusQuery) (*tsdb.QueryResult
}
for _, v := range data {
series := tsdb.TimeSeries{
series := plugins.DataTimeSeries{
Name: formatLegend(v.Metric, query),
Tags: make(map[string]string, len(v.Metric)),
Points: make([]tsdb.TimePoint, 0, len(v.Values)),
Points: make([]plugins.DataTimePoint, 0, len(v.Values)),
}
for k, v := range v.Metric {
@@ -209,10 +211,13 @@ func parseResponse(value model.Value, query *PrometheusQuery) (*tsdb.QueryResult
}
for _, k := range v.Values {
series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(float64(k.Value)), float64(k.Timestamp.Unix()*1000)))
series.Points = append(series.Points, plugins.DataTimePoint{
null.FloatFrom(float64(k.Value)),
null.FloatFrom(float64(k.Timestamp.Unix() * 1000)),
})
}
queryRes.Series = append(queryRes.Series, &series)
queryRes.Series = append(queryRes.Series, series)
}
return queryRes, nil
pkg/tsdb/prometheus/prometheus_test.go
+35 -19
View File
@@ -6,7 +6,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
p "github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
)
@@ -15,6 +15,9 @@ func TestPrometheus(t *testing.T) {
dsInfo := &models.DataSource{
JsonData: simplejson.New(),
}
plug, err := NewExecutor(dsInfo)
executor := plug.(*PrometheusExecutor)
require.NoError(t, err)
t.Run("converting metric name", func(t *testing.T) {
metric := map[p.LabelName]p.LabelValue{
@@ -50,14 +53,17 @@ func TestPrometheus(t *testing.T) {
"refId": "A"
}`
jsonModel, _ := simplejson.NewJson([]byte(json))
queryContext := &tsdb.TsdbQuery{}
queryModels := []*tsdb.Query{
queryModels := []plugins.DataSubQuery{
{Model: jsonModel},
}
queryContext.TimeRange = tsdb.NewTimeRange("12h", "now")
timeRange := plugins.NewDataTimeRange("12h", "now")
queryContext := plugins.DataQuery{
Queries: queryModels,
TimeRange: &timeRange,
}
models, err := parseQuery(dsInfo, queryModels, queryContext)
models, err := executor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Second*30, models[0].Step)
})
@@ -70,18 +76,22 @@ func TestPrometheus(t *testing.T) {
"refId": "A"
}`
jsonModel, _ := simplejson.NewJson([]byte(json))
queryContext := &tsdb.TsdbQuery{}
queryModels := []*tsdb.Query{
queryModels := []plugins.DataSubQuery{
{Model: jsonModel},
}
queryContext.TimeRange = tsdb.NewTimeRange("48h", "now")
models, err := parseQuery(dsInfo, queryModels, queryContext)
timeRange := plugins.NewDataTimeRange("48h", "now")
queryContext := plugins.DataQuery{
Queries: queryModels,
TimeRange: &timeRange,
}
models, err := executor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Minute*2, models[0].Step)
queryContext.TimeRange = tsdb.NewTimeRange("1h", "now")
models, err = parseQuery(dsInfo, queryModels, queryContext)
timeRange = plugins.NewDataTimeRange("1h", "now")
queryContext.TimeRange = &timeRange
models, err = executor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Second*15, models[0].Step)
})
@@ -94,14 +104,17 @@ func TestPrometheus(t *testing.T) {
"refId": "A"
}`
jsonModel, _ := simplejson.NewJson([]byte(json))
queryContext := &tsdb.TsdbQuery{}
queryModels := []*tsdb.Query{
queryModels := []plugins.DataSubQuery{
{Model: jsonModel},
}
queryContext.TimeRange = tsdb.NewTimeRange("48h", "now")
timeRange := plugins.NewDataTimeRange("48h", "now")
queryContext := plugins.DataQuery{
TimeRange: &timeRange,
Queries: queryModels,
}
models, err := parseQuery(dsInfo, queryModels, queryContext)
models, err := executor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Minute*20, models[0].Step)
})
@@ -114,14 +127,17 @@ func TestPrometheus(t *testing.T) {
"refId": "A"
}`
jsonModel, _ := simplejson.NewJson([]byte(json))
queryContext := &tsdb.TsdbQuery{}
queryModels := []*tsdb.Query{
queryModels := []plugins.DataSubQuery{
{Model: jsonModel},
}
queryContext.TimeRange = tsdb.NewTimeRange("48h", "now")
timeRange := plugins.NewDataTimeRange("48h", "now")
queryContext := plugins.DataQuery{
TimeRange: &timeRange,
Queries: queryModels,
}
models, err := parseQuery(dsInfo, queryModels, queryContext)
models, err := executor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Minute*2, models[0].Step)
})