Introduce TSDB service (#31520)

* Introduce TSDB service

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.org>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Zoltán Bedi <zoltan.bedi@gmail.com>
This commit is contained in:
Arve Knudsen
2021-03-08 07:02:49 +01:00
committed by GitHub
co-authored by Erik Sundell Will Browne Torkel Ödegaard Will Browne Zoltán Bedi
parent c899bf3592
commit b79e61656a
203 changed files with 5269 additions and 4776 deletions
+18 -17
View File
@@ -7,7 +7,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
"github.com/influxdata/influxdb-client-go/v2/api"
)
@@ -21,21 +21,22 @@ func init() {
}
// Query builds flux queries, executes them, and returns the results.
func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
glog.Debug("Received a query", "query", *tsdbQuery)
tRes := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult),
func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
plugins.DataResponse, error) {
glog.Debug("Received a query", "query", tsdbQuery)
tRes := plugins.DataResponse{
Results: make(map[string]plugins.DataQueryResult),
}
r, err := runnerFromDataSource(dsInfo)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
defer r.client.Close()
for _, query := range tsdbQuery.Queries {
qm, err := getQueryModelTSDB(query, tsdbQuery.TimeRange, dsInfo)
qm, err := getQueryModelTSDB(query, *tsdbQuery.TimeRange, dsInfo)
if err != nil {
tRes.Results[query.RefId] = &tsdb.QueryResult{Error: err}
tRes.Results[query.RefID] = plugins.DataQueryResult{Error: err}
continue
}
@@ -43,7 +44,7 @@ func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQ
maxSeries := dsInfo.JsonData.Get("maxSeries").MustInt(1000)
res := executeQuery(ctx, *qm, r, maxSeries)
tRes.Results[query.RefId] = backendDataResponseToTSDBResponse(&res, query.RefId)
tRes.Results[query.RefID] = backendDataResponseToDataResponse(&res, query.RefID)
}
return tRes, nil
}
@@ -94,16 +95,16 @@ func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) {
}, nil
}
// backendDataResponseToTSDBResponse takes the SDK's style response and changes it into a
// tsdb.QueryResult. This is a wrapper so less of existing code needs to be changed. This should
// backendDataResponseToDataResponse takes the SDK's style response and changes it into a
// plugins.DataQueryResult. This is a wrapper so less of existing code needs to be changed. This should
// be able to be removed in the near future https://github.com/grafana/grafana/pull/25472.
func backendDataResponseToTSDBResponse(dr *backend.DataResponse, refID string) *tsdb.QueryResult {
qr := &tsdb.QueryResult{RefId: refID}
qr.Error = dr.Error
func backendDataResponseToDataResponse(dr *backend.DataResponse, refID string) plugins.DataQueryResult {
qr := plugins.DataQueryResult{
RefID: refID,
Error: dr.Error,
}
if dr.Frames != nil {
qr.Dataframes = tsdb.NewDecodedDataFrames(dr.Frames)
qr.Dataframes = plugins.NewDecodedDataFrames(dr.Frames)
}
return qr
}
+5 -4
View File
@@ -7,7 +7,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
)
// queryOptions represents datasource configuration options
@@ -46,8 +46,9 @@ type queryModel struct {
// return model, nil
// }
// getQueryModelTSDB builds a queryModel from tsdb.Query information and datasource configuration (dsInfo).
func getQueryModelTSDB(query *tsdb.Query, timeRange *tsdb.TimeRange, dsInfo *models.DataSource) (*queryModel, error) {
// getQueryModelTSDB builds a queryModel from plugins.DataQuery information and datasource configuration (dsInfo).
func getQueryModelTSDB(query plugins.DataSubQuery, timeRange plugins.DataTimeRange,
dsInfo *models.DataSource) (*queryModel, error) {
model := &queryModel{}
queryBytes, err := query.Model.Encode()
if err != nil {
@@ -86,7 +87,7 @@ func getQueryModelTSDB(query *tsdb.Query, timeRange *tsdb.TimeRange, dsInfo *mod
if model.MaxDataPoints == 0 {
model.MaxDataPoints = 10000 // 10k/series should be a reasonable place to abort!
}
model.Interval = time.Millisecond * time.Duration(query.IntervalMs)
model.Interval = time.Millisecond * time.Duration(query.IntervalMS)
if model.Interval.Milliseconds() == 0 {
model.Interval = time.Millisecond // 1ms
}
+24 -26
View File
@@ -12,19 +12,19 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/influxdb/flux"
)
type InfluxDBExecutor struct {
type Executor struct {
// *models.DataSource
QueryParser *InfluxdbQueryParser
ResponseParser *ResponseParser
}
func NewInfluxDBExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return &InfluxDBExecutor{
func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
return &Executor{
QueryParser: &InfluxdbQueryParser{},
ResponseParser: &ResponseParser{},
}, nil
@@ -38,10 +38,10 @@ var ErrInvalidHttpMode error = errors.New("'httpMode' should be either 'GET' or
func init() {
glog = log.New("tsdb.influxdb")
tsdb.RegisterTsdbQueryEndpoint("influxdb", NewInfluxDBExecutor)
}
func (e *InfluxDBExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
plugins.DataResponse, error) {
glog.Debug("Received a query request", "numQueries", len(tsdbQuery.Queries))
version := dsInfo.JsonData.Get("version").MustString("")
@@ -54,14 +54,14 @@ func (e *InfluxDBExecutor) Query(ctx context.Context, dsInfo *models.DataSource,
// NOTE: the following path is currently only called from alerting queries
// In dashboards, the request runs through proxy and are managed in the frontend
query, err := e.getQuery(dsInfo, tsdbQuery.Queries, tsdbQuery)
query, err := e.getQuery(dsInfo, tsdbQuery)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
rawQuery, err := query.Build(tsdbQuery)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
if setting.Env == setting.Dev {
@@ -70,17 +70,17 @@ func (e *InfluxDBExecutor) Query(ctx context.Context, dsInfo *models.DataSource,
req, err := e.createRequest(ctx, dsInfo, rawQuery)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
httpClient, err := dsInfo.GetHttpClient()
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
@@ -88,41 +88,39 @@ func (e *InfluxDBExecutor) Query(ctx context.Context, dsInfo *models.DataSource,
}
}()
if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("InfluxDB returned error status: %s", resp.Status)
return plugins.DataResponse{}, fmt.Errorf("InfluxDB returned error status: %s", resp.Status)
}
var response Response
dec := json.NewDecoder(resp.Body)
dec.UseNumber()
if err := dec.Decode(&response); err != nil {
return nil, err
return plugins.DataResponse{}, err
}
if response.Err != nil {
return nil, response.Err
return plugins.DataResponse{}, response.Err
}
result := &tsdb.Response{}
result.Results = make(map[string]*tsdb.QueryResult)
result.Results["A"] = e.ResponseParser.Parse(&response, query)
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{
"A": e.ResponseParser.Parse(&response, query),
},
}
return result, nil
}
func (e *InfluxDBExecutor) getQuery(dsInfo *models.DataSource, queries []*tsdb.Query, context *tsdb.TsdbQuery) (*Query, error) {
if len(queries) == 0 {
func (e *Executor) getQuery(dsInfo *models.DataSource, query plugins.DataQuery) (*Query, error) {
if len(query.Queries) == 0 {
return nil, fmt.Errorf("query request contains no queries")
}
// The model supports multiple queries, but right now this is only used from
// alerting so we only needed to support batch executing 1 query at a time.
query, err := e.QueryParser.Parse(queries[0].Model, dsInfo)
if err != nil {
return nil, err
}
return query, nil
return e.QueryParser.Parse(query.Queries[0].Model, dsInfo)
}
func (e *InfluxDBExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource, query string) (*http.Request, error) {
func (e *Executor) createRequest(ctx context.Context, dsInfo *models.DataSource, query string) (*http.Request, error) {
u, err := url.Parse(dsInfo.Url)
if err != nil {
return nil, err
+2 -2
View File
@@ -12,14 +12,14 @@ import (
"github.com/stretchr/testify/require"
)
func TestInfluxDBExecutor_createRequest(t *testing.T) {
func TestExecutor_createRequest(t *testing.T) {
datasource := &models.DataSource{
Url: "http://awesome-influxdb:1337",
Database: "awesome-db",
JsonData: simplejson.New(),
}
query := "SELECT awesomeness FROM somewhere"
e := &InfluxDBExecutor{
e := &Executor{
QueryParser: &InfluxdbQueryParser{},
ResponseParser: &ResponseParser{},
}
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
type InfluxdbQueryParser struct{}
@@ -40,7 +40,7 @@ func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.Data
return nil, err
}
parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond*1)
parsedInterval, err := interval.GetIntervalFrom(dsInfo, model, time.Millisecond*1)
if err != nil {
return nil, err
}
+11 -10
View File
@@ -6,7 +6,8 @@ import (
"strconv"
"strings"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
var (
@@ -14,7 +15,7 @@ var (
regexpMeasurementPattern = regexp.MustCompile(`^\/.*\/$`)
)
func (query *Query) Build(queryContext *tsdb.TsdbQuery) (string, error) {
func (query *Query) Build(queryContext plugins.DataQuery) (string, error) {
var res string
if query.UseRawQuery && query.RawQuery != "" {
res = query.RawQuery
@@ -27,13 +28,13 @@ func (query *Query) Build(queryContext *tsdb.TsdbQuery) (string, error) {
res += query.renderTz()
}
calculator := tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{})
interval := calculator.Calculate(queryContext.TimeRange, query.Interval)
calculator := interval.NewCalculator(interval.CalculatorOptions{})
i := calculator.Calculate(*queryContext.TimeRange, query.Interval)
res = strings.ReplaceAll(res, "$timeFilter", query.renderTimeFilter(queryContext))
res = strings.ReplaceAll(res, "$interval", interval.Text)
res = strings.ReplaceAll(res, "$__interval_ms", strconv.FormatInt(interval.Milliseconds(), 10))
res = strings.ReplaceAll(res, "$__interval", interval.Text)
res = strings.ReplaceAll(res, "$interval", i.Text)
res = strings.ReplaceAll(res, "$__interval_ms", strconv.FormatInt(i.Milliseconds(), 10))
res = strings.ReplaceAll(res, "$__interval", i.Text)
return res, nil
}
@@ -77,7 +78,7 @@ func (query *Query) renderTags() []string {
return res
}
func (query *Query) renderTimeFilter(queryContext *tsdb.TsdbQuery) string {
func (query *Query) renderTimeFilter(queryContext plugins.DataQuery) string {
from := "now() - " + queryContext.TimeRange.From
to := ""
@@ -88,7 +89,7 @@ func (query *Query) renderTimeFilter(queryContext *tsdb.TsdbQuery) string {
return fmt.Sprintf("time > %s%s", from, to)
}
func (query *Query) renderSelectors(queryContext *tsdb.TsdbQuery) string {
func (query *Query) renderSelectors(queryContext plugins.DataQuery) string {
res := "SELECT "
var selectors []string
@@ -135,7 +136,7 @@ func (query *Query) renderWhereClause() string {
return res
}
func (query *Query) renderGroupBy(queryContext *tsdb.TsdbQuery) string {
func (query *Query) renderGroupBy(queryContext plugins.DataQuery) string {
groupBy := ""
for i, group := range query.GroupBy {
if i == 0 {
+7 -7
View File
@@ -4,7 +4,7 @@ import (
"fmt"
"strings"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
)
var renders map[string]QueryDefinition
@@ -15,7 +15,7 @@ type DefinitionParameters struct {
}
type QueryDefinition struct {
Renderer func(query *Query, queryContext *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string
Renderer func(query *Query, queryContext plugins.DataQuery, part *QueryPart, innerExpr string) string
Params []DefinitionParameters
}
@@ -97,14 +97,14 @@ func init() {
renders["alias"] = QueryDefinition{Renderer: aliasRenderer}
}
func fieldRenderer(query *Query, queryContext *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
func fieldRenderer(query *Query, queryContext plugins.DataQuery, part *QueryPart, innerExpr string) string {
if part.Params[0] == "*" {
return "*"
}
return fmt.Sprintf(`"%s"`, part.Params[0])
}
func functionRenderer(query *Query, queryContext *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
func functionRenderer(query *Query, queryContext plugins.DataQuery, part *QueryPart, innerExpr string) string {
for i, param := range part.Params {
if part.Type == "time" && param == "auto" {
part.Params[i] = "$__interval"
@@ -120,11 +120,11 @@ func functionRenderer(query *Query, queryContext *tsdb.TsdbQuery, part *QueryPar
return fmt.Sprintf("%s(%s)", part.Type, params)
}
func suffixRenderer(query *Query, queryContext *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
func suffixRenderer(query *Query, queryContext plugins.DataQuery, part *QueryPart, innerExpr string) string {
return fmt.Sprintf("%s %s", innerExpr, part.Params[0])
}
func aliasRenderer(query *Query, queryContext *tsdb.TsdbQuery, part *QueryPart, innerExpr string) string {
func aliasRenderer(query *Query, queryContext plugins.DataQuery, part *QueryPart, innerExpr string) string {
return fmt.Sprintf(`%s AS "%s"`, innerExpr, part.Params[0])
}
@@ -147,6 +147,6 @@ type QueryPart struct {
Params []string
}
func (qp *QueryPart) Render(query *Query, queryContext *tsdb.TsdbQuery, expr string) string {
func (qp *QueryPart) Render(query *Query, queryContext plugins.DataQuery, expr string) string {
return qp.Def.Renderer(query, queryContext, qp, expr)
}
+3 -2
View File
@@ -3,7 +3,7 @@ package influxdb
import (
"testing"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
)
func TestInfluxdbQueryPart(t *testing.T) {
@@ -27,7 +27,8 @@ func TestInfluxdbQueryPart(t *testing.T) {
{mode: "non_negative_difference", params: []string{}, input: "max(value)", expected: `non_negative_difference(max(value))`},
}
queryContext := &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("5m", "now")}
timeRange := plugins.NewDataTimeRange("5m", "now")
queryContext := plugins.DataQuery{TimeRange: &timeRange}
query := &Query{}
for _, tc := range tcs {
+8 -5
View File
@@ -6,7 +6,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
. "github.com/smartystreets/goconvey/convey"
)
@@ -27,8 +27,9 @@ func TestInfluxdbQueryBuilder(t *testing.T) {
tag1 := &Tag{Key: "hostname", Value: "server1", Operator: "="}
tag2 := &Tag{Key: "hostname", Value: "server2", Operator: "=", Condition: "OR"}
queryContext := &tsdb.TsdbQuery{
TimeRange: tsdb.NewTimeRange("5m", "now"),
timeRange := plugins.NewDataTimeRange("5m", "now")
queryContext := plugins.DataQuery{
TimeRange: &timeRange,
}
Convey("can build simple query", func() {
@@ -114,12 +115,14 @@ func TestInfluxdbQueryBuilder(t *testing.T) {
query := Query{}
Convey("render from: 2h to now-1h", func() {
query := Query{}
queryContext := &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("2h", "now-1h")}
timeRange := plugins.NewDataTimeRange("2h", "now-1h")
queryContext := plugins.DataQuery{TimeRange: &timeRange}
So(query.renderTimeFilter(queryContext), ShouldEqual, "time > now() - 2h and time < now() - 1h")
})
Convey("render from: 10m", func() {
queryContext := &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("10m", "now")}
timeRange := plugins.NewDataTimeRange("10m", "now")
queryContext := plugins.DataQuery{TimeRange: &timeRange}
So(query.renderTimeFilter(queryContext), ShouldEqual, "time > now() - 10m")
})
})
+11 -11
View File
@@ -8,7 +8,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
)
type ResponseParser struct{}
@@ -21,8 +21,8 @@ func init() {
legendFormat = regexp.MustCompile(`\[\[(\w+)(\.\w+)*\]\]*|\$\s*(\w+?)*`)
}
func (rp *ResponseParser) Parse(response *Response, query *Query) *tsdb.QueryResult {
queryRes := tsdb.NewQueryResult()
func (rp *ResponseParser) Parse(response *Response, query *Query) plugins.DataQueryResult {
var queryRes plugins.DataQueryResult
for _, result := range response.Results {
queryRes.Series = append(queryRes.Series, rp.transformRows(result.Series, queryRes, query)...)
@@ -34,22 +34,22 @@ func (rp *ResponseParser) Parse(response *Response, query *Query) *tsdb.QueryRes
return queryRes
}
func (rp *ResponseParser) transformRows(rows []Row, queryResult *tsdb.QueryResult, query *Query) tsdb.TimeSeriesSlice {
var result tsdb.TimeSeriesSlice
func (rp *ResponseParser) transformRows(rows []Row, queryResult plugins.DataQueryResult, query *Query) plugins.DataTimeSeriesSlice {
var result plugins.DataTimeSeriesSlice
for _, row := range rows {
for columnIndex, column := range row.Columns {
if column == "time" {
continue
}
var points tsdb.TimeSeriesPoints
var points plugins.DataTimeSeriesPoints
for _, valuePair := range row.Values {
point, err := rp.parseTimepoint(valuePair, columnIndex)
if err == nil {
points = append(points, point)
}
}
result = append(result, &tsdb.TimeSeries{
result = append(result, plugins.DataTimeSeries{
Name: rp.formatSeriesName(row, column, query),
Points: points,
Tags: row.Tags,
@@ -115,19 +115,19 @@ func (rp *ResponseParser) buildSeriesNameFromQuery(row Row, column string) strin
return fmt.Sprintf("%s.%s%s", row.Name, column, tagText)
}
func (rp *ResponseParser) parseTimepoint(valuePair []interface{}, valuePosition int) (tsdb.TimePoint, error) {
func (rp *ResponseParser) parseTimepoint(valuePair []interface{}, valuePosition int) (plugins.DataTimePoint, error) {
value := rp.parseValue(valuePair[valuePosition])
timestampNumber, ok := valuePair[0].(json.Number)
if !ok {
return tsdb.TimePoint{}, fmt.Errorf("valuePair[0] has invalid type: %#v", valuePair[0])
return plugins.DataTimePoint{}, fmt.Errorf("valuePair[0] has invalid type: %#v", valuePair[0])
}
timestamp, err := timestampNumber.Float64()
if err != nil {
return tsdb.TimePoint{}, err
return plugins.DataTimePoint{}, err
}
return tsdb.NewTimePoint(value, timestamp), nil
return plugins.DataTimePoint{value, null.FloatFrom(timestamp)}, nil
}
func (rp *ResponseParser) parseValue(value interface{}) null.Float {