public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce TSDB service (#31520)

Browse Source 
* Introduce TSDB service

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.org>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Zoltán Bedi <zoltan.bedi@gmail.com>
This commit is contained in:
Arve Knudsen
2021-03-08 07:02:49 +01:00
committed by GitHub
co-authored by Erik Sundell Will Browne Torkel Ödegaard Will Browne Zoltán Bedi
parent c899bf3592
commit b79e61656a
203 changed files with 5269 additions and 4776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/expr/graph.go
+9 -9
View File
@@ -28,7 +28,7 @@ type Node interface {
ID() int64 // ID() allows the gonum graph node interface to be fulfilled
NodeType() NodeType
RefID() string
Execute(c context.Context, vars mathexp.Vars) (mathexp.Results, error)
Execute(c context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error)
String() string
}
@@ -37,10 +37,10 @@ type DataPipeline []Node
// execute runs all the command/datasource requests in the pipeline return a
// map of the refId of the of each command
func (dp *DataPipeline) execute(c context.Context) (mathexp.Vars, error) {
func (dp *DataPipeline) execute(c context.Context, s *Service) (mathexp.Vars, error) {
vars := make(mathexp.Vars)
for _, node := range *dp {
res, err := node.Execute(c, vars)
res, err := node.Execute(c, vars, s)
if err != nil {
return nil, err
}
@@ -52,8 +52,8 @@ func (dp *DataPipeline) execute(c context.Context) (mathexp.Vars, error) {
// BuildPipeline builds a graph of the nodes, and returns the nodes in an
// executable order.
func buildPipeline(req *backend.QueryDataRequest) (DataPipeline, error) {
graph, err := buildDependencyGraph(req)
func (s *Service) buildPipeline(req *backend.QueryDataRequest) (DataPipeline, error) {
graph, err := s.buildDependencyGraph(req)
if err != nil {
return nil, err
}
@@ -67,8 +67,8 @@ func buildPipeline(req *backend.QueryDataRequest) (DataPipeline, error) {
}
// buildDependencyGraph returns a dependency graph for a set of queries.
func buildDependencyGraph(req *backend.QueryDataRequest) (*simple.DirectedGraph, error) {
graph, err := buildGraph(req)
func (s *Service) buildDependencyGraph(req *backend.QueryDataRequest) (*simple.DirectedGraph, error) {
graph, err := s.buildGraph(req)
if err != nil {
return nil, err
}
@@ -113,7 +113,7 @@ func buildNodeRegistry(g *simple.DirectedGraph) map[string]Node {
}
// buildGraph creates a new graph populated with nodes for every query.
func buildGraph(req *backend.QueryDataRequest) (*simple.DirectedGraph, error) {
func (s *Service) buildGraph(req *backend.QueryDataRequest) (*simple.DirectedGraph, error) {
dp := simple.NewDirectedGraph()
for _, query := range req.Queries {
@@ -139,7 +139,7 @@ func buildGraph(req *backend.QueryDataRequest) (*simple.DirectedGraph, error) {
case DatasourceName:
node, err = buildCMDNode(dp, rn)
default: // If it's not an expression query, it's a data source query.
node, err = buildDSNode(dp, rn, req.PluginContext.OrgID)
node, err = s.buildDSNode(dp, rn, req.PluginContext.OrgID)
}
if err != nil {
return nil, err
pkg/expr/nodes.go
+4 -4
View File
@@ -82,7 +82,7 @@ func (gn *CMDNode) NodeType() NodeType {
// Execute runs the node and adds the results to vars. If the node requires
// other nodes they must have already been executed and their results must
// already by in vars.
func (gn *CMDNode) Execute(ctx context.Context, vars mathexp.Vars) (mathexp.Results, error) {
func (gn *CMDNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error) {
return gn.Command.Execute(ctx, vars)
}
@@ -142,7 +142,7 @@ func (dn *DSNode) NodeType() NodeType {
return TypeDatasourceNode
}
func buildDSNode(dp *simple.DirectedGraph, rn *rawNode, orgID int64) (*DSNode, error) {
func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, orgID int64) (*DSNode, error) {
encodedQuery, err := json.Marshal(rn.Query)
if err != nil {
return nil, err
@@ -203,7 +203,7 @@ func buildDSNode(dp *simple.DirectedGraph, rn *rawNode, orgID int64) (*DSNode, e
// Execute runs the node and adds the results to vars. If the node requires
// other nodes they must have already been executed and their results must
// already by in vars.
func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars) (mathexp.Results, error) {
func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (mathexp.Results, error) {
pc := backend.PluginContext{
OrgID: dn.orgID,
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
@@ -223,7 +223,7 @@ func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars) (mathexp.Resul
},
}
resp, err := QueryData(ctx, &backend.QueryDataRequest{
resp, err := s.queryData(ctx, &backend.QueryDataRequest{
PluginContext: pc,
Queries: q,
})
pkg/expr/service.go
+5 -3
View File
@@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
)
// DatasourceName is the string constant used as the datasource name in requests
@@ -21,7 +22,8 @@ const DatasourceUID = "-100"
// Service is service representation for expression handling.
type Service struct {
Cfg *setting.Cfg
Cfg *setting.Cfg
DataService *tsdb.Service
}
func (s *Service) isDisabled() bool {
@@ -33,13 +35,13 @@ func (s *Service) isDisabled() bool {
// BuildPipeline builds a pipeline from a request.
func (s *Service) BuildPipeline(req *backend.QueryDataRequest) (DataPipeline, error) {
return buildPipeline(req)
return s.buildPipeline(req)
}
// ExecutePipeline executes an expression pipeline and returns all the results.
func (s *Service) ExecutePipeline(ctx context.Context, pipeline DataPipeline) (*backend.QueryDataResponse, error) {
res := backend.NewQueryDataResponse()
vars, err := pipeline.execute(ctx)
vars, err := pipeline.execute(ctx, s)
if err != nil {
return nil, err
}
pkg/expr/service_test.go
+28 -19
View File
@@ -12,6 +12,9 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/stretchr/testify/require"
)
@@ -21,9 +24,21 @@ func TestService(t *testing.T) {
data.NewField("time", nil, []*time.Time{utp(1)}),
data.NewField("value", nil, []*float64{fp(2)}))
registerEndPoint(dsDF)
s := Service{}
dataSvc := tsdb.NewService()
dataSvc.PluginManager = &manager.PluginManager{
BackendPluginManager: fakeBackendPM{},
}
s := Service{DataService: &dataSvc}
me := &mockEndpoint{
Frames: []*data.Frame{dsDF},
}
s.DataService.RegisterQueryHandler("test", func(*models.DataSource) (plugins.DataPlugin, error) {
return me, nil
})
bus.AddHandler("test", func(query *models.GetDataSourceQuery) error {
query.Result = &models.DataSource{Id: 1, OrgId: 1, Type: "test"}
return nil
})
queries := []backend.DataQuery{
{
@@ -87,27 +102,21 @@ type mockEndpoint struct {
Frames data.Frames
}
func (me *mockEndpoint) Query(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
return &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
func (me *mockEndpoint) DataQuery(ctx context.Context, ds *models.DataSource, query plugins.DataQuery) (
plugins.DataResponse, error) {
return plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{
"A": {
Dataframes: tsdb.NewDecodedDataFrames(me.Frames),
Dataframes: plugins.NewDecodedDataFrames(me.Frames),
},
},
}, nil
}
func registerEndPoint(df ...*data.Frame) {
me := &mockEndpoint{
Frames: df,
}
endpoint := func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return me, nil
}
type fakeBackendPM struct {
backendplugin.Manager
}
tsdb.RegisterTsdbQueryEndpoint("test", endpoint)
bus.AddHandler("test", func(query *models.GetDataSourceQuery) error {
query.Result = &models.DataSource{Id: 1, OrgId: 1, Type: "test"}
return nil
})
func (pm fakeBackendPM) GetDataPlugin(string) interface{} {
return nil
}
pkg/expr/transform.go
+24 -23
View File
@@ -10,7 +10,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
@@ -35,7 +35,7 @@ func init() {
}
// WrapTransformData creates and executes transform requests
func (s *Service) WrapTransformData(ctx context.Context, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
func (s *Service) WrapTransformData(ctx context.Context, query plugins.DataQuery) (plugins.DataResponse, error) {
sdkReq := &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
OrgID: query.User.OrgId,
@@ -46,12 +46,12 @@ func (s *Service) WrapTransformData(ctx context.Context, query *tsdb.TsdbQuery)
for _, q := range query.Queries {
modelJSON, err := q.Model.MarshalJSON()
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
sdkReq.Queries = append(sdkReq.Queries, backend.DataQuery{
JSON: modelJSON,
Interval: time.Duration(q.IntervalMs) * time.Millisecond,
RefID: q.RefId,
Interval: time.Duration(q.IntervalMS) * time.Millisecond,
RefID: q.RefID,
MaxDataPoints: q.MaxDataPoints,
QueryType: q.QueryType,
TimeRange: backend.TimeRange{
@@ -62,16 +62,16 @@ func (s *Service) WrapTransformData(ctx context.Context, query *tsdb.TsdbQuery)
}
pbRes, err := s.TransformData(ctx, sdkReq)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
tR := &tsdb.Response{
Results: make(map[string]*tsdb.QueryResult, len(pbRes.Responses)),
tR := plugins.DataResponse{
Results: make(map[string]plugins.DataQueryResult, len(pbRes.Responses)),
}
for refID, res := range pbRes.Responses {
tRes := &tsdb.QueryResult{
RefId: refID,
Dataframes: tsdb.NewDecodedDataFrames(res.Frames),
tRes := plugins.DataQueryResult{
RefID: refID,
Dataframes: plugins.NewDecodedDataFrames(res.Frames),
}
// if len(res.JsonMeta) != 0 {
// tRes.Meta = simplejson.NewFromAny(res.JsonMeta)
@@ -158,9 +158,9 @@ func hiddenRefIDs(queries []backend.DataQuery) (map[string]struct{}, error) {
return hidden, nil
}
// QueryData is called used to query datasources that are not expression commands, but are used
// queryData is called used to query datasources that are not expression commands, but are used
// alongside expressions and/or are the input of an expression command.
func QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
func (s *Service) queryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
if len(req.Queries) == 0 {
return nil, fmt.Errorf("zero queries found in datasource request")
}
@@ -184,15 +184,15 @@ func QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.Que
}
// Convert plugin-model (datasource) queries to tsdb queries
queries := make([]*tsdb.Query, len(req.Queries))
queries := make([]plugins.DataSubQuery, len(req.Queries))
for i, query := range req.Queries {
sj, err := simplejson.NewJson(query.JSON)
if err != nil {
return nil, err
}
queries[i] = &tsdb.Query{
RefId: query.RefID,
IntervalMs: query.Interval.Milliseconds(),
queries[i] = plugins.DataSubQuery{
RefID: query.RefID,
IntervalMS: query.Interval.Milliseconds(),
MaxDataPoints: query.MaxDataPoints,
QueryType: query.QueryType,
DataSource: getDsInfo.Result,
@@ -201,20 +201,21 @@ func QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.Que
}
// For now take Time Range from first query.
timeRange := tsdb.NewTimeRange(strconv.FormatInt(req.Queries[0].TimeRange.From.Unix()*1000, 10), strconv.FormatInt(req.Queries[0].TimeRange.To.Unix()*1000, 10))
timeRange := plugins.NewDataTimeRange(strconv.FormatInt(req.Queries[0].TimeRange.From.Unix()*1000, 10),
strconv.FormatInt(req.Queries[0].TimeRange.To.Unix()*1000, 10))
tQ := &tsdb.TsdbQuery{
TimeRange: timeRange,
tQ := plugins.DataQuery{
TimeRange: &timeRange,
Queries: queries,
}
// Execute the converted queries
tsdbRes, err := tsdb.HandleRequest(ctx, getDsInfo.Result, tQ)
tsdbRes, err := s.DataService.HandleRequest(ctx, getDsInfo.Result, tQ)
if err != nil {
return nil, err
}
// Convert tsdb results (map) to plugin-model/datasource (slice) results.
// Only error, tsdb.Series, and encoded Dataframes responses are mapped.
// Only error, Series, and encoded Dataframes responses are mapped.
responses := make(map[string]backend.DataResponse, len(tsdbRes.Results))
for refID, res := range tsdbRes.Results {
pRes := backend.DataResponse{}
@@ -233,7 +234,7 @@ func QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.Que
}
for _, series := range res.Series {
frame, err := tsdb.SeriesToFrame(series)
frame, err := plugins.SeriesToFrame(series)
frame.RefID = refID
if err != nil {
return nil, err