public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

api/ds/query: simplify data sources lookup for queries and expressions (#41172)

Browse Source
This commit is contained in:
Ryan McKinley
2021-11-05 08:12:55 -07:00
committed by GitHub
parent c8b7373016
commit 3489721ed6
16 changed files with 255 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/metrics.go
+76 -81
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb/grafanads"
"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -33,42 +34,23 @@ func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext, reqDTO dtos.MetricReq
Queries: make([]plugins.DataSubQuery, 0, len(reqDTO.Queries)),
}
// Loop to see if we have an expression.
prevType := ""
var ds *models.DataSource
// Parse the queries
hasExpression := false
datasources := make(map[string]*models.DataSource, len(reqDTO.Queries))
for _, query := range reqDTO.Queries {
dsType := query.Get("datasource").MustString("")
if dsType == expr.DatasourceName {
return hs.handleExpressions(c, reqDTO)
ds, errRsp := hs.getDataSourceFromQuery(c, query, datasources)
if errRsp != nil {
return errRsp
}
if prevType != "" && prevType != dsType {
// For mixed datasource case, each data source is sent in a single request.
// So only the datasource from the first query is needed. As all requests
// should be the same data source.
hs.log.Debug("Can't process query since it's missing data source ID")
return response.Error(http.StatusBadRequest, "All queries must use the same datasource", nil)
}
if ds == nil {
// require ID for everything
dsID, err := query.Get("datasourceId").Int64()
if err != nil {
hs.log.Debug("Can't process query since it's missing data source ID")
return response.Error(http.StatusBadRequest, "Query missing data source ID", nil)
}
if dsID == grafanads.DatasourceID {
ds = grafanads.DataSourceModel(c.OrgId)
} else {
ds, err = hs.DataSourceCache.GetDatasource(dsID, c.SignedInUser, c.SkipCache)
if err != nil {
return hs.handleGetDataSourceError(err, dsID)
}
}
return response.Error(http.StatusBadRequest, "Datasource not found for query", nil)
}
datasources[ds.Uid] = ds
if expr.IsDataSource(ds.Uid) {
hasExpression = true
}
prevType = dsType
}
for _, query := range reqDTO.Queries {
hs.log.Debug("Processing metrics query", "query", query)
request.Queries = append(request.Queries, plugins.DataSubQuery{
@@ -81,6 +63,24 @@ func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext, reqDTO dtos.MetricReq
})
}
if hasExpression {
exprService := expr.Service{
Cfg: hs.Cfg,
DataService: hs.DataService,
}
qdr, err := exprService.WrapTransformData(c.Req.Context(), request)
if err != nil {
return response.Error(500, "expression request error", err)
}
return toMacronResponse(qdr)
}
ds := request.Queries[0].DataSource
if len(datasources) > 1 {
// We do not (yet) support mixed query type
return response.Error(http.StatusBadRequest, "All queries must use the same datasource", nil)
}
err := hs.PluginRequestValidator.Validate(ds.Url, nil)
if err != nil {
return response.Error(http.StatusForbidden, "Access denied", err)
@@ -99,6 +99,49 @@ func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext, reqDTO dtos.MetricReq
return toMacronResponse(resp)
}
func (hs *HTTPServer) getDataSourceFromQuery(c *models.ReqContext, query *simplejson.Json, history map[string]*models.DataSource) (*models.DataSource, response.Response) {
var err error
uid := query.Get("datasource").Get("uid").MustString()
// before 8.3 special types could be sent as datasource (expr)
if uid == "" {
uid = query.Get("datasource").MustString()
}
// check cache value
ds, ok := history[uid]
if ok {
return ds, nil
}
if expr.IsDataSource(uid) {
return expr.DataSourceModel(), nil
}
if uid == grafanads.DatasourceUID {
return grafanads.DataSourceModel(c.OrgId), nil
}
if uid != "" {
ds, err = hs.DataSourceCache.GetDatasourceByUID(uid, c.SignedInUser, c.SkipCache)
if err != nil {
return nil, hs.handleGetDataSourceError(err, uid)
}
return ds, nil
}
// Fallback to the datasourceId
id, err := query.Get("datasourceId").Int64()
if err != nil {
return nil, response.Error(http.StatusBadRequest, "Query missing data source ID/UID", nil)
}
ds, err = hs.DataSourceCache.GetDatasource(id, c.SignedInUser, c.SkipCache)
if err != nil {
return nil, hs.handleGetDataSourceError(err, id)
}
return ds, nil
}
func toMacronResponse(qdr *backend.QueryDataResponse) response.Response {
statusCode := http.StatusOK
for _, res := range qdr.Responses {
@@ -110,56 +153,8 @@ func toMacronResponse(qdr *backend.QueryDataResponse) response.Response {
return response.JSONStreaming(statusCode, qdr)
}
// handleExpressions handles POST /api/ds/query when there is an expression.
func (hs *HTTPServer) handleExpressions(c *models.ReqContext, reqDTO dtos.MetricRequest) response.Response {
timeRange := plugins.NewDataTimeRange(reqDTO.From, reqDTO.To)
request := plugins.DataQuery{
TimeRange: &timeRange,
Debug: reqDTO.Debug,
User: c.SignedInUser,
Queries: make([]plugins.DataSubQuery, 0, len(reqDTO.Queries)),
}
for _, query := range reqDTO.Queries {
hs.log.Debug("Processing metrics query", "query", query)
name := query.Get("datasource").MustString("")
datasourceID, err := query.Get("datasourceId").Int64()
if err != nil {
hs.log.Debug("Can't process query since it's missing data source ID")
return response.Error(400, "Query missing data source ID", nil)
}
if name != expr.DatasourceName {
// Expression requests have everything in one request, so need to check
// all data source queries for possible permission / not found issues.
if _, err = hs.DataSourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache); err != nil {
return hs.handleGetDataSourceError(err, datasourceID)
}
}
request.Queries = append(request.Queries, plugins.DataSubQuery{
RefID: query.Get("refId").MustString("A"),
MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
IntervalMS: query.Get("intervalMs").MustInt64(1000),
QueryType: query.Get("queryType").MustString(""),
Model: query,
})
}
exprService := expr.Service{
Cfg: hs.Cfg,
DataService: hs.DataService,
}
qdr, err := exprService.WrapTransformData(c.Req.Context(), request)
if err != nil {
return response.Error(500, "expression request error", err)
}
return toMacronResponse(qdr)
}
func (hs *HTTPServer) handleGetDataSourceError(err error, datasourceID int64) *response.NormalResponse {
hs.log.Debug("Encountered error getting data source", "err", err, "id", datasourceID)
func (hs *HTTPServer) handleGetDataSourceError(err error, datasourceRef interface{}) *response.NormalResponse {
hs.log.Debug("Encountered error getting data source", "err", err, "ref", datasourceRef)
if errors.Is(err, models.ErrDataSourceAccessDenied) {
return response.Error(403, "Access denied to data source", err)
}
pkg/expr/graph.go
+18 -5
View File
@@ -143,17 +143,30 @@ func (s *Service) buildGraph(req *Request) (*simple.DirectedGraph, error) {
RefID: query.RefID,
TimeRange: query.TimeRange,
QueryType: query.QueryType,
DatasourceUID: query.DatasourceUID,
DatasourceUID: query.GetDatasourceUID(),
}
isExpr, err := rn.IsExpressionQuery()
if err != nil {
return nil, err
numericDSID := float64(0) // legacy
if rn.DatasourceUID == "" {
if rv, ok := rn.Query["datasourceId"]; ok {
if sv, ok := rv.(float64); ok {
if sv == DatasourceID {
rn.DatasourceUID = DatasourceUID
}
if sv > 0 {
numericDSID = sv
}
}
}
}
if rn.DatasourceUID == "" && numericDSID == 0 {
return nil, fmt.Errorf("missing datasource uid in query with refId %v", query.RefID)
}
var node Node
if isExpr {
if rn.IsExpressionQuery() {
node, err = buildCMDNode(dp, rn)
} else {
node, err = s.buildDSNode(dp, rn, req)
pkg/expr/graph_test.go
+34 -36
View File
@@ -20,7 +20,7 @@ func TestServicebuildPipeLine(t *testing.T) {
Queries: []Query{
{
RefID: "A",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "B",
"reducer": "mean",
@@ -41,7 +41,7 @@ func TestServicebuildPipeLine(t *testing.T) {
Queries: []Query{
{
RefID: "A",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "$B",
"type": "math"
@@ -49,7 +49,7 @@ func TestServicebuildPipeLine(t *testing.T) {
},
{
RefID: "B",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "$A",
"type": "math"
@@ -65,7 +65,7 @@ func TestServicebuildPipeLine(t *testing.T) {
Queries: []Query{
{
RefID: "A",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "$A",
"type": "math"
@@ -81,7 +81,7 @@ func TestServicebuildPipeLine(t *testing.T) {
Queries: []Query{
{
RefID: "A",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "$B",
"type": "math"
@@ -97,7 +97,7 @@ func TestServicebuildPipeLine(t *testing.T) {
Queries: []Query{
{
RefID: "A",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"type": "classic_conditions",
"conditions": [
@@ -128,7 +128,7 @@ func TestServicebuildPipeLine(t *testing.T) {
},
{
RefID: "B",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "C",
"reducer": "mean",
@@ -149,7 +149,7 @@ func TestServicebuildPipeLine(t *testing.T) {
Queries: []Query{
{
RefID: "A",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"type": "classic_conditions",
"conditions": [
@@ -180,7 +180,7 @@ func TestServicebuildPipeLine(t *testing.T) {
},
{
RefID: "B",
DatasourceUID: DatasourceUID,
DatasourceUID: OldDatasourceUID,
JSON: json.RawMessage(`{
"expression": "A",
"reducer": "mean",
@@ -195,33 +195,31 @@ func TestServicebuildPipeLine(t *testing.T) {
},
expectErrContains: "classic conditions may not be the input for other expressions",
},
//{
// name: "Queries with new datasource ref object",
// req: &Request{
// Queries: []Query{
// {
// RefID: "A",
// JSON: json.RawMessage(`{
// "datasource": {
// "uid": "MyDS"
// }
// }`),
// },
// {
// RefID: "B",
// JSON: json.RawMessage(`{
// "datasource": {
// "uid": "MyDS"
// },
// "expression": "A",
// "reducer": "mean",
// "type": "reduce"
// }`),
// },
// },
// },
// expectedOrder: []string{"B", "A"},
//},
{
name: "Queries with new datasource ref object",
req: &Request{
Queries: []Query{
{
RefID: "A",
Datasource: DataSourceRef{
UID: DatasourceUID,
},
JSON: json.RawMessage(`{
"expression": "B",
"reducer": "mean",
"type": "reduce"
}`),
},
{
RefID: "B",
Datasource: DataSourceRef{
UID: "Fake",
},
},
},
},
expectedOrder: []string{"B", "A"},
},
}
s := Service{}
for _, tt := range tests {
pkg/expr/nodes.go
+11 -57
View File
@@ -30,62 +30,19 @@ type rawNode struct {
Query map[string]interface{}
QueryType string
TimeRange TimeRange
DatasourceUID string
DatasourceUID string // Gets populated from Either DatasourceUID or Datasource.UID
}
func (rn *rawNode) GetDatasourceUID() (string, error) {
if rn.DatasourceUID != "" {
return rn.DatasourceUID, nil
func (rn *rawNode) IsExpressionQuery() bool {
if IsDataSource(rn.DatasourceUID) {
return true
}
rawDs, ok := rn.Query["datasource"]
if !ok {
return "", fmt.Errorf("no datasource property found in query model")
if v, ok := rn.Query["datasourceId"]; ok {
if v == OldDatasourceUID {
return true
}
}
// For old queries with string datasource prop representing data source name
if dsName, ok := rawDs.(string); ok {
return dsName, nil
}
dsRef, ok := rawDs.(map[string]interface{})
if !ok {
return "", fmt.Errorf("data source property is not an object nor string, got %T", rawDs)
}
if dsUid, ok := dsRef["uid"].(string); ok {
return dsUid, nil
}
return "", fmt.Errorf("no datasource uid found for query, got %T", rn.Query)
}
func (rn *rawNode) IsExpressionQuery() (bool, error) {
if rn.DatasourceUID != "" {
return rn.DatasourceUID == DatasourceUID, nil
}
rawDs, ok := rn.Query["datasource"]
if !ok {
return false, fmt.Errorf("no datasource property found in query model")
}
// For old queries with string datasource prop representing data source name
dsName, ok := rawDs.(string)
if ok && dsName == DatasourceName {
return true, nil
}
dsRef, ok := rawDs.(map[string]interface{})
if !ok {
return false, nil
}
if dsRef["uid"].(string) == DatasourceUID {
return true, nil
}
return false, nil
return false
}
func (rn *rawNode) GetCommandType() (c CommandType, err error) {
@@ -213,6 +170,7 @@ func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, req *Reques
request: *req,
}
// support old datasourceId property
rawDsID, ok := rn.Query["datasourceId"]
if ok {
floatDsID, ok := rawDsID.(float64)
@@ -221,11 +179,7 @@ func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, req *Reques
}
dsNode.datasourceID = int64(floatDsID)
} else {
dsUid, err := rn.GetDatasourceUID()
if err != nil {
return nil, fmt.Errorf("neither datasourceId or datasourceUid in expression data source request for refId %v", rn.RefID)
}
dsNode.datasourceUID = dsUid
dsNode.datasourceUID = rn.DatasourceUID
}
var floatIntervalMS float64
pkg/expr/service.go
+30 -6
View File
@@ -4,21 +4,34 @@ import (
"context"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
)
// DatasourceName is the string constant used as the datasource name in requests
// to identify it as an expression command.
const DatasourceName = "__expr__"
// DatasourceType is the string constant used as the datasource when the property is in Datasource.Type.
// Type in requests is used to identify what type of data source plugin the request belongs to.
const DatasourceType = "__expr__"
// DatasourceUID is the string constant used as the datasource name in requests
// to identify it as an expression command when use in Datasource.UID.
const DatasourceUID = DatasourceType
// DatasourceID is the fake datasource id used in requests to identify it as an
// expression command.
const DatasourceID = -100
// DatasourceUID is the fake datasource uid used in requests to identify it as an
// expression command.
const DatasourceUID = "-100"
// OldDatasourceUID is the datasource uid used in requests to identify it as an
// expression command. It goes with the query root level datasourceUID property. It was accidentally
// set to the Id and is now kept for backwards compatibility. The newer Datasource.UID property
// should be used instead and should be set to "__expr__".
const OldDatasourceUID = "-100"
// IsDataSource checks if the uid points to an expression query
func IsDataSource(uid string) bool {
return uid == DatasourceUID || uid == OldDatasourceUID
}
// Service is service representation for expression handling.
type Service struct {
@@ -52,3 +65,14 @@ func (s *Service) ExecutePipeline(ctx context.Context, pipeline DataPipeline) (*
}
return res, nil
}
func DataSourceModel() *models.DataSource {
return &models.DataSource{
Id: DatasourceID,
Uid: DatasourceUID,
Name: DatasourceUID,
Type: DatasourceType,
JsonData: simplejson.New(),
SecureJsonData: make(map[string][]byte),
}
}
pkg/expr/transform.go
+25 -1
View File
@@ -40,6 +40,9 @@ func (s *Service) WrapTransformData(ctx context.Context, query plugins.DataQuery
}
for _, q := range query.Queries {
if q.DataSource == nil {
return nil, fmt.Errorf("mising datasource info: " + q.RefID)
}
modelJSON, err := q.Model.MarshalJSON()
if err != nil {
return nil, err
@@ -50,6 +53,10 @@ func (s *Service) WrapTransformData(ctx context.Context, query plugins.DataQuery
RefID: q.RefID,
MaxDataPoints: q.MaxDataPoints,
QueryType: q.QueryType,
Datasource: DataSourceRef{
Type: q.DataSource.Type,
UID: q.DataSource.Uid,
},
TimeRange: TimeRange{
From: query.TimeRange.GetFromAsTimeUTC(),
To: query.TimeRange.GetToAsTimeUTC(),
@@ -72,13 +79,30 @@ type Request struct {
type Query struct {
RefID string
TimeRange TimeRange
DatasourceUID string
DatasourceUID string // deprecated, value -100 when expressions
Datasource DataSourceRef `json:"datasource"`
JSON json.RawMessage
Interval time.Duration
QueryType string
MaxDataPoints int64
}
type DataSourceRef struct {
Type string `json:"type"` // value should be __expr__
UID string `json:"uid"` // value should be __expr__
}
func (q *Query) GetDatasourceUID() string {
if q.DatasourceUID != "" {
return q.DatasourceUID // backwards compatibility gets precedence
}
if q.Datasource.UID != "" {
return q.Datasource.UID
}
return ""
}
// TimeRange is a time.Time based TimeRange.
type TimeRange struct {
From time.Time
pkg/expr/translate/translate.go
+1 -1
View File
@@ -259,7 +259,7 @@ func (dc *dashConditionsJSON) GetNew(orgID int64) (*ngmodels.Condition, error) {
ccAlertQuery := ngmodels.AlertQuery{
RefID: ccRefID,
Model: exprModelJSON,
DatasourceUID: expr.DatasourceUID,
DatasourceUID: expr.OldDatasourceUID,
}
ngCond.Data = append(ngCond.Data, ccAlertQuery)
pkg/services/ngalert/models/alert_query.go
+1 -1
View File
@@ -89,7 +89,7 @@ func (aq *AlertQuery) setModelProps() error {
// IsExpression returns true if the alert query is an expression.
func (aq *AlertQuery) IsExpression() (bool, error) {
return aq.DatasourceUID == expr.DatasourceUID, nil
return expr.IsDataSource(aq.DatasourceUID), nil
}
// setMaxDatapoints sets the model maxDataPoints if it's missing or invalid