Live: support query execution with live RPC (#43118)
Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
This commit is contained in:
Ryan McKinley
co-authored by
Alexander Emelin
Alexander Emelin
parent
f4cc353225
commit
c0ff685d3b
@@ -89,7 +89,7 @@ type testState struct {
|
||||
|
||||
func newTestLive(t *testing.T) *live.GrafanaLive {
|
||||
cfg := &setting.Cfg{AppURL: "http://localhost:3000/"}
|
||||
gLive, err := live.ProvideService(nil, cfg, routing.NewRouteRegister(), nil, nil, nil, nil, sqlstore.InitTestDB(t), nil, &usagestats.UsageStatsMock{T: t})
|
||||
gLive, err := live.ProvideService(nil, cfg, routing.NewRouteRegister(), nil, nil, nil, nil, sqlstore.InitTestDB(t), nil, &usagestats.UsageStatsMock{T: t}, nil)
|
||||
require.NoError(t, err)
|
||||
return gLive
|
||||
}
|
||||
|
||||
+7
-16
@@ -13,14 +13,14 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/query"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/routing"
|
||||
httpstatic "github.com/grafana/grafana/pkg/api/static"
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/expr"
|
||||
"github.com/grafana/grafana/pkg/infra/localcache"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/infra/metrics"
|
||||
"github.com/grafana/grafana/pkg/infra/remotecache"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/login/social"
|
||||
@@ -43,8 +43,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/live/pushhttp"
|
||||
"github.com/grafana/grafana/pkg/services/login"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert"
|
||||
"github.com/grafana/grafana/pkg/services/notifications"
|
||||
"github.com/grafana/grafana/pkg/services/oauthtoken"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/quota"
|
||||
"github.com/grafana/grafana/pkg/services/rendering"
|
||||
@@ -103,19 +101,16 @@ type HTTPServer struct {
|
||||
AlertNG *ngalert.AlertNG
|
||||
LibraryPanelService librarypanels.Service
|
||||
LibraryElementService libraryelements.Service
|
||||
notificationService *notifications.NotificationService
|
||||
SocialService social.Service
|
||||
OAuthTokenService oauthtoken.OAuthTokenService
|
||||
Listener net.Listener
|
||||
EncryptionService encryption.Internal
|
||||
SecretsService secrets.Service
|
||||
DataSourcesService *datasources.Service
|
||||
cleanUpService *cleanup.CleanUpService
|
||||
tracingService tracing.Tracer
|
||||
internalMetricsSvc *metrics.InternalMetricsService
|
||||
updateChecker *updatechecker.Service
|
||||
searchUsersService searchusers.Service
|
||||
expressionService *expr.Service
|
||||
queryDataService *query.Service
|
||||
}
|
||||
|
||||
type ServerOptions struct {
|
||||
@@ -137,11 +132,10 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
|
||||
contextHandler *contexthandler.ContextHandler,
|
||||
schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG,
|
||||
libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service,
|
||||
notificationService *notifications.NotificationService, tracingService tracing.Tracer,
|
||||
internalMetricsSvc *metrics.InternalMetricsService, quotaService *quota.QuotaService,
|
||||
socialService social.Service, oauthTokenService oauthtoken.OAuthTokenService,
|
||||
quotaService *quota.QuotaService, socialService social.Service, tracingService tracing.Tracer,
|
||||
encryptionService encryption.Internal, updateChecker *updatechecker.Service, searchUsersService searchusers.Service,
|
||||
dataSourcesService *datasources.Service, secretsService secrets.Service, expressionService *expr.Service) (*HTTPServer, error) {
|
||||
dataSourcesService *datasources.Service, secretsService secrets.Service,
|
||||
queryDataService *query.Service) (*HTTPServer, error) {
|
||||
web.Env = cfg.Env
|
||||
m := web.New()
|
||||
|
||||
@@ -182,19 +176,16 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
|
||||
LibraryPanelService: libraryPanelService,
|
||||
LibraryElementService: libraryElementService,
|
||||
QuotaService: quotaService,
|
||||
notificationService: notificationService,
|
||||
tracingService: tracingService,
|
||||
internalMetricsSvc: internalMetricsSvc,
|
||||
log: log.New("http.server"),
|
||||
web: m,
|
||||
Listener: opts.Listener,
|
||||
SocialService: socialService,
|
||||
OAuthTokenService: oauthTokenService,
|
||||
EncryptionService: encryptionService,
|
||||
SecretsService: secretsService,
|
||||
DataSourcesService: dataSourcesService,
|
||||
searchUsersService: searchUsersService,
|
||||
expressionService: expressionService,
|
||||
queryDataService: queryDataService,
|
||||
}
|
||||
if hs.Listener != nil {
|
||||
hs.log.Debug("Using provided listener")
|
||||
|
||||
+4
-225
@@ -1,44 +1,24 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
"github.com/grafana/grafana/pkg/api/dtos"
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/expr"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/plugins/adapters"
|
||||
"github.com/grafana/grafana/pkg/tsdb/grafanads"
|
||||
"github.com/grafana/grafana/pkg/services/query"
|
||||
"github.com/grafana/grafana/pkg/tsdb/legacydata"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
"github.com/grafana/grafana/pkg/web"
|
||||
)
|
||||
|
||||
// ErrBadQuery returned whenever request is malformed and must contain a message
|
||||
// suitable to return in API response.
|
||||
type ErrBadQuery struct {
|
||||
Message string
|
||||
}
|
||||
|
||||
func NewErrBadQuery(msg string) *ErrBadQuery {
|
||||
return &ErrBadQuery{Message: msg}
|
||||
}
|
||||
|
||||
func (e ErrBadQuery) Error() string {
|
||||
return fmt.Sprintf("bad query: %s", e.Message)
|
||||
}
|
||||
|
||||
func (hs *HTTPServer) handleQueryMetricsError(err error) *response.NormalResponse {
|
||||
if errors.Is(err, models.ErrDataSourceAccessDenied) {
|
||||
return response.Error(http.StatusForbidden, "Access denied to data source", err)
|
||||
}
|
||||
var badQuery *ErrBadQuery
|
||||
var badQuery *query.ErrBadQuery
|
||||
if errors.As(err, &badQuery) {
|
||||
return response.Error(http.StatusBadRequest, util.Capitalize(badQuery.Message), err)
|
||||
}
|
||||
@@ -53,7 +33,7 @@ func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext) response.Response {
|
||||
return response.Error(http.StatusBadRequest, "bad request data", err)
|
||||
}
|
||||
|
||||
resp, err := hs.queryMetrics(c.Req.Context(), c.SignedInUser, c.SkipCache, reqDTO, true)
|
||||
resp, err := hs.queryDataService.QueryData(c.Req.Context(), c.SignedInUser, c.SkipCache, reqDTO, true)
|
||||
if err != nil {
|
||||
return hs.handleQueryMetricsError(err)
|
||||
}
|
||||
@@ -70,7 +50,7 @@ func (hs *HTTPServer) QueryMetrics(c *models.ReqContext) response.Response {
|
||||
return response.Error(http.StatusBadRequest, "bad request data", err)
|
||||
}
|
||||
|
||||
sdkResp, err := hs.queryMetrics(c.Req.Context(), c.SignedInUser, c.SkipCache, reqDto, false)
|
||||
sdkResp, err := hs.queryDataService.QueryData(c.Req.Context(), c.SignedInUser, c.SkipCache, reqDto, false)
|
||||
if err != nil {
|
||||
return hs.handleQueryMetricsError(err)
|
||||
}
|
||||
@@ -107,207 +87,6 @@ func (hs *HTTPServer) QueryMetrics(c *models.ReqContext) response.Response {
|
||||
return response.JSON(statusCode, &legacyResp)
|
||||
}
|
||||
|
||||
func (hs *HTTPServer) queryMetrics(ctx context.Context, user *models.SignedInUser, skipCache bool, reqDTO dtos.MetricRequest, handleExpressions bool) (*backend.QueryDataResponse, error) {
|
||||
parsedReq, err := hs.parseMetricRequest(user, skipCache, reqDTO)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if handleExpressions && parsedReq.hasExpression {
|
||||
return hs.handleExpressions(ctx, user, parsedReq)
|
||||
}
|
||||
return hs.handleQueryData(ctx, user, parsedReq)
|
||||
}
|
||||
|
||||
// handleExpressions handles POST /api/ds/query when there is an expression.
|
||||
func (hs *HTTPServer) handleExpressions(ctx context.Context, user *models.SignedInUser, parsedReq *parsedRequest) (*backend.QueryDataResponse, error) {
|
||||
exprReq := expr.Request{
|
||||
OrgId: user.OrgId,
|
||||
Queries: []expr.Query{},
|
||||
}
|
||||
|
||||
for _, pq := range parsedReq.parsedQueries {
|
||||
if pq.datasource == nil {
|
||||
return nil, NewErrBadQuery(fmt.Sprintf("query mising datasource info: %s", pq.query.RefID))
|
||||
}
|
||||
|
||||
exprReq.Queries = append(exprReq.Queries, expr.Query{
|
||||
JSON: pq.query.JSON,
|
||||
Interval: pq.query.Interval,
|
||||
RefID: pq.query.RefID,
|
||||
MaxDataPoints: pq.query.MaxDataPoints,
|
||||
QueryType: pq.query.QueryType,
|
||||
Datasource: expr.DataSourceRef{
|
||||
Type: pq.datasource.Type,
|
||||
UID: pq.datasource.Uid,
|
||||
},
|
||||
TimeRange: expr.TimeRange{
|
||||
From: pq.query.TimeRange.From,
|
||||
To: pq.query.TimeRange.To,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
qdr, err := hs.expressionService.TransformData(ctx, &exprReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("expression request error: %w", err)
|
||||
}
|
||||
return qdr, nil
|
||||
}
|
||||
|
||||
func (hs *HTTPServer) handleQueryData(ctx context.Context, user *models.SignedInUser, parsedReq *parsedRequest) (*backend.QueryDataResponse, error) {
|
||||
ds := parsedReq.parsedQueries[0].datasource
|
||||
if err := hs.PluginRequestValidator.Validate(ds.Url, nil); err != nil {
|
||||
return nil, models.ErrDataSourceAccessDenied
|
||||
}
|
||||
|
||||
instanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert data source to instance settings")
|
||||
}
|
||||
|
||||
req := &backend.QueryDataRequest{
|
||||
PluginContext: backend.PluginContext{
|
||||
OrgID: ds.OrgId,
|
||||
PluginID: ds.Type,
|
||||
User: adapters.BackendUserFromSignedInUser(user),
|
||||
DataSourceInstanceSettings: instanceSettings,
|
||||
},
|
||||
Headers: map[string]string{},
|
||||
Queries: []backend.DataQuery{},
|
||||
}
|
||||
|
||||
if hs.OAuthTokenService.IsOAuthPassThruEnabled(ds) {
|
||||
if token := hs.OAuthTokenService.GetCurrentOAuthToken(ctx, user); token != nil {
|
||||
req.Headers["Authorization"] = fmt.Sprintf("%s %s", token.Type(), token.AccessToken)
|
||||
|
||||
idToken, ok := token.Extra("id_token").(string)
|
||||
if ok && idToken != "" {
|
||||
req.Headers["X-ID-Token"] = idToken
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, q := range parsedReq.parsedQueries {
|
||||
req.Queries = append(req.Queries, q.query)
|
||||
}
|
||||
|
||||
return hs.pluginClient.QueryData(ctx, req)
|
||||
}
|
||||
|
||||
type parsedQuery struct {
|
||||
datasource *models.DataSource
|
||||
query backend.DataQuery
|
||||
}
|
||||
|
||||
type parsedRequest struct {
|
||||
hasExpression bool
|
||||
parsedQueries []parsedQuery
|
||||
}
|
||||
|
||||
func (hs *HTTPServer) parseMetricRequest(user *models.SignedInUser, skipCache bool, reqDTO dtos.MetricRequest) (*parsedRequest, error) {
|
||||
if len(reqDTO.Queries) == 0 {
|
||||
return nil, NewErrBadQuery("no queries found")
|
||||
}
|
||||
|
||||
timeRange := legacydata.NewDataTimeRange(reqDTO.From, reqDTO.To)
|
||||
req := &parsedRequest{
|
||||
hasExpression: false,
|
||||
parsedQueries: []parsedQuery{},
|
||||
}
|
||||
|
||||
// Parse the queries
|
||||
datasources := map[string]*models.DataSource{}
|
||||
for _, query := range reqDTO.Queries {
|
||||
ds, err := hs.getDataSourceFromQuery(user, skipCache, query, datasources)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ds == nil {
|
||||
return nil, NewErrBadQuery("invalid data source ID")
|
||||
}
|
||||
|
||||
datasources[ds.Uid] = ds
|
||||
if expr.IsDataSource(ds.Uid) {
|
||||
req.hasExpression = true
|
||||
}
|
||||
|
||||
hs.log.Debug("Processing metrics query", "query", query)
|
||||
|
||||
modelJSON, err := query.MarshalJSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.parsedQueries = append(req.parsedQueries, parsedQuery{
|
||||
datasource: ds,
|
||||
query: backend.DataQuery{
|
||||
TimeRange: backend.TimeRange{
|
||||
From: timeRange.GetFromAsTimeUTC(),
|
||||
To: timeRange.GetToAsTimeUTC(),
|
||||
},
|
||||
RefID: query.Get("refId").MustString("A"),
|
||||
MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
|
||||
Interval: time.Duration(query.Get("intervalMs").MustInt64(1000)) * time.Millisecond,
|
||||
QueryType: query.Get("queryType").MustString(""),
|
||||
JSON: modelJSON,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if !req.hasExpression {
|
||||
if len(datasources) > 1 {
|
||||
// We do not (yet) support mixed query type
|
||||
return nil, NewErrBadQuery("all queries must use the same datasource")
|
||||
}
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (hs *HTTPServer) getDataSourceFromQuery(user *models.SignedInUser, skipCache bool, query *simplejson.Json, history map[string]*models.DataSource) (*models.DataSource, error) {
|
||||
var err error
|
||||
uid := query.Get("datasource").Get("uid").MustString()
|
||||
|
||||
// before 8.3 special types could be sent as datasource (expr)
|
||||
if uid == "" {
|
||||
uid = query.Get("datasource").MustString()
|
||||
}
|
||||
|
||||
// check cache value
|
||||
ds, ok := history[uid]
|
||||
if ok {
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
if expr.IsDataSource(uid) {
|
||||
return expr.DataSourceModel(), nil
|
||||
}
|
||||
|
||||
if uid == grafanads.DatasourceUID {
|
||||
return grafanads.DataSourceModel(user.OrgId), nil
|
||||
}
|
||||
|
||||
// use datasourceId if it exists
|
||||
id := query.Get("datasourceId").MustInt64(0)
|
||||
if id > 0 {
|
||||
ds, err = hs.DataSourceCache.GetDatasource(id, user, skipCache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
if uid != "" {
|
||||
ds, err = hs.DataSourceCache.GetDatasourceByUID(uid, user, skipCache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
return nil, NewErrBadQuery("missing data source ID/UID")
|
||||
}
|
||||
|
||||
func toJsonStreamingResponse(qdr *backend.QueryDataResponse) response.Response {
|
||||
statusCode := http.StatusOK
|
||||
for _, res := range qdr.Responses {
|
||||
|
||||
Reference in New Issue
Block a user