From 5cc737bb24f80514d35b1f02c7113e106a277dc6 Mon Sep 17 00:00:00 2001 From: Kyle Brandt Date: Thu, 7 Sep 2023 16:02:07 -0400 Subject: [PATCH] SSE: Put data source query grouping behind feature flag (#74551) change was originally merged in commit: 720d716 via PR: https://github.com/grafana/grafana/pull/72935 with no flag flag is: sseGroupByDatasource --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/expr/graph.go | 21 ++++--- pkg/expr/nodes.go | 61 ++++++++++++++++++- pkg/services/featuremgmt/registry.go | 6 ++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 ++ 7 files changed, 84 insertions(+), 11 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 56a8ee3bf47..dd442cbee01 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -133,6 +133,7 @@ Experimental features might be changed or removed without prior notice. | `noBasicRole` | Enables a new role that has no permissions by default | | `angularDeprecationUI` | Display new Angular deprecation-related UI features | | `dashgpt` | Enable AI powered features in dashboards | +| `sseGroupByDatasource` | Send query to the same datasource in a single request when using server side expressions | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 8c46d2e8dea..69e7a1f4416 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -123,4 +123,5 @@ export interface FeatureToggles { dashgpt?: boolean; reportingRetries?: boolean; newBrowseDashboards?: boolean; + sseGroupByDatasource?: boolean; } diff --git a/pkg/expr/graph.go b/pkg/expr/graph.go index bd02ee20392..ae2773f9282 100644 --- a/pkg/expr/graph.go +++ b/pkg/expr/graph.go @@ -56,21 +56,24 @@ type DataPipeline []Node func (dp *DataPipeline) execute(c context.Context, now time.Time, s *Service) (mathexp.Vars, error) { vars := make(mathexp.Vars) + groupByDSFlag := s.features.IsEnabled(featuremgmt.FlagSseGroupByDatasource) // Execute datasource nodes first, and grouped by datasource. - dsNodes := []*DSNode{} - for _, node := range *dp { - if node.NodeType() != TypeDatasourceNode { - continue + if groupByDSFlag { + dsNodes := []*DSNode{} + for _, node := range *dp { + if node.NodeType() != TypeDatasourceNode { + continue + } + dsNodes = append(dsNodes, node.(*DSNode)) } - dsNodes = append(dsNodes, node.(*DSNode)) - } - if err := executeDSNodesGrouped(c, now, vars, s, dsNodes); err != nil { - return nil, err + if err := executeDSNodesGrouped(c, now, vars, s, dsNodes); err != nil { + return nil, err + } } for _, node := range *dp { - if node.NodeType() == TypeDatasourceNode { + if groupByDSFlag && node.NodeType() == TypeDatasourceNode { continue // already executed via executeDSNodesGrouped } c, span := s.tracer.Start(c, "SSE.ExecuteNode") diff --git a/pkg/expr/nodes.go b/pkg/expr/nodes.go index 6101e1f8d76..f9601bccfd9 100644 --- a/pkg/expr/nodes.go +++ b/pkg/expr/nodes.go @@ -290,8 +290,65 @@ func executeDSNodesGrouped(ctx context.Context, now time.Time, vars mathexp.Vars // other nodes they must have already been executed and their results must // already by in vars. func (dn *DSNode) Execute(ctx context.Context, now time.Time, _ mathexp.Vars, s *Service) (r mathexp.Results, e error) { - panic("Execute called on DSNode and should not be") - // Datasource queries are sent as a group to the datasource, see executeDSNodesGrouped. + logger := logger.FromContext(ctx).New("datasourceType", dn.datasource.Type, "queryRefId", dn.refID, "datasourceUid", dn.datasource.UID, "datasourceVersion", dn.datasource.Version) + ctx, span := s.tracer.Start(ctx, "SSE.ExecuteDatasourceQuery") + defer span.End() + + pCtx, err := s.pCtxProvider.GetWithDataSource(ctx, dn.datasource.Type, dn.request.User, dn.datasource) + if err != nil { + return mathexp.Results{}, err + } + span.SetAttributes("datasource.type", dn.datasource.Type, attribute.Key("datasource.type").String(dn.datasource.Type)) + span.SetAttributes("datasource.uid", dn.datasource.UID, attribute.Key("datasource.uid").String(dn.datasource.UID)) + + req := &backend.QueryDataRequest{ + PluginContext: pCtx, + Queries: []backend.DataQuery{ + { + RefID: dn.refID, + MaxDataPoints: dn.maxDP, + Interval: time.Duration(int64(time.Millisecond) * dn.intervalMS), + JSON: dn.query, + TimeRange: dn.timeRange.AbsoluteTime(now), + QueryType: dn.queryType, + }, + }, + Headers: dn.request.Headers, + } + + responseType := "unknown" + respStatus := "success" + defer func() { + if e != nil { + responseType = "error" + respStatus = "failure" + span.AddEvents([]string{"error", "message"}, + []tracing.EventValue{ + {Str: fmt.Sprintf("%v", err)}, + {Str: "failed to query data source"}, + }) + } + logger.Debug("Data source queried", "responseType", responseType) + useDataplane := strings.HasPrefix(responseType, "dataplane-") + s.metrics.dsRequests.WithLabelValues(respStatus, fmt.Sprintf("%t", useDataplane), dn.datasource.Type).Inc() + }() + + resp, err := s.dataService.QueryData(ctx, req) + if err != nil { + return mathexp.Results{}, MakeQueryError(dn.refID, dn.datasource.UID, err) + } + + dataFrames, err := getResponseFrame(resp, dn.refID) + if err != nil { + return mathexp.Results{}, MakeQueryError(dn.refID, dn.datasource.UID, err) + } + + var result mathexp.Results + responseType, result, err = convertDataFramesToResults(ctx, dataFrames, dn.datasource.Type, s, logger) + if err != nil { + err = MakeConversionError(dn.refID, err) + } + return result, err } func getResponseFrame(resp *backend.QueryDataResponse, refID string) (data.Frames, error) { diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 03b139d333e..8d32f59a469 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -731,5 +731,11 @@ var ( Owner: grafanaFrontendPlatformSquad, FrontendOnly: true, }, + { + Name: "sseGroupByDatasource", + Description: "Send query to the same datasource in a single request when using server side expressions", + Stage: FeatureStageExperimental, + Owner: grafanaObservabilityMetricsSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 278a58b2dae..bf1633ffc23 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -104,3 +104,4 @@ angularDeprecationUI,experimental,@grafana/plugins-platform-backend,false,false, dashgpt,experimental,@grafana/dashboards-squad,false,false,false,true reportingRetries,preview,@grafana/sharing-squad,false,false,true,false newBrowseDashboards,preview,@grafana/grafana-frontend-platform,false,false,false,true +sseGroupByDatasource,experimental,@grafana/observability-metrics,false,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 135d1eaf670..be0c3910f0e 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -426,4 +426,8 @@ const ( // FlagNewBrowseDashboards // New browse/manage dashboards UI FlagNewBrowseDashboards = "newBrowseDashboards" + + // FlagSseGroupByDatasource + // Send query to the same datasource in a single request when using server side expressions + FlagSseGroupByDatasource = "sseGroupByDatasource" )