From 674144c8e84cb4aca3bfc448f08b88bb516d3b77 Mon Sep 17 00:00:00 2001 From: Kyle Brandt Date: Wed, 29 Mar 2023 11:26:32 -0400 Subject: [PATCH] Prometheus: Update FrameType and make __name__ the field name (#62694) These changes would make the response more in line with the dataplane contract, changes are under the feature toggle prometheusDataplane --- .../feature-toggles/index.md | 89 ++++++++++--------- .../src/types/featureToggles.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 6 ++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/tsdb/prometheus/querydata/request.go | 2 + pkg/tsdb/prometheus/querydata/response.go | 1 + pkg/util/converter/prom.go | 42 +++++++-- 8 files changed, 94 insertions(+), 52 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index b0e2c69f0f4..dfbb468eacb 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -54,50 +54,51 @@ Some stable features are enabled by default. You can disable a stable feature by These features are early in their development lifecycle and so are not yet supported in Grafana Cloud. Alpha features might be changed or removed without prior notice. -| Feature toggle name | Description | -| ---------------------------------- | --------------------------------------------------------------------------------------------------------- | -| `alertingBigTransactions` | Use big transactions for alerting database writes | -| `dashboardPreviews` | Create and show thumbnails for dashboard search results | -| `live-service-web-worker` | This will use a webworker thread to processes events rather than the main thread | -| `queryOverLive` | Use Grafana Live WebSocket to execute backend queries | -| `publicDashboards` | Enables public access to dashboards | -| `publicDashboardsEmailSharing` | Enables public dashboard sharing to be restricted to only allowed emails | -| `lokiLive` | Support WebSocket streaming for loki (early prototype) | -| `lokiDataframeApi` | Use experimental loki api for WebSocket streaming (early prototype) | -| `storage` | Configurable storage for dashboards, datasources, and resources | -| `exploreMixedDatasource` | Enable mixed datasource in Explore | -| `newTraceView` | Shows the new trace view design | -| `correlations` | Correlations page | -| `datasourceQueryMultiStatus` | Introduce HTTP 207 Multi Status for api/ds/query | -| `traceToMetrics` | Enable trace to metrics links | -| `prometheusWideSeries` | Enable wide series responses in the Prometheus datasource | -| `canvasPanelNesting` | Allow elements nesting | -| `scenes` | Experimental framework to build interactive dashboards | -| `disableSecretsCompatibility` | Disable duplicated secret storage in legacy tables | -| `logRequestsInstrumentedAsUnknown` | Logs the path for requests that are instrumented as unknown | -| `redshiftAsyncQueryDataSupport` | Enable async query data support for Redshift | -| `athenaAsyncQueryDataSupport` | Enable async query data support for Athena | -| `newPanelChromeUI` | Show updated look and feel of grafana-ui PanelChrome: panel header, icons, and menu | -| `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema | -| `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query | -| `elasticsearchBackendMigration` | Use Elasticsearch as backend data source | -| `datasourceOnboarding` | Enable data source onboarding page | -| `emptyDashboardPage` | Enable the redesigned user interface of a dashboard page that includes no panels | -| `secureSocksDatasourceProxy` | Enable secure socks tunneling for supported core datasources | -| `authnService` | Use new auth service to perform authentication | -| `alertingBacktesting` | Rule backtesting API for alerting | -| `editPanelCSVDragAndDrop` | Enables drag and drop for CSV and Excel files | -| `logsContextDatasourceUi` | Allow datasource to provide custom UI for context view | -| `lokiQuerySplitting` | Split large interval queries into subqueries with smaller time intervals | -| `lokiQuerySplittingConfig` | Give users the option to configure split durations for Loki queries | -| `individualCookiePreferences` | Support overriding cookie preferences per user | -| `onlyExternalOrgRoleSync` | Prohibits a user from changing organization roles synced with external auth providers | -| `drawerDataSourcePicker` | Changes the user experience for data source selection to a drawer. | -| `traceqlSearch` | Enables the 'TraceQL Search' tab for the Tempo datasource which provides a UI to generate TraceQL queries | -| `prometheusMetricEncyclopedia` | Replaces the Prometheus query builder metric select option with a paginated and filterable component | -| `timeSeriesTable` | Enable time series table transformer & sparkline cell type | -| `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy | -| `clientTokenRotation` | Replaces the current in-request token rotation so that the client initiates the rotation | +| Feature toggle name | Description | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `alertingBigTransactions` | Use big transactions for alerting database writes | +| `dashboardPreviews` | Create and show thumbnails for dashboard search results | +| `live-service-web-worker` | This will use a webworker thread to processes events rather than the main thread | +| `queryOverLive` | Use Grafana Live WebSocket to execute backend queries | +| `publicDashboards` | Enables public access to dashboards | +| `publicDashboardsEmailSharing` | Enables public dashboard sharing to be restricted to only allowed emails | +| `lokiLive` | Support WebSocket streaming for loki (early prototype) | +| `lokiDataframeApi` | Use experimental loki api for WebSocket streaming (early prototype) | +| `storage` | Configurable storage for dashboards, datasources, and resources | +| `exploreMixedDatasource` | Enable mixed datasource in Explore | +| `newTraceView` | Shows the new trace view design | +| `correlations` | Correlations page | +| `datasourceQueryMultiStatus` | Introduce HTTP 207 Multi Status for api/ds/query | +| `traceToMetrics` | Enable trace to metrics links | +| `prometheusWideSeries` | Enable wide series responses in the Prometheus datasource | +| `canvasPanelNesting` | Allow elements nesting | +| `scenes` | Experimental framework to build interactive dashboards | +| `disableSecretsCompatibility` | Disable duplicated secret storage in legacy tables | +| `logRequestsInstrumentedAsUnknown` | Logs the path for requests that are instrumented as unknown | +| `redshiftAsyncQueryDataSupport` | Enable async query data support for Redshift | +| `athenaAsyncQueryDataSupport` | Enable async query data support for Athena | +| `newPanelChromeUI` | Show updated look and feel of grafana-ui PanelChrome: panel header, icons, and menu | +| `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema | +| `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query | +| `elasticsearchBackendMigration` | Use Elasticsearch as backend data source | +| `datasourceOnboarding` | Enable data source onboarding page | +| `emptyDashboardPage` | Enable the redesigned user interface of a dashboard page that includes no panels | +| `secureSocksDatasourceProxy` | Enable secure socks tunneling for supported core datasources | +| `authnService` | Use new auth service to perform authentication | +| `alertingBacktesting` | Rule backtesting API for alerting | +| `editPanelCSVDragAndDrop` | Enables drag and drop for CSV and Excel files | +| `logsContextDatasourceUi` | Allow datasource to provide custom UI for context view | +| `lokiQuerySplitting` | Split large interval queries into subqueries with smaller time intervals | +| `lokiQuerySplittingConfig` | Give users the option to configure split durations for Loki queries | +| `individualCookiePreferences` | Support overriding cookie preferences per user | +| `onlyExternalOrgRoleSync` | Prohibits a user from changing organization roles synced with external auth providers | +| `drawerDataSourcePicker` | Changes the user experience for data source selection to a drawer. | +| `traceqlSearch` | Enables the 'TraceQL Search' tab for the Tempo datasource which provides a UI to generate TraceQL queries | +| `prometheusMetricEncyclopedia` | Replaces the Prometheus query builder metric select option with a paginated and filterable component | +| `timeSeriesTable` | Enable time series table transformer & sparkline cell type | +| `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy | +| `clientTokenRotation` | Replaces the current in-request token rotation so that the client initiates the rotation | +| `prometheusDataplane` | Changes responses to from Prometheus to be compliant with the dataplane specification. In particular it sets the numeric Field.Name from 'Value' to the value of the `__name__` label when present. | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index b36cca8b73b..4ffbae94787 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -86,4 +86,5 @@ export interface FeatureToggles { influxdbBackendMigration?: boolean; clientTokenRotation?: boolean; disableElasticsearchBackendExploreQuery?: boolean; + prometheusDataplane?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index e7401c126db..47109985ba7 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -454,5 +454,11 @@ var ( State: FeatureStateBeta, Owner: grafanaObservabilityLogsSquad, }, + { + Name: "prometheusDataplane", + Description: "Changes responses to from Prometheus to be compliant with the dataplane specification. In particular it sets the numeric Field.Name from 'Value' to the value of the `__name__` label when present.", + State: FeatureStateAlpha, + Owner: grafanaObservabilityMetricsSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 349877154ec..926e97f184f 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -67,3 +67,4 @@ timeSeriesTable,alpha,@grafana/app-o11y,false,false,false,true influxdbBackendMigration,alpha,@grafana/observability-metrics,false,false,false,true clientTokenRotation,alpha,@grafana/grafana-authnz-team,false,false,false,false disableElasticsearchBackendExploreQuery,beta,@grafana/observability-logs,false,false,false,false +prometheusDataplane,alpha,@grafana/observability-metrics,false,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 8c47ffe4e02..1dec03e028f 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -278,4 +278,8 @@ const ( // FlagDisableElasticsearchBackendExploreQuery // Disable executing of Elasticsearch Explore queries trough backend FlagDisableElasticsearchBackendExploreQuery = "disableElasticsearchBackendExploreQuery" + + // FlagPrometheusDataplane + // Changes responses to from Prometheus to be compliant with the dataplane specification. In particular it sets the numeric Field.Name from 'Value' to the value of the `__name__` label when present. + FlagPrometheusDataplane = "prometheusDataplane" ) diff --git a/pkg/tsdb/prometheus/querydata/request.go b/pkg/tsdb/prometheus/querydata/request.go index dace8ed30c3..130863fbd46 100644 --- a/pkg/tsdb/prometheus/querydata/request.go +++ b/pkg/tsdb/prometheus/querydata/request.go @@ -43,6 +43,7 @@ type QueryData struct { URL string TimeInterval string enableWideSeries bool + enableDataplane bool exemplarSampler func() exemplar.Sampler } @@ -82,6 +83,7 @@ func New( ID: settings.ID, URL: settings.URL, enableWideSeries: features.IsEnabled(featuremgmt.FlagPrometheusWideSeries), + enableDataplane: features.IsEnabled(featuremgmt.FlagPrometheusDataplane), exemplarSampler: exemplarSampler, }, nil } diff --git a/pkg/tsdb/prometheus/querydata/response.go b/pkg/tsdb/prometheus/querydata/response.go index b26c53b3167..63211a227d5 100644 --- a/pkg/tsdb/prometheus/querydata/response.go +++ b/pkg/tsdb/prometheus/querydata/response.go @@ -28,6 +28,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt r := converter.ReadPrometheusStyleResult(iter, converter.Options{ MatrixWideSeries: s.enableWideSeries, VectorWideSeries: s.enableWideSeries, + Dataplane: s.enableDataplane, }) // Add frame to attach metadata diff --git a/pkg/util/converter/prom.go b/pkg/util/converter/prom.go index 1196b94cda8..702f12c0ffb 100644 --- a/pkg/util/converter/prom.go +++ b/pkg/util/converter/prom.go @@ -21,6 +21,7 @@ func logf(format string, a ...interface{}) { type Options struct { MatrixWideSeries bool VectorWideSeries bool + Dataplane bool } // ReadPrometheusStyleResult will read results from a prometheus or loki server and return data frames @@ -115,15 +116,15 @@ func readPrometheusData(iter *jsoniter.Iterator, opt Options) backend.DataRespon switch resultType { case "matrix": if opt.MatrixWideSeries { - rsp = readMatrixOrVectorWide(iter, resultType) + rsp = readMatrixOrVectorWide(iter, resultType, opt) } else { - rsp = readMatrixOrVectorMulti(iter, resultType) + rsp = readMatrixOrVectorMulti(iter, resultType, opt) } case "vector": if opt.VectorWideSeries { - rsp = readMatrixOrVectorWide(iter, resultType) + rsp = readMatrixOrVectorWide(iter, resultType, opt) } else { - rsp = readMatrixOrVectorMulti(iter, resultType) + rsp = readMatrixOrVectorMulti(iter, resultType, opt) } case "streams": rsp = readStream(iter) @@ -355,7 +356,7 @@ func readScalar(iter *jsoniter.Iterator) backend.DataResponse { frame := data.NewFrame("", timeField, valueField) frame.Meta = &data.FrameMeta{ - Type: data.FrameTypeTimeSeriesMulti, + Type: data.FrameTypeNumericMulti, Custom: resultTypeToCustomMeta("scalar"), } @@ -364,16 +365,25 @@ func readScalar(iter *jsoniter.Iterator) backend.DataResponse { } } -func readMatrixOrVectorWide(iter *jsoniter.Iterator, resultType string) backend.DataResponse { +func readMatrixOrVectorWide(iter *jsoniter.Iterator, resultType string, opt Options) backend.DataResponse { rowIdx := 0 timeMap := map[int64]int{} timeField := data.NewFieldFromFieldType(data.FieldTypeTime, 0) timeField.Name = data.TimeSeriesTimeFieldName frame := data.NewFrame("", timeField) - frame.Meta = &data.FrameMeta{ + + frame.Meta = &data.FrameMeta{ // Overwritten if histogram Type: data.FrameTypeTimeSeriesWide, Custom: resultTypeToCustomMeta(resultType), } + + if opt.Dataplane && resultType == "vector" { + frame.Meta.Type = data.FrameTypeNumericWide + } + if opt.Dataplane { + frame.Meta.TypeVersion = data.FrameTypeVersion{0, 1} + } + rsp := backend.DataResponse{ Frames: []*data.Frame{}, } @@ -390,6 +400,11 @@ func readMatrixOrVectorWide(iter *jsoniter.Iterator, resultType string) backend. switch l1Field { case "metric": iter.ReadVal(&valueField.Labels) + if opt.Dataplane { + if n, ok := valueField.Labels["__name__"]; ok { + valueField.Name = n + } + } case "value": timeMap, rowIdx = addValuePairToFrame(frame, timeMap, rowIdx, iter) @@ -472,7 +487,7 @@ func addValuePairToFrame(frame *data.Frame, timeMap map[int64]int, rowIdx int, i return timeMap, rowIdx } -func readMatrixOrVectorMulti(iter *jsoniter.Iterator, resultType string) backend.DataResponse { +func readMatrixOrVectorMulti(iter *jsoniter.Iterator, resultType string, opt Options) backend.DataResponse { rsp := backend.DataResponse{} for iter.ReadArray() { @@ -488,6 +503,11 @@ func readMatrixOrVectorMulti(iter *jsoniter.Iterator, resultType string) backend switch l1Field { case "metric": iter.ReadVal(&valueField.Labels) + if opt.Dataplane { + if n, ok := valueField.Labels["__name__"]; ok { + valueField.Name = n + } + } case "value": t, v, err := readTimeValuePair(iter) @@ -548,6 +568,12 @@ func readMatrixOrVectorMulti(iter *jsoniter.Iterator, resultType string) backend Type: data.FrameTypeTimeSeriesMulti, Custom: resultTypeToCustomMeta(resultType), } + if opt.Dataplane && resultType == "vector" { + frame.Meta.Type = data.FrameTypeNumericMulti + } + if opt.Dataplane { + frame.Meta.TypeVersion = data.FrameTypeVersion{0, 1} + } rsp.Frames = append(rsp.Frames, frame) } }