From 0fc403d116b9cbd0b93e497cff0ddded55f9fe60 Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Tue, 3 Oct 2023 20:23:55 +0200 Subject: [PATCH] Instrumentation: Enable native histograms for HTTP requests (#75731) Signed-off-by: bergquist --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/middleware/request_metrics.go | 33 ++++++++++++++----- pkg/services/featuremgmt/registry.go | 7 ++++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 +++ 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 71ff650d3be..90295652945 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -142,6 +142,7 @@ Experimental features might be changed or removed without prior notice. | `pluginsAPIMetrics` | Sends metrics of public grafana packages usage by plugins | | `httpSLOLevels` | Adds SLO level to http request metrics | | `alertingModifiedExport` | Enables using UI for provisioned rules modification and export | +| `enableNativeHTTPHistogram` | Enables native HTTP Histograms | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index a7b6d374bd9..a3f181c3601 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -135,4 +135,5 @@ export interface FeatureToggles { cloudWatchWildCardDimensionValues?: boolean; externalServiceAccounts?: boolean; alertingModifiedExport?: boolean; + enableNativeHTTPHistogram?: boolean; } diff --git a/pkg/middleware/request_metrics.go b/pkg/middleware/request_metrics.go index 56af07206cb..ad25e314be0 100644 --- a/pkg/middleware/request_metrics.go +++ b/pkg/middleware/request_metrics.go @@ -49,13 +49,26 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR histogramLabels = append(histogramLabels, "slo_group") } + histogramOptions := prometheus.HistogramOpts{ + Namespace: "grafana", + Name: "http_request_duration_seconds", + Help: "Histogram of latencies for HTTP requests.", + Buckets: defBuckets, + } + + if features.IsEnabled(featuremgmt.FlagEnableNativeHTTPHistogram) { + // the recommended default value from the prom_client + // https://github.com/prometheus/client_golang/blob/main/prometheus/histogram.go#L411 + // Giving this variable an value means the client will expose the histograms as an + // native histogram instead of normal a normal histogram. + histogramOptions.NativeHistogramBucketFactor = 1.1 + // The default value in OTel. It probably good enough for us as well. + histogramOptions.NativeHistogramMaxBucketNumber = 160 + histogramOptions.NativeHistogramMinResetDuration = time.Hour + } + httpRequestDurationHistogram := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "grafana", - Name: "http_request_duration_seconds", - Help: "Histogram of latencies for HTTP requests.", - Buckets: defBuckets, - }, + histogramOptions, histogramLabels, ) @@ -108,16 +121,18 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR histogram := httpRequestDurationHistogram. WithLabelValues(labelValues...) + elapsedTime := time.Since(now).Seconds() + if traceID := tracing.TraceIDFromContext(r.Context(), true); traceID != "" { // Need to type-convert the Observer to an // ExemplarObserver. This will always work for a // HistogramVec. histogram.(prometheus.ExemplarObserver).ObserveWithExemplar( - time.Since(now).Seconds(), prometheus.Labels{"traceID": traceID}, + elapsedTime, prometheus.Labels{"traceID": traceID}, ) - return + } else { + histogram.Observe(elapsedTime) } - histogram.Observe(time.Since(now).Seconds()) switch { case strings.HasPrefix(r.RequestURI, "/api/datasources/proxy"): diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index c1d7adcc972..c41abfb6ff3 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -816,5 +816,12 @@ var ( FrontendOnly: false, Owner: grafanaAlertingSquad, }, + { + Name: "enableNativeHTTPHistogram", + Description: "Enables native HTTP Histograms", + Stage: FeatureStageExperimental, + FrontendOnly: false, + Owner: hostedGrafanaTeam, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index f9e321a5c75..18eff8b92a5 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -116,3 +116,4 @@ idForwarding,experimental,@grafana/grafana-authnz-team,true,false,false,false cloudWatchWildCardDimensionValues,GA,@grafana/aws-datasources,false,false,false,false externalServiceAccounts,experimental,@grafana/grafana-authnz-team,true,false,false,false alertingModifiedExport,experimental,@grafana/alerting-squad,false,false,false,false +enableNativeHTTPHistogram,experimental,@grafana/hosted-grafana-team,false,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 047df4ba9d0..5b88d2a8172 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -474,4 +474,8 @@ const ( // FlagAlertingModifiedExport // Enables using UI for provisioned rules modification and export FlagAlertingModifiedExport = "alertingModifiedExport" + + // FlagEnableNativeHTTPHistogram + // Enables native HTTP Histograms + FlagEnableNativeHTTPHistogram = "enableNativeHTTPHistogram" )