From c04f6bedecb5bed63150386fbcad7b010848a8bf Mon Sep 17 00:00:00 2001 From: Costa Alexoglou Date: Mon, 20 Oct 2025 13:17:56 +0200 Subject: [PATCH] fix: apiserver metrics in custom handlers (#112508) * fix: apiserver metrics in custom handlers * chore: review feedback --- .../builder/CUSTOM_ROUTES_METRICS.md | 162 ++++++++++++++++++ .../apiserver/builder/custom_route_metrics.go | 100 +++++++++++ pkg/services/apiserver/builder/helper.go | 10 +- .../apiserver/builder/request_handler.go | 25 ++- pkg/services/apiserver/service.go | 1 + 5 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 pkg/services/apiserver/builder/CUSTOM_ROUTES_METRICS.md create mode 100644 pkg/services/apiserver/builder/custom_route_metrics.go diff --git a/pkg/services/apiserver/builder/CUSTOM_ROUTES_METRICS.md b/pkg/services/apiserver/builder/CUSTOM_ROUTES_METRICS.md new file mode 100644 index 00000000000..00e26aea782 --- /dev/null +++ b/pkg/services/apiserver/builder/CUSTOM_ROUTES_METRICS.md @@ -0,0 +1,162 @@ +## Custom Route Metrics in Grafana API Server + +### Problem + +Custom API routes registered via `APIGroupRouteProvider.GetAPIRoutes()` bypass the standard Kubernetes apiserver metrics recording middleware. Requests to these routes were not being recorded in `apiserver_request_total`. + +**Why this happens:** + +``` +HTTP Request + ↓ +Gorilla Mux (custom routes) + ├─ [MATCHED] → Your custom handler → Response ❌ (no metrics recorded) + │ + └─ [NOT MATCHED] → k8s DefaultBuildHandlerChain + ├─ WithRequestMetrics (records apiserver_request_total) ✅ + └─ Standard REST storage handlers +``` + +Custom routes are served by Gorilla Mux and return immediately, never reaching the k8s metrics middleware. + +### Solution + +We've implemented **centralized automatic instrumentation** that records custom route requests in the **same `apiserver_request_total` metric** used by standard Kubernetes API calls. This means: + +- ✅ **Single metric** for all API requests (standard + custom routes) +- ✅ **Consistent labels** matching Kubernetes conventions +- ✅ **No dashboard changes** needed - existing queries work +- ✅ **Automatic** for all API servers with zero code changes + +#### Key Components + +1. **`pkg/services/apiserver/builder/custom_route_metrics.go`** + - Defines `CustomRouteMetrics` + - Uses the existing `apiserver_request_total` metric (via `metrics.MonitorRequest`) + - Provides `responseWriterWithStatus` to capture HTTP status codes + - Provides `InstrumentHandler()` to wrap route handlers + +2. **`pkg/services/apiserver/builder/request_handler.go`** + - Updated `GetCustomRoutesHandler()` to accept `prometheus.Registerer` + - Automatically instruments ALL custom routes (both root and namespace) + - Works transparently for all API servers + +3. **Updated signatures** + - `BuildHandlerChainFuncFromBuilders`: now accepts `prometheus.Registerer` + - `GetDefaultBuildHandlerChainFunc`: now accepts `prometheus.Registerer` + - `SetupConfig`: now accepts `prometheus.Registerer` + - Factory interfaces updated to pass registry through + +### Usage + +**For new API servers:** No action required! Custom routes are automatically instrumented. + +**For existing API servers:** No changes needed. All API servers using `APIGroupRouteProvider` automatically get metrics. + +### Metrics + +Query in Prometheus - **same metric as standard API calls**: + +```promql +# Total requests by status code (includes both standard REST and custom routes) +sum by (code)(rate(apiserver_request_total{group="provisioning.grafana.app"}[$__rate_interval])) + +# Requests by resource and status +sum by (resource, code)(rate(apiserver_request_total{group="provisioning.grafana.app"}[$__rate_interval])) + +# Error rate (4xx + 5xx) +sum(rate(apiserver_request_total{group="provisioning.grafana.app", code=~"[45].."}[$__rate_interval])) +/ +sum(rate(apiserver_request_total{group="provisioning.grafana.app"}[$__rate_interval])) + +# Custom routes specifically (filter by resource name) +sum by (resource, code)(rate(apiserver_request_total{ + group="provisioning.grafana.app", + resource=~"stats|settings" # custom route resources +}[$__rate_interval])) +``` + +### Example + +For a custom route like `/apis/provisioning.grafana.app/v0alpha1/namespaces/default/settings`: + +```go +func (b *APIBuilder) GetAPIRoutes(gv schema.GroupVersion) *builder.APIRoutes { + return &builder.APIRoutes{ + Namespace: []builder.APIRouteHandler{ + { + Path: "settings", + Handler: b.handleSettings, // Automatically instrumented! + }, + }, + } +} +``` + +Metrics recorded in `apiserver_request_total`: +``` +apiserver_request_total{ + verb="GET", + dry_run="", + group="provisioning.grafana.app", + version="v0alpha1", + resource="settings", + subresource="", + scope="namespace", + component="", + code="200" +} 1 + +apiserver_request_total{ + verb="GET", + dry_run="", + group="provisioning.grafana.app", + version="v0alpha1", + resource="settings", + subresource="", + scope="namespace", + component="", + code="401" +} 1 +``` + +### Implementation Details + +The `responseWriterWithStatus` wrapper: +- Wraps `http.ResponseWriter` to intercept `WriteHeader()` and `Write()` calls +- Captures the status code before the response is sent +- Defaults to 200 if `WriteHeader()` is never called +- Works with `errhttp.Write()`, manual `WriteHeader()`, and implicit status codes + +### API Servers with Custom Routes + +Currently instrumented (as of October 2025): +1. `provisioning.grafana.app` - stats, settings +2. `querylibrary.grafana.app` +3. `queries.grafana.app` +4. `alertenrichment.grafana.app` +5. `scim.grafana.app` +6. `dashboard.grafana.app` +7. `iam.grafana.app` +8. `ofrep.grafana.app` +9. `dashboardsnapshot.grafana.app` + +All of these automatically get metrics with no code changes required. + +### Testing + +To verify metrics are working: + +1. Make requests to custom routes +2. Check Prometheus: + ```promql + apiserver_custom_route_requests_total + ``` +3. You should see metrics with proper status codes (200, 401, 404, 500, etc.) + +### Future Improvements + +- Add request duration histogram +- Add request size histogram +- Add response size histogram +- Align label names with standard Kubernetes metrics diff --git a/pkg/services/apiserver/builder/custom_route_metrics.go b/pkg/services/apiserver/builder/custom_route_metrics.go new file mode 100644 index 00000000000..25799b87605 --- /dev/null +++ b/pkg/services/apiserver/builder/custom_route_metrics.go @@ -0,0 +1,100 @@ +package builder + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apiserver/pkg/endpoints/metrics" + "k8s.io/apiserver/pkg/endpoints/request" +) + +// CustomRouteMetrics provides metrics for custom API routes that bypass the standard +// Kubernetes REST storage path. It reuses the standard apiserver_request_total metric +// to ensure consistency with other apiserver metrics. +type CustomRouteMetrics struct { + // We don't store anything here since we use the global k8s metrics +} + +// NewCustomRouteMetrics creates a new CustomRouteMetrics. +// Note: This doesn't register any new metrics, it reuses the existing +// `apiserver_request_total` metric that's already registered by Kubernetes. +func NewCustomRouteMetrics(_ prometheus.Registerer) *CustomRouteMetrics { + // No need to register anything - we'll use the existing k8s metrics + return &CustomRouteMetrics{} +} + +type responseWriterWithStatus struct { + http.ResponseWriter + statusCode int + written bool +} + +func newResponseWriterWithStatus(w http.ResponseWriter) *responseWriterWithStatus { + return &responseWriterWithStatus{ + ResponseWriter: w, + statusCode: http.StatusOK, + written: false, + } +} + +func (w *responseWriterWithStatus) WriteHeader(statusCode int) { + if !w.written { + w.statusCode = statusCode + w.written = true + } + w.ResponseWriter.WriteHeader(statusCode) +} + +func (w *responseWriterWithStatus) Write(b []byte) (int, error) { + if !w.written { + // If WriteHeader hasn't been called, + // this is a StatusOK (default) + w.written = true + } + return w.ResponseWriter.Write(b) +} + +func (w *responseWriterWithStatus) StatusCode() int { + return w.statusCode +} + +// InstrumentHandler wraps an HTTP handler to record metrics for custom routes. +// It captures the status code and records it using the standard apiserver_request_total metric, +// making custom routes appear alongside regular Kubernetes API metrics. +func (m *CustomRouteMetrics) InstrumentHandler(group, version, resource string, handler http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + wrappedWriter := newResponseWriterWithStatus(w) + startTime := time.Now() + + handler(wrappedWriter, r) + + // Determine scope + // See: + // https://github.com/kubernetes/kubernetes/blob/3828756d90cf28e0ab5e0ccd550041b70c642b91/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go#L326 + scope := "cluster" + if reqInfo, ok := request.RequestInfoFrom(r.Context()); ok && reqInfo != nil { + scope = metrics.CleanScope(reqInfo) + } + + // Record using the standard Kubernetes apiserver_request_total metric + // This makes custom routes appear in the same metric as standard REST API calls + // See: + // https://github.com/kubernetes/kubernetes/blob/3828756d90cf28e0ab5e0ccd550041b70c642b91/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go#L78-L86 + metrics.MonitorRequest( + r, + r.Method, // verb (HTTP method) + group, // API group + version, // API version + resource, // resource name (e.g., "stats", "settings") + "", // subresource (empty for custom routes) + scope, // scope (cluster, namespace, or resource) + "", // not sure if component is neeeded for custom resources + false, // if endpoint is deprecated, default to false + "", // `removedRelease` is unused with `deprecated=false` + wrappedWriter.StatusCode(), // HTTP status code + 0, // respSize (not tracked) + time.Since(startTime), + ) + } +} diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index 2b775335b9a..dbff00737c9 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -37,7 +37,7 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/apistore" ) -type BuildHandlerChainFuncFromBuilders = func([]APIGroupBuilder) BuildHandlerChainFunc +type BuildHandlerChainFuncFromBuilders = func([]APIGroupBuilder, prometheus.Registerer) BuildHandlerChainFunc type BuildHandlerChainFunc = func(delegateHandler http.Handler, c *genericapiserver.Config) http.Handler func ProvideDefaultBuildHandlerChainFuncFromBuilders() BuildHandlerChainFuncFromBuilders { @@ -66,12 +66,13 @@ var PathRewriters = []filters.PathRewriter{ }, } -func GetDefaultBuildHandlerChainFunc(builders []APIGroupBuilder) BuildHandlerChainFunc { +func GetDefaultBuildHandlerChainFunc(builders []APIGroupBuilder, reg prometheus.Registerer) BuildHandlerChainFunc { return func(delegateHandler http.Handler, c *genericapiserver.Config) http.Handler { requestHandler, err := GetCustomRoutesHandler( delegateHandler, c.LoopbackClientConfig, - builders) + builders, + reg) if err != nil { panic(fmt.Sprintf("could not build the request handler for specified API builders: %s", err.Error())) } @@ -106,6 +107,7 @@ func SetupConfig( buildHandlerChainFuncFromBuilders BuildHandlerChainFuncFromBuilders, gvs []schema.GroupVersion, additionalOpenAPIDefGetters []common.GetOpenAPIDefinitions, + reg prometheus.Registerer, ) error { serverConfig.AdmissionControl = NewAdmissionFromBuilders(builders) defsGetter := GetOpenAPIDefinitions(builders, additionalOpenAPIDefGetters...) @@ -229,7 +231,7 @@ func SetupConfig( serverConfig.OpenAPIV3Config.Info.Version = buildVersion serverConfig.SkipOpenAPIInstallation = false - serverConfig.BuildHandlerChainFunc = buildHandlerChainFuncFromBuilders(builders) + serverConfig.BuildHandlerChainFunc = buildHandlerChainFuncFromBuilders(builders, reg) // set priority for aggregated discovery for i, b := range builders { diff --git a/pkg/services/apiserver/builder/request_handler.go b/pkg/services/apiserver/builder/request_handler.go index abeed273df8..80fbf6d0eaa 100644 --- a/pkg/services/apiserver/builder/request_handler.go +++ b/pkg/services/apiserver/builder/request_handler.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" restclient "k8s.io/client-go/rest" "k8s.io/kube-openapi/pkg/spec3" ) @@ -13,10 +14,12 @@ type requestHandler struct { router *mux.Router } -func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient.Config, builders []APIGroupBuilder) (http.Handler, error) { +func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient.Config, builders []APIGroupBuilder, metricsRegistry prometheus.Registerer) (http.Handler, error) { useful := false // only true if any routes exist anywhere router := mux.NewRouter() + metrics := NewCustomRouteMetrics(metricsRegistry) + for _, builder := range builders { provider, ok := builder.(APIGroupRouteProvider) if !ok || provider == nil { @@ -44,7 +47,15 @@ func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient if err != nil { return nil, err } - sub.HandleFunc("/"+route.Path, route.Handler). + + instrumentedHandler := metrics.InstrumentHandler( + gv.Group, + gv.Version, + route.Path, // Use path as resource identifier + route.Handler, + ) + + sub.HandleFunc("/"+route.Path, instrumentedHandler). Methods(methods...) } @@ -62,7 +73,15 @@ func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient if err != nil { return nil, err } - sub.HandleFunc("/"+route.Path, route.Handler). + + instrumentedHandler := metrics.InstrumentHandler( + gv.Group, + gv.Version, + route.Path, // Use path as resource identifier + route.Handler, + ) + + sub.HandleFunc("/"+route.Path, instrumentedHandler). Methods(methods...) } } diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 86706dcbb48..77acc027a6f 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -354,6 +354,7 @@ func (s *service) start(ctx context.Context) error { s.buildHandlerChainFuncFromBuilders, groupVersions, defGetters, + s.metrics, ) if err != nil { return err