fix: apiserver metrics in custom handlers (#112508)

* fix: apiserver metrics in custom handlers * chore: review feedback
2025-10-20 13:17:56 +02:00
parent b0acfd1189
commit c04f6bedec
5 changed files with 291 additions and 7 deletions
@@ -0,0 +1,162 @@
+## Custom Route Metrics in Grafana API Server
+
+### Problem
+
+Custom API routes registered via `APIGroupRouteProvider.GetAPIRoutes()` bypass the standard Kubernetes apiserver metrics recording middleware. Requests to these routes were not being recorded in `apiserver_request_total`.
+
+**Why this happens:**
+
+```
+HTTP Request
+    ↓
+Gorilla Mux (custom routes) 
+    ├─ [MATCHED] → Your custom handler → Response ❌ (no metrics recorded)
+    │
+    └─ [NOT MATCHED] → k8s DefaultBuildHandlerChain
+                           ├─ WithRequestMetrics (records apiserver_request_total) ✅
+                           └─ Standard REST storage handlers
+```
+
+Custom routes are served by Gorilla Mux and return immediately, never reaching the k8s metrics middleware.
+
+### Solution
+
+We've implemented **centralized automatic instrumentation** that records custom route requests in the **same `apiserver_request_total` metric** used by standard Kubernetes API calls. This means:
+
+- ✅ **Single metric** for all API requests (standard + custom routes)
+- ✅ **Consistent labels** matching Kubernetes conventions
+- ✅ **No dashboard changes** needed - existing queries work
+- ✅ **Automatic** for all API servers with zero code changes
+
+#### Key Components
+
+1. **`pkg/services/apiserver/builder/custom_route_metrics.go`**
+   - Defines `CustomRouteMetrics` 
+   - Uses the existing `apiserver_request_total` metric (via `metrics.MonitorRequest`)
+   - Provides `responseWriterWithStatus` to capture HTTP status codes
+   - Provides `InstrumentHandler()` to wrap route handlers
+
+2. **`pkg/services/apiserver/builder/request_handler.go`**
+   - Updated `GetCustomRoutesHandler()` to accept `prometheus.Registerer`
+   - Automatically instruments ALL custom routes (both root and namespace)
+   - Works transparently for all API servers
+
+3. **Updated signatures**
+   - `BuildHandlerChainFuncFromBuilders`: now accepts `prometheus.Registerer`
+   - `GetDefaultBuildHandlerChainFunc`: now accepts `prometheus.Registerer`
+   - `SetupConfig`: now accepts `prometheus.Registerer`
+   - Factory interfaces updated to pass registry through
+
+### Usage
+
+**For new API servers:** No action required! Custom routes are automatically instrumented.
+
+**For existing API servers:** No changes needed. All API servers using `APIGroupRouteProvider` automatically get metrics.
+
+### Metrics
+
+Query in Prometheus - **same metric as standard API calls**:
+
+```promql
+# Total requests by status code (includes both standard REST and custom routes)
+sum by (code)(rate(apiserver_request_total{group="provisioning.grafana.app"}[$__rate_interval]))
+
+# Requests by resource and status
+sum by (resource, code)(rate(apiserver_request_total{group="provisioning.grafana.app"}[$__rate_interval]))
+
+# Error rate (4xx + 5xx)
+sum(rate(apiserver_request_total{group="provisioning.grafana.app", code=~"[45].."}[$__rate_interval]))
+/ 
+sum(rate(apiserver_request_total{group="provisioning.grafana.app"}[$__rate_interval]))
+
+# Custom routes specifically (filter by resource name)
+sum by (resource, code)(rate(apiserver_request_total{
+  group="provisioning.grafana.app",
+  resource=~"stats|settings"  # custom route resources
+}[$__rate_interval]))
+```
+
+### Example
+
+For a custom route like `/apis/provisioning.grafana.app/v0alpha1/namespaces/default/settings`:
+
+```go
+func (b *APIBuilder) GetAPIRoutes(gv schema.GroupVersion) *builder.APIRoutes {
+    return &builder.APIRoutes{
+        Namespace: []builder.APIRouteHandler{
+            {
+                Path: "settings",
+                Handler: b.handleSettings, // Automatically instrumented!
+            },
+        },
+    }
+}
+```
+
+Metrics recorded in `apiserver_request_total`:
+```
+apiserver_request_total{
+  verb="GET",
+  dry_run="",
+  group="provisioning.grafana.app",
+  version="v0alpha1",
+  resource="settings",
+  subresource="",
+  scope="namespace",
+  component="",
+  code="200"
+} 1
+
+apiserver_request_total{
+  verb="GET",
+  dry_run="",
+  group="provisioning.grafana.app",
+  version="v0alpha1",
+  resource="settings",
+  subresource="",
+  scope="namespace",
+  component="",
+  code="401"
+} 1
+```
+
+### Implementation Details
+
+The `responseWriterWithStatus` wrapper:
+- Wraps `http.ResponseWriter` to intercept `WriteHeader()` and `Write()` calls
+- Captures the status code before the response is sent
+- Defaults to 200 if `WriteHeader()` is never called
+- Works with `errhttp.Write()`, manual `WriteHeader()`, and implicit status codes
+
+### API Servers with Custom Routes
+
+Currently instrumented (as of October 2025):
+1. `provisioning.grafana.app` - stats, settings
+2. `querylibrary.grafana.app`
+3. `queries.grafana.app`
+4. `alertenrichment.grafana.app`
+5. `scim.grafana.app`
+6. `dashboard.grafana.app`
+7. `iam.grafana.app`
+8. `ofrep.grafana.app`
+9. `dashboardsnapshot.grafana.app`
+
+All of these automatically get metrics with no code changes required.
+
+### Testing
+
+To verify metrics are working:
+
+1. Make requests to custom routes
+2. Check Prometheus:
+   ```promql
+   apiserver_custom_route_requests_total
+   ```
+3. You should see metrics with proper status codes (200, 401, 404, 500, etc.)
+
+### Future Improvements
+
+- Add request duration histogram
+- Add request size histogram
+- Add response size histogram
+- Align label names with standard Kubernetes metrics
@@ -0,0 +1,100 @@
+package builder
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// CustomRouteMetrics provides metrics for custom API routes that bypass the standard
+// Kubernetes REST storage path. It reuses the standard apiserver_request_total metric
+// to ensure consistency with other apiserver metrics.
+type CustomRouteMetrics struct {
+	// We don't store anything here since we use the global k8s metrics
+}
+
+// NewCustomRouteMetrics creates a new CustomRouteMetrics.
+// Note: This doesn't register any new metrics, it reuses the existing
+// `apiserver_request_total` metric that's already registered by Kubernetes.
+func NewCustomRouteMetrics(_ prometheus.Registerer) *CustomRouteMetrics {
+	// No need to register anything - we'll use the existing k8s metrics
+	return &CustomRouteMetrics{}
+}
+
+type responseWriterWithStatus struct {
+	http.ResponseWriter
+	statusCode int
+	written    bool
+}
+
+func newResponseWriterWithStatus(w http.ResponseWriter) *responseWriterWithStatus {
+	return &responseWriterWithStatus{
+		ResponseWriter: w,
+		statusCode:     http.StatusOK,
+		written:        false,
+	}
+}
+
+func (w *responseWriterWithStatus) WriteHeader(statusCode int) {
+	if !w.written {
+		w.statusCode = statusCode
+		w.written = true
+	}
+	w.ResponseWriter.WriteHeader(statusCode)
+}
+
+func (w *responseWriterWithStatus) Write(b []byte) (int, error) {
+	if !w.written {
+		// If WriteHeader hasn't been called,
+		// this is a StatusOK (default)
+		w.written = true
+	}
+	return w.ResponseWriter.Write(b)
+}
+
+func (w *responseWriterWithStatus) StatusCode() int {
+	return w.statusCode
+}
+
+// InstrumentHandler wraps an HTTP handler to record metrics for custom routes.
+// It captures the status code and records it using the standard apiserver_request_total metric,
+// making custom routes appear alongside regular Kubernetes API metrics.
+func (m *CustomRouteMetrics) InstrumentHandler(group, version, resource string, handler http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		wrappedWriter := newResponseWriterWithStatus(w)
+		startTime := time.Now()
+
+		handler(wrappedWriter, r)
+
+		// Determine scope
+		// See:
+		// https://github.com/kubernetes/kubernetes/blob/3828756d90cf28e0ab5e0ccd550041b70c642b91/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go#L326
+		scope := "cluster"
+		if reqInfo, ok := request.RequestInfoFrom(r.Context()); ok && reqInfo != nil {
+			scope = metrics.CleanScope(reqInfo)
+		}
+
+		// Record using the standard Kubernetes apiserver_request_total metric
+		// This makes custom routes appear in the same metric as standard REST API calls
+		// See:
+		// https://github.com/kubernetes/kubernetes/blob/3828756d90cf28e0ab5e0ccd550041b70c642b91/staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go#L78-L86
+		metrics.MonitorRequest(
+			r,
+			r.Method,                   // verb (HTTP method)
+			group,                      // API group
+			version,                    // API version
+			resource,                   // resource name (e.g., "stats", "settings")
+			"",                         // subresource (empty for custom routes)
+			scope,                      // scope (cluster, namespace, or resource)
+			"",                         // not sure if component is neeeded for custom resources
+			false,                      // if endpoint is deprecated, default to false
+			"",                         // `removedRelease` is unused with `deprecated=false`
+			wrappedWriter.StatusCode(), // HTTP status code
+			0,                          // respSize (not tracked)
+			time.Since(startTime),
+		)
+	}
+}
@@ -37,7 +37,7 @@ import (
 	"github.com/grafana/grafana/pkg/storage/unified/apistore"
 )

-type BuildHandlerChainFuncFromBuilders = func([]APIGroupBuilder) BuildHandlerChainFunc
+type BuildHandlerChainFuncFromBuilders = func([]APIGroupBuilder, prometheus.Registerer) BuildHandlerChainFunc
 type BuildHandlerChainFunc = func(delegateHandler http.Handler, c *genericapiserver.Config) http.Handler

 func ProvideDefaultBuildHandlerChainFuncFromBuilders() BuildHandlerChainFuncFromBuilders {
@@ -66,12 +66,13 @@ var PathRewriters = []filters.PathRewriter{
 	},
 }

-func GetDefaultBuildHandlerChainFunc(builders []APIGroupBuilder) BuildHandlerChainFunc {
+func GetDefaultBuildHandlerChainFunc(builders []APIGroupBuilder, reg prometheus.Registerer) BuildHandlerChainFunc {
 	return func(delegateHandler http.Handler, c *genericapiserver.Config) http.Handler {
 		requestHandler, err := GetCustomRoutesHandler(
 			delegateHandler,
 			c.LoopbackClientConfig,
-			builders)
+			builders,
+			reg)
 		if err != nil {
 			panic(fmt.Sprintf("could not build the request handler for specified API builders: %s", err.Error()))
 		}
@@ -106,6 +107,7 @@ func SetupConfig(
 	buildHandlerChainFuncFromBuilders BuildHandlerChainFuncFromBuilders,
 	gvs []schema.GroupVersion,
 	additionalOpenAPIDefGetters []common.GetOpenAPIDefinitions,
+	reg prometheus.Registerer,
 ) error {
 	serverConfig.AdmissionControl = NewAdmissionFromBuilders(builders)
 	defsGetter := GetOpenAPIDefinitions(builders, additionalOpenAPIDefGetters...)
@@ -229,7 +231,7 @@ func SetupConfig(
 	serverConfig.OpenAPIV3Config.Info.Version = buildVersion

 	serverConfig.SkipOpenAPIInstallation = false
-	serverConfig.BuildHandlerChainFunc = buildHandlerChainFuncFromBuilders(builders)
+	serverConfig.BuildHandlerChainFunc = buildHandlerChainFuncFromBuilders(builders, reg)

 	// set priority for aggregated discovery
 	for i, b := range builders {
@@ -5,6 +5,7 @@ import (
 	"net/http"

 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/kube-openapi/pkg/spec3"
 )
@@ -13,10 +14,12 @@ type requestHandler struct {
 	router *mux.Router
 }

-func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient.Config, builders []APIGroupBuilder) (http.Handler, error) {
+func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient.Config, builders []APIGroupBuilder, metricsRegistry prometheus.Registerer) (http.Handler, error) {
 	useful := false // only true if any routes exist anywhere
 	router := mux.NewRouter()

+	metrics := NewCustomRouteMetrics(metricsRegistry)
+
 	for _, builder := range builders {
 		provider, ok := builder.(APIGroupRouteProvider)
 		if !ok || provider == nil {
@@ -44,7 +47,15 @@ func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient
 				if err != nil {
 					return nil, err
 				}
-				sub.HandleFunc("/"+route.Path, route.Handler).
+
+				instrumentedHandler := metrics.InstrumentHandler(
+					gv.Group,
+					gv.Version,
+					route.Path, // Use path as resource identifier
+					route.Handler,
+				)
+
+				sub.HandleFunc("/"+route.Path, instrumentedHandler).
 					Methods(methods...)
 			}

@@ -62,7 +73,15 @@ func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient
 				if err != nil {
 					return nil, err
 				}
-				sub.HandleFunc("/"+route.Path, route.Handler).
+
+				instrumentedHandler := metrics.InstrumentHandler(
+					gv.Group,
+					gv.Version,
+					route.Path, // Use path as resource identifier
+					route.Handler,
+				)
+
+				sub.HandleFunc("/"+route.Path, instrumentedHandler).
 					Methods(methods...)
 			}
 		}
@@ -354,6 +354,7 @@ func (s *service) start(ctx context.Context) error {
 		s.buildHandlerChainFuncFromBuilders,
 		groupVersions,
 		defGetters,
+		s.metrics,
 	)
 	if err != nil {
 		return err