API Server: Include traceID in HTTP request logs (#88179)

pkg/apiserver/builder/helper.go

@@ -91,7 +91,9 @@ func SetupConfig(
 			panic(fmt.Sprintf("could not build handler chain func: %s", err.Error()))
 		}
 
-		handler := genericapiserver.DefaultBuildHandlerChain(requestHandler, c)
+		// Needs to run last in request chain to function as expected, hence we register it first.
+		handler := filters.WithTracingHTTPLoggingAttributes(requestHandler)
+		handler = genericapiserver.DefaultBuildHandlerChain(handler, c)
 		handler = filters.WithAcceptHeader(handler)
 		handler = filters.WithPathRewriters(handler, pathRewriters)
 		handler = k8stracing.WithTracing(handler, serverConfig.TracerProvider, "KubernetesAPI")

pkg/apiserver/endpoints/filters/tracing_log.go

@@ -0,0 +1,26 @@
+package filters
+
+import (
+	"net/http"
+
+	"go.opentelemetry.io/otel/trace"
+	"k8s.io/apiserver/pkg/server/httplog"
+)
+
+// WithTracingHTTPLoggingAttributes adds tracing attributes to HTTP request logs.
+func WithTracingHTTPLoggingAttributes(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		spanCtx := trace.SpanContextFromContext(r.Context())
+		if spanCtx.IsValid() {
+			if spanCtx.HasTraceID() {
+				httplog.AddKeyValue(r.Context(), "traceID", spanCtx.TraceID().String())
+			}
+
+			if spanCtx.HasSpanID() {
+				httplog.AddKeyValue(r.Context(), "spanID", spanCtx.SpanID().String())
+			}
+		}
+
+		handler.ServeHTTP(w, r)
+	})
+}

pkg/apiserver/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240409140820-518d3341d58f
 	github.com/prometheus/client_golang v1.19.0
 	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/otel/trace v1.26.0
 	golang.org/x/mod v0.15.0
 	k8s.io/apimachinery v0.29.2
 	k8s.io/apiserver v0.29.2
@@ -122,7 +123,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 // indirect
 	go.opentelemetry.io/otel/metric v1.26.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.26.0 // indirect
-	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect