From 2d5300ce3aac7c7af9d781aa7a40ca562e8fd458 Mon Sep 17 00:00:00 2001 From: Steve Simpson Date: Wed, 19 Jul 2023 16:01:54 +0200 Subject: [PATCH] gRPC Server: Instrument requests made to the server. (#71914) * gRPC Server: Instrument requests made to the server. Expose metrics from the gRPC server in order to monitor for failed responses and response latency. Uses code from the already vendored weaveworks/common. * Review comments. --- go.mod | 3 +++ go.sum | 5 +++++ pkg/services/grpcserver/service.go | 26 +++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1763048e56c..212fe5ce9c7 100644 --- a/go.mod +++ b/go.mod @@ -327,6 +327,8 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/getsentry/sentry-go v0.12.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect + github.com/gogo/googleapis v1.4.1 // indirect + github.com/gogo/status v1.1.1 // indirect github.com/google/cel-go v0.12.6 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -356,6 +358,7 @@ require ( github.com/onsi/gomega v1.27.6 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20220512140940-7b36cea86235 // indirect + github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect github.com/ory/go-acc v0.2.6 // indirect github.com/ory/go-convenience v0.1.0 // indirect github.com/ory/viper v1.7.5 // indirect diff --git a/go.sum b/go.sum index e41cac64f03..83a539d545f 100644 --- a/go.sum +++ b/go.sum @@ -1110,6 +1110,7 @@ github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gofrs/uuid/v3 v3.1.2/go.mod h1:xPwMqoocQ1L5G6pXX5BcE7N5jlzn2o19oqAKxwZW/kI= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -1120,6 +1121,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg= +github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -1944,6 +1947,8 @@ github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rm github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= +github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w= +github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= diff --git a/pkg/services/grpcserver/service.go b/pkg/services/grpcserver/service.go index bf267fb420e..ea742bf8483 100644 --- a/pkg/services/grpcserver/service.go +++ b/pkg/services/grpcserver/service.go @@ -8,6 +8,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpcAuth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + "github.com/prometheus/client_golang/prometheus" + "github.com/weaveworks/common/instrument" + "github.com/weaveworks/common/middleware" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -19,6 +22,10 @@ import ( "github.com/grafana/grafana/pkg/setting" ) +var ( + grpcRequestDuration *prometheus.HistogramVec +) + type Provider interface { registry.BackgroundService GetServer() *grpc.Server @@ -32,12 +39,27 @@ type GPRCServerService struct { address string } -func ProvideService(cfg *setting.Cfg, authenticator interceptors.Authenticator, tracer tracing.Tracer) (Provider, error) { +func ProvideService(cfg *setting.Cfg, authenticator interceptors.Authenticator, tracer tracing.Tracer, registerer prometheus.Registerer) (Provider, error) { s := &GPRCServerService{ cfg: cfg, logger: log.New("grpc-server"), } + // Register the metric here instead of an init() function so that we do + // nothing unless the feature is actually enabled. + if grpcRequestDuration == nil { + grpcRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "grafana", + Name: "grpc_request_duration_seconds", + Help: "Time (in seconds) spent serving HTTP requests.", + Buckets: instrument.DefBuckets, + }, []string{"method", "route", "status_code", "ws"}) + + if err := registerer.Register(grpcRequestDuration); err != nil { + return nil, err + } + } + var opts []grpc.ServerOption // Default auth is admin token check, but this can be overridden by @@ -48,12 +70,14 @@ func ProvideService(cfg *setting.Cfg, authenticator interceptors.Authenticator, grpc_middleware.ChainUnaryServer( grpcAuth.UnaryServerInterceptor(authenticator.Authenticate), interceptors.TracingUnaryInterceptor(tracer), + middleware.UnaryServerInstrumentInterceptor(grpcRequestDuration), ), ), grpc.StreamInterceptor( grpc_middleware.ChainStreamServer( interceptors.TracingStreamInterceptor(tracer), grpcAuth.StreamServerInterceptor(authenticator.Authenticate), + middleware.StreamServerInstrumentInterceptor(grpcRequestDuration), ), ), }...)