From 868e3a5e8e52a3eed320760a19be22c7e1f90873 Mon Sep 17 00:00:00 2001 From: Eric Leijonmarck Date: Tue, 16 Sep 2025 09:49:37 +0100 Subject: [PATCH] `grafana-iam`: Adds config opts for RBACRemoteClient for load balancing (#110819) --- conf/defaults.ini | 10 +++++++++ conf/sample.ini | 6 +++++ pkg/services/authz/README.md | 7 ++++++ pkg/services/authz/rbac.go | 10 +++++++++ pkg/services/authz/rbac_settings.go | 8 ++++--- pkg/services/grpcserver/README.md | 33 ++++++++++++++++++++++++---- pkg/services/grpcserver/service.go | 34 ++++++++++++++++++++++++++++- pkg/setting/setting_grpc.go | 23 +++++++++++++++++++ 8 files changed, 123 insertions(+), 8 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index bca78940a73..bc8a5168967 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -123,6 +123,16 @@ max_recv_msg_size = # Maximum size of a message that can be sent in bytes. If not set, uses the gRPC default (unlimited). max_send_msg_size = +# Maximum amount of time a connection may exist before it will be closed +max_connection_age = +max_connection_age_grace = +max_connection_idle = + +# Frequency of server-to-client pings to check if a connection is still active +keepalive_time = +keepalive_timeout = +keepalive_min_time = + #################################### Database ############################ [database] # You can configure the database connection by specifying type, host, name, user and password diff --git a/conf/sample.ini b/conf/sample.ini index bda3c51b01f..3d0f1d6dc32 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -119,6 +119,12 @@ ;max_send_msg_size = # this will log the request and response for each unary gRPC call ;enable_logging = false +;max_connection_age = +;max_connection_age_grace = +;max_connection_idle = +;keepalive_time = +;keepalive_timeout = +;keepalive_min_time = #################################### Database #################################### [database] diff --git a/pkg/services/authz/README.md b/pkg/services/authz/README.md index 27d4dda2dfd..5b7d672aec7 100644 --- a/pkg/services/authz/README.md +++ b/pkg/services/authz/README.md @@ -31,6 +31,13 @@ listen = false mode = "inproc" ``` +For load balancing you would want to enable the load balancing configuration. This sets sane default for multiple pods to be evenly distributed with load across the different pods. + +```ini +[authorization] +load_balancing_enabled = true +``` + ### Example Here is an example to connect the authorization client to a remote grpc server. diff --git a/pkg/services/authz/rbac.go b/pkg/services/authz/rbac.go index e713ceae2be..753d3934c62 100644 --- a/pkg/services/authz/rbac.go +++ b/pkg/services/authz/rbac.go @@ -180,6 +180,16 @@ func newRemoteRBACClient(clientCfg *authzClientSettings, tracer trace.Tracer, re grpc.WithChainStreamInterceptor(streamInterceptors...), } + // // if we serve the client as a load balancer + if clientCfg.loadBalancingEnabled { + // Use round_robin to balances requests more evenly over the available Grafana replicas. + opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy": "round_robin"}`)) + + // Disable looking up service config from TXT DNS records. + // This reduces the number of requests made to the DNS servers. + opts = append(opts, grpc.WithDisableServiceConfig()) + } + conn, err := grpc.NewClient(clientCfg.remoteAddress, opts...) if err != nil { return nil, fmt.Errorf("failed to create authz client to remote server: %w", err) diff --git a/pkg/services/authz/rbac_settings.go b/pkg/services/authz/rbac_settings.go index a4a48a2e7f1..d3027ce756c 100644 --- a/pkg/services/authz/rbac_settings.go +++ b/pkg/services/authz/rbac_settings.go @@ -23,9 +23,10 @@ const ( ) type authzClientSettings struct { - remoteAddress string - certFile string - mode clientMode + remoteAddress string + certFile string + mode clientMode + loadBalancingEnabled bool token string tokenExchangeURL string @@ -51,6 +52,7 @@ func readAuthzClientSettings(cfg *setting.Cfg) (*authzClientSettings, error) { if s.mode == clientModeInproc { return s, nil } + s.loadBalancingEnabled = authzSection.Key("load_balancing_enabled").MustBool(false) s.remoteAddress = authzSection.Key("remote_address").MustString("") s.certFile = authzSection.Key("cert_file").MustString("") diff --git a/pkg/services/grpcserver/README.md b/pkg/services/grpcserver/README.md index f96f64072aa..16d14751189 100644 --- a/pkg/services/grpcserver/README.md +++ b/pkg/services/grpcserver/README.md @@ -12,16 +12,16 @@ The `grpcserver` package provides the implementation of the gRPC server for hand ## Usage -Enable the gRPC server in Grafana by setting the `grpcServer` feature toggle to `true` in your `custom.ini` configuration file. +Enable the gRPC server in Grafana by setting the `grpcServer` feature toggle to `true` in your `custom.ini` configuration file. -``` ini +```ini [feature_toggles] grpcServer = true ``` You can specify the gRPC server specific settings in the `grpc_server` section of the configuration file. -``` ini +```ini [grpc_server] network = "tcp" address = "127.0.0.1:10000" @@ -36,6 +36,31 @@ max_recv_msg_size = max_send_msg_size = ``` +### Optional: Connection Management and Load Balancing + +These settings help with: + +- **Resource management**: Prevent resource leaks from idle connections +- **Connection health**: Detect and clean up dead connections +- **Load balancing**: Force connection recycling for better distribution across multiple server instances +- **DoS protection**: Rate limit keepalive pings from clients + +```ini +# Connection management options +# Maximum amount of time a connection may exist before it will be closed +max_connection_age = 300s +# Additional time to allow for pending RPCs to complete before forcibly closing connections +max_connection_age_grace = 10s +# Maximum amount of idle time before a connection is closed +max_connection_idle = 300s +# Frequency of server-to-client pings to check if a connection is still active +keepalive_time = 30s +# Amount of time the server waits for a response to keepalive pings before closing the connection +keepalive_timeout = 5s +# Minimum amount of time a client should wait before sending a keepalive ping +keepalive_min_time = 5s +``` + ## Example Services -View [health.go] and [reflection.go] for examples of how to implement gRPC services in Grafana. These services are currently initialized by the [background service registry](../../registry/backgroundsvcs/background_services.go). \ No newline at end of file +View [health.go] and [reflection.go] for examples of how to implement gRPC services in Grafana. These services are currently initialized by the [background service registry](../../registry/backgroundsvcs/background_services.go). diff --git a/pkg/services/grpcserver/service.go b/pkg/services/grpcserver/service.go index e42fb60b4a0..a1a1be8f51e 100644 --- a/pkg/services/grpcserver/service.go +++ b/pkg/services/grpcserver/service.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/registry" @@ -104,12 +105,43 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, authe opts = append(opts, grpc.MaxSendMsgSize(s.cfg.MaxSendMsgSize)) } + // Apply connection management settings + keepaliveParams := keepalive.ServerParameters{ + MaxConnectionAge: s.cfg.MaxConnectionAge, + MaxConnectionAgeGrace: s.cfg.MaxConnectionAgeGrace, + MaxConnectionIdle: s.cfg.MaxConnectionIdle, + Time: s.cfg.KeepaliveTime, + Timeout: s.cfg.KeepaliveTimeout, + } + keepalivePolicy := keepalive.EnforcementPolicy{ + MinTime: s.cfg.KeepaliveMinTime, + } + + // Only add keepalive options if any values are configured + if s.cfg.MaxConnectionAge > 0 || s.cfg.MaxConnectionAgeGrace > 0 || s.cfg.MaxConnectionIdle > 0 || + s.cfg.KeepaliveTime > 0 || s.cfg.KeepaliveTimeout > 0 { + opts = append(opts, grpc.KeepaliveParams(keepaliveParams)) + } + if s.cfg.KeepaliveMinTime > 0 { + opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalivePolicy)) + } + s.server = grpc.NewServer(opts...) return s, nil } func (s *gPRCServerService) Run(ctx context.Context) error { - s.logger.Info("Running GRPC server", "address", s.cfg.Address, "network", s.cfg.Network, "tls", s.cfg.TLSConfig != nil, "max_recv_msg_size", s.cfg.MaxRecvMsgSize, "max_send_msg_size", s.cfg.MaxSendMsgSize) + s.logger.Info("Running GRPC server", + "address", s.cfg.Address, + "network", s.cfg.Network, + "tls", s.cfg.TLSConfig != nil, + "max_recv_msg_size", s.cfg.MaxRecvMsgSize, + "max_send_msg_size", s.cfg.MaxSendMsgSize, + "max_connection_age", s.cfg.MaxConnectionAge, + "max_connection_idle", s.cfg.MaxConnectionIdle, + "keepalive_time", s.cfg.KeepaliveTime, + "keepalive_timeout", s.cfg.KeepaliveTimeout, + "keepalive_min_time", s.cfg.KeepaliveMinTime) listener, err := net.Listen(s.cfg.Network, s.cfg.Address) if err != nil { diff --git a/pkg/setting/setting_grpc.go b/pkg/setting/setting_grpc.go index a28c262b93c..c747a42c29d 100644 --- a/pkg/setting/setting_grpc.go +++ b/pkg/setting/setting_grpc.go @@ -5,6 +5,7 @@ import ( "fmt" "io/fs" "os" + "time" "github.com/spf13/pflag" @@ -20,6 +21,12 @@ type GRPCServerSettings struct { MaxRecvMsgSize int MaxSendMsgSize int + MaxConnectionAge time.Duration + MaxConnectionAgeGrace time.Duration + KeepaliveTime time.Duration + KeepaliveTimeout time.Duration + KeepaliveMinTime time.Duration + MaxConnectionIdle time.Duration // Internal fields useTLS bool certFile string @@ -119,6 +126,14 @@ func readGRPCServerSettings(cfg *Cfg, iniFile *ini.File) error { cfg.GRPCServer.MaxRecvMsgSize = server.Key("max_recv_msg_size").MustInt(0) cfg.GRPCServer.MaxSendMsgSize = server.Key("max_send_msg_size").MustInt(0) + // Read connection management options from INI file + cfg.GRPCServer.MaxConnectionAge = server.Key("max_connection_age").MustDuration(0) + cfg.GRPCServer.MaxConnectionAgeGrace = server.Key("max_connection_age_grace").MustDuration(0) + cfg.GRPCServer.MaxConnectionIdle = server.Key("max_connection_idle").MustDuration(0) + cfg.GRPCServer.KeepaliveTime = server.Key("keepalive_time").MustDuration(0) + cfg.GRPCServer.KeepaliveTimeout = server.Key("keepalive_timeout").MustDuration(0) + cfg.GRPCServer.KeepaliveMinTime = server.Key("keepalive_min_time").MustDuration(0) + return cfg.GRPCServer.processAddress() } @@ -134,4 +149,12 @@ func (c *GRPCServerSettings) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&c.useTLS, "grpc-server-use-tls", false, "Enable TLS for the gRPC server") fs.StringVar(&c.certFile, "grpc-server-cert-file", "", "Path to the certificate file for the gRPC server") fs.StringVar(&c.keyFile, "grpc-server-key-file", "", "Path to the certificate key file for the gRPC server") + + // Connection management options + fs.DurationVar(&c.MaxConnectionAge, "grpc-server-max-connection-age", 0, "Maximum amount of time a connection may exist before it will be closed (e.g. 30s)") + fs.DurationVar(&c.MaxConnectionAgeGrace, "grpc-server-max-connection-age-grace", 0, "Additional time to allow for pending RPCs to complete before forcibly closing connections (e.g. 5s)") + fs.DurationVar(&c.MaxConnectionIdle, "grpc-server-max-connection-idle", 0, "Maximum amount of idle time before a connection is closed (e.g. 15s)") + fs.DurationVar(&c.KeepaliveTime, "grpc-server-keepalive-time", 0, "Frequency of server-to-client pings to check if a connection is still active (e.g. 10s)") + fs.DurationVar(&c.KeepaliveTimeout, "grpc-server-keepalive-timeout", 0, "Amount of time the server waits for a response to keepalive pings before closing the connection (e.g. 3s)") + fs.DurationVar(&c.KeepaliveMinTime, "grpc-server-keepalive-min-time", 0, "Minimum amount of time a client should wait before sending a keepalive ping (e.g. 5s)") }