public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

grafana-iam: Adds config opts for RBACRemoteClient for load balancing (#110819)

Browse Source
This commit is contained in:
Eric Leijonmarck
2025-09-16 09:49:37 +01:00
committed by GitHub
parent 22b96c7c3e
commit 868e3a5e8e
8 changed files with 123 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conf/defaults.ini
+10
View File
@@ -123,6 +123,16 @@ max_recv_msg_size =
# Maximum size of a message that can be sent in bytes. If not set, uses the gRPC default (unlimited).
max_send_msg_size =
# Maximum amount of time a connection may exist before it will be closed
max_connection_age =
max_connection_age_grace =
max_connection_idle =
# Frequency of server-to-client pings to check if a connection is still active
keepalive_time =
keepalive_timeout =
keepalive_min_time =
#################################### Database ############################
[database]
# You can configure the database connection by specifying type, host, name, user and password
conf/sample.ini
+6
View File
@@ -119,6 +119,12 @@
;max_send_msg_size =
# this will log the request and response for each unary gRPC call
;enable_logging = false
;max_connection_age =
;max_connection_age_grace =
;max_connection_idle =
;keepalive_time =
;keepalive_timeout =
;keepalive_min_time =
#################################### Database ####################################
[database]
pkg/services/authz/README.md
+7
View File
@@ -31,6 +31,13 @@ listen = false
mode = "inproc"
```
For load balancing you would want to enable the load balancing configuration. This sets sane default for multiple pods to be evenly distributed with load across the different pods.
```ini
[authorization]
load_balancing_enabled = true
```
### Example
Here is an example to connect the authorization client to a remote grpc server.
pkg/services/authz/rbac.go
+10
View File
@@ -180,6 +180,16 @@ func newRemoteRBACClient(clientCfg *authzClientSettings, tracer trace.Tracer, re
grpc.WithChainStreamInterceptor(streamInterceptors...),
}
// // if we serve the client as a load balancer
if clientCfg.loadBalancingEnabled {
// Use round_robin to balances requests more evenly over the available Grafana replicas.
opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy": "round_robin"}`))
// Disable looking up service config from TXT DNS records.
// This reduces the number of requests made to the DNS servers.
opts = append(opts, grpc.WithDisableServiceConfig())
}
conn, err := grpc.NewClient(clientCfg.remoteAddress, opts...)
if err != nil {
return nil, fmt.Errorf("failed to create authz client to remote server: %w", err)
pkg/services/authz/rbac_settings.go
+5 -3
View File
@@ -23,9 +23,10 @@ const (
)
type authzClientSettings struct {
remoteAddress string
certFile string
mode clientMode
remoteAddress string
certFile string
mode clientMode
loadBalancingEnabled bool
token string
tokenExchangeURL string
@@ -51,6 +52,7 @@ func readAuthzClientSettings(cfg *setting.Cfg) (*authzClientSettings, error) {
if s.mode == clientModeInproc {
return s, nil
}
s.loadBalancingEnabled = authzSection.Key("load_balancing_enabled").MustBool(false)
s.remoteAddress = authzSection.Key("remote_address").MustString("")
s.certFile = authzSection.Key("cert_file").MustString("")
pkg/services/grpcserver/README.md
+29 -4
View File
@@ -12,16 +12,16 @@ The `grpcserver` package provides the implementation of the gRPC server for hand
## Usage
Enable the gRPC server in Grafana by setting the `grpcServer` feature toggle to `true` in your `custom.ini` configuration file.
Enable the gRPC server in Grafana by setting the `grpcServer` feature toggle to `true` in your `custom.ini` configuration file.
``` ini
```ini
[feature_toggles]
grpcServer = true
```
You can specify the gRPC server specific settings in the `grpc_server` section of the configuration file.
``` ini
```ini
[grpc_server]
network = "tcp"
address = "127.0.0.1:10000"
@@ -36,6 +36,31 @@ max_recv_msg_size =
max_send_msg_size =
```
### Optional: Connection Management and Load Balancing
These settings help with:
- **Resource management**: Prevent resource leaks from idle connections
- **Connection health**: Detect and clean up dead connections
- **Load balancing**: Force connection recycling for better distribution across multiple server instances
- **DoS protection**: Rate limit keepalive pings from clients
```ini
# Connection management options
# Maximum amount of time a connection may exist before it will be closed
max_connection_age = 300s
# Additional time to allow for pending RPCs to complete before forcibly closing connections
max_connection_age_grace = 10s
# Maximum amount of idle time before a connection is closed
max_connection_idle = 300s
# Frequency of server-to-client pings to check if a connection is still active
keepalive_time = 30s
# Amount of time the server waits for a response to keepalive pings before closing the connection
keepalive_timeout = 5s
# Minimum amount of time a client should wait before sending a keepalive ping
keepalive_min_time = 5s
```
## Example Services
View [health.go] and [reflection.go] for examples of how to implement gRPC services in Grafana. These services are currently initialized by the [background service registry](../../registry/backgroundsvcs/background_services.go).
View [health.go] and [reflection.go] for examples of how to implement gRPC services in Grafana. These services are currently initialized by the [background service registry](../../registry/backgroundsvcs/background_services.go).
pkg/services/grpcserver/service.go
+33 -1
View File
@@ -14,6 +14,7 @@ import (
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/keepalive"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/registry"
@@ -104,12 +105,43 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, authe
opts = append(opts, grpc.MaxSendMsgSize(s.cfg.MaxSendMsgSize))
}
// Apply connection management settings
keepaliveParams := keepalive.ServerParameters{
MaxConnectionAge: s.cfg.MaxConnectionAge,
MaxConnectionAgeGrace: s.cfg.MaxConnectionAgeGrace,
MaxConnectionIdle: s.cfg.MaxConnectionIdle,
Time: s.cfg.KeepaliveTime,
Timeout: s.cfg.KeepaliveTimeout,
}
keepalivePolicy := keepalive.EnforcementPolicy{
MinTime: s.cfg.KeepaliveMinTime,
}
// Only add keepalive options if any values are configured
if s.cfg.MaxConnectionAge > 0 || s.cfg.MaxConnectionAgeGrace > 0 || s.cfg.MaxConnectionIdle > 0 ||
s.cfg.KeepaliveTime > 0 || s.cfg.KeepaliveTimeout > 0 {
opts = append(opts, grpc.KeepaliveParams(keepaliveParams))
}
if s.cfg.KeepaliveMinTime > 0 {
opts = append(opts, grpc.KeepaliveEnforcementPolicy(keepalivePolicy))
}
s.server = grpc.NewServer(opts...)
return s, nil
}
func (s *gPRCServerService) Run(ctx context.Context) error {
s.logger.Info("Running GRPC server", "address", s.cfg.Address, "network", s.cfg.Network, "tls", s.cfg.TLSConfig != nil, "max_recv_msg_size", s.cfg.MaxRecvMsgSize, "max_send_msg_size", s.cfg.MaxSendMsgSize)
s.logger.Info("Running GRPC server",
"address", s.cfg.Address,
"network", s.cfg.Network,
"tls", s.cfg.TLSConfig != nil,
"max_recv_msg_size", s.cfg.MaxRecvMsgSize,
"max_send_msg_size", s.cfg.MaxSendMsgSize,
"max_connection_age", s.cfg.MaxConnectionAge,
"max_connection_idle", s.cfg.MaxConnectionIdle,
"keepalive_time", s.cfg.KeepaliveTime,
"keepalive_timeout", s.cfg.KeepaliveTimeout,
"keepalive_min_time", s.cfg.KeepaliveMinTime)
listener, err := net.Listen(s.cfg.Network, s.cfg.Address)
if err != nil {
pkg/setting/setting_grpc.go
+23
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"io/fs"
"os"
"time"
"github.com/spf13/pflag"
@@ -20,6 +21,12 @@ type GRPCServerSettings struct {
MaxRecvMsgSize int
MaxSendMsgSize int
MaxConnectionAge time.Duration
MaxConnectionAgeGrace time.Duration
KeepaliveTime time.Duration
KeepaliveTimeout time.Duration
KeepaliveMinTime time.Duration
MaxConnectionIdle time.Duration
// Internal fields
useTLS bool
certFile string
@@ -119,6 +126,14 @@ func readGRPCServerSettings(cfg *Cfg, iniFile *ini.File) error {
cfg.GRPCServer.MaxRecvMsgSize = server.Key("max_recv_msg_size").MustInt(0)
cfg.GRPCServer.MaxSendMsgSize = server.Key("max_send_msg_size").MustInt(0)
// Read connection management options from INI file
cfg.GRPCServer.MaxConnectionAge = server.Key("max_connection_age").MustDuration(0)
cfg.GRPCServer.MaxConnectionAgeGrace = server.Key("max_connection_age_grace").MustDuration(0)
cfg.GRPCServer.MaxConnectionIdle = server.Key("max_connection_idle").MustDuration(0)
cfg.GRPCServer.KeepaliveTime = server.Key("keepalive_time").MustDuration(0)
cfg.GRPCServer.KeepaliveTimeout = server.Key("keepalive_timeout").MustDuration(0)
cfg.GRPCServer.KeepaliveMinTime = server.Key("keepalive_min_time").MustDuration(0)
return cfg.GRPCServer.processAddress()
}
@@ -134,4 +149,12 @@ func (c *GRPCServerSettings) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&c.useTLS, "grpc-server-use-tls", false, "Enable TLS for the gRPC server")
fs.StringVar(&c.certFile, "grpc-server-cert-file", "", "Path to the certificate file for the gRPC server")
fs.StringVar(&c.keyFile, "grpc-server-key-file", "", "Path to the certificate key file for the gRPC server")
// Connection management options
fs.DurationVar(&c.MaxConnectionAge, "grpc-server-max-connection-age", 0, "Maximum amount of time a connection may exist before it will be closed (e.g. 30s)")
fs.DurationVar(&c.MaxConnectionAgeGrace, "grpc-server-max-connection-age-grace", 0, "Additional time to allow for pending RPCs to complete before forcibly closing connections (e.g. 5s)")
fs.DurationVar(&c.MaxConnectionIdle, "grpc-server-max-connection-idle", 0, "Maximum amount of idle time before a connection is closed (e.g. 15s)")
fs.DurationVar(&c.KeepaliveTime, "grpc-server-keepalive-time", 0, "Frequency of server-to-client pings to check if a connection is still active (e.g. 10s)")
fs.DurationVar(&c.KeepaliveTimeout, "grpc-server-keepalive-timeout", 0, "Amount of time the server waits for a response to keepalive pings before closing the connection (e.g. 3s)")
fs.DurationVar(&c.KeepaliveMinTime, "grpc-server-keepalive-min-time", 0, "Minimum amount of time a client should wait before sending a keepalive ping (e.g. 5s)")
}