From 93af167f9c8e4c37c5e02e6dcb2253751a7509bc Mon Sep 17 00:00:00 2001 From: Matheus Macabu Date: Mon, 20 Oct 2025 14:04:59 +0200 Subject: [PATCH] Secrets: Add config for enabling gRPC client-side load balancing (#111989) --- pkg/operators/provisioning/config.go | 1 + pkg/registry/apis/secret/decrypt/grpc_client.go | 14 ++++++++++++-- pkg/registry/apis/secret/decrypt/service.go | 2 +- pkg/registry/apis/secret/inline/grpc_client.go | 11 ++++++++++- pkg/registry/apis/secret/inline/service.go | 7 +++++-- pkg/services/apiserver/options/storage.go | 3 +++ pkg/setting/setting_secrets_manager.go | 2 ++ 7 files changed, 34 insertions(+), 6 deletions(-) diff --git a/pkg/operators/provisioning/config.go b/pkg/operators/provisioning/config.go index 0cf3465823b..ae5cd957492 100644 --- a/pkg/operators/provisioning/config.go +++ b/pkg/operators/provisioning/config.go @@ -311,6 +311,7 @@ func setupDecrypter(cfg *setting.Cfg, tracer tracing.Tracer, tokenExchangeClient tracer, address, secretsTls, + secretsSec.Key("grpc_client_load_balancing").MustBool(false), ) if err != nil { return nil, fmt.Errorf("create decrypt service: %w", err) diff --git a/pkg/registry/apis/secret/decrypt/grpc_client.go b/pkg/registry/apis/secret/decrypt/grpc_client.go index d41ab267db3..f892de5ebe5 100644 --- a/pkg/registry/apis/secret/decrypt/grpc_client.go +++ b/pkg/registry/apis/secret/decrypt/grpc_client.go @@ -40,8 +40,8 @@ type TLSConfig struct { InsecureSkipVerify bool } -func NewGRPCDecryptClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string) (*GRPCDecryptClient, error) { - return NewGRPCDecryptClientWithTLS(tokenExchanger, tracer, address, TLSConfig{}) +func NewGRPCDecryptClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string, clientLoadBalancingEnabled bool) (*GRPCDecryptClient, error) { + return NewGRPCDecryptClientWithTLS(tokenExchanger, tracer, address, TLSConfig{}, clientLoadBalancingEnabled) } func NewGRPCDecryptClientWithTLS( @@ -49,6 +49,7 @@ func NewGRPCDecryptClientWithTLS( tracer trace.Tracer, address string, tlsConfig TLSConfig, + clientLoadBalancingEnabled bool, ) (*GRPCDecryptClient, error) { var opts []grpc.DialOption if tlsConfig.UseTLS { @@ -62,6 +63,15 @@ func NewGRPCDecryptClientWithTLS( opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } + if clientLoadBalancingEnabled { + // Use round_robin to balances requests more evenly over the available replicas. + opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`)) + + // Disable looking up service config from TXT DNS records. + // This reduces the number of requests made to the DNS servers. + opts = append(opts, grpc.WithDisableServiceConfig()) + } + conn, err := grpc.NewClient(address, opts...) if err != nil { return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err) diff --git a/pkg/registry/apis/secret/decrypt/service.go b/pkg/registry/apis/secret/decrypt/service.go index e725331cfa7..db53b963769 100644 --- a/pkg/registry/apis/secret/decrypt/service.go +++ b/pkg/registry/apis/secret/decrypt/service.go @@ -34,7 +34,7 @@ func ProvideDecryptService(cfg *setting.Cfg, tracer trace.Tracer, decryptStorage tlsConfig := readTLSFromConfig(cfg) - client, err := NewGRPCDecryptClientWithTLS(tokenExchangeClient, tracer, cfg.SecretsManagement.GrpcServerAddress, tlsConfig) + client, err := NewGRPCDecryptClientWithTLS(tokenExchangeClient, tracer, cfg.SecretsManagement.GrpcServerAddress, tlsConfig, cfg.SecretsManagement.GrpcClientLoadBalancing) if err != nil { return nil, fmt.Errorf("failed to create grpc decrypt client: %w", err) } diff --git a/pkg/registry/apis/secret/inline/grpc_client.go b/pkg/registry/apis/secret/inline/grpc_client.go index 7b9157aef80..e465c24ac7a 100644 --- a/pkg/registry/apis/secret/inline/grpc_client.go +++ b/pkg/registry/apis/secret/inline/grpc_client.go @@ -36,7 +36,7 @@ type TLSConfig struct { InsecureSkipVerify bool } -func NewGRPCInlineClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string, tlsConfig TLSConfig) (*GRPCInlineClient, error) { +func NewGRPCInlineClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string, tlsConfig TLSConfig, clientLoadBalancingEnabled bool) (*GRPCInlineClient, error) { var opts []grpc.DialOption if tlsConfig.UseTLS { creds, err := createTLSCredentials(tlsConfig) @@ -49,6 +49,15 @@ func NewGRPCInlineClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tr opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } + if clientLoadBalancingEnabled { + // Use round_robin to balances requests more evenly over the available replicas. + opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`)) + + // Disable looking up service config from TXT DNS records. + // This reduces the number of requests made to the DNS servers. + opts = append(opts, grpc.WithDisableServiceConfig()) + } + conn, err := grpc.NewClient(address, opts...) if err != nil { return nil, fmt.Errorf("failed to connect to grpc server at %s: %w", address, err) diff --git a/pkg/registry/apis/secret/inline/service.go b/pkg/registry/apis/secret/inline/service.go index 73b6798a7de..5a9c24d05dd 100644 --- a/pkg/registry/apis/secret/inline/service.go +++ b/pkg/registry/apis/secret/inline/service.go @@ -24,16 +24,19 @@ func ProvideInlineSecureValueService( cfg.SecretsManagement.GrpcServerAddress, readTLSFromConfig(cfg), tracer, + cfg.SecretsManagement.GrpcClientLoadBalancing, ) } return NewLocalInlineSecureValueService(tracer, secureValueService, accessClient), nil } -func NewGRPCSecureValueService(tokenCfg *grpcutils.GrpcClientConfig, +func NewGRPCSecureValueService( + tokenCfg *grpcutils.GrpcClientConfig, address string, tlsCfg TLSConfig, tracer trace.Tracer, + clientLoadBalancingEnabled bool, ) (contracts.InlineSecureValueSupport, error) { if address == "" { return nil, fmt.Errorf("grpc_server_address is required when grpc client is enabled") @@ -51,7 +54,7 @@ func NewGRPCSecureValueService(tokenCfg *grpcutils.GrpcClientConfig, return nil, fmt.Errorf("failed to create token exchange client: %w", err) } - client, err := NewGRPCInlineClient(tokenExchangeClient, tracer, address, tlsCfg) + client, err := NewGRPCInlineClient(tokenExchangeClient, tracer, address, tlsCfg, clientLoadBalancingEnabled) if err != nil { return nil, fmt.Errorf("failed to create grpc inline secure value client: %w", err) } diff --git a/pkg/services/apiserver/options/storage.go b/pkg/services/apiserver/options/storage.go index 97b95a45b26..ee91ff8eb89 100644 --- a/pkg/services/apiserver/options/storage.go +++ b/pkg/services/apiserver/options/storage.go @@ -57,6 +57,7 @@ type StorageOptions struct { // Secrets Manager Configuration for InlineSecureValueSupport SecretsManagerGrpcClientEnable bool + SecretsManagerGrpcClientLoadBalancing bool SecretsManagerGrpcServerAddress string SecretsManagerGrpcServerUseTLS bool SecretsManagerGrpcServerTLSSkipVerify bool @@ -115,6 +116,7 @@ func (o *StorageOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.SecretsManagerGrpcServerTLSSkipVerify, "grafana.secrets-manager.grpc-server-tls-skip-verify", false, "Skip TLS verification for gRPC server") fs.StringVar(&o.SecretsManagerGrpcServerTLSServerName, "grafana.secrets-manager.grpc-server-tls-server-name", "", "Server name for TLS verification") fs.StringVar(&o.SecretsManagerGrpcServerTLSCAFile, "grafana.secrets-manager.grpc-server-tls-ca-file", "", "CA file for TLS verification") + fs.BoolVar(&o.SecretsManagerGrpcClientLoadBalancing, "grafana.secrets-manager.grpc-client-load-balancing", false, "Enable client-side load balancing for gRPC client") } func (o *StorageOptions) Validate() []error { @@ -218,6 +220,7 @@ func (o *StorageOptions) ApplyTo(serverConfig *genericapiserver.RecommendedConfi o.SecretsManagerGrpcServerAddress, tlsCfg, tracer, + o.SecretsManagerGrpcClientLoadBalancing, ) if err != nil { return fmt.Errorf("failed to create inline secure value service: %w", err) diff --git a/pkg/setting/setting_secrets_manager.go b/pkg/setting/setting_secrets_manager.go index 72ff3a32361..926a0c4b962 100644 --- a/pkg/setting/setting_secrets_manager.go +++ b/pkg/setting/setting_secrets_manager.go @@ -18,6 +18,7 @@ type SecretsManagerSettings struct { ConfiguredKMSProviders map[string]map[string]string GrpcClientEnable bool // Whether to enable the gRPC client. If disabled, it will use the in-process services implementations. + GrpcClientLoadBalancing bool // Whether to enable gRPC client-side load balancing GrpcServerUseTLS bool // Whether to use TLS when communicating with the gRPC server GrpcServerTLSSkipVerify bool // Whether to skip TLS verification when communicating with the gRPC server GrpcServerTLSServerName string // Server name to use for TLS verification @@ -43,6 +44,7 @@ func (cfg *Cfg) readSecretsManagerSettings() { cfg.SecretsManagement.CurrentEncryptionProvider = secretsMgmt.Key("encryption_provider").MustString(MisconfiguredProvider) cfg.SecretsManagement.GrpcClientEnable = secretsMgmt.Key("grpc_client_enable").MustBool(false) + cfg.SecretsManagement.GrpcClientLoadBalancing = secretsMgmt.Key("grpc_client_load_balancing").MustBool(false) cfg.SecretsManagement.GrpcServerUseTLS = secretsMgmt.Key("grpc_server_use_tls").MustBool(false) cfg.SecretsManagement.GrpcServerTLSSkipVerify = secretsMgmt.Key("grpc_server_tls_skip_verify").MustBool(false) cfg.SecretsManagement.GrpcServerTLSServerName = valueAsString(secretsMgmt, "grpc_server_tls_server_name", "")