Secrets: Add config for enabling gRPC client-side load balancing (#111989)

pkg/operators/provisioning/config.go

@@ -311,6 +311,7 @@ func setupDecrypter(cfg *setting.Cfg, tracer tracing.Tracer, tokenExchangeClient
 		tracer,
 		address,
 		secretsTls,
+		secretsSec.Key("grpc_client_load_balancing").MustBool(false),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("create decrypt service: %w", err)

pkg/registry/apis/secret/decrypt/grpc_client.go

@@ -40,8 +40,8 @@ type TLSConfig struct {
 	InsecureSkipVerify bool
 }
 
-func NewGRPCDecryptClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string) (*GRPCDecryptClient, error) {
-	return NewGRPCDecryptClientWithTLS(tokenExchanger, tracer, address, TLSConfig{})
+func NewGRPCDecryptClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string, clientLoadBalancingEnabled bool) (*GRPCDecryptClient, error) {
+	return NewGRPCDecryptClientWithTLS(tokenExchanger, tracer, address, TLSConfig{}, clientLoadBalancingEnabled)
 }
 
 func NewGRPCDecryptClientWithTLS(
@@ -49,6 +49,7 @@ func NewGRPCDecryptClientWithTLS(
 	tracer trace.Tracer,
 	address string,
 	tlsConfig TLSConfig,
+	clientLoadBalancingEnabled bool,
 ) (*GRPCDecryptClient, error) {
 	var opts []grpc.DialOption
 	if tlsConfig.UseTLS {
@@ -62,6 +63,15 @@ func NewGRPCDecryptClientWithTLS(
 		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	}
 
+	if clientLoadBalancingEnabled {
+		// Use round_robin to balances requests more evenly over the available replicas.
+		opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`))
+
+		// Disable looking up service config from TXT DNS records.
+		// This reduces the number of requests made to the DNS servers.
+		opts = append(opts, grpc.WithDisableServiceConfig())
+	}
+
 	conn, err := grpc.NewClient(address, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to grpc decrypt server at %s: %w", address, err)

pkg/registry/apis/secret/decrypt/service.go

@@ -34,7 +34,7 @@ func ProvideDecryptService(cfg *setting.Cfg, tracer trace.Tracer, decryptStorage
 
 		tlsConfig := readTLSFromConfig(cfg)
 
-		client, err := NewGRPCDecryptClientWithTLS(tokenExchangeClient, tracer, cfg.SecretsManagement.GrpcServerAddress, tlsConfig)
+		client, err := NewGRPCDecryptClientWithTLS(tokenExchangeClient, tracer, cfg.SecretsManagement.GrpcServerAddress, tlsConfig, cfg.SecretsManagement.GrpcClientLoadBalancing)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create grpc decrypt client: %w", err)
 		}

pkg/registry/apis/secret/inline/grpc_client.go

@@ -36,7 +36,7 @@ type TLSConfig struct {
 	InsecureSkipVerify bool
 }
 
-func NewGRPCInlineClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string, tlsConfig TLSConfig) (*GRPCInlineClient, error) {
+func NewGRPCInlineClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tracer, address string, tlsConfig TLSConfig, clientLoadBalancingEnabled bool) (*GRPCInlineClient, error) {
 	var opts []grpc.DialOption
 	if tlsConfig.UseTLS {
 		creds, err := createTLSCredentials(tlsConfig)
@@ -49,6 +49,15 @@ func NewGRPCInlineClient(tokenExchanger authnlib.TokenExchanger, tracer trace.Tr
 		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	}
 
+	if clientLoadBalancingEnabled {
+		// Use round_robin to balances requests more evenly over the available replicas.
+		opts = append(opts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`))
+
+		// Disable looking up service config from TXT DNS records.
+		// This reduces the number of requests made to the DNS servers.
+		opts = append(opts, grpc.WithDisableServiceConfig())
+	}
+
 	conn, err := grpc.NewClient(address, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to grpc server at %s: %w", address, err)

pkg/registry/apis/secret/inline/service.go

@@ -24,16 +24,19 @@ func ProvideInlineSecureValueService(
 			cfg.SecretsManagement.GrpcServerAddress,
 			readTLSFromConfig(cfg),
 			tracer,
+			cfg.SecretsManagement.GrpcClientLoadBalancing,
 		)
 	}
 
 	return NewLocalInlineSecureValueService(tracer, secureValueService, accessClient), nil
 }
 
-func NewGRPCSecureValueService(tokenCfg *grpcutils.GrpcClientConfig,
+func NewGRPCSecureValueService(
+	tokenCfg *grpcutils.GrpcClientConfig,
 	address string,
 	tlsCfg TLSConfig,
 	tracer trace.Tracer,
+	clientLoadBalancingEnabled bool,
 ) (contracts.InlineSecureValueSupport, error) {
 	if address == "" {
 		return nil, fmt.Errorf("grpc_server_address is required when grpc client is enabled")
@@ -51,7 +54,7 @@ func NewGRPCSecureValueService(tokenCfg *grpcutils.GrpcClientConfig,
 		return nil, fmt.Errorf("failed to create token exchange client: %w", err)
 	}
 
-	client, err := NewGRPCInlineClient(tokenExchangeClient, tracer, address, tlsCfg)
+	client, err := NewGRPCInlineClient(tokenExchangeClient, tracer, address, tlsCfg, clientLoadBalancingEnabled)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create grpc inline secure value client: %w", err)
 	}

pkg/services/apiserver/options/storage.go

@@ -57,6 +57,7 @@ type StorageOptions struct {
 
 	// Secrets Manager Configuration for InlineSecureValueSupport
 	SecretsManagerGrpcClientEnable        bool
+	SecretsManagerGrpcClientLoadBalancing bool
 	SecretsManagerGrpcServerAddress       string
 	SecretsManagerGrpcServerUseTLS        bool
 	SecretsManagerGrpcServerTLSSkipVerify bool
@@ -115,6 +116,7 @@ func (o *StorageOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&o.SecretsManagerGrpcServerTLSSkipVerify, "grafana.secrets-manager.grpc-server-tls-skip-verify", false, "Skip TLS verification for gRPC server")
 	fs.StringVar(&o.SecretsManagerGrpcServerTLSServerName, "grafana.secrets-manager.grpc-server-tls-server-name", "", "Server name for TLS verification")
 	fs.StringVar(&o.SecretsManagerGrpcServerTLSCAFile, "grafana.secrets-manager.grpc-server-tls-ca-file", "", "CA file for TLS verification")
+	fs.BoolVar(&o.SecretsManagerGrpcClientLoadBalancing, "grafana.secrets-manager.grpc-client-load-balancing", false, "Enable client-side load balancing for gRPC client")
 }
 
 func (o *StorageOptions) Validate() []error {
@@ -218,6 +220,7 @@ func (o *StorageOptions) ApplyTo(serverConfig *genericapiserver.RecommendedConfi
 			o.SecretsManagerGrpcServerAddress,
 			tlsCfg,
 			tracer,
+			o.SecretsManagerGrpcClientLoadBalancing,
 		)
 		if err != nil {
 			return fmt.Errorf("failed to create inline secure value service: %w", err)

pkg/setting/setting_secrets_manager.go

@@ -18,6 +18,7 @@ type SecretsManagerSettings struct {
 	ConfiguredKMSProviders map[string]map[string]string
 
 	GrpcClientEnable        bool   // Whether to enable the gRPC client. If disabled, it will use the in-process services implementations.
+	GrpcClientLoadBalancing bool   // Whether to enable gRPC client-side load balancing
 	GrpcServerUseTLS        bool   // Whether to use TLS when communicating with the gRPC server
 	GrpcServerTLSSkipVerify bool   // Whether to skip TLS verification when communicating with the gRPC server
 	GrpcServerTLSServerName string // Server name to use for TLS verification
@@ -43,6 +44,7 @@ func (cfg *Cfg) readSecretsManagerSettings() {
 	cfg.SecretsManagement.CurrentEncryptionProvider = secretsMgmt.Key("encryption_provider").MustString(MisconfiguredProvider)
 
 	cfg.SecretsManagement.GrpcClientEnable = secretsMgmt.Key("grpc_client_enable").MustBool(false)
+	cfg.SecretsManagement.GrpcClientLoadBalancing = secretsMgmt.Key("grpc_client_load_balancing").MustBool(false)
 	cfg.SecretsManagement.GrpcServerUseTLS = secretsMgmt.Key("grpc_server_use_tls").MustBool(false)
 	cfg.SecretsManagement.GrpcServerTLSSkipVerify = secretsMgmt.Key("grpc_server_tls_skip_verify").MustBool(false)
 	cfg.SecretsManagement.GrpcServerTLSServerName = valueAsString(secretsMgmt, "grpc_server_tls_server_name", "")