Zanzana: Refactor remote client initialization (#114142)

* Zanzana: Refactor remote client

* rename config field URL to Addr

* Instrument grpc queries

* fix duplicated field

apps/iam/pkg/app/app.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/grafana/grafana-app-sdk/app"
 	"github.com/grafana/grafana-app-sdk/logging"
 	"github.com/grafana/grafana-app-sdk/operator"
@@ -12,7 +14,6 @@ import (
 	foldersKind "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
 	"github.com/grafana/grafana/apps/iam/pkg/reconcilers"
 	"github.com/grafana/grafana/pkg/services/authz"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 var appManifestData = app.ManifestData{
@@ -78,7 +79,7 @@ func New(cfg app.Config) (app.App, error) {
 	folderReconciler, err := reconcilers.NewFolderReconciler(reconcilers.ReconcilerConfig{
 		ZanzanaCfg: appSpecificConfig.ZanzanaClientCfg,
 		Metrics:    metrics,
-	})
+	}, appSpecificConfig.MetricsRegisterer)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create FolderReconciler: %w", err)
 	}

apps/iam/pkg/reconcilers/folder_reconciler.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
@@ -35,9 +36,9 @@ type FolderReconciler struct {
 	metrics         *ReconcilerMetrics
 }
 
-func NewFolderReconciler(cfg ReconcilerConfig) (operator.Reconciler, error) {
+func NewFolderReconciler(cfg ReconcilerConfig, reg prometheus.Registerer) (operator.Reconciler, error) {
 	// Create Zanzana client
-	zanzanaClient, err := authz.NewRemoteZanzanaClient("*", cfg.ZanzanaCfg)
+	zanzanaClient, err := authz.NewRemoteZanzanaClient(cfg.ZanzanaCfg, reg)
 
 	if err != nil {
 		return nil, fmt.Errorf("unable to create zanzana client: %w", err)

pkg/operators/iam/zanzana_folder_reconciler.go

@@ -11,15 +11,16 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
+
 	"github.com/grafana/grafana-app-sdk/logging"
 	"github.com/grafana/grafana-app-sdk/operator"
 	folder "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
 	"github.com/grafana/grafana/apps/iam/pkg/app"
 	"github.com/grafana/grafana/pkg/server"
 	"github.com/grafana/grafana/pkg/setting"
-	"github.com/prometheus/client_golang/prometheus"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/transport"
 
 	"github.com/grafana/authlib/authn"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -95,7 +96,7 @@ func buildIAMConfigFromSettings(cfg *setting.Cfg, registerer prometheus.Register
 	if zanzanaURL == "" {
 		return nil, fmt.Errorf("zanzana_url is required in [operator] section")
 	}
-	iamCfg.AppConfig.ZanzanaClientCfg.URL = zanzanaURL
+	iamCfg.AppConfig.ZanzanaClientCfg.Addr = zanzanaURL
 
 	iamCfg.AppConfig.InformerConfig.MaxConcurrentWorkers = operatorSec.Key("max_concurrent_workers").MustUint64(20)
 

pkg/services/authz/rbac.go

@@ -152,7 +152,7 @@ func ProvideStandaloneAuthZClient(
 	//nolint:staticcheck // not yet migrated to OpenFeature
 	zanzanaEnabled := features.IsEnabledGlobally(featuremgmt.FlagZanzana)
 
-	zanzanaClient, err := ProvideStandaloneZanzanaClient(cfg, features)
+	zanzanaClient, err := ProvideStandaloneZanzanaClient(cfg, features, reg)
 	if err != nil {
 		return nil, err
 	}

pkg/services/authz/zanzana.go

@@ -4,16 +4,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/fullstorydev/grpchan/inprocgrpc"
 	authnlib "github.com/grafana/authlib/authn"
 	authzv1 "github.com/grafana/authlib/authz/proto/v1"
 	"github.com/grafana/authlib/grpcutils"
 	"github.com/grafana/authlib/types"
+	"github.com/grafana/dskit/middleware"
 	"github.com/grafana/dskit/services"
 	grpcAuth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth"
 	openfgav1 "github.com/openfga/api/proto/openfga/v1"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
@@ -43,14 +46,14 @@ func ProvideZanzanaClient(cfg *setting.Cfg, db db.DB, tracer tracing.Tracer, fea
 
 	switch cfg.ZanzanaClient.Mode {
 	case setting.ZanzanaModeClient:
-		return NewRemoteZanzanaClient(
+		zanzanaConfig := ZanzanaClientConfig{
-			fmt.Sprintf("stacks-%s", cfg.StackID),
+			Addr:             cfg.ZanzanaClient.Addr,
-			ZanzanaClientConfig{
+			Token:            cfg.ZanzanaClient.Token,
-				URL:              cfg.ZanzanaClient.Addr,
+			TokenExchangeURL: cfg.ZanzanaClient.TokenExchangeURL,
-				Token:            cfg.ZanzanaClient.Token,
+			TokenNamespace:   cfg.ZanzanaClient.TokenNamespace,
-				TokenExchangeURL: cfg.ZanzanaClient.TokenExchangeURL,
+			ServerCertFile:   cfg.ZanzanaClient.ServerCertFile,
-				ServerCertFile:   cfg.ZanzanaClient.ServerCertFile,
+		}
-			})
+		return NewRemoteZanzanaClient(zanzanaConfig, reg)
 
 	case setting.ZanzanaModeEmbedded:
 		logger := log.New("zanzana.server")
@@ -97,32 +100,33 @@ func ProvideZanzanaClient(cfg *setting.Cfg, db db.DB, tracer tracing.Tracer, fea
 
 // ProvideStandaloneZanzanaClient provides a standalone Zanzana client, without registering the Zanzana service.
 // Client connects to a remote Zanzana server specified in the configuration.
-func ProvideStandaloneZanzanaClient(cfg *setting.Cfg, features featuremgmt.FeatureToggles) (zanzana.Client, error) {
+func ProvideStandaloneZanzanaClient(cfg *setting.Cfg, features featuremgmt.FeatureToggles, reg prometheus.Registerer) (zanzana.Client, error) {
 	//nolint:staticcheck // not yet migrated to OpenFeature
 	if !features.IsEnabledGlobally(featuremgmt.FlagZanzana) {
 		return zClient.NewNoopClient(), nil
 	}
 
 	zanzanaConfig := ZanzanaClientConfig{
-		URL:              cfg.ZanzanaClient.Addr,
+		Addr:             cfg.ZanzanaClient.Addr,
 		Token:            cfg.ZanzanaClient.Token,
 		TokenExchangeURL: cfg.ZanzanaClient.TokenExchangeURL,
+		TokenNamespace:   cfg.ZanzanaClient.TokenNamespace,
 		ServerCertFile:   cfg.ZanzanaClient.ServerCertFile,
 	}
 
-	return NewRemoteZanzanaClient(cfg.ZanzanaClient.TokenNamespace, zanzanaConfig)
+	return NewRemoteZanzanaClient(zanzanaConfig, reg)
 }
 
 type ZanzanaClientConfig struct {
-	URL              string
+	Addr             string
 	Token            string
 	TokenExchangeURL string
-	ServerCertFile   string
 	TokenNamespace   string
+	ServerCertFile   string
 }
 
 // NewRemoteZanzanaClient creates a new Zanzana client that connects to remote Zanzana server.
-func NewRemoteZanzanaClient(namespace string, cfg ZanzanaClientConfig) (zanzana.Client, error) {
+func NewRemoteZanzanaClient(cfg ZanzanaClientConfig, reg prometheus.Registerer) (zanzana.Client, error) {
 	tokenClient, err := authnlib.NewTokenExchangeClient(authnlib.TokenExchangeConfig{
 		Token:            cfg.Token,
 		TokenExchangeURL: cfg.TokenExchangeURL,
@@ -139,18 +143,25 @@ func NewRemoteZanzanaClient(namespace string, cfg ZanzanaClientConfig) (zanzana.
 		}
 	}
 
+	authzRequestDuration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
+		Name:                            "authz_zanzana_client_request_duration_seconds",
+		Help:                            "Time spent executing requests to zanzana server.",
+		NativeHistogramBucketFactor:     1.1,
+		NativeHistogramMaxBucketNumber:  160,
+		NativeHistogramMinResetDuration: time.Hour,
+	}, []string{"operation", "status_code"})
+	unaryInterceptors, streamInterceptors := instrument(authzRequestDuration, middleware.ReportGRPCStatusOption)
+
 	dialOptions := []grpc.DialOption{
 		grpc.WithTransportCredentials(transportCredentials),
 		grpc.WithPerRPCCredentials(
-			NewGRPCTokenAuth(
+			NewGRPCTokenAuth(AuthzServiceAudience, cfg.TokenNamespace, tokenClient),
-				AuthzServiceAudience,
-				namespace,
-				tokenClient,
-			),
 		),
+		grpc.WithChainUnaryInterceptor(unaryInterceptors...),
+		grpc.WithChainStreamInterceptor(streamInterceptors...),
 	}
 
-	conn, err := grpc.NewClient(cfg.URL, dialOptions...)
+	conn, err := grpc.NewClient(cfg.Addr, dialOptions...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create zanzana client to remote server: %w", err)
 	}

pkg/setting/settings_zanzana.go

@@ -110,15 +110,24 @@ func (cfg *Cfg) readZanzanaSettings() {
 		zc.Mode = "embedded"
 	}
 
-	zc.Token = clientSec.Key("token").MustString("")
-	zc.TokenExchangeURL = clientSec.Key("token_exchange_url").MustString("")
 	zc.Addr = clientSec.Key("address").MustString("")
 	zc.ServerCertFile = clientSec.Key("tls_cert").MustString("")
 
-	// TODO: read Token and TokenExchangeURL from grpc_client_authentication section
 	grpcClientAuthSection := cfg.SectionWithEnvOverrides("grpc_client_authentication")
+	zc.Token = grpcClientAuthSection.Key("token").MustString("")
+	zc.TokenExchangeURL = grpcClientAuthSection.Key("token_exchange_url").MustString("")
 	zc.TokenNamespace = grpcClientAuthSection.Key("token_namespace").MustString("stacks-" + cfg.StackID)
 
+	// TODO: remove old settings when migrated
+	token := clientSec.Key("token").MustString("")
+	tokenExchangeURL := clientSec.Key("token_exchange_url").MustString("")
+	if token != "" {
+		zc.Token = token
+	}
+	if tokenExchangeURL != "" {
+		zc.TokenExchangeURL = tokenExchangeURL
+	}
+
 	cfg.ZanzanaClient = zc
 
 	zs := ZanzanaServerSettings{}