Zanzana: Add remote client metrics (#116012)

* Zanzana: Add remote client metrics

* fix linter

pkg/services/authz/zanzana.go

@@ -90,7 +90,7 @@ func ProvideZanzanaClient(cfg *setting.Cfg, db db.DB, tracer tracing.Tracer, fea
 		authzv1.RegisterAuthzServiceServer(channel, srv)
 		authzextv1.RegisterAuthzExtentionServiceServer(channel, srv)
 
-		client, err := zClient.New(channel)
+		client, err := zClient.New(channel, reg)
 		if err != nil {
 			return nil, fmt.Errorf("failed to initialize zanzana client: %w", err)
 		}
@@ -169,7 +169,7 @@ func NewRemoteZanzanaClient(cfg ZanzanaClientConfig, reg prometheus.Registerer)
 		return nil, fmt.Errorf("failed to create zanzana client to remote server: %w", err)
 	}
 
-	client, err := zClient.New(conn)
+	client, err := zClient.New(conn, reg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize zanzana client: %w", err)
 	}

pkg/services/authz/zanzana/client/client.go

@@ -9,6 +9,7 @@ import (
 	authzlib "github.com/grafana/authlib/authz"
 	authzv1 "github.com/grafana/authlib/authz/proto/v1"
 	authlib "github.com/grafana/authlib/types"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1"
@@ -25,15 +26,17 @@ type Client struct {
 	authz          authzv1.AuthzServiceClient
 	authzext       authzextv1.AuthzExtentionServiceClient
 	authzlibclient *authzlib.ClientImpl
+	metrics        *clientMetrics
 }
 
-func New(cc grpc.ClientConnInterface) (*Client, error) {
+func New(cc grpc.ClientConnInterface, reg prometheus.Registerer) (*Client, error) {
 	authzlibclient := authzlib.NewClient(cc, authzlib.WithTracerClientOption(tracer))
 	c := &Client{
 		authzlibclient: authzlibclient,
 		authz:          authzv1.NewAuthzServiceClient(cc),
 		authzext:       authzextv1.NewAuthzExtentionServiceClient(cc),
 		logger:         log.New("zanzana.client"),
+		metrics:        newClientMetrics(reg),
 	}
 
 	return c, nil
@@ -43,6 +46,9 @@ func (c *Client) Check(ctx context.Context, id authlib.AuthInfo, req authlib.Che
 	ctx, span := tracer.Start(ctx, "authlib.zanzana.client.Check")
 	defer span.End()
 
+	timer := prometheus.NewTimer(c.metrics.requestDurationSeconds.WithLabelValues("Check", req.Namespace))
+	defer timer.ObserveDuration()
+
 	return c.authzlibclient.Check(ctx, id, req, folder)
 }
 
@@ -50,6 +56,9 @@ func (c *Client) Compile(ctx context.Context, id authlib.AuthInfo, req authlib.L
 	ctx, span := tracer.Start(ctx, "authlib.zanzana.client.Compile")
 	defer span.End()
 
+	timer := prometheus.NewTimer(c.metrics.requestDurationSeconds.WithLabelValues("Compile", req.Namespace))
+	defer timer.ObserveDuration()
+
 	return c.authzlibclient.Compile(ctx, id, req)
 }
 
@@ -64,6 +73,9 @@ func (c *Client) Write(ctx context.Context, req *authzextv1.WriteRequest) error
 	ctx, span := tracer.Start(ctx, "authlib.zanzana.client.Write")
 	defer span.End()
 
+	timer := prometheus.NewTimer(c.metrics.requestDurationSeconds.WithLabelValues("Write", req.Namespace))
+	defer timer.ObserveDuration()
+
 	_, err := c.authzext.Write(ctx, req)
 	return err
 }
@@ -72,6 +84,9 @@ func (c *Client) BatchCheck(ctx context.Context, req *authzextv1.BatchCheckReque
 	ctx, span := tracer.Start(ctx, "authlib.zanzana.client.Check")
 	defer span.End()
 
+	timer := prometheus.NewTimer(c.metrics.requestDurationSeconds.WithLabelValues("BatchCheck", req.Namespace))
+	defer timer.ObserveDuration()
+
 	return c.authzext.BatchCheck(ctx, req)
 }
 
@@ -87,6 +102,9 @@ func (c *Client) Mutate(ctx context.Context, req *authzextv1.MutateRequest) erro
 	ctx, span := tracer.Start(ctx, "authlib.zanzana.client.Mutate")
 	defer span.End()
 
+	timer := prometheus.NewTimer(c.metrics.requestDurationSeconds.WithLabelValues("Mutate", req.Namespace))
+	defer timer.ObserveDuration()
+
 	_, err := c.authzext.Mutate(ctx, req)
 	return err
 }
@@ -95,5 +113,8 @@ func (c *Client) Query(ctx context.Context, req *authzextv1.QueryRequest) (*auth
 	ctx, span := tracer.Start(ctx, "authlib.zanzana.client.Query")
 	defer span.End()
 
+	timer := prometheus.NewTimer(c.metrics.requestDurationSeconds.WithLabelValues("Query", req.Namespace))
+	defer timer.ObserveDuration()
+
 	return c.authzext.Query(ctx, req)
 }

pkg/services/authz/zanzana/client/metrics.go

@@ -7,10 +7,10 @@ import (
 
 const (
 	metricsNamespace = "iam"
-	metricsSubSystem = "authz_zanzana"
+	metricsSubSystem = "authz_zanzana_client"
 )
 
-type metrics struct {
+type shadowClientMetrics struct {
 	// evaluationsSeconds is a summary for evaluating access for a specific engine (RBAC and zanzana)
 	evaluationsSeconds *prometheus.HistogramVec
 	// compileSeconds is a summary for compiling item checker for a specific engine (RBAC and zanzana)
@@ -19,8 +19,13 @@ type metrics struct {
 	evaluationStatusTotal *prometheus.CounterVec
 }
 
-func newShadowClientMetrics(reg prometheus.Registerer) *metrics {
-	return &metrics{
+type clientMetrics struct {
+	// requestDurationSeconds is a summary for zanzana client request duration
+	requestDurationSeconds *prometheus.HistogramVec
+}
+
+func newShadowClientMetrics(reg prometheus.Registerer) *shadowClientMetrics {
+	return &shadowClientMetrics{
 		evaluationsSeconds: promauto.With(reg).NewHistogramVec(
 			prometheus.HistogramOpts{
 				Name:      "engine_evaluations_seconds",
@@ -52,3 +57,18 @@ func newShadowClientMetrics(reg prometheus.Registerer) *metrics {
 		),
 	}
 }
+
+func newClientMetrics(reg prometheus.Registerer) *clientMetrics {
+	return &clientMetrics{
+		requestDurationSeconds: promauto.With(reg).NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name:      "request_duration_seconds",
+				Help:      "Histogram for zanzana client request duration",
+				Namespace: metricsNamespace,
+				Subsystem: metricsSubSystem,
+				Buckets:   prometheus.ExponentialBuckets(0.00001, 4, 10),
+			},
+			[]string{"method", "request_namespace"},
+		),
+	}
+}

pkg/services/authz/zanzana/client/shadow_client.go

@@ -20,7 +20,7 @@ type ShadowClient struct {
 	logger        log.Logger
 	accessClient  authlib.AccessClient
 	zanzanaClient authlib.AccessClient
-	metrics       *metrics
+	metrics       *shadowClientMetrics
 }
 
 // WithShadowClient returns a new access client that runs zanzana checks in the background.