From 2b254ed62301243ec6ab9a15a1e1eddbb158a33b Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Thu, 21 Aug 2025 15:20:07 +0200 Subject: [PATCH] Zanzana: Add server side metrics (#109923) * Zanzana: Add metrics to server side * Zanzana: Collect check duration * add metrics for other methods --- pkg/server/module_server.go | 2 +- pkg/services/authz/zanzana.go | 8 +++-- pkg/services/authz/zanzana/server.go | 5 +-- pkg/services/authz/zanzana/server/metrics.go | 31 +++++++++++++++++++ pkg/services/authz/zanzana/server/server.go | 9 ++++-- .../authz/zanzana/server/server_check.go | 9 ++++-- .../authz/zanzana/server/server_list.go | 9 ++++-- .../authz/zanzana/server/server_read.go | 5 +++ .../authz/zanzana/server/server_test.go | 3 +- .../authz/zanzana/server/server_write.go | 5 +++ 10 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 pkg/services/authz/zanzana/server/metrics.go diff --git a/pkg/server/module_server.go b/pkg/server/module_server.go index 19cb24db04b..38e3c3c16e3 100644 --- a/pkg/server/module_server.go +++ b/pkg/server/module_server.go @@ -189,7 +189,7 @@ func (s *ModuleServer) Run() error { }) m.RegisterModule(modules.ZanzanaServer, func() (services.Service, error) { - return authz.ProvideZanzanaService(s.cfg, s.features) + return authz.ProvideZanzanaService(s.cfg, s.features, s.registerer) }) m.RegisterModule(modules.FrontendServer, func() (services.Service, error) { diff --git a/pkg/services/authz/zanzana.go b/pkg/services/authz/zanzana.go index 35ac04151f2..c42befe8d12 100644 --- a/pkg/services/authz/zanzana.go +++ b/pkg/services/authz/zanzana.go @@ -88,7 +88,7 @@ func ProvideZanzana(cfg *setting.Cfg, db db.DB, tracer tracing.Tracer, features return nil, fmt.Errorf("failed to start zanzana: %w", err) } - srv, err := zanzana.NewServer(cfg.ZanzanaServer, openfga, logger, tracer) + srv, err := zanzana.NewServer(cfg.ZanzanaServer, openfga, logger, tracer, reg) if err != nil { return nil, fmt.Errorf("failed to start zanzana: %w", err) } @@ -127,11 +127,12 @@ type ZanzanaService interface { var _ ZanzanaService = (*Zanzana)(nil) // ProvideZanzanaService is used to register zanzana as a module so we can run it seperatly from grafana. -func ProvideZanzanaService(cfg *setting.Cfg, features featuremgmt.FeatureToggles) (*Zanzana, error) { +func ProvideZanzanaService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, reg prometheus.Registerer) (*Zanzana, error) { s := &Zanzana{ cfg: cfg, features: features, logger: log.New("zanzana.server"), + reg: reg, } s.BasicService = services.NewBasicService(s.start, s.running, s.stopping).WithName("zanzana") @@ -147,6 +148,7 @@ type Zanzana struct { logger log.Logger handle grpcserver.Provider features featuremgmt.FeatureToggles + reg prometheus.Registerer } func (z *Zanzana) start(ctx context.Context) error { @@ -172,7 +174,7 @@ func (z *Zanzana) start(ctx context.Context) error { return fmt.Errorf("failed to start zanzana: %w", err) } - zanzanaServer, err := zanzana.NewServer(z.cfg.ZanzanaServer, openfgaServer, z.logger, tracer) + zanzanaServer, err := zanzana.NewServer(z.cfg.ZanzanaServer, openfgaServer, z.logger, tracer, z.reg) if err != nil { return fmt.Errorf("failed to start zanzana: %w", err) } diff --git a/pkg/services/authz/zanzana/server.go b/pkg/services/authz/zanzana/server.go index 1e46aa52324..91a9800efbd 100644 --- a/pkg/services/authz/zanzana/server.go +++ b/pkg/services/authz/zanzana/server.go @@ -5,6 +5,7 @@ import ( openfgaserver "github.com/openfga/openfga/pkg/server" openfgastorage "github.com/openfga/openfga/pkg/storage" + "github.com/prometheus/client_golang/prometheus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" @@ -13,8 +14,8 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -func NewServer(cfg setting.ZanzanaServerSettings, openfga server.OpenFGAServer, logger log.Logger, tracer tracing.Tracer) (*server.Server, error) { - return server.NewServer(cfg, openfga, logger, tracer) +func NewServer(cfg setting.ZanzanaServerSettings, openfga server.OpenFGAServer, logger log.Logger, tracer tracing.Tracer, reg prometheus.Registerer) (*server.Server, error) { + return server.NewServer(cfg, openfga, logger, tracer, reg) } func NewHealthServer(target server.DiagnosticServer) *server.HealthServer { diff --git a/pkg/services/authz/zanzana/server/metrics.go b/pkg/services/authz/zanzana/server/metrics.go new file mode 100644 index 00000000000..7df91aed169 --- /dev/null +++ b/pkg/services/authz/zanzana/server/metrics.go @@ -0,0 +1,31 @@ +package server + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricsNamespace = "iam" + metricsSubSystem = "authz_zanzana_server" +) + +type metrics struct { + // requestDurationSeconds is a summary for zanzana server request duration + requestDurationSeconds *prometheus.HistogramVec +} + +func newZanzanaServerMetrics(reg prometheus.Registerer) *metrics { + return &metrics{ + requestDurationSeconds: promauto.With(reg).NewHistogramVec( + prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "Histogram for zanzana server request duration", + Namespace: metricsNamespace, + Subsystem: metricsSubSystem, + Buckets: prometheus.ExponentialBuckets(0.00001, 4, 10), + }, + []string{"method", "namespace"}, + ), + } +} diff --git a/pkg/services/authz/zanzana/server/server.go b/pkg/services/authz/zanzana/server/server.go index f599fa6ae57..85635260c26 100644 --- a/pkg/services/authz/zanzana/server/server.go +++ b/pkg/services/authz/zanzana/server/server.go @@ -9,6 +9,7 @@ import ( "github.com/fullstorydev/grpchan/inprocgrpc" authzv1 "github.com/grafana/authlib/authz/proto/v1" openfgav1 "github.com/openfga/api/proto/openfga/v1" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/protobuf/types/known/wrapperspb" dashboardV2alpha1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2alpha1" @@ -43,8 +44,9 @@ type Server struct { storesMU *sync.Mutex cache *localcache.CacheService - logger log.Logger - tracer tracing.Tracer + logger log.Logger + tracer tracing.Tracer + metrics *metrics } type storeInfo struct { @@ -52,7 +54,7 @@ type storeInfo struct { ModelID string } -func NewServer(cfg setting.ZanzanaServerSettings, openfga OpenFGAServer, logger log.Logger, tracer tracing.Tracer) (*Server, error) { +func NewServer(cfg setting.ZanzanaServerSettings, openfga OpenFGAServer, logger log.Logger, tracer tracing.Tracer, reg prometheus.Registerer) (*Server, error) { channel := &inprocgrpc.Channel{} openfgav1.RegisterOpenFGAServiceServer(channel, openfga) openFGAClient := openfgav1.NewOpenFGAServiceClient(channel) @@ -66,6 +68,7 @@ func NewServer(cfg setting.ZanzanaServerSettings, openfga OpenFGAServer, logger cache: localcache.New(cfg.CacheSettings.CheckQueryCacheTTL, cacheCleanInterval), logger: logger, tracer: tracer, + metrics: newZanzanaServerMetrics(reg), } return s, nil diff --git a/pkg/services/authz/zanzana/server/server_check.go b/pkg/services/authz/zanzana/server/server_check.go index 0faadea7d0a..e9712b6fe21 100644 --- a/pkg/services/authz/zanzana/server/server_check.go +++ b/pkg/services/authz/zanzana/server/server_check.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" authzv1 "github.com/grafana/authlib/authz/proto/v1" openfgav1 "github.com/openfga/api/proto/openfga/v1" @@ -16,9 +17,11 @@ import ( func (s *Server) Check(ctx context.Context, r *authzv1.CheckRequest) (*authzv1.CheckResponse, error) { ctx, span := s.tracer.Start(ctx, "server.Check") defer span.End() - span.SetAttributes( - attribute.String("namespace", r.GetNamespace()), - ) + span.SetAttributes(attribute.String("namespace", r.GetNamespace())) + + defer func(t time.Time) { + s.metrics.requestDurationSeconds.WithLabelValues("server.Check", r.GetNamespace()).Observe(time.Since(t).Seconds()) + }(time.Now()) res, err := s.check(ctx, r) if err != nil { diff --git a/pkg/services/authz/zanzana/server/server_list.go b/pkg/services/authz/zanzana/server/server_list.go index b8dfcfab4a5..a5fa19e3896 100644 --- a/pkg/services/authz/zanzana/server/server_list.go +++ b/pkg/services/authz/zanzana/server/server_list.go @@ -8,6 +8,7 @@ import ( "hash/fnv" "io" "strings" + "time" authzv1 "github.com/grafana/authlib/authz/proto/v1" openfgav1 "github.com/openfga/api/proto/openfga/v1" @@ -19,9 +20,11 @@ import ( func (s *Server) List(ctx context.Context, r *authzv1.ListRequest) (*authzv1.ListResponse, error) { ctx, span := s.tracer.Start(ctx, "server.List") defer span.End() - span.SetAttributes( - attribute.String("namespace", r.GetNamespace()), - ) + span.SetAttributes(attribute.String("namespace", r.GetNamespace())) + + defer func(t time.Time) { + s.metrics.requestDurationSeconds.WithLabelValues("server.List", r.GetNamespace()).Observe(time.Since(t).Seconds()) + }(time.Now()) res, err := s.list(ctx, r) if err != nil { diff --git a/pkg/services/authz/zanzana/server/server_read.go b/pkg/services/authz/zanzana/server/server_read.go index da4141dce1c..d78d2fcbc0b 100644 --- a/pkg/services/authz/zanzana/server/server_read.go +++ b/pkg/services/authz/zanzana/server/server_read.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" openfgav1 "github.com/openfga/api/proto/openfga/v1" @@ -15,6 +16,10 @@ func (s *Server) Read(ctx context.Context, req *authzextv1.ReadRequest) (*authze ctx, span := s.tracer.Start(ctx, "server.Read") defer span.End() + defer func(t time.Time) { + s.metrics.requestDurationSeconds.WithLabelValues("server.Read", req.GetNamespace()).Observe(time.Since(t).Seconds()) + }(time.Now()) + res, err := s.read(ctx, req) if err != nil { s.logger.Error("failed to perform read request", "error", err, "namespace", req.GetNamespace()) diff --git a/pkg/services/authz/zanzana/server/server_test.go b/pkg/services/authz/zanzana/server/server_test.go index 7b3c57155de..5d700c96a17 100644 --- a/pkg/services/authz/zanzana/server/server_test.go +++ b/pkg/services/authz/zanzana/server/server_test.go @@ -5,6 +5,7 @@ import ( "testing" openfgav1 "github.com/openfga/api/proto/openfga/v1" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" authnlib "github.com/grafana/authlib/authn" @@ -92,7 +93,7 @@ func setup(t *testing.T, testDB db.DB, cfg *setting.Cfg) *Server { openfga, err := NewOpenFGAServer(cfg.ZanzanaServer, store) require.NoError(t, err) - srv, err := NewServer(cfg.ZanzanaServer, openfga, log.NewNopLogger(), tracing.NewNoopTracerService()) + srv, err := NewServer(cfg.ZanzanaServer, openfga, log.NewNopLogger(), tracing.NewNoopTracerService(), prometheus.NewRegistry()) require.NoError(t, err) storeInf, err := srv.getStoreInfo(context.Background(), namespace) diff --git a/pkg/services/authz/zanzana/server/server_write.go b/pkg/services/authz/zanzana/server/server_write.go index 9ca10a38ec5..3cbbc6a5a32 100644 --- a/pkg/services/authz/zanzana/server/server_write.go +++ b/pkg/services/authz/zanzana/server/server_write.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" openfgav1 "github.com/openfga/api/proto/openfga/v1" @@ -15,6 +16,10 @@ func (s *Server) Write(ctx context.Context, req *authzextv1.WriteRequest) (*auth ctx, span := s.tracer.Start(ctx, "server.Write") defer span.End() + defer func(t time.Time) { + s.metrics.requestDurationSeconds.WithLabelValues("server.Write", req.GetNamespace()).Observe(time.Since(t).Seconds()) + }(time.Now()) + res, err := s.write(ctx, req) if err != nil { s.logger.Error("failed to perform write request", "error", err, "namespace", req.GetNamespace())