public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Zanzana: Perform shadow requests (#103444)

Browse Source 
* Zanzana: Execute checks in the background

* add metrics

* collect metrics

* cleanup

* shadow compile checker

* add time metrics for compiler

* run compile in parallel

* prevent deadlock
This commit is contained in:
Alexander Zobnin
2025-04-08 10:03:35 +02:00
committed by GitHub
parent 54192e3308
commit 4bc9203cf6
6 changed files with 193 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/authz/rbac.go
+19 -7
View File
@@ -8,6 +8,11 @@ import (
"time"
"github.com/fullstorydev/grpchan/inprocgrpc"
authnlib "github.com/grafana/authlib/authn"
authzlib "github.com/grafana/authlib/authz"
authzv1 "github.com/grafana/authlib/authz/proto/v1"
"github.com/grafana/authlib/cache"
authlib "github.com/grafana/authlib/types"
grpcAuth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/trace"
@@ -16,11 +21,6 @@ import (
"google.golang.org/grpc/credentials/insecure"
"k8s.io/client-go/rest"
authnlib "github.com/grafana/authlib/authn"
authzlib "github.com/grafana/authlib/authz"
authzv1 "github.com/grafana/authlib/authz/proto/v1"
"github.com/grafana/authlib/cache"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -29,6 +29,7 @@ import (
"github.com/grafana/grafana/pkg/services/apiserver"
"github.com/grafana/grafana/pkg/services/authz/rbac"
"github.com/grafana/grafana/pkg/services/authz/rbac/store"
"github.com/grafana/grafana/pkg/services/authz/zanzana"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/grpcserver"
"github.com/grafana/grafana/pkg/setting"
@@ -47,6 +48,7 @@ func ProvideAuthZClient(
reg prometheus.Registerer,
db db.DB,
acService accesscontrol.Service,
zanzanaClient zanzana.Client,
restConfig apiserver.RestConfigProvider,
) (authlib.AccessClient, error) {
authCfg, err := readAuthzClientSettings(cfg)
@@ -60,7 +62,11 @@ func ProvideAuthZClient(
switch authCfg.mode {
case clientModeCloud:
return newRemoteRBACClient(authCfg, tracer)
rbacClient, err := newRemoteRBACClient(authCfg, tracer)
if features.IsEnabledGlobally(featuremgmt.FlagZanzana) {
return zanzana.WithShadowClient(rbacClient, zanzanaClient, reg)
}
return rbacClient, err
default:
sql := legacysql.NewDatabaseProvider(db)
@@ -92,7 +98,13 @@ func ProvideAuthZClient(
return ctx, nil
}))
authzv1.RegisterAuthzServiceServer(channel, server)
return newRBACClient(channel, tracer), nil
rbacClient := newRBACClient(channel, tracer)
if features.IsEnabledGlobally(featuremgmt.FlagZanzana) {
return zanzana.WithShadowClient(rbacClient, zanzanaClient, reg)
}
return rbacClient, nil
}
}
pkg/services/authz/zanzana.go
+1 -1
View File
@@ -34,7 +34,7 @@ import (
// ProvideZanzana used to register ZanzanaClient.
// It will also start an embedded ZanzanaSever if mode is set to "embedded".
func ProvideZanzana(cfg *setting.Cfg, db db.DB, tracer tracing.Tracer, features featuremgmt.FeatureToggles) (zanzana.Client, error) {
func ProvideZanzana(cfg *setting.Cfg, db db.DB, tracer tracing.Tracer, features featuremgmt.FeatureToggles, reg prometheus.Registerer) (zanzana.Client, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagZanzana) {
return zanzana.NewNoopClient(), nil
}
pkg/services/authz/zanzana/client.go
+6
View File
@@ -6,6 +6,8 @@ import (
"google.golang.org/grpc"
authlib "github.com/grafana/authlib/types"
"github.com/prometheus/client_golang/prometheus"
authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1"
"github.com/grafana/grafana/pkg/services/authz/zanzana/client"
)
@@ -22,6 +24,10 @@ func NewClient(cc grpc.ClientConnInterface) (*client.Client, error) {
return client.New(cc)
}
func WithShadowClient(accessClient authlib.AccessClient, zanzanaClient authlib.AccessClient, reg prometheus.Registerer) (authlib.AccessClient, error) {
return client.WithShadowClient(accessClient, zanzanaClient, reg), nil
}
func NewNoopClient() *client.NoopClient {
return client.NewNoop()
}
pkg/services/authz/zanzana/client/client.go
+2 -3
View File
@@ -3,11 +3,10 @@ package client
import (
"context"
"go.opentelemetry.io/otel"
"google.golang.org/grpc"
authzv1 "github.com/grafana/authlib/authz/proto/v1"
authlib "github.com/grafana/authlib/types"
"go.opentelemetry.io/otel"
"google.golang.org/grpc"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/log"
pkg/services/authz/zanzana/client/metrics.go
+54
View File
@@ -0,0 +1,54 @@
package client
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
const (
metricsNamespace = "iam"
metricsSubSystem = "authz_zanzana"
)
type metrics struct {
// evaluationsSeconds is a summary for evaluating access for a specific engine (RBAC and zanzana)
evaluationsSeconds *prometheus.HistogramVec
// compileSeconds is a summary for compiling item checker for a specific engine (RBAC and zanzana)
compileSeconds *prometheus.HistogramVec
// evaluationStatusTotal is a metric for zanzana evaluation status
evaluationStatusTotal *prometheus.CounterVec
}
func newShadowClientMetrics(reg prometheus.Registerer) *metrics {
return &metrics{
evaluationsSeconds: promauto.With(reg).NewHistogramVec(
prometheus.HistogramOpts{
Name: "engine_evaluations_seconds",
Help: "Histogram for evaluation time for the specific access control engine (RBAC and zanzana).",
Namespace: metricsNamespace,
Subsystem: metricsSubSystem,
Buckets: prometheus.ExponentialBuckets(0.00001, 4, 10),
},
[]string{"engine"},
),
compileSeconds: promauto.With(reg).NewHistogramVec(
prometheus.HistogramOpts{
Name: "compile_seconds",
Help: "Histogram for item checker compilation time for the specific access control engine (RBAC and zanzana).",
Namespace: metricsNamespace,
Subsystem: metricsSubSystem,
Buckets: prometheus.ExponentialBuckets(0.00001, 4, 10),
},
[]string{"engine"},
),
evaluationStatusTotal: promauto.With(reg).NewCounterVec(
prometheus.CounterOpts{
Name: "evaluation_status_total",
Help: "evaluation status (success or error) for zanzana",
Namespace: metricsNamespace,
Subsystem: metricsSubSystem,
},
[]string{"status"},
),
}
}
pkg/services/authz/zanzana/client/shadow_client.go
+111
View File
@@ -0,0 +1,111 @@
package client
import (
"context"
authlib "github.com/grafana/authlib/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/grafana/grafana/pkg/infra/log"
)
type ShadowClient struct {
logger log.Logger
accessClient authlib.AccessClient
zanzanaClient authlib.AccessClient
metrics *metrics
}
// WithShadowClient returns a new access client that runs zanzana checks in the background.
func WithShadowClient(accessClient authlib.AccessClient, zanzanaClient authlib.AccessClient, reg prometheus.Registerer) authlib.AccessClient {
client := &ShadowClient{
logger: log.New("zanzana-shadow-client"),
accessClient: accessClient,
zanzanaClient: zanzanaClient,
metrics: newShadowClientMetrics(reg),
}
return client
}
func (c *ShadowClient) Check(ctx context.Context, id authlib.AuthInfo, req authlib.CheckRequest) (authlib.CheckResponse, error) {
acResChan := make(chan authlib.CheckResponse, 1)
acErrChan := make(chan error, 1)
go func() {
if c.zanzanaClient == nil {
return
}
timer := prometheus.NewTimer(c.metrics.evaluationsSeconds.WithLabelValues("zanzana"))
defer timer.ObserveDuration()
zanzanaCtx := context.WithoutCancel(ctx)
res, err := c.zanzanaClient.Check(zanzanaCtx, id, req)
if err != nil {
c.logger.Error("Failed to run zanzana check", "error", err)
}
acRes := <-acResChan
acErr := <-acErrChan
if acErr == nil {
if res.Allowed != acRes.Allowed {
c.metrics.evaluationStatusTotal.WithLabelValues("error").Inc()
c.logger.Warn("Zanzana check result does not match", "expected", acRes.Allowed, "actual", res.Allowed)
} else {
c.metrics.evaluationStatusTotal.WithLabelValues("success").Inc()
}
}
}()
timer := prometheus.NewTimer(c.metrics.evaluationsSeconds.WithLabelValues("rbac"))
res, err := c.accessClient.Check(ctx, id, req)
timer.ObserveDuration()
acResChan <- res
acErrChan <- err
return res, err
}
func (c *ShadowClient) Compile(ctx context.Context, id authlib.AuthInfo, req authlib.ListRequest) (authlib.ItemChecker, error) {
zanzanaItemCheckerChan := make(chan authlib.ItemChecker, 1)
go func() {
if c.zanzanaClient == nil {
zanzanaItemCheckerChan <- nil
return
}
timer := prometheus.NewTimer(c.metrics.compileSeconds.WithLabelValues("zanzana"))
itemChecker, err := c.zanzanaClient.Compile(ctx, id, req)
timer.ObserveDuration()
if err != nil {
c.logger.Warn("Failed to compile zanzana item checker", "error", err)
}
zanzanaItemCheckerChan <- itemChecker
}()
timer := prometheus.NewTimer(c.metrics.compileSeconds.WithLabelValues("rbac"))
rbacItemChecker, err := c.accessClient.Compile(ctx, id, req)
timer.ObserveDuration()
if err != nil {
return nil, err
}
zanzanaItemChecker := <-zanzanaItemCheckerChan
shadowItemChecker := func(name, folder string) bool {
rbacRes := rbacItemChecker(name, folder)
if zanzanaItemChecker != nil {
zanzanaRes := zanzanaItemChecker(name, folder)
if zanzanaRes != rbacRes {
c.metrics.evaluationStatusTotal.WithLabelValues("error").Inc()
c.logger.Warn("Zanzana compile result does not match", "expected", rbacRes, "actual", zanzanaRes)
} else {
c.metrics.evaluationStatusTotal.WithLabelValues("success").Inc()
}
}
return rbacRes
}
return shadowItemChecker, err
}