public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more metrics to the IAM Folder Reconciler (#111275)

Browse Source 
* Add more metrics to the operator

* Add namespace when logging metrics

* Skip flaky test
This commit is contained in:
Mihai Turdean
2025-09-18 09:03:32 -06:00
committed by GitHub
parent 28952cc490
commit aaa8094a53
13 changed files with 471 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/iam/pkg/app/app.go
+23 -17
View File
@@ -12,6 +12,7 @@ import (
foldersKind "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
"github.com/grafana/grafana/apps/iam/pkg/reconcilers"
"github.com/grafana/grafana/pkg/services/authz"
"github.com/prometheus/client_golang/prometheus"
)
var appManifestData = app.ManifestData{
@@ -24,16 +25,17 @@ type InformerConfig struct {
}
type AppConfig struct {
ZanzanaClientCfg authz.ZanzanaClientConfig
InformerConfig InformerConfig
FolderReconcilerNamespace string
ZanzanaClientCfg authz.ZanzanaClientConfig
InformerConfig InformerConfig
Namespace string
MetricsRegisterer prometheus.Registerer
}
func Provider(appCfg app.SpecificConfig) app.Provider {
return simple.NewAppProvider(app.NewEmbeddedManifest(appManifestData), appCfg, New)
}
func generateInformerSupplier(informerConfig InformerConfig) simple.InformerSupplier {
func generateInformerSupplier(informerConfig InformerConfig, metrics *reconcilers.ReconcilerMetrics) simple.InformerSupplier {
return func(kind resource.Kind, clients resource.ClientGenerator, options operator.ListWatchOptions) (operator.Informer, error) {
client, err := clients.ClientFor(kind)
if err != nil {
@@ -56,6 +58,10 @@ func generateInformerSupplier(informerConfig InformerConfig) simple.InformerSupp
MaxConcurrentWorkers: informerConfig.MaxConcurrentWorkers,
ErrorHandler: func(ctx context.Context, err error) {
logging.FromContext(ctx).With("error", err).Error("ConcurrentInformer processing error")
if metrics != nil {
// Use "unknown" for action since informer errors don't have specific actions
metrics.RecordReconcileFailure("unknown", "informer")
}
},
},
)
@@ -68,10 +74,13 @@ func New(cfg app.Config) (app.App, error) {
return nil, fmt.Errorf("invalid config type: expected AppConfig, got %T", cfg.SpecificConfig)
}
// Initialize metrics first so they can be shared across components
metrics := reconcilers.NewReconcilerMetrics(appSpecificConfig.MetricsRegisterer, appSpecificConfig.Namespace)
folderReconciler, err := reconcilers.NewFolderReconciler(reconcilers.ReconcilerConfig{
ZanzanaCfg: appSpecificConfig.ZanzanaClientCfg,
KubeConfig: &cfg.KubeConfig,
FolderReconcilerNamespace: appSpecificConfig.FolderReconcilerNamespace,
ZanzanaCfg: appSpecificConfig.ZanzanaClientCfg,
KubeConfig: &cfg.KubeConfig,
Metrics: metrics,
})
if err != nil {
return nil, fmt.Errorf("unable to create FolderReconciler: %w", err)
@@ -79,26 +88,23 @@ func New(cfg app.Config) (app.App, error) {
logging.DefaultLogger.Info("FolderReconciler created")
reconcilerOptions := simple.UnmanagedKindReconcileOptions{}
if cfg.SpecificConfig.(AppConfig).FolderReconcilerNamespace != "" {
reconcilerOptions.Namespace = cfg.SpecificConfig.(AppConfig).FolderReconcilerNamespace
}
config := simple.AppConfig{
Name: cfg.ManifestData.AppName,
KubeConfig: cfg.KubeConfig,
InformerConfig: simple.AppInformerConfig{
InformerSupplier: generateInformerSupplier(appSpecificConfig.InformerConfig),
InformerSupplier: generateInformerSupplier(appSpecificConfig.InformerConfig, metrics),
ErrorHandler: func(ctx context.Context, err error) {
logging.FromContext(ctx).With("error", err).Error("Informer processing error")
if metrics != nil {
// Use "unknown" for action since top-level informer errors don't have specific actions
metrics.RecordReconcileFailure("unknown", "informer")
}
},
},
UnmanagedKinds: []simple.AppUnmanagedKind{
{
Kind: foldersKind.FolderKind(),
Reconciler: folderReconciler,
ReconcileOptions: reconcilerOptions,
Kind: foldersKind.FolderKind(),
Reconciler: folderReconciler,
},
},
}
apps/iam/pkg/reconcilers/folder_reconciler.go
+46 -13
View File
@@ -28,16 +28,17 @@ type PermissionStore interface {
DeleteFolderParents(ctx context.Context, namespace, folderUID string) error
}
// AppConfig represents the app-specific configuration
// ReconcilerConfig represents the app-specific configuration
type ReconcilerConfig struct {
ZanzanaCfg authz.ZanzanaClientConfig
KubeConfig *rest.Config
FolderReconcilerNamespace string
ZanzanaCfg authz.ZanzanaClientConfig
KubeConfig *rest.Config
Metrics *ReconcilerMetrics
}
type FolderReconciler struct {
permissionStore PermissionStore
folderStore FolderStore
metrics *ReconcilerMetrics
}
func NewFolderReconciler(cfg ReconcilerConfig) (operator.Reconciler, error) {
@@ -55,6 +56,7 @@ func NewFolderReconciler(cfg ReconcilerConfig) (operator.Reconciler, error) {
folderReconciler := &FolderReconciler{
permissionStore: permissionStore,
folderStore: folderStore,
metrics: cfg.Metrics,
}
reconciler := &operator.TypedReconciler[*foldersKind.Folder]{
@@ -68,13 +70,13 @@ func NewFolderReconciler(cfg ReconcilerConfig) (operator.Reconciler, error) {
func actionToString(action operator.ReconcileAction) string {
switch action {
case operator.ReconcileActionCreated:
return "CREATE"
return "create"
case operator.ReconcileActionUpdated:
return "UPDATE"
return "update"
case operator.ReconcileActionDeleted:
return "DELETE"
return "delete"
default:
return "UNKNOWN"
return "unknown"
}
}
@@ -94,22 +96,30 @@ func (r *FolderReconciler) reconcile(ctx context.Context, req operator.TypedReco
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
action := actionToString(req.Action)
err := validateFolder(req.Object)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, "validation failed")
if r.metrics != nil {
r.metrics.RecordReconcileFailure(action, "informer")
}
return operator.ReconcileResult{}, err
}
var result operator.ReconcileResult
switch req.Action {
case operator.ReconcileActionCreated:
result, err = r.handleUpdateFolder(ctx, req.Object)
result, err = r.handleUpdateFolder(ctx, req.Object, action)
case operator.ReconcileActionUpdated:
result, err = r.handleUpdateFolder(ctx, req.Object)
result, err = r.handleUpdateFolder(ctx, req.Object, action)
case operator.ReconcileActionDeleted:
result, err = r.handleDeleteFolder(ctx, req.Object)
result, err = r.handleDeleteFolder(ctx, req.Object, action)
default:
if r.metrics != nil {
r.metrics.RecordReconcileSuccess(action, "no_changes_needed")
}
return operator.ReconcileResult{}, nil
}
@@ -121,7 +131,7 @@ func (r *FolderReconciler) reconcile(ctx context.Context, req operator.TypedReco
return result, err
}
func (r *FolderReconciler) handleUpdateFolder(ctx context.Context, folder *foldersKind.Folder) (operator.ReconcileResult, error) {
func (r *FolderReconciler) handleUpdateFolder(ctx context.Context, folder *foldersKind.Folder, action string) (operator.ReconcileResult, error) {
logger := logging.FromContext(ctx)
folderUID := folder.Name
@@ -130,32 +140,48 @@ func (r *FolderReconciler) handleUpdateFolder(ctx context.Context, folder *folde
parentUID, err := r.folderStore.GetFolderParent(ctx, namespace, folderUID)
if err != nil {
logger.Error("Error getting folder parent", "error", err)
if r.metrics != nil {
r.metrics.RecordReconcileFailure(action, "folder_store")
}
return operator.ReconcileResult{}, err
}
parents, err := r.permissionStore.GetFolderParents(ctx, namespace, folderUID)
if err != nil {
logger.Error("Error getting folder parents", "error", err)
if r.metrics != nil {
r.metrics.RecordReconcileFailure(action, "permission_store")
}
return operator.ReconcileResult{}, err
}
if (len(parents) == 0 && parentUID == "") || (len(parents) == 1 && parents[0] == parentUID) {
logger.Info("Folder is already reconciled", "folder", folderUID, "parent", parentUID, "namespace", namespace)
if r.metrics != nil {
r.metrics.RecordReconcileSuccess(action, "no_changes_needed")
}
return operator.ReconcileResult{}, nil
}
err = r.permissionStore.SetFolderParent(ctx, namespace, folderUID, parentUID)
if err != nil {
logger.Error("Error setting folder parent", "error", err)
if r.metrics != nil {
r.metrics.RecordReconcileFailure(action, "permission_store")
}
return operator.ReconcileResult{}, err
}
logger.Info("Folder parent set in permission store", "folder", folderUID, "parent", parentUID, "namespace", namespace)
if r.metrics != nil {
r.metrics.RecordReconcileSuccess(action, "changes_made")
}
return operator.ReconcileResult{}, nil
}
func (r *FolderReconciler) handleDeleteFolder(ctx context.Context, folder *foldersKind.Folder) (operator.ReconcileResult, error) {
func (r *FolderReconciler) handleDeleteFolder(ctx context.Context, folder *foldersKind.Folder, action string) (operator.ReconcileResult, error) {
logger := logging.FromContext(ctx)
namespace := folder.Namespace
@@ -164,11 +190,18 @@ func (r *FolderReconciler) handleDeleteFolder(ctx context.Context, folder *folde
err := r.permissionStore.DeleteFolderParents(ctx, namespace, folderUID)
if err != nil {
logger.Error("Error deleting folder parents", "error", err)
if r.metrics != nil {
r.metrics.RecordReconcileFailure(action, "permission_store")
}
return operator.ReconcileResult{}, err
}
logger.Info("Folder deleted from permission store", "folder", folderUID, "namespace", namespace)
if r.metrics != nil {
r.metrics.RecordReconcileSuccess(action, "changes_made")
}
return operator.ReconcileResult{}, nil
}
apps/iam/pkg/reconcilers/metrics.go
+75
View File
@@ -0,0 +1,75 @@
package reconcilers
import (
"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
"github.com/prometheus/client_golang/prometheus"
)
// ReconcilerMetrics holds all the metrics for the IAM folder reconciler
type ReconcilerMetrics struct {
// Comprehensive reconcile operations counter with labels for action, status, and detailed outcome
ReconcileOperations *prometheus.CounterVec
}
// NewReconcilerMetrics creates a new set of metrics for the reconciler
func NewReconcilerMetrics(registerer prometheus.Registerer, namespace string) *ReconcilerMetrics {
// Labels for comprehensive reconcile tracking
operationLabels := []string{"action", "outcome"}
operationValues := map[string][]string{
"action": {"create", "update", "delete", "unknown"},
"outcome": {
"success_changes_made",
"success_no_changes_needed",
"failure_informer",
"failure_permission_store",
"failure_folder_store",
"failure_unknown",
},
}
metrics := &ReconcilerMetrics{
ReconcileOperations: metricutil.NewCounterVecStartingAtZero(
prometheus.CounterOpts{
Namespace: namespace,
Subsystem: "iam_folder_reconciler",
Name: "folder_reconcile_operations_total",
Help: "Total number of folder reconcile operations by action and outcome",
},
operationLabels,
operationValues,
),
}
// Register the metric
if registerer != nil {
registerer.MustRegister(
metrics.ReconcileOperations,
)
}
return metrics
}
// RecordReconcileSuccess records a successful reconcile operation
func (m *ReconcilerMetrics) RecordReconcileSuccess(action, result string) {
if m.ReconcileOperations != nil {
// Validate parameters
if action == "" || result == "" {
return
}
outcome := "success_" + result // result should be "changes_made" or "no_changes_needed"
m.ReconcileOperations.WithLabelValues(action, outcome).Inc()
}
}
// RecordReconcileFailure records a failed reconcile operation
func (m *ReconcilerMetrics) RecordReconcileFailure(action, source string) {
if m.ReconcileOperations != nil {
// Validate parameters
if action == "" || source == "" {
return
}
outcome := "failure_" + source // source should be "informer", "permission_store", "folder_store", or "unknown"
m.ReconcileOperations.WithLabelValues(action, outcome).Inc()
}
}