From 03abe18bb291204c034d61f612203159c43caf45 Mon Sep 17 00:00:00 2001 From: Mihai Turdean <6640685+mihai-turdean@users.noreply.github.com> Date: Thu, 11 Sep 2025 09:21:44 -0600 Subject: [PATCH] Instrument IAM Folder Operator with Tracing (#110923) --- apps/iam/go.mod | 4 +- apps/iam/pkg/reconcilers/folder_reconciler.go | 45 ++++++++- apps/iam/pkg/reconcilers/folder_service.go | 28 +++++- apps/iam/pkg/reconcilers/zanzana_service.go | 93 ++++++++++++++++++- .../iam/zanzana_folder_reconciler.go | 1 + 5 files changed, 162 insertions(+), 9 deletions(-) diff --git a/apps/iam/go.mod b/apps/iam/go.mod index dd7ea5260e6..54ae9805548 100644 --- a/apps/iam/go.mod +++ b/apps/iam/go.mod @@ -28,6 +28,8 @@ require ( github.com/grafana/grafana-app-sdk/plugin v0.40.3 github.com/grafana/grafana/apps/folder v0.0.0 github.com/grafana/grafana/pkg/apimachinery v0.0.0 + go.opentelemetry.io/otel v1.37.0 + go.opentelemetry.io/otel/trace v1.37.0 k8s.io/apimachinery v0.33.3 k8s.io/client-go v0.33.3 k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff @@ -377,7 +379,6 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 // indirect go.opentelemetry.io/contrib/samplers/jaegerremote v0.30.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 // indirect @@ -395,7 +396,6 @@ require ( go.opentelemetry.io/otel/sdk v1.37.0 // indirect go.opentelemetry.io/otel/sdk/log v0.12.2 // indirect go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/mock v0.5.2 // indirect diff --git a/apps/iam/pkg/reconcilers/folder_reconciler.go b/apps/iam/pkg/reconcilers/folder_reconciler.go index 93ac7264e0c..b4150e8f37a 100644 --- a/apps/iam/pkg/reconcilers/folder_reconciler.go +++ b/apps/iam/pkg/reconcilers/folder_reconciler.go @@ -9,6 +9,10 @@ import ( "github.com/grafana/grafana-app-sdk/operator" foldersKind "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" "github.com/grafana/grafana/pkg/services/authz" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "k8s.io/client-go/rest" ) @@ -60,26 +64,61 @@ func NewFolderReconciler(cfg ReconcilerConfig) (operator.Reconciler, error) { return reconciler, nil } +// actionToString converts a ReconcileAction to a human-readable string +func actionToString(action operator.ReconcileAction) string { + switch action { + case operator.ReconcileActionCreated: + return "CREATE" + case operator.ReconcileActionUpdated: + return "UPDATE" + case operator.ReconcileActionDeleted: + return "DELETE" + default: + return "UNKNOWN" + } +} + func (r *FolderReconciler) reconcile(ctx context.Context, req operator.TypedReconcileRequest[*foldersKind.Folder]) (operator.ReconcileResult, error) { + // Create root span for the entire reconciliation process + tracer := otel.GetTracerProvider().Tracer("iam-folder-reconciler") + ctx, span := tracer.Start(ctx, "folder.reconcile", + trace.WithAttributes( + attribute.String("action", actionToString(req.Action)), + attribute.String("folder.uid", req.Object.Name), + attribute.String("namespace", req.Object.Namespace), + ), + ) + defer span.End() + // Add timeout to prevent hanging operations ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() err := validateFolder(req.Object) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "validation failed") return operator.ReconcileResult{}, err } + var result operator.ReconcileResult switch req.Action { case operator.ReconcileActionCreated: - return r.handleUpdateFolder(ctx, req.Object) + result, err = r.handleUpdateFolder(ctx, req.Object) case operator.ReconcileActionUpdated: - return r.handleUpdateFolder(ctx, req.Object) + result, err = r.handleUpdateFolder(ctx, req.Object) case operator.ReconcileActionDeleted: - return r.handleDeleteFolder(ctx, req.Object) + result, err = r.handleDeleteFolder(ctx, req.Object) default: return operator.ReconcileResult{}, nil } + + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "reconciliation failed") + } + + return result, err } func (r *FolderReconciler) handleUpdateFolder(ctx context.Context, folder *foldersKind.Folder) (operator.ReconcileResult, error) { diff --git a/apps/iam/pkg/reconcilers/folder_service.go b/apps/iam/pkg/reconcilers/folder_service.go index 00ed59c75b1..0a1a74a4257 100644 --- a/apps/iam/pkg/reconcilers/folder_service.go +++ b/apps/iam/pkg/reconcilers/folder_service.go @@ -6,6 +6,10 @@ import ( foldersKind "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" "github.com/grafana/grafana/pkg/apimachinery/utils" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -22,23 +26,45 @@ type APIFolderStore struct { } func (s *APIFolderStore) GetFolderParent(ctx context.Context, namespace, uid string) (string, error) { + tracer := otel.GetTracerProvider().Tracer("iam-folder-reconciler") + ctx, span := tracer.Start(ctx, "APIFolderStore.GetFolderParent", + trace.WithAttributes( + attribute.String("folder.uid", uid), + attribute.String("folder.namespace", namespace), + ), + ) + defer span.End() + client, err := s.client(namespace) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to create kubernetes client") return "", fmt.Errorf("create resource client: %w", err) } // Get the folder by UID unstructuredObj, err := client.Get(ctx, uid, metav1.GetOptions{}) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to get folder from kubernetes API") return "", fmt.Errorf("get folder %s: %w", uid, err) } object, err := utils.MetaAccessor(unstructuredObj) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to get meta accessor from folder object") return "", fmt.Errorf("get meta accessor: %w", err) } - return object.GetFolder(), nil + parentUID := object.GetFolder() + span.SetAttributes(attribute.String("folder.parent_uid", parentUID)) + span.SetStatus(codes.Ok, "successfully retrieved folder parent") + span.AddEvent("folder.parent.retrieved", trace.WithAttributes( + attribute.String("parent.uid", parentUID), + )) + + return parentUID, nil } func (s *APIFolderStore) client(namespace string) (dynamic.ResourceInterface, error) { diff --git a/apps/iam/pkg/reconcilers/zanzana_service.go b/apps/iam/pkg/reconcilers/zanzana_service.go index a62cec83b80..10897a446c7 100644 --- a/apps/iam/pkg/reconcilers/zanzana_service.go +++ b/apps/iam/pkg/reconcilers/zanzana_service.go @@ -7,6 +7,10 @@ import ( authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1" "github.com/grafana/grafana/pkg/services/authz/zanzana" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) type ZanzanaPermissionStore struct { @@ -20,8 +24,20 @@ func NewZanzanaPermissionStore(zanzanaClient zanzana.Client) PermissionStore { } func (c *ZanzanaPermissionStore) SetFolderParent(ctx context.Context, namespace, folderUID, parentUID string) error { + tracer := otel.GetTracerProvider().Tracer("iam-folder-reconciler") + ctx, span := tracer.Start(ctx, "zanzana-permission-store.set-folder-parent", + trace.WithAttributes( + attribute.String("folder.uid", folderUID), + attribute.String("folder.namespace", namespace), + attribute.String("parent.uid", parentUID), + ), + ) + defer span.End() + err := c.DeleteFolderParents(ctx, namespace, folderUID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to delete existing folder parents") return err } @@ -32,11 +48,15 @@ func (c *ZanzanaPermissionStore) SetFolderParent(ctx context.Context, namespace, user, err := toFolderTuple(parentUID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to create parent tuple") return err } object, err := toFolderTuple(folderUID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to create folder tuple") return err } @@ -50,6 +70,8 @@ func (c *ZanzanaPermissionStore) SetFolderParent(ctx context.Context, namespace, }}, }, }); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to write parent tuple to zanzana") return err } @@ -57,11 +79,23 @@ func (c *ZanzanaPermissionStore) SetFolderParent(ctx context.Context, namespace, } func (c *ZanzanaPermissionStore) GetFolderParents(ctx context.Context, namespace, folderUID string) ([]string, error) { + tracer := otel.GetTracerProvider().Tracer("iam-folder-reconciler") + ctx, span := tracer.Start(ctx, "ZanzanaPermissionStore.GetFolderParents", + trace.WithAttributes( + attribute.String("folder.uid", folderUID), + attribute.String("folder.namespace", namespace), + ), + ) + defer span.End() + tuples, err := c.listFolderParentRelations(ctx, namespace, folderUID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to list folder parent relations") return nil, err } + span.SetAttributes(attribute.Int("tuples.count", len(tuples))) parents := make([]string, 0, len(tuples)) for _, t := range tuples { @@ -73,10 +107,16 @@ func (c *ZanzanaPermissionStore) GetFolderParents(ctx context.Context, namespace if len(uidAndRelationParts) > 0 { parents = append(parents, uidAndRelationParts[0]) } else { - return nil, fmt.Errorf("invalid user format: %s, expected format: folder:UID or folder:UID#relation", t.Key.User) + err := fmt.Errorf("invalid user format: %s, expected format: folder:UID or folder:UID#relation", t.Key.User) + span.RecordError(err) + span.SetStatus(codes.Error, "invalid tuple user format") + return nil, err } } else { - return nil, fmt.Errorf("invalid user format: %s, expected format: folder:UID or folder:UID#relation", t.Key.User) + err := fmt.Errorf("invalid user format: %s, expected format: folder:UID or folder:UID#relation", t.Key.User) + span.RecordError(err) + span.SetStatus(codes.Error, "invalid tuple user format") + return nil, err } } @@ -84,14 +124,29 @@ func (c *ZanzanaPermissionStore) GetFolderParents(ctx context.Context, namespace } func (c *ZanzanaPermissionStore) DeleteFolderParents(ctx context.Context, namespace, folderUID string) error { + tracer := otel.GetTracerProvider().Tracer("iam-folder-reconciler") + ctx, span := tracer.Start(ctx, "ZanzanaPermissionStore.DeleteFolderParents", + trace.WithAttributes( + attribute.String("folder.uid", folderUID), + attribute.String("folder.namespace", namespace), + ), + ) + defer span.End() + tuples, err := c.listFolderParentRelations(ctx, namespace, folderUID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to list folder parent relations") return err } + span.SetAttributes(attribute.Int("tuples.toDelete.count", len(tuples))) + if len(tuples) > 0 { err = c.deleteTuples(ctx, namespace, tuples) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to delete tuples") return err } } @@ -102,8 +157,19 @@ func (c *ZanzanaPermissionStore) DeleteFolderParents(ctx context.Context, namesp // listFolderParentRelations lists parent relations where the given folder is the object. // It returns tuples where other folders are parents of this folder, not children. func (c *ZanzanaPermissionStore) listFolderParentRelations(ctx context.Context, namespace, folderUID string) ([]*authzextv1.Tuple, error) { + tracer := otel.GetTracerProvider().Tracer("iam-folder-reconciler") + ctx, span := tracer.Start(ctx, "ZanzanaPermissionStore.listFolderParentRelations", + trace.WithAttributes( + attribute.String("folder.uid", folderUID), + attribute.String("folder.namespace", namespace), + ), + ) + defer span.End() + object, err := toFolderTuple(folderUID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to create folder tuple") return nil, err } @@ -118,6 +184,8 @@ func (c *ZanzanaPermissionStore) listFolderParentRelations(ctx context.Context, }) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to read tuples from zanzana") return nil, err } @@ -132,6 +200,8 @@ func (c *ZanzanaPermissionStore) listFolderParentRelations(ctx context.Context, }, }) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to read tuples from zanzana") return nil, err } @@ -143,6 +213,15 @@ func (c *ZanzanaPermissionStore) listFolderParentRelations(ctx context.Context, } func (c *ZanzanaPermissionStore) deleteTuples(ctx context.Context, namespace string, tuples []*authzextv1.Tuple) error { + tracer := otel.GetTracerProvider().Tracer("zanzana-folder-reconciler") + ctx, span := tracer.Start(ctx, "zanzana-permission-store.delete-tuples", + trace.WithAttributes( + attribute.String("namespace", namespace), + attribute.Int("tuples.count", len(tuples)), + ), + ) + defer span.End() + tupleKeys := make([]*authzextv1.TupleKeyWithoutCondition, 0, len(tuples)) for _, t := range tuples { tupleKeys = append(tupleKeys, &authzextv1.TupleKeyWithoutCondition{ @@ -152,12 +231,20 @@ func (c *ZanzanaPermissionStore) deleteTuples(ctx context.Context, namespace str }) } - return c.zanzanaClient.Write(ctx, &authzextv1.WriteRequest{ + err := c.zanzanaClient.Write(ctx, &authzextv1.WriteRequest{ Namespace: namespace, Deletes: &authzextv1.WriteRequestDeletes{ TupleKeys: tupleKeys, }, }) + + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "failed to delete tuples in zanzana") + return err + } + + return nil } func toFolderTuple(UID string) (string, error) { diff --git a/pkg/operators/iam/zanzana_folder_reconciler.go b/pkg/operators/iam/zanzana_folder_reconciler.go index c857e99ac2c..68974df7c92 100644 --- a/pkg/operators/iam/zanzana_folder_reconciler.go +++ b/pkg/operators/iam/zanzana_folder_reconciler.go @@ -129,6 +129,7 @@ func buildIAMConfigFromSettings(cfg *setting.Cfg) (*iamConfig, error) { metricsSection := cfg.SectionWithEnvOverrides("metrics") iamCfg.RunnerConfig.MetricsConfig.Enabled = metricsSection.Key("enabled").MustBool(true) + iamCfg.RunnerConfig.MetricsConfig.Namespace = metricsSection.Key("namespace").MustString("grafana-iam") return &iamCfg, nil }