package controller import ( "context" "errors" "fmt" "time" "go.opentelemetry.io/otel/attribute" apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" informer "github.com/grafana/grafana/apps/provisioning/pkg/generated/informers/externalversions/provisioning/v0alpha1" listers "github.com/grafana/grafana/apps/provisioning/pkg/generated/listers/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/prometheus/client_golang/prometheus" ) const loggerName = "provisioning-repository-controller" const ( maxAttempts = 3 ) type queueItem struct { key string obj interface{} attempts int } //go:generate mockery --name finalizerProcessor --structname MockFinalizerProcessor --inpackage --filename finalizer_mock.go --with-expecter type finalizerProcessor interface { process(ctx context.Context, repo repository.Repository, finalizers []string) error } // RepositoryController controls how and when CRD is established. type RepositoryController struct { client client.ProvisioningV0alpha1Interface repoLister listers.RepositoryLister repoSynced cache.InformerSynced logger logging.Logger jobs interface { jobs.Queue jobs.Store } finalizer finalizerProcessor statusPatcher StatusPatcher repoFactory repository.Factory healthChecker *HealthChecker // To allow injection for testing. processFn func(item *queueItem) error enqueueRepository func(obj any) keyFunc func(obj any) (string, error) queue workqueue.TypedRateLimitingInterface[*queueItem] registry prometheus.Registerer tracer tracing.Tracer } // NewRepositoryController creates new RepositoryController. func NewRepositoryController( provisioningClient client.ProvisioningV0alpha1Interface, repoInformer informer.RepositoryInformer, repoFactory repository.Factory, resourceLister resources.ResourceLister, clients resources.ClientFactory, jobs interface { jobs.Queue jobs.Store }, healthChecker *HealthChecker, statusPatcher StatusPatcher, registry prometheus.Registerer, tracer tracing.Tracer, parallelOperations int, ) (*RepositoryController, error) { finalizerMetrics := registerFinalizerMetrics(registry) rc := &RepositoryController{ client: provisioningClient, repoLister: repoInformer.Lister(), repoSynced: repoInformer.Informer().HasSynced, queue: workqueue.NewTypedRateLimitingQueueWithConfig( workqueue.DefaultTypedControllerRateLimiter[*queueItem](), workqueue.TypedRateLimitingQueueConfig[*queueItem]{ Name: "provisioningRepositoryController", }, ), repoFactory: repoFactory, healthChecker: healthChecker, statusPatcher: statusPatcher, finalizer: &finalizer{ lister: resourceLister, clientFactory: clients, metrics: &finalizerMetrics, maxWorkers: parallelOperations, }, jobs: jobs, logger: logging.DefaultLogger.With("logger", loggerName), registry: registry, tracer: tracer, } _, err := repoInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: rc.enqueue, UpdateFunc: func(oldObj, newObj interface{}) { rc.enqueue(newObj) }, }) if err != nil { return nil, err } rc.processFn = rc.process rc.enqueueRepository = rc.enqueue rc.keyFunc = repoKeyFunc return rc, nil } func repoKeyFunc(obj any) (string, error) { repo, ok := obj.(*provisioning.Repository) if !ok { return "", fmt.Errorf("expected a Repository but got %T", obj) } return cache.DeletionHandlingMetaNamespaceKeyFunc(repo) } // Run starts the RepositoryController. // // Note: This function intentionally does NOT create a tracing span because it runs indefinitely // until shutdown. Individual processing operations already have their own spans. func (rc *RepositoryController) Run(ctx context.Context, workerCount int) { defer utilruntime.HandleCrash() defer rc.queue.ShutDown() logger := rc.logger ctx = logging.Context(ctx, logger) logger.Info("Starting RepositoryController") defer logger.Info("Shutting down RepositoryController") if !cache.WaitForCacheSync(ctx.Done(), rc.repoSynced) { return } logger.Info("Starting workers", "count", workerCount) for i := 0; i < workerCount; i++ { go wait.UntilWithContext(ctx, rc.runWorker, time.Second) } logger.Info("Started workers") <-ctx.Done() logger.Info("Shutting down workers") } func (rc *RepositoryController) runWorker(ctx context.Context) { for rc.processNextWorkItem(ctx) { } } func (rc *RepositoryController) enqueue(obj interface{}) { key, err := rc.keyFunc(obj) if err != nil { utilruntime.HandleError(fmt.Errorf("couldn't get key for object: %v", err)) return } item := queueItem{key: key, obj: obj} rc.queue.Add(&item) } // processNextWorkItem deals with one key off the queue. // It returns false when it's time to quit. func (rc *RepositoryController) processNextWorkItem(ctx context.Context) bool { item, quit := rc.queue.Get() if quit { return false } defer rc.queue.Done(item) // TODO: should we move tracking work to trace ids instead? logger := logging.FromContext(ctx).With("work_key", item.key) logger.Info("RepositoryController processing key") err := rc.processFn(item) if err == nil { rc.queue.Forget(item) return true } item.attempts++ logger = logger.With("error", err, "attempts", item.attempts) logger.Error("RepositoryController failed to process key") if item.attempts >= maxAttempts { logger.Error("RepositoryController failed too many times") rc.queue.Forget(item) return true } if !apierrors.IsServiceUnavailable(err) { logger.Info("RepositoryController will not retry") rc.queue.Forget(item) return true } else { logger.Info("RepositoryController will retry as service is unavailable") } utilruntime.HandleError(fmt.Errorf("%v failed with: %v", item, err)) rc.queue.AddRateLimited(item) return true } func (rc *RepositoryController) handleDelete(ctx context.Context, obj *provisioning.Repository) error { logger := logging.FromContext(ctx) logger.Info("handle repository delete") // Process any finalizers if len(obj.Finalizers) > 0 { repo, err := rc.repoFactory.Build(ctx, obj) if err != nil { return fmt.Errorf("create repository from configuration: %w", err) } err = rc.finalizer.process(ctx, repo, obj.Finalizers) if err != nil { if statusErr := rc.updateDeleteStatus(ctx, obj, fmt.Errorf("remove finalizers: %w", err)); statusErr != nil { logger.Error("failed to update repository status after finalizer removal error", "error", statusErr) } return fmt.Errorf("process finalizers: %w", err) } // remove the finalizers _, err = rc.client.Repositories(obj.GetNamespace()). Patch(ctx, obj.Name, types.JSONPatchType, []byte(`[ { "op": "remove", "path": "/metadata/finalizers" } ]`), v1.PatchOptions{ FieldManager: "provisioning-controller", }) if err != nil { return fmt.Errorf("remove finalizers: %w", err) } return nil } else { logger.Info("no finalizers to process") } return nil } func (rc *RepositoryController) updateDeleteStatus(ctx context.Context, obj *provisioning.Repository, err error) error { logger := logging.FromContext(ctx) logger.Info("updating repository status with deletion error", "error", err.Error()) return rc.statusPatcher.Patch(ctx, obj, map[string]interface{}{ "op": "replace", "path": "/status/deleteError", "value": err.Error(), }) } func (rc *RepositoryController) shouldResync(ctx context.Context, obj *provisioning.Repository) bool { // don't trigger resync if a sync was never started if obj.Status.Sync.Finished == 0 && obj.Status.Sync.State == "" { return false } syncAge := time.Since(time.UnixMilli(obj.Status.Sync.Finished)) syncInterval := time.Duration(obj.Spec.Sync.IntervalSeconds) * time.Second tolerance := time.Second // Check for stale sync status - if sync status indicates a job is running but the job no longer exists // Only check if Finished is set (meaning a sync has completed before) to avoid interfering with initial syncs // Only trigger resync if sync is enabled and sync interval has elapsed (to avoid unnecessary operations) if obj.Status.Sync.Finished > 0 && obj.Spec.Sync.Enabled && (obj.Status.Sync.State == provisioning.JobStatePending || obj.Status.Sync.State == provisioning.JobStateWorking) && obj.Status.Sync.JobID != "" { _, err := rc.jobs.Get(ctx, obj.Namespace, obj.Status.Sync.JobID) if apierrors.IsNotFound(err) { // Job was cleaned up but sync status wasn't updated - trigger resync to reconcile // Only trigger if sync interval has elapsed to avoid unnecessary operations if syncAge >= (syncInterval - tolerance) { logger := logging.FromContext(ctx) logger.Info("detected stale sync status", "job_id", obj.Status.Sync.JobID) return true } } // For other errors, log but continue with normal logic if err != nil { logger := logging.FromContext(ctx) logger.Warn("failed to check job existence for stale sync status", "error", err, "job_id", obj.Status.Sync.JobID) } } // HACK: how would this work in a multi-tenant world or under heavy load? // It will start queueing up jobs and we will have to deal with that pendingForTooLong := syncAge >= syncInterval/2 && obj.Status.Sync.State == provisioning.JobStatePending isRunning := obj.Status.Sync.State == provisioning.JobStateWorking return obj.Spec.Sync.Enabled && syncAge >= (syncInterval-tolerance) && !pendingForTooLong && !isRunning } func (rc *RepositoryController) runHooks(ctx context.Context, repo repository.Repository, obj *provisioning.Repository) ([]map[string]interface{}, error) { logger := logging.FromContext(ctx) hooks, _ := repo.(repository.Hooks) if hooks == nil { return nil, nil } if obj.Status.ObservedGeneration < 1 { logger.Info("handle repository create") patchOperations, err := hooks.OnCreate(ctx) if err != nil { return nil, fmt.Errorf("error running OnCreate: %w", err) } return patchOperations, nil } logger.Info("handle repository spec update", "Generation", obj.Generation, "ObservedGeneration", obj.Status.ObservedGeneration) patchOperations, err := hooks.OnUpdate(ctx) if err != nil { return nil, fmt.Errorf("error running OnUpdate: %w", err) } return patchOperations, nil } func (rc *RepositoryController) determineSyncStrategy(ctx context.Context, obj *provisioning.Repository, repo repository.Repository, shouldResync bool, healthStatus provisioning.HealthStatus) *provisioning.SyncJobOptions { logger := logging.FromContext(ctx) switch { case !obj.Spec.Sync.Enabled: logger.Info("skip sync as it's disabled") return nil case !healthStatus.Healthy: logger.Info("skip sync for unhealthy repository") return nil case healthStatus.Healthy != obj.Status.Health.Healthy: logger.Info("repository became healthy, full resync") return &provisioning.SyncJobOptions{} case obj.Status.ObservedGeneration < 1: logger.Info("full sync for new repository") return &provisioning.SyncJobOptions{} case obj.Generation != obj.Status.ObservedGeneration: logger.Info("full sync for spec change") return &provisioning.SyncJobOptions{} case shouldResync: // Continue to see if we could skip for other reasons versioned, ok := repo.(repository.Versioned) // If the repository is not versioned, we don't have a way to check for incremental updates if !ok { logger.Info("full sync on interval for non-versioned repository") return &provisioning.SyncJobOptions{} } latestRef, err := versioned.LatestRef(ctx) if err != nil { logger.Warn("incremental sync on interval without knowing if ref has actually changed", "error", err) return &provisioning.SyncJobOptions{Incremental: true} } // Only resync if the latest ref is different from the last synced ref if latestRef == obj.Status.Sync.LastRef { logger.Info("skip incremental sync as reference is the same") return nil } // Whenever possible, we try to keep it as an incremental sync to keep things performant. // However, if there are any .keep file deletions inside a folder with no other deletions, we need // to do a full sync to see if the folder was deleted as well in git. incremental, err := shouldUseIncrementalSync(ctx, versioned, obj, latestRef) if err != nil { logger.Warn("unable to compare files for incremental sync, doing full sync", "error", err) return &provisioning.SyncJobOptions{} } logger.Info("sync on interval", "incremental", incremental) return &provisioning.SyncJobOptions{Incremental: incremental} default: return nil } } func shouldUseIncrementalSync(ctx context.Context, versioned repository.Versioned, obj *provisioning.Repository, latestRef string) (bool, error) { changes, err := versioned.CompareFiles(ctx, obj.Status.Sync.LastRef, latestRef) if err != nil { return false, err } var deletedPaths []string for _, change := range changes { if change.Action == repository.FileActionDeleted { deletedPaths = append(deletedPaths, change.Path) } } return repository.CanUseIncrementalSync(deletedPaths), nil } func (rc *RepositoryController) addSyncJob(ctx context.Context, obj *provisioning.Repository, syncOptions *provisioning.SyncJobOptions) error { ctx, span := rc.tracer.Start(ctx, "provisioning.controller.add_sync_job") defer span.End() span.SetAttributes( attribute.String("repository", obj.GetName()), attribute.String("namespace", obj.Namespace), attribute.Bool("incremental", syncOptions != nil && syncOptions.Incremental), ) job, err := rc.jobs.Insert(ctx, obj.Namespace, provisioning.JobSpec{ Repository: obj.GetName(), Action: provisioning.JobActionPull, Pull: syncOptions, }) if apierrors.IsAlreadyExists(err) { logging.FromContext(ctx).Info("sync job already exists") return nil } if err != nil { span.RecordError(err) // FIXME: should we update the status of the repository if we fail to add the job? return fmt.Errorf("error adding sync job: %w", err) } span.SetAttributes(attribute.String("job.name", job.Name)) return nil } func (rc *RepositoryController) determineSyncStatusOps(obj *provisioning.Repository, syncOptions *provisioning.SyncJobOptions, healthStatus provisioning.HealthStatus) []map[string]interface{} { const unhealthyMessage = "Repository is unhealthy" hasUnhealthyMessage := len(obj.Status.Sync.Message) > 0 && obj.Status.Sync.Message[0] == unhealthyMessage var patchOperations []map[string]interface{} switch { case syncOptions != nil: // We will try to trigger a new sync job if we have sync options patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/sync/state", "value": provisioning.JobStatePending, }) patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/sync/started", "value": int64(0), }) case healthStatus.Healthy && hasUnhealthyMessage: // if the repository is healthy and the message is set, clear it // FIXME: is this the clearest way to do this? Should we introduce another status or way of way of handling more // specific errors? patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/sync/message", "value": []string{}, }) case !healthStatus.Healthy && !hasUnhealthyMessage: // if the repository is unhealthy and the message is not already set, set it patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/sync/state", "value": provisioning.JobStateError, }) patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/sync/message", "value": []string{unhealthyMessage}, }) } return patchOperations } //nolint:gocyclo func (rc *RepositoryController) process(item *queueItem) error { logger := rc.logger.With("key", item.key) ctx := logging.Context(context.Background(), logger) namespace, name, err := cache.SplitMetaNamespaceKey(item.key) if err != nil { return err } obj, err := rc.repoLister.Repositories(namespace).Get(name) switch { case apierrors.IsNotFound(err): return errors.New("repository not found in cache") case err != nil: return err } ctx, _, err = identity.WithProvisioningIdentity(ctx, namespace) if err != nil { return err } ctx = request.WithNamespace(ctx, namespace) logger = logger.WithContext(ctx) if obj.DeletionTimestamp != nil { return rc.handleDelete(ctx, obj) } shouldResync := rc.shouldResync(ctx, obj) shouldCheckHealth := rc.healthChecker.ShouldCheckHealth(obj) hasSpecChanged := obj.Generation != obj.Status.ObservedGeneration patchOperations := []map[string]interface{}{} // Determine the main triggering condition switch { case hasSpecChanged: logger.Info("spec changed", "Generation", obj.Generation, "ObservedGeneration", obj.Status.ObservedGeneration) patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", "path": "/status/observedGeneration", "value": obj.Generation, }) case shouldResync: logger.Info("sync interval triggered", "sync_interval", time.Duration(obj.Spec.Sync.IntervalSeconds)*time.Second, "sync_status", obj.Status.Sync) case shouldCheckHealth: logger.Info("health is stale", "health_status", obj.Status.Health.Healthy) default: logger.Info("skipping as conditions are not met", "status", obj.Status, "generation", obj.Generation, "sync_spec", obj.Spec.Sync) return nil } repo, err := rc.repoFactory.Build(ctx, obj) if err != nil { return fmt.Errorf("unable to create repository from configuration: %w", err) } // Handle hooks - may return early if hooks fail hookOps, shouldContinue, err := rc.processHooks(ctx, repo, obj) if err != nil { return fmt.Errorf("process hooks: %w", err) } if !shouldContinue { return nil // Hook handling already updated status and returned early } if len(hookOps) > 0 { patchOperations = append(patchOperations, hookOps...) } // Handle health checks using the health checker _, healthStatus, healthPatchOps, err := rc.healthChecker.RefreshHealthWithPatchOps(ctx, repo) if err != nil { return fmt.Errorf("update health status: %w", err) } // Add health patch operations first if len(healthPatchOps) > 0 { patchOperations = append(patchOperations, healthPatchOps...) } // determine the sync strategy and sync status to apply syncOptions := rc.determineSyncStrategy(ctx, obj, repo, shouldResync, healthStatus) patchOperations = append(patchOperations, rc.determineSyncStatusOps(obj, syncOptions, healthStatus)...) // Apply all patch operations if len(patchOperations) > 0 { err := rc.statusPatcher.Patch(ctx, obj, patchOperations...) if err != nil { return fmt.Errorf("status patch operations failed: %w", err) } } // QUESTION: should we trigger the sync job after we have applied all patch operations or before? // Is there are risk of race condition here? // Trigger sync job after we have applied all patch operations if syncOptions != nil { if err := rc.addSyncJob(ctx, obj, syncOptions); err != nil { return err } } return nil } // processHooks handles hook execution with intelligent retry logic // Returns hook operations, whether processing should continue, and any error func (rc *RepositoryController) processHooks(ctx context.Context, repo repository.Repository, obj *provisioning.Repository) ([]map[string]interface{}, bool, error) { shouldRunHooks := obj.Generation != obj.Status.ObservedGeneration // Skip hooks if status already indicates recent hook failure to avoid infinite retry if shouldRunHooks && rc.healthChecker.HasRecentFailure(obj.Status.Health, provisioning.HealthFailureHook) { shouldRunHooks = false } if !shouldRunHooks { return nil, true, nil } hookOps, err := rc.runHooks(ctx, repo, obj) if err != nil { if err := rc.healthChecker.RecordFailure(ctx, provisioning.HealthFailureHook, err, obj); err != nil { return nil, false, fmt.Errorf("update status after hook failure: %w", err) } return nil, false, err } return hookOps, true, nil }