[Provisioning] Fix on config update and regular health check updates (#100181)

Fix health check updates

pkg/registry/apis/provisioning/repository_controller.go

@@ -256,14 +256,7 @@ func (rc *RepositoryController) process(item *queueItem) error {
 	obj = obj.DeepCopy()
 	hooks, _ := repo.(repository.RepositoryHooks)
 
-	var (
-		sync          *provisioning.SyncJobOptions
-		webhookStatus *provisioning.WebhookStatus
-	)
-
-	switch {
-	// Delete (note this switch does not fallthrough to the health check)
-	case obj.DeletionTimestamp != nil:
+	if obj.DeletionTimestamp != nil {
 		logger.Info("handle repository delete")
 
 		// Process any finalizers
@@ -281,9 +274,71 @@ func (rc *RepositoryController) process(item *queueItem) error {
 					FieldManager: "repository-controller",
 				})
 		}
-		return err // delete will be called again
 
-	// Create
+		return err // delete will be called again
+	}
+
+	// Test configuration before anything else
+	health := obj.Status.Health
+	if obj.DeletionTimestamp == nil && healthAge > 200*time.Millisecond {
+		logger.Info("running health check")
+		res, err := rc.tester.TestRepository(ctx, repo)
+		if err != nil {
+			res = &provisioning.TestResults{
+				Success: false,
+				Errors: []string{
+					"error running test repository",
+					err.Error(),
+				},
+			}
+		}
+
+		health = provisioning.HealthStatus{
+			Healthy: res.Success,
+			Checked: time.Now().UnixMilli(),
+			Message: res.Errors,
+		}
+		patch, err := json.Marshal(map[string]any{
+			"status": map[string]any{
+				"health": health,
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("error encoding health patch: %w", err)
+		}
+
+		if _, err := rc.client.Repositories(obj.GetNamespace()).
+			Patch(ctx, obj.GetName(), types.MergePatchType, patch, v1.PatchOptions{}, "status"); err != nil {
+			return fmt.Errorf("update health status: %w", err)
+		}
+	}
+
+	// Do not continue if provided invalid configuration because sync and hooks won't probably work
+	if !health.Healthy {
+		logger.Error("repository is unhealthy", "errors", health.Message)
+		patch, err := json.Marshal(map[string]any{
+			"status": map[string]any{
+				"observedGeneration": obj.Generation,
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("error encoding health patch: %w", err)
+		}
+
+		if _, err := rc.client.Repositories(obj.GetNamespace()).
+			Patch(ctx, obj.GetName(), types.MergePatchType, patch, v1.PatchOptions{}, "status"); err != nil {
+			return fmt.Errorf("update health status: %w", err)
+		}
+
+		return nil
+	}
+
+	var (
+		sync          *provisioning.SyncJobOptions
+		webhookStatus *provisioning.WebhookStatus
+	)
+
+	switch {
 	case obj.Status.ObservedGeneration < 1:
 		logger.Info("handle repository create")
 		if hooks != nil {
@@ -306,52 +361,7 @@ func (rc *RepositoryController) process(item *queueItem) error {
 		return fmt.Errorf("error running hooks: %w", err)
 	}
 
-	health := obj.Status.Health
-
-	if obj.DeletionTimestamp == nil && healthAge > 200*time.Millisecond {
-		logger.Info("running health check")
-		res, err := rc.tester.TestRepository(ctx, repo)
-		if err != nil {
-			res = &provisioning.TestResults{
-				Success: false,
-				Errors: []string{
-					"error running test repository",
-					err.Error(),
-				},
-			}
-		}
-
-		health = provisioning.HealthStatus{
-			Healthy: res.Success,
-			Checked: time.Now().UnixMilli(),
-			Message: res.Errors,
-		}
-		if !res.Success {
-			logger.Error("repository is unhealthy", "errors", res.Errors)
-		} else {
-			logger.Info("repository is healthy")
-		}
-
-		patch, err := json.Marshal(map[string]any{
-			"status": map[string]any{
-				"health": health,
-			},
-		})
-		if err != nil {
-			return fmt.Errorf("error encoding health patch: %w", err)
-		}
-
-		if _, err := rc.client.Repositories(obj.GetNamespace()).
-			Patch(ctx, obj.GetName(), types.MergePatchType, patch, v1.PatchOptions{}, "status"); err != nil {
-			return fmt.Errorf("update health status: %w", err)
-		}
-	}
-
-	// Maybe add a new sync job
-	if health.Healthy &&
-		obj.Spec.Sync.Enabled && sync != nil &&
-		obj.Generation > 0 &&
-		!isSyncJobPendingOrRunning {
+	if obj.Spec.Sync.Enabled && sync != nil && obj.Generation > 0 && !isSyncJobPendingOrRunning {
 		job, err := rc.jobs.Add(ctx, &provisioning.Job{
 			ObjectMeta: v1.ObjectMeta{
 				Namespace: obj.Namespace,