Provisioning: Add connection operator with health check updates (#116028)

* Add connection operator with health check updates

- Add ConnectionController to watch and reconcile Connection resources
- Add ConnectionStatusPatcher for updating connection status
- Add connection_operator.go entry point for standalone operator
- Register connection operator in pkg/operators/register.go
- Add connection controller to in-process setup in register.go
- Add unit tests for connection controller
- Add integration tests for health check updates

* Fix integration test: get latest version before update to avoid conflicts

* refactor: move repoFactory to operator-specific configs

- Remove repoFactory from shared provisioningControllerConfig
- Add repoFactory to repoControllerConfig and jobsControllerConfig
- This allows connection operator to run without repository setup

* Remove unneccesary comments

pkg/operators/provisioning/config.go

@@ -36,7 +36,6 @@ import (
 type provisioningControllerConfig struct {
 	provisioningClient  *client.Clientset
 	resyncInterval      time.Duration
-	repoFactory         repository.Factory
 	unified             resources.ResourceStore
 	clients             resources.ClientFactory
 	tokenExchangeClient *authn.TokenExchangeClient
@@ -129,16 +128,6 @@ func setupFromConfig(cfg *setting.Cfg, registry prometheus.Registerer) (controll
 		return nil, fmt.Errorf("failed to create provisioning client: %w", err)
 	}
 
-	decrypter, err := setupDecrypter(cfg, tracer, tokenExchangeClient)
-	if err != nil {
-		return nil, fmt.Errorf("failed to setup decrypter: %w", err)
-	}
-
-	repoFactory, err := setupRepoFactory(cfg, decrypter, provisioningClient, registry)
-	if err != nil {
-		return nil, fmt.Errorf("failed to setup repository getter: %w", err)
-	}
-
 	// HACK: This logic directly connects to unified storage. We are doing this for now as there is no global
 	// search endpoint. But controllers, in general, should not connect directly to unified storage and instead
 	// go through the api server. Once there is a global search endpoint, we will switch to that here as well.
@@ -195,7 +184,6 @@ func setupFromConfig(cfg *setting.Cfg, registry prometheus.Registerer) (controll
 
 	return &provisioningControllerConfig{
 		provisioningClient:  provisioningClient,
-		repoFactory:         repoFactory,
 		unified:             unified,
 		clients:             clients,
 		resyncInterval:      operatorSec.Key("resync_interval").MustDuration(60 * time.Second),

pkg/operators/provisioning/connection_operator.go

@@ -0,0 +1,86 @@
+package provisioning
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/grafana/grafana-app-sdk/logging"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/client-go/tools/cache"
+
+	appcontroller "github.com/grafana/grafana/apps/provisioning/pkg/controller"
+	informer "github.com/grafana/grafana/apps/provisioning/pkg/generated/informers/externalversions"
+	"github.com/grafana/grafana/pkg/registry/apis/provisioning/controller"
+	"github.com/grafana/grafana/pkg/server"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+// RunConnectionController starts the connection controller operator.
+func RunConnectionController(deps server.OperatorDependencies) error {
+	logger := logging.NewSLogLogger(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	})).With("logger", "provisioning-connection-controller")
+	logger.Info("Starting provisioning connection controller")
+
+	controllerCfg, err := getConnectionControllerConfig(deps.Config, deps.Registerer)
+	if err != nil {
+		return fmt.Errorf("failed to setup operator: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		fmt.Println("Received shutdown signal, stopping controllers")
+		cancel()
+	}()
+
+	informerFactory := informer.NewSharedInformerFactoryWithOptions(
+		controllerCfg.provisioningClient,
+		controllerCfg.resyncInterval,
+	)
+
+	statusPatcher := appcontroller.NewConnectionStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1())
+	connInformer := informerFactory.Provisioning().V0alpha1().Connections()
+
+	connController, err := controller.NewConnectionController(
+		controllerCfg.provisioningClient.ProvisioningV0alpha1(),
+		connInformer,
+		statusPatcher,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create connection controller: %w", err)
+	}
+
+	informerFactory.Start(ctx.Done())
+	if !cache.WaitForCacheSync(ctx.Done(), connInformer.Informer().HasSynced) {
+		return fmt.Errorf("failed to sync informer cache")
+	}
+
+	connController.Run(ctx, controllerCfg.workerCount)
+	return nil
+}
+
+type connectionControllerConfig struct {
+	provisioningControllerConfig
+	workerCount int
+}
+
+func getConnectionControllerConfig(cfg *setting.Cfg, registry prometheus.Registerer) (*connectionControllerConfig, error) {
+	controllerCfg, err := setupFromConfig(cfg, registry)
+	if err != nil {
+		return nil, err
+	}
+
+	return &connectionControllerConfig{
+		provisioningControllerConfig: *controllerCfg,
+		workerCount:                  cfg.SectionWithEnvOverrides("operator").Key("worker_count").MustInt(1),
+	}, nil
+}

pkg/operators/provisioning/repo_operator.go

@@ -106,6 +106,7 @@ func RunRepoController(deps server.OperatorDependencies) error {
 
 type repoControllerConfig struct {
 	provisioningControllerConfig
+	repoFactory         repository.Factory
 	workerCount         int
 	parallelOperations  int
 	allowedTargets      []string
@@ -119,6 +120,17 @@ func getRepoControllerConfig(cfg *setting.Cfg, registry prometheus.Registerer) (
 		return nil, err
 	}
 
+	// Setup repository factory for repo controller
+	decrypter, err := setupDecrypter(cfg, tracing.NewNoopTracerService(), controllerCfg.tokenExchangeClient)
+	if err != nil {
+		return nil, fmt.Errorf("failed to setup decrypter: %w", err)
+	}
+
+	repoFactory, err := setupRepoFactory(cfg, decrypter, controllerCfg.provisioningClient, registry)
+	if err != nil {
+		return nil, fmt.Errorf("failed to setup repository factory: %w", err)
+	}
+
 	allowedTargets := []string{}
 	cfg.SectionWithEnvOverrides("provisioning").Key("allowed_targets").Strings("|")
 	if len(allowedTargets) == 0 {
@@ -127,6 +139,7 @@ func getRepoControllerConfig(cfg *setting.Cfg, registry prometheus.Registerer) (
 
 	return &repoControllerConfig{
 		provisioningControllerConfig: *controllerCfg,
+		repoFactory:                  repoFactory,
 		allowedTargets:               allowedTargets,
 		workerCount:                  cfg.SectionWithEnvOverrides("operator").Key("worker_count").MustInt(1),
 		parallelOperations:           cfg.SectionWithEnvOverrides("operator").Key("parallel_operations").MustInt(10),

pkg/operators/register.go

@@ -13,6 +13,12 @@ func init() {
 		RunFunc:     provisioning.RunRepoController,
 	})
 
+	server.RegisterOperator(server.Operator{
+		Name:        "provisioning-connection",
+		Description: "Watch provisioning connections",
+		RunFunc:     provisioning.RunConnectionController,
+	})
+
 	server.RegisterOperator(server.Operator{
 		Name:        "iam-folder-reconciler",
 		Description: "Reconcile folder resources into Zanzana",