Provisioning: Add Standalone Job Controller Without Job Processing (#109610)

* Add standalone job controller * Add makefile * Add limit on the current implementation * Move job controllers to app package * Add TLS flags
2025-08-25 08:48:40 +00:00
parent 9204a08207
commit e7ccefcf92
24 changed files with 665 additions and 25 deletions
@@ -0,0 +1,28 @@
+.PHONY: build clean test
+BINARY_NAME=job-controller
+BUILD_DIR=bin
+LDFLAGS=-w -s
+
+build:
+	@echo "Building $(BINARY_NAME)..."
+	@mkdir -p $(BUILD_DIR)
+	go build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(BINARY_NAME) .
+
+clean:
+	@echo "Cleaning..."
+	@rm -rf $(BUILD_DIR)
+run:
+	@echo "Running $(BINARY_NAME)..."
+	./$(BUILD_DIR)/$(BINARY_NAME)
+
+install:
+	@echo "Installing $(BINARY_NAME)..."
+	go install .
+
+help:
+	@echo "Available targets:"
+	@echo "  build   - Build the binary"
+	@echo "  clean   - Clean build artifacts"
+	@echo "  run     - Run the binary"
+	@echo "  install - Install the binary"
+	@echo "  help    - Show this help"
@@ -0,0 +1,138 @@
+# Jobs Controller
+
+> [!WARNING]
+> This controller has current limitations:
+>
+> - This binary does not start the ConcurrentJobDriver yet. Notifications are logged but not consumed by workers here.
+> - Job processing (claim/renew/update/complete) isn't implemented yet as it requires refactoring of some components.
+
+### Behavior
+
+- Watches provisioning `Jobs` and emits notifications on job creation.
+- Optionally cleans up `HistoricJobs` after a configurable expiration. Disable when job history is stored in Loki.
+
+- Queueing and claiming:
+  - Creating a `Job` enqueues work. Drivers “claim” one job at a time under a time-bound lease so only one worker processes it at once.
+  - If a driver crashes or loses its lease, cleanup makes the job eligible to be claimed again. This yields at-least-once processing.
+  - New job notifications reduce latency; periodic ticks ensure progress even without notifications.
+
+- Processing and status:
+  - A supporting worker processes the job, renewing the lease in the background. If lease renewal fails or expires, processing aborts.
+  - Status updates are persisted with conflict-aware retries. Progress is throttled to avoid excessive writes while still providing timely feedback.
+  - When processing finishes, the job is marked complete and a copy is written to history.
+
+- Historic jobs role:
+  - Historic jobs are a read-only audit trail and UX surface for recent job outcomes, progress summaries, errors, and reference URLs.
+- Retention is implementation-dependent: this controller can prune old history objects periodically, or history can be stored in Loki; when using Loki, disable local cleanup with `--history-expiration=0`.
+
+This binary currently wires informers and emits job-create notifications. In the full setup, concurrent drivers consume notifications and execute workers to process jobs using the behavior above.
+
+### Flags
+
+- `--token` (string): Token to use for authentication against the provisioning API.
+- `--token-exchange-url` (string): Token exchange endpoint used to mint the access token for the provisioning API.
+- `--provisioning-server-url` (string): Base URL to the provisioning API server (e.g., `https://localhost:6446`).
+- `--history-expiration` (duration): If greater than zero, enables HistoricJobs cleanup and sets the retention window (e.g., `30s`, `15m`, `24h`). If `0`, cleanup is disabled.
+
+#### TLS Configuration
+
+- `--tls-insecure` (bool): Skip TLS certificate verification. Default: `true` (for development/testing).
+- `--tls-cert-file` (string): Path to TLS client certificate file for mutual TLS authentication.
+- `--tls-key-file` (string): Path to TLS client private key file for mutual TLS authentication.
+- `--tls-ca-file` (string): Path to TLS CA certificate file for server certificate verification.
+
+### How to run
+
+1. Build from this folder:
+   - `make build`
+2. Ensure the following services are running locally: provisioning API server, secrets service API server, repository controller, unified storage, and auth.
+3. Start the controller:
+   - Using Loki for job history:
+     - Ensure the Provisioning API is configured with Loki for job history (see `createJobHistoryConfigFromSettings` in `pkg/registry/apis/provisioning/register.go`).
+     - Run without history cleanup:
+       - `./bin/job-controller --token-exchange-url=http://localhost:6481/sign/access-token --token=ProvisioningAdminToken --provisioning-server-url=https://localhost:6446`
+   - Without Loki (local/dev or when Loki is unavailable):
+     - Run without cleanup:
+       - `./bin/job-controller --token-exchange-url=http://localhost:6481/sign/access-token --token=ProvisioningAdminToken --provisioning-server-url=https://localhost:6446`
+     - Or enable local HistoricJobs cleanup with a retention window:
+       - `./bin/job-controller --token-exchange-url=http://localhost:6481/sign/access-token --token=ProvisioningAdminToken --provisioning-server-url=https://localhost:6446 --history-expiration=30s`
+
+#### TLS Configuration Examples
+
+- **Production with proper TLS verification**:
+
+  ```bash
+  ./bin/job-controller \
+    --token-exchange-url=http://localhost:6481/sign/access-token \
+    --token=ProvisioningAdminToken \
+    --provisioning-server-url=https://provisioning.example.com:6446 \
+    --tls-insecure=false \
+    --tls-ca-file=/path/to/ca-cert.pem
+  ```
+
+- **Mutual TLS authentication**:
+
+  ```bash
+  ./bin/job-controller \
+    --token-exchange-url=http://localhost:6481/sign/access-token \
+    --token=ProvisioningAdminToken \
+    --provisioning-server-url=https://provisioning.example.com:6446 \
+    --tls-insecure=false \
+    --tls-ca-file=/path/to/ca-cert.pem \
+    --tls-cert-file=/path/to/client-cert.pem \
+    --tls-key-file=/path/to/client-key.pem
+  ```
+
+- **Development with self-signed certificates (insecure)**:
+
+  ```bash
+  ./bin/job-controller \
+    --token-exchange-url=http://localhost:6481/sign/access-token \
+    --token=ProvisioningAdminToken \
+    --provisioning-server-url=https://localhost:6446 \
+    --tls-insecure=true
+  ```
+
+### Expected behavior
+
+1. Create a repository and enqueue a job (note that the repository must be marked as healthy):
+
+```curl
+
+export ACCESS_TOKEN=$(curl -X POST http://localhost:6481/sign/access-token \
+  -H "X-Realms: [{\"type\":\"system\",\"identifier\":\"system\"}]" \
+  -H "X-Org-ID: 0" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer ProvisioningAdminToken" \
+  -d '{
+    "namespace": "*",
+    "audiences": ["provisioning.grafana.app"]
+  }' | jq -r '.data.token')
+```
+
+```curl
+
+curl -X POST https://localhost:6446/apis/provisioning.grafana.app/v0alpha1/namespaces/default/repositories/test6/jobs \
+  -H "Content-Type: application/json" --insecure \
+  -H "X-Access-Token: Bearer $ACCESS_TOKEN" \
+  -d '{
+    "action": "pull",
+    "pull": {
+      "incremental": false
+    }
+  }'
+```
+
+2. The controller emits a notification on job creation.
+
+```
+➜  job-controller git:(feature/standalone-job-controller) ✗ ./bin/job-controller --token-exchange-url=http://localhost:6481/sign/access-token --token=ProvisioningAdminToken --provisioning-server-url=https://localhost:6446
+{"time":"2025-08-21T14:27:03.789337+02:00","level":"INFO","msg":"job create notification received","logger":"provisioning-job-controller"}
+```
+
+```
+
+```
+
+3. In a full setup with the concurrent driver, workers claim and process jobs, updating status and writing history.
+4. Entries move to `HistoricJobs`; if cleanup is enabled, older entries are pruned based on `--history-expiration`.
@@ -0,0 +1,234 @@
+package main
+
+import (
+	"context"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/grafana/authlib/authn"
+	"github.com/grafana/grafana-app-sdk/logging"
+	"github.com/urfave/cli/v2"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/transport"
+
+	authrt "github.com/grafana/grafana/apps/provisioning/pkg/auth"
+	"github.com/grafana/grafana/apps/provisioning/pkg/controller"
+	client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned"
+	informer "github.com/grafana/grafana/apps/provisioning/pkg/generated/informers/externalversions"
+)
+
+var (
+	token                 = flag.String("token", "", "Token to use for authentication")
+	tokenExchangeURL      = flag.String("token-exchange-url", "", "Token exchange URL")
+	provisioningServerURL = flag.String("provisioning-server-url", "", "Provisioning server URL")
+	tlsInsecure           = flag.Bool("tls-insecure", true, "Skip TLS certificate verification")
+	tlsCertFile           = flag.String("tls-cert-file", "", "Path to TLS certificate file")
+	tlsKeyFile            = flag.String("tls-key-file", "", "Path to TLS private key file")
+	tlsCAFile             = flag.String("tls-ca-file", "", "Path to TLS CA certificate file")
+)
+
+func main() {
+	app := &cli.App{
+		Name:  "job-controller",
+		Usage: "Watch provisioning jobs and manage job history cleanup",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "token",
+				Usage:       "Token to use for authentication",
+				Value:       "",
+				Destination: token,
+			},
+			&cli.StringFlag{
+				Name:        "token-exchange-url",
+				Usage:       "Token exchange URL",
+				Value:       "",
+				Destination: tokenExchangeURL,
+			},
+			&cli.StringFlag{
+				Name:        "provisioning-server-url",
+				Usage:       "Provisioning server URL",
+				Value:       "",
+				Destination: provisioningServerURL,
+			},
+			&cli.BoolFlag{
+				Name:        "tls-insecure",
+				Usage:       "Skip TLS certificate verification",
+				Value:       true,
+				Destination: tlsInsecure,
+			},
+			&cli.StringFlag{
+				Name:        "tls-cert-file",
+				Usage:       "Path to TLS certificate file",
+				Value:       "",
+				Destination: tlsCertFile,
+			},
+			&cli.StringFlag{
+				Name:        "tls-key-file",
+				Usage:       "Path to TLS private key file",
+				Value:       "",
+				Destination: tlsKeyFile,
+			},
+			&cli.StringFlag{
+				Name:        "tls-ca-file",
+				Usage:       "Path to TLS CA certificate file",
+				Value:       "",
+				Destination: tlsCAFile,
+			},
+			&cli.DurationFlag{
+				Name:  "history-expiration",
+				Usage: "Duration after which HistoricJobs are deleted; 0 disables cleanup. When the Provisioning API is configured to use Loki for job history, leave this at 0.",
+				Value: 0,
+			},
+		},
+		Action: runJobController,
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func runJobController(c *cli.Context) error {
+	// TODO: Wire notifications into a ConcurrentJobDriver when a client-backed Store and Workers are available.
+	// For now, just log notifications to verify events end-to-end.
+	logger := logging.NewSLogLogger(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	})).With("logger", "provisioning-job-controller")
+	logger.Info("Starting provisioning job controller")
+
+	tokenExchangeClient, err := authn.NewTokenExchangeClient(authn.TokenExchangeConfig{
+		TokenExchangeURL: *tokenExchangeURL,
+		Token:            *token,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create token exchange client: %w", err)
+	}
+
+	tlsConfig, err := buildTLSConfig()
+	if err != nil {
+		return fmt.Errorf("failed to build TLS configuration: %w", err)
+	}
+
+	config := &rest.Config{
+		APIPath: "/apis",
+		Host:    *provisioningServerURL,
+		WrapTransport: transport.WrapperFunc(func(rt http.RoundTripper) http.RoundTripper {
+			return authrt.NewRoundTripper(tokenExchangeClient, rt)
+		}),
+		TLSClientConfig: tlsConfig,
+	}
+
+	provisioningClient, err := client.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to create provisioning client: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		fmt.Println("Received shutdown signal, stopping controllers")
+		cancel()
+	}()
+
+	// Jobs informer and controller (resync ~60s like in register.go)
+	jobInformerFactory := informer.NewSharedInformerFactoryWithOptions(
+		provisioningClient,
+		60*time.Second,
+	)
+	jobInformer := jobInformerFactory.Provisioning().V0alpha1().Jobs()
+	jobController, err := controller.NewJobController(jobInformer)
+	if err != nil {
+		return fmt.Errorf("failed to create job controller: %w", err)
+	}
+
+	logger.Info("jobs controller started")
+	notifications := jobController.InsertNotifications()
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-notifications:
+				logger.Info("job create notification received")
+			}
+		}
+	}()
+
+	// Optionally enable history cleanup if a positive expiration is provided
+	historyExpiration := c.Duration("history-expiration")
+	var startHistoryInformers func()
+	if historyExpiration > 0 {
+		// History jobs informer and controller (separate factory with resync == expiration)
+		historyInformerFactory := informer.NewSharedInformerFactoryWithOptions(
+			provisioningClient,
+			historyExpiration,
+		)
+		historyJobInformer := historyInformerFactory.Provisioning().V0alpha1().HistoricJobs()
+		_, err = controller.NewHistoryJobController(
+			provisioningClient.ProvisioningV0alpha1(),
+			historyJobInformer,
+			historyExpiration,
+		)
+		if err != nil {
+			return fmt.Errorf("failed to create history job controller: %w", err)
+		}
+		logger.Info("history cleanup enabled", "expiration", historyExpiration.String())
+		startHistoryInformers = func() { historyInformerFactory.Start(ctx.Done()) }
+	} else {
+		startHistoryInformers = func() {}
+	}
+
+	// Start informers
+	go jobInformerFactory.Start(ctx.Done())
+	go startHistoryInformers()
+
+	// Optionally wait for job cache sync; history cleanup can rely on resync events
+	if !cache.WaitForCacheSync(ctx.Done(), jobInformer.Informer().HasSynced) {
+		return fmt.Errorf("failed to sync job informer cache")
+	}
+
+	<-ctx.Done()
+	return nil
+}
+
+func buildTLSConfig() (rest.TLSClientConfig, error) {
+	tlsConfig := rest.TLSClientConfig{
+		Insecure: *tlsInsecure,
+	}
+
+	// If client certificate and key are provided
+	if *tlsCertFile != "" && *tlsKeyFile != "" {
+		tlsConfig.CertFile = *tlsCertFile
+		tlsConfig.KeyFile = *tlsKeyFile
+	}
+
+	// If CA certificate is provided
+	if *tlsCAFile != "" {
+		caCert, err := os.ReadFile(*tlsCAFile)
+		if err != nil {
+			return tlsConfig, fmt.Errorf("failed to read CA certificate file: %w", err)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(caCert) {
+			return tlsConfig, fmt.Errorf("failed to parse CA certificate")
+		}
+
+		tlsConfig.CAData = caCert
+	}
+
+	return tlsConfig, nil
+}