Provisioning: Add metrics for repo controller (#111450)

This commit is contained in:
Stephanie Hingtgen
2025-09-22 20:14:03 +00:00
committed by GitHub
parent a2d09490ae
commit 8b1caccc72
17 changed files with 187 additions and 66 deletions
+1 -1
View File
@@ -11,7 +11,6 @@ require (
github.com/grafana/grafana/pkg/apimachinery v0.0.0-20250804150913-990f1c69ecc2
github.com/grafana/nanogit v0.0.0-20250723104447-68f58f5ecec0
github.com/migueleliasweb/go-github-mock v1.1.0
github.com/prometheus/client_golang v1.23.2
github.com/stretchr/testify v1.11.1
golang.org/x/oauth2 v0.30.0
k8s.io/apimachinery v0.34.1
@@ -54,6 +53,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
+3 -6
View File
@@ -7,20 +7,17 @@ import (
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1"
"github.com/prometheus/client_golang/prometheus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
type RepositoryStatusPatcher struct {
client client.ProvisioningV0alpha1Interface
registry prometheus.Registerer
client client.ProvisioningV0alpha1Interface
}
func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface, registry prometheus.Registerer) *RepositoryStatusPatcher {
func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *RepositoryStatusPatcher {
return &RepositoryStatusPatcher{
client: client,
registry: registry,
client: client,
}
}
@@ -8,7 +8,6 @@ import (
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1/fake"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -17,7 +16,7 @@ import (
func TestNewRepositoryStatusPatcher(t *testing.T) {
client := &fake.FakeProvisioningV0alpha1{}
patcher := NewRepositoryStatusPatcher(client, prometheus.DefaultRegisterer)
patcher := NewRepositoryStatusPatcher(client)
require.NotNil(t, patcher)
require.Equal(t, client, patcher.client)
}
@@ -104,7 +103,7 @@ func TestRepositoryStatusPatcher_Patch(t *testing.T) {
client.AddReactor("patch", "repositories", tt.reactorFunc)
}
patcher := NewRepositoryStatusPatcher(&client, prometheus.DefaultRegisterer)
patcher := NewRepositoryStatusPatcher(&client)
err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations...)
if tt.expectedError != "" {
+1 -1
View File
@@ -180,7 +180,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis
parsers := resources.NewParserFactory(clients)
resourceLister := resources.NewResourceLister(controllerCfg.unified)
repositoryResources := resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister)
statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), registry)
statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1())
workers := make([]jobs.Worker, 0)
+1 -1
View File
@@ -55,7 +55,7 @@ func RunRepoController(deps server.OperatorDependencies) error {
if err != nil {
return fmt.Errorf("create API client job store: %w", err)
}
statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), deps.Registerer)
statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1())
healthChecker := controller.NewHealthChecker(statusPatcher, deps.Registerer)
repoInformer := informerFactory.Provisioning().V0alpha1().Repositories()
@@ -5,6 +5,7 @@ import (
"encoding/json"
"sort"
"strings"
"time"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -17,11 +18,13 @@ import (
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
metricutils "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
)
type finalizer struct {
lister resources.ResourceLister
clientFactory resources.ClientFactory
metrics *finalizerMetrics
}
func (f *finalizer) process(ctx context.Context,
@@ -31,18 +34,24 @@ func (f *finalizer) process(ctx context.Context,
logger := logging.FromContext(ctx)
for _, finalizer := range finalizers {
var err error
var count int
start := time.Now()
outcome := metricutils.SuccessOutcome
switch finalizer {
case repository.CleanFinalizer:
// NOTE: the controller loop will never get run unless a finalizer is set
hooks, ok := repo.(repository.Hooks)
if ok {
if err := hooks.OnDelete(ctx); err != nil {
if err = hooks.OnDelete(ctx); err != nil {
logger.Warn("Error running deletion hooks", "err", err)
outcome = metricutils.ErrorOutcome
}
}
case repository.ReleaseOrphanResourcesFinalizer:
err := f.processExistingItems(ctx, repo.Config(),
count, err = f.processExistingItems(ctx, repo.Config(),
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
patchAnnotations, err := getPatchedAnnotations(item)
if err != nil {
@@ -55,20 +64,29 @@ func (f *finalizer) process(ctx context.Context,
return err
})
if err != nil {
return err
outcome = metricutils.ErrorOutcome
logger.Warn("Error processing release orphan resources finalizer", "err", err)
}
case repository.RemoveOrphanResourcesFinalizer:
err := f.processExistingItems(ctx, repo.Config(),
count, err = f.processExistingItems(ctx, repo.Config(),
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
return client.Delete(ctx, item.Name, v1.DeleteOptions{})
})
if err != nil {
return err
outcome = metricutils.ErrorOutcome
logger.Warn("Error processing remove orphan resources finalizer", "err", err)
}
default:
logger.Warn("skipping unknown finalizer", "finalizer", finalizer)
continue
}
f.metrics.RecordFinalizer(finalizer, outcome, count, time.Since(start).Seconds())
if err != nil {
return err
}
}
return nil
@@ -79,17 +97,17 @@ func (f *finalizer) processExistingItems(
ctx context.Context,
repo *provisioning.Repository,
cb func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error,
) error {
) (int, error) {
logger := logging.FromContext(ctx)
clients, err := f.clientFactory.Clients(ctx, repo.Namespace)
if err != nil {
return err
return 0, err
}
items, err := f.lister.List(ctx, repo.Namespace, repo.Name)
if err != nil {
logger.Warn("error listing resources", "error", err)
return err
return 0, err
}
// Safe deletion order
@@ -103,7 +121,7 @@ func (f *finalizer) processExistingItems(
Resource: item.Resource,
})
if err != nil {
return err
return count, err
}
err = cb(res, &item)
@@ -115,7 +133,7 @@ func (f *finalizer) processExistingItems(
}
}
logger.Info("processed orphan items", "items", count, "errors", errors)
return nil
return count, nil
}
type jsonPatchOperation struct {
@@ -5,8 +5,10 @@ import (
"fmt"
"time"
"github.com/grafana/grafana-app-sdk/logging"
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
"github.com/prometheus/client_golang/prometheus"
)
@@ -20,14 +22,15 @@ type StatusPatcher interface {
// HealthChecker provides unified health checking for repositories
type HealthChecker struct {
statusPatcher StatusPatcher
registry prometheus.Registerer
healthMetrics healthMetrics
}
// NewHealthChecker creates a new health checker
func NewHealthChecker(statusPatcher StatusPatcher, registry prometheus.Registerer) *HealthChecker {
healthMetrics := registerHealthMetrics(registry)
return &HealthChecker{
statusPatcher: statusPatcher,
registry: registry,
healthMetrics: healthMetrics,
}
}
@@ -166,8 +169,17 @@ func (hc *HealthChecker) RefreshTimestamp(ctx context.Context, repo *provisionin
// refreshHealth performs a comprehensive health check
// Returns test results, health status, and any error
func (hc *HealthChecker) refreshHealth(ctx context.Context, repo repository.Repository, existingStatus provisioning.HealthStatus) (*provisioning.TestResults, provisioning.HealthStatus, error) {
logger := logging.FromContext(ctx).With("repo", repo.Config().GetName(), "namespace", repo.Config().GetNamespace())
start := time.Now()
outcome := utils.SuccessOutcome
defer func() {
hc.healthMetrics.RecordHealthCheck(outcome, time.Since(start).Seconds())
}()
res, err := repository.TestRepository(ctx, repo)
if err != nil {
outcome = utils.ErrorOutcome
logger.Error("failed to test repository", "error", err)
return nil, existingStatus, fmt.Errorf("failed to test repository: %w", err)
}
@@ -19,7 +19,7 @@ import (
func TestNewHealthChecker(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
assert.NotNil(t, hc)
assert.Equal(t, mockPatcher, hc.statusPatcher)
@@ -136,7 +136,7 @@ func TestShouldCheckHealth(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
result := hc.ShouldCheckHealth(tt.repo)
assert.Equal(t, tt.expected, result)
@@ -223,7 +223,7 @@ func TestHasRecentFailure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
result := hc.HasRecentFailure(tt.healthStatus, tt.failureType)
assert.Equal(t, tt.expected, result)
@@ -265,7 +265,7 @@ func TestRecordFailure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
repo := &provisioning.Repository{
Status: provisioning.RepositoryStatus{
@@ -310,7 +310,7 @@ func TestRecordFailure(t *testing.T) {
func TestRecordFailureFunction(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
testErr := errors.New("test error")
result := hc.recordFailure(provisioning.HealthFailureHook, testErr)
@@ -447,7 +447,7 @@ func TestRefreshHealth(t *testing.T) {
testError: tt.testError,
}
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
if tt.expectPatch {
if tt.patchError != nil {
@@ -557,7 +557,7 @@ func TestHasHealthStatusChanged(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
result := hc.hasHealthStatusChanged(tt.old, tt.new)
assert.Equal(t, tt.expected, result)
@@ -0,0 +1,84 @@
package controller
import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
"github.com/prometheus/client_golang/prometheus"
)
type finalizerMetrics struct {
registry prometheus.Registerer
finalizerProcessedTotal *prometheus.CounterVec
finalizerDuration *prometheus.HistogramVec
}
func registerFinalizerMetrics(registry prometheus.Registerer) finalizerMetrics {
finalizerProcessedTotal := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "grafana_provisioning_finalizers_processed_total",
Help: "Total number of finalizers processed",
},
[]string{"finalizer_type", "outcome"},
)
registry.MustRegister(finalizerProcessedTotal)
finalizerDuration := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "grafana_provisioning_finalizers_duration_seconds",
Help: "Duration of processing the finalizers",
Buckets: []float64{0.5, 1.0, 2.0, 5.0, 10.0, 30.0},
},
[]string{"finalizer_type", "resource_count_bucket"},
)
registry.MustRegister(finalizerDuration)
return finalizerMetrics{
registry: registry,
finalizerProcessedTotal: finalizerProcessedTotal,
finalizerDuration: finalizerDuration,
}
}
func (m *finalizerMetrics) RecordFinalizer(finalizerType string, outcome string, resourceCountChanged int, duration float64) {
m.finalizerProcessedTotal.WithLabelValues(finalizerType, outcome).Inc()
if outcome == utils.SuccessOutcome {
m.finalizerDuration.WithLabelValues(finalizerType, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration)
}
}
type healthMetrics struct {
registry prometheus.Registerer
healthCheckedTotal *prometheus.CounterVec
healthCheckedDuration *prometheus.HistogramVec
}
func registerHealthMetrics(registry prometheus.Registerer) healthMetrics {
healthCheckedTotal := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "grafana_provisioning_health_checked_total",
Help: "Total number of health checks performed",
},
[]string{"outcome"},
)
registry.MustRegister(healthCheckedTotal)
healthCheckedDuration := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "grafana_provisioning_health_checked_duration_seconds",
Help: "Duration of health checks",
Buckets: []float64{0.1, 0.2, 0.5, 1.0, 2.0, 5.0},
},
[]string{},
)
registry.MustRegister(healthCheckedDuration)
return healthMetrics{
registry: registry,
healthCheckedTotal: healthCheckedTotal,
healthCheckedDuration: healthCheckedDuration,
}
}
func (m *healthMetrics) RecordHealthCheck(outcome string, duration float64) {
m.healthCheckedTotal.WithLabelValues(outcome).Inc()
m.healthCheckedDuration.WithLabelValues().Observe(duration)
}
@@ -77,6 +77,8 @@ func NewRepositoryController(
statusPatcher StatusPatcher,
registry prometheus.Registerer,
) (*RepositoryController, error) {
finalizerMetrics := registerFinalizerMetrics(registry)
rc := &RepositoryController{
client: provisioningClient,
repoLister: repoInformer.Lister(),
@@ -93,6 +95,7 @@ func NewRepositoryController(
finalizer: &finalizer{
lister: resourceLister,
clientFactory: clients,
metrics: &finalizerMetrics,
},
jobs: jobs,
logger: logging.DefaultLogger.With("logger", loggerName),
@@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
)
type Worker struct {
@@ -44,7 +45,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
opts := *job.Spec.Delete
paths := opts.Paths
start := time.Now()
outcome := jobs.ErrorOutcome
outcome := utils.ErrorOutcome
resourcesDeleted := 0
defer func() {
w.metrics.RecordJob(string(provisioning.JobActionDelete), outcome, resourcesDeleted, time.Since(start).Seconds())
@@ -119,7 +120,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
}
}
outcome = jobs.SuccessOutcome
outcome = utils.SuccessOutcome
jobStatus := progress.Complete(ctx, nil)
for _, summary := range jobStatus.Summary {
resourcesDeleted += int(summary.Delete)
@@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
)
//go:generate mockery --name ExportFn --structname MockExportFn --inpackage --filename mock_export_fn.go --with-expecter
@@ -56,7 +57,7 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository,
logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace())
start := time.Now()
outcome := jobs.ErrorOutcome
outcome := utils.ErrorOutcome
resourcesExported := 0
defer func() {
r.metrics.RecordJob(string(provisioning.JobActionPush), outcome, resourcesExported, time.Since(start).Seconds())
@@ -118,7 +119,7 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository,
return err
}
outcome = jobs.SuccessOutcome
outcome = utils.SuccessOutcome
jobStatus := progress.Complete(ctx, nil)
for _, summary := range jobStatus.Summary {
resourcesExported += int(summary.Write)
+5 -26
View File
@@ -1,10 +1,8 @@
package jobs
import "github.com/prometheus/client_golang/prometheus"
const (
SuccessOutcome = "success"
ErrorOutcome = "error"
import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
"github.com/prometheus/client_golang/prometheus"
)
type JobMetrics struct {
@@ -87,27 +85,8 @@ func (m *JobMetrics) RecordJob(jobAction string, outcome string, resourceCountCh
m.processedTotal.WithLabelValues(jobAction, outcome).Inc()
// only record duration when the job was successful. otherwise resource count will be incorrect
if outcome == SuccessOutcome {
m.durationHist.WithLabelValues(jobAction, getResourceCountBucket(resourceCountChanged)).Observe(duration)
}
}
func getResourceCountBucket(count int) string {
switch {
case count == 0:
return "0"
case count <= 10:
return "1-10"
case count <= 50:
return "11-50"
case count <= 100:
return "51-100"
case count <= 500:
return "101-500"
case count <= 1000:
return "501-1000"
default:
return "1000+"
if outcome == utils.SuccessOutcome {
m.durationHist.WithLabelValues(jobAction, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration)
}
}
@@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana/apps/provisioning/pkg/safepath"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
)
type Worker struct {
@@ -43,7 +44,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
}
opts := *job.Spec.Move
logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace())
outcome := jobs.ErrorOutcome
outcome := utils.ErrorOutcome
start := time.Now()
resourcesMoved := 0
defer func() {
@@ -129,7 +130,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
}
}
outcome = jobs.SuccessOutcome
outcome = utils.SuccessOutcome
jobStatus := progress.Complete(ctx, nil)
for _, summary := range jobStatus.Summary {
// FileActionRenamed increments both delete & create, use create here
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
)
@@ -64,7 +65,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace())
start := time.Now()
outcome := jobs.ErrorOutcome
outcome := utils.ErrorOutcome
totalChangesMade := 0
defer func() {
r.metrics.RecordJob(string(provisioning.JobActionPull), outcome, totalChangesMade, time.Since(start).Seconds())
@@ -123,7 +124,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
if syncError != nil {
logger.Debug("failed to sync the repository", "error", syncError)
} else {
outcome = jobs.SuccessOutcome
outcome = utils.SuccessOutcome
for _, summary := range jobStatus.Summary {
totalChangesMade += int(summary.Create + summary.Update + summary.Delete)
}
+1 -1
View File
@@ -716,7 +716,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
return fmt.Errorf("create API client job store: %w", err)
}
b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient(), b.registry)
b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient())
b.healthChecker = controller.NewHealthChecker(b.statusPatcher, b.registry)
// if running solely CRUD, skip the rest of the setup
@@ -0,0 +1,25 @@
package utils
const (
SuccessOutcome = "success"
ErrorOutcome = "error"
)
func GetResourceCountBucket(count int) string {
switch {
case count == 0:
return "0"
case count <= 10:
return "1-10"
case count <= 50:
return "11-50"
case count <= 100:
return "51-100"
case count <= 500:
return "101-500"
case count <= 1000:
return "501-1000"
default:
return "1000+"
}
}