public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provisioning: Add metrics for repo controller (#111450)

Browse Source
This commit is contained in:
Stephanie Hingtgen
2025-09-22 20:14:03 +00:00
committed by GitHub
parent a2d09490ae
commit 8b1caccc72
17 changed files with 187 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/registry/apis/provisioning/controller/finalizers.go
+28 -10
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"sort"
"strings"
"time"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -17,11 +18,13 @@ import (
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
metricutils "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
)
type finalizer struct {
lister resources.ResourceLister
clientFactory resources.ClientFactory
metrics *finalizerMetrics
}
func (f *finalizer) process(ctx context.Context,
@@ -31,18 +34,24 @@ func (f *finalizer) process(ctx context.Context,
logger := logging.FromContext(ctx)
for _, finalizer := range finalizers {
var err error
var count int
start := time.Now()
outcome := metricutils.SuccessOutcome
switch finalizer {
case repository.CleanFinalizer:
// NOTE: the controller loop will never get run unless a finalizer is set
hooks, ok := repo.(repository.Hooks)
if ok {
if err := hooks.OnDelete(ctx); err != nil {
if err = hooks.OnDelete(ctx); err != nil {
logger.Warn("Error running deletion hooks", "err", err)
outcome = metricutils.ErrorOutcome
}
}
case repository.ReleaseOrphanResourcesFinalizer:
err := f.processExistingItems(ctx, repo.Config(),
count, err = f.processExistingItems(ctx, repo.Config(),
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
patchAnnotations, err := getPatchedAnnotations(item)
if err != nil {
@@ -55,20 +64,29 @@ func (f *finalizer) process(ctx context.Context,
return err
})
if err != nil {
return err
outcome = metricutils.ErrorOutcome
logger.Warn("Error processing release orphan resources finalizer", "err", err)
}
case repository.RemoveOrphanResourcesFinalizer:
err := f.processExistingItems(ctx, repo.Config(),
count, err = f.processExistingItems(ctx, repo.Config(),
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
return client.Delete(ctx, item.Name, v1.DeleteOptions{})
})
if err != nil {
return err
outcome = metricutils.ErrorOutcome
logger.Warn("Error processing remove orphan resources finalizer", "err", err)
}
default:
logger.Warn("skipping unknown finalizer", "finalizer", finalizer)
continue
}
f.metrics.RecordFinalizer(finalizer, outcome, count, time.Since(start).Seconds())
if err != nil {
return err
}
}
return nil
@@ -79,17 +97,17 @@ func (f *finalizer) processExistingItems(
ctx context.Context,
repo *provisioning.Repository,
cb func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error,
) error {
) (int, error) {
logger := logging.FromContext(ctx)
clients, err := f.clientFactory.Clients(ctx, repo.Namespace)
if err != nil {
return err
return 0, err
}
items, err := f.lister.List(ctx, repo.Namespace, repo.Name)
if err != nil {
logger.Warn("error listing resources", "error", err)
return err
return 0, err
}
// Safe deletion order
@@ -103,7 +121,7 @@ func (f *finalizer) processExistingItems(
Resource: item.Resource,
})
if err != nil {
return err
return count, err
}
err = cb(res, &item)
@@ -115,7 +133,7 @@ func (f *finalizer) processExistingItems(
}
}
logger.Info("processed orphan items", "items", count, "errors", errors)
return nil
return count, nil
}
type jsonPatchOperation struct {
pkg/registry/apis/provisioning/controller/health.go
+14 -2
View File
@@ -5,8 +5,10 @@ import (
"fmt"
"time"
"github.com/grafana/grafana-app-sdk/logging"
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
"github.com/prometheus/client_golang/prometheus"
)
@@ -20,14 +22,15 @@ type StatusPatcher interface {
// HealthChecker provides unified health checking for repositories
type HealthChecker struct {
statusPatcher StatusPatcher
registry prometheus.Registerer
healthMetrics healthMetrics
}
// NewHealthChecker creates a new health checker
func NewHealthChecker(statusPatcher StatusPatcher, registry prometheus.Registerer) *HealthChecker {
healthMetrics := registerHealthMetrics(registry)
return &HealthChecker{
statusPatcher: statusPatcher,
registry: registry,
healthMetrics: healthMetrics,
}
}
@@ -166,8 +169,17 @@ func (hc *HealthChecker) RefreshTimestamp(ctx context.Context, repo *provisionin
// refreshHealth performs a comprehensive health check
// Returns test results, health status, and any error
func (hc *HealthChecker) refreshHealth(ctx context.Context, repo repository.Repository, existingStatus provisioning.HealthStatus) (*provisioning.TestResults, provisioning.HealthStatus, error) {
logger := logging.FromContext(ctx).With("repo", repo.Config().GetName(), "namespace", repo.Config().GetNamespace())
start := time.Now()
outcome := utils.SuccessOutcome
defer func() {
hc.healthMetrics.RecordHealthCheck(outcome, time.Since(start).Seconds())
}()
res, err := repository.TestRepository(ctx, repo)
if err != nil {
outcome = utils.ErrorOutcome
logger.Error("failed to test repository", "error", err)
return nil, existingStatus, fmt.Errorf("failed to test repository: %w", err)
}
pkg/registry/apis/provisioning/controller/health_test.go
+7 -7
View File
@@ -19,7 +19,7 @@ import (
func TestNewHealthChecker(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
assert.NotNil(t, hc)
assert.Equal(t, mockPatcher, hc.statusPatcher)
@@ -136,7 +136,7 @@ func TestShouldCheckHealth(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
result := hc.ShouldCheckHealth(tt.repo)
assert.Equal(t, tt.expected, result)
@@ -223,7 +223,7 @@ func TestHasRecentFailure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
result := hc.HasRecentFailure(tt.healthStatus, tt.failureType)
assert.Equal(t, tt.expected, result)
@@ -265,7 +265,7 @@ func TestRecordFailure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
repo := &provisioning.Repository{
Status: provisioning.RepositoryStatus{
@@ -310,7 +310,7 @@ func TestRecordFailure(t *testing.T) {
func TestRecordFailureFunction(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
testErr := errors.New("test error")
result := hc.recordFailure(provisioning.HealthFailureHook, testErr)
@@ -447,7 +447,7 @@ func TestRefreshHealth(t *testing.T) {
testError: tt.testError,
}
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
if tt.expectPatch {
if tt.patchError != nil {
@@ -557,7 +557,7 @@ func TestHasHealthStatusChanged(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockPatcher := mocks.NewStatusPatcher(t)
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
result := hc.hasHealthStatusChanged(tt.old, tt.new)
assert.Equal(t, tt.expected, result)
pkg/registry/apis/provisioning/controller/metrics.go
+84
View File
@@ -0,0 +1,84 @@
package controller
import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
"github.com/prometheus/client_golang/prometheus"
)
type finalizerMetrics struct {
registry prometheus.Registerer
finalizerProcessedTotal *prometheus.CounterVec
finalizerDuration *prometheus.HistogramVec
}
func registerFinalizerMetrics(registry prometheus.Registerer) finalizerMetrics {
finalizerProcessedTotal := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "grafana_provisioning_finalizers_processed_total",
Help: "Total number of finalizers processed",
},
[]string{"finalizer_type", "outcome"},
)
registry.MustRegister(finalizerProcessedTotal)
finalizerDuration := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "grafana_provisioning_finalizers_duration_seconds",
Help: "Duration of processing the finalizers",
Buckets: []float64{0.5, 1.0, 2.0, 5.0, 10.0, 30.0},
},
[]string{"finalizer_type", "resource_count_bucket"},
)
registry.MustRegister(finalizerDuration)
return finalizerMetrics{
registry: registry,
finalizerProcessedTotal: finalizerProcessedTotal,
finalizerDuration: finalizerDuration,
}
}
func (m *finalizerMetrics) RecordFinalizer(finalizerType string, outcome string, resourceCountChanged int, duration float64) {
m.finalizerProcessedTotal.WithLabelValues(finalizerType, outcome).Inc()
if outcome == utils.SuccessOutcome {
m.finalizerDuration.WithLabelValues(finalizerType, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration)
}
}
type healthMetrics struct {
registry prometheus.Registerer
healthCheckedTotal *prometheus.CounterVec
healthCheckedDuration *prometheus.HistogramVec
}
func registerHealthMetrics(registry prometheus.Registerer) healthMetrics {
healthCheckedTotal := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "grafana_provisioning_health_checked_total",
Help: "Total number of health checks performed",
},
[]string{"outcome"},
)
registry.MustRegister(healthCheckedTotal)
healthCheckedDuration := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "grafana_provisioning_health_checked_duration_seconds",
Help: "Duration of health checks",
Buckets: []float64{0.1, 0.2, 0.5, 1.0, 2.0, 5.0},
},
[]string{},
)
registry.MustRegister(healthCheckedDuration)
return healthMetrics{
registry: registry,
healthCheckedTotal: healthCheckedTotal,
healthCheckedDuration: healthCheckedDuration,
}
}
func (m *healthMetrics) RecordHealthCheck(outcome string, duration float64) {
m.healthCheckedTotal.WithLabelValues(outcome).Inc()
m.healthCheckedDuration.WithLabelValues().Observe(duration)
}
pkg/registry/apis/provisioning/controller/repository.go
+3
View File
@@ -77,6 +77,8 @@ func NewRepositoryController(
statusPatcher StatusPatcher,
registry prometheus.Registerer,
) (*RepositoryController, error) {
finalizerMetrics := registerFinalizerMetrics(registry)
rc := &RepositoryController{
client: provisioningClient,
repoLister: repoInformer.Lister(),
@@ -93,6 +95,7 @@ func NewRepositoryController(
finalizer: &finalizer{
lister: resourceLister,
clientFactory: clients,
metrics: &finalizerMetrics,
},
jobs: jobs,
logger: logging.DefaultLogger.With("logger", loggerName),