Provisioning: Add metrics for repo controller (#111450)
This commit is contained in:
@@ -11,7 +11,6 @@ require (
|
||||
github.com/grafana/grafana/pkg/apimachinery v0.0.0-20250804150913-990f1c69ecc2
|
||||
github.com/grafana/nanogit v0.0.0-20250723104447-68f58f5ecec0
|
||||
github.com/migueleliasweb/go-github-mock v1.1.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/oauth2 v0.30.0
|
||||
k8s.io/apimachinery v0.34.1
|
||||
@@ -54,6 +53,7 @@ require (
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
|
||||
@@ -7,20 +7,17 @@ import (
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
type RepositoryStatusPatcher struct {
|
||||
client client.ProvisioningV0alpha1Interface
|
||||
registry prometheus.Registerer
|
||||
client client.ProvisioningV0alpha1Interface
|
||||
}
|
||||
|
||||
func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface, registry prometheus.Registerer) *RepositoryStatusPatcher {
|
||||
func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *RepositoryStatusPatcher {
|
||||
return &RepositoryStatusPatcher{
|
||||
client: client,
|
||||
registry: registry,
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1/fake"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/require"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
@@ -17,7 +16,7 @@ import (
|
||||
|
||||
func TestNewRepositoryStatusPatcher(t *testing.T) {
|
||||
client := &fake.FakeProvisioningV0alpha1{}
|
||||
patcher := NewRepositoryStatusPatcher(client, prometheus.DefaultRegisterer)
|
||||
patcher := NewRepositoryStatusPatcher(client)
|
||||
require.NotNil(t, patcher)
|
||||
require.Equal(t, client, patcher.client)
|
||||
}
|
||||
@@ -104,7 +103,7 @@ func TestRepositoryStatusPatcher_Patch(t *testing.T) {
|
||||
client.AddReactor("patch", "repositories", tt.reactorFunc)
|
||||
}
|
||||
|
||||
patcher := NewRepositoryStatusPatcher(&client, prometheus.DefaultRegisterer)
|
||||
patcher := NewRepositoryStatusPatcher(&client)
|
||||
err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations...)
|
||||
|
||||
if tt.expectedError != "" {
|
||||
|
||||
@@ -180,7 +180,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis
|
||||
parsers := resources.NewParserFactory(clients)
|
||||
resourceLister := resources.NewResourceLister(controllerCfg.unified)
|
||||
repositoryResources := resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister)
|
||||
statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), registry)
|
||||
statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1())
|
||||
|
||||
workers := make([]jobs.Worker, 0)
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ func RunRepoController(deps server.OperatorDependencies) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("create API client job store: %w", err)
|
||||
}
|
||||
statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), deps.Registerer)
|
||||
statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1())
|
||||
healthChecker := controller.NewHealthChecker(statusPatcher, deps.Registerer)
|
||||
|
||||
repoInformer := informerFactory.Provisioning().V0alpha1().Repositories()
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
@@ -17,11 +18,13 @@ import (
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
metricutils "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
)
|
||||
|
||||
type finalizer struct {
|
||||
lister resources.ResourceLister
|
||||
clientFactory resources.ClientFactory
|
||||
metrics *finalizerMetrics
|
||||
}
|
||||
|
||||
func (f *finalizer) process(ctx context.Context,
|
||||
@@ -31,18 +34,24 @@ func (f *finalizer) process(ctx context.Context,
|
||||
logger := logging.FromContext(ctx)
|
||||
|
||||
for _, finalizer := range finalizers {
|
||||
var err error
|
||||
var count int
|
||||
start := time.Now()
|
||||
outcome := metricutils.SuccessOutcome
|
||||
|
||||
switch finalizer {
|
||||
case repository.CleanFinalizer:
|
||||
// NOTE: the controller loop will never get run unless a finalizer is set
|
||||
hooks, ok := repo.(repository.Hooks)
|
||||
if ok {
|
||||
if err := hooks.OnDelete(ctx); err != nil {
|
||||
if err = hooks.OnDelete(ctx); err != nil {
|
||||
logger.Warn("Error running deletion hooks", "err", err)
|
||||
outcome = metricutils.ErrorOutcome
|
||||
}
|
||||
}
|
||||
|
||||
case repository.ReleaseOrphanResourcesFinalizer:
|
||||
err := f.processExistingItems(ctx, repo.Config(),
|
||||
count, err = f.processExistingItems(ctx, repo.Config(),
|
||||
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
patchAnnotations, err := getPatchedAnnotations(item)
|
||||
if err != nil {
|
||||
@@ -55,20 +64,29 @@ func (f *finalizer) process(ctx context.Context,
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
outcome = metricutils.ErrorOutcome
|
||||
logger.Warn("Error processing release orphan resources finalizer", "err", err)
|
||||
}
|
||||
|
||||
case repository.RemoveOrphanResourcesFinalizer:
|
||||
err := f.processExistingItems(ctx, repo.Config(),
|
||||
count, err = f.processExistingItems(ctx, repo.Config(),
|
||||
func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
|
||||
return client.Delete(ctx, item.Name, v1.DeleteOptions{})
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
outcome = metricutils.ErrorOutcome
|
||||
logger.Warn("Error processing remove orphan resources finalizer", "err", err)
|
||||
}
|
||||
|
||||
default:
|
||||
logger.Warn("skipping unknown finalizer", "finalizer", finalizer)
|
||||
continue
|
||||
}
|
||||
|
||||
f.metrics.RecordFinalizer(finalizer, outcome, count, time.Since(start).Seconds())
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -79,17 +97,17 @@ func (f *finalizer) processExistingItems(
|
||||
ctx context.Context,
|
||||
repo *provisioning.Repository,
|
||||
cb func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error,
|
||||
) error {
|
||||
) (int, error) {
|
||||
logger := logging.FromContext(ctx)
|
||||
clients, err := f.clientFactory.Clients(ctx, repo.Namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
|
||||
items, err := f.lister.List(ctx, repo.Namespace, repo.Name)
|
||||
if err != nil {
|
||||
logger.Warn("error listing resources", "error", err)
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Safe deletion order
|
||||
@@ -103,7 +121,7 @@ func (f *finalizer) processExistingItems(
|
||||
Resource: item.Resource,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return count, err
|
||||
}
|
||||
|
||||
err = cb(res, &item)
|
||||
@@ -115,7 +133,7 @@ func (f *finalizer) processExistingItems(
|
||||
}
|
||||
}
|
||||
logger.Info("processed orphan items", "items", count, "errors", errors)
|
||||
return nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
type jsonPatchOperation struct {
|
||||
|
||||
@@ -5,8 +5,10 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana-app-sdk/logging"
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
@@ -20,14 +22,15 @@ type StatusPatcher interface {
|
||||
// HealthChecker provides unified health checking for repositories
|
||||
type HealthChecker struct {
|
||||
statusPatcher StatusPatcher
|
||||
registry prometheus.Registerer
|
||||
healthMetrics healthMetrics
|
||||
}
|
||||
|
||||
// NewHealthChecker creates a new health checker
|
||||
func NewHealthChecker(statusPatcher StatusPatcher, registry prometheus.Registerer) *HealthChecker {
|
||||
healthMetrics := registerHealthMetrics(registry)
|
||||
return &HealthChecker{
|
||||
statusPatcher: statusPatcher,
|
||||
registry: registry,
|
||||
healthMetrics: healthMetrics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,8 +169,17 @@ func (hc *HealthChecker) RefreshTimestamp(ctx context.Context, repo *provisionin
|
||||
// refreshHealth performs a comprehensive health check
|
||||
// Returns test results, health status, and any error
|
||||
func (hc *HealthChecker) refreshHealth(ctx context.Context, repo repository.Repository, existingStatus provisioning.HealthStatus) (*provisioning.TestResults, provisioning.HealthStatus, error) {
|
||||
logger := logging.FromContext(ctx).With("repo", repo.Config().GetName(), "namespace", repo.Config().GetNamespace())
|
||||
start := time.Now()
|
||||
outcome := utils.SuccessOutcome
|
||||
defer func() {
|
||||
hc.healthMetrics.RecordHealthCheck(outcome, time.Since(start).Seconds())
|
||||
}()
|
||||
|
||||
res, err := repository.TestRepository(ctx, repo)
|
||||
if err != nil {
|
||||
outcome = utils.ErrorOutcome
|
||||
logger.Error("failed to test repository", "error", err)
|
||||
return nil, existingStatus, fmt.Errorf("failed to test repository: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestNewHealthChecker(t *testing.T) {
|
||||
mockPatcher := mocks.NewStatusPatcher(t)
|
||||
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
assert.NotNil(t, hc)
|
||||
assert.Equal(t, mockPatcher, hc.statusPatcher)
|
||||
@@ -136,7 +136,7 @@ func TestShouldCheckHealth(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mockPatcher := mocks.NewStatusPatcher(t)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
result := hc.ShouldCheckHealth(tt.repo)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
@@ -223,7 +223,7 @@ func TestHasRecentFailure(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mockPatcher := mocks.NewStatusPatcher(t)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
result := hc.HasRecentFailure(tt.healthStatus, tt.failureType)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
@@ -265,7 +265,7 @@ func TestRecordFailure(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mockPatcher := mocks.NewStatusPatcher(t)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
repo := &provisioning.Repository{
|
||||
Status: provisioning.RepositoryStatus{
|
||||
@@ -310,7 +310,7 @@ func TestRecordFailure(t *testing.T) {
|
||||
|
||||
func TestRecordFailureFunction(t *testing.T) {
|
||||
mockPatcher := mocks.NewStatusPatcher(t)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
testErr := errors.New("test error")
|
||||
result := hc.recordFailure(provisioning.HealthFailureHook, testErr)
|
||||
@@ -447,7 +447,7 @@ func TestRefreshHealth(t *testing.T) {
|
||||
testError: tt.testError,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
if tt.expectPatch {
|
||||
if tt.patchError != nil {
|
||||
@@ -557,7 +557,7 @@ func TestHasHealthStatusChanged(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mockPatcher := mocks.NewStatusPatcher(t)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer)
|
||||
hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry())
|
||||
|
||||
result := hc.hasHealthStatusChanged(tt.old, tt.new)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type finalizerMetrics struct {
|
||||
registry prometheus.Registerer
|
||||
finalizerProcessedTotal *prometheus.CounterVec
|
||||
finalizerDuration *prometheus.HistogramVec
|
||||
}
|
||||
|
||||
func registerFinalizerMetrics(registry prometheus.Registerer) finalizerMetrics {
|
||||
finalizerProcessedTotal := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "grafana_provisioning_finalizers_processed_total",
|
||||
Help: "Total number of finalizers processed",
|
||||
},
|
||||
[]string{"finalizer_type", "outcome"},
|
||||
)
|
||||
registry.MustRegister(finalizerProcessedTotal)
|
||||
|
||||
finalizerDuration := prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "grafana_provisioning_finalizers_duration_seconds",
|
||||
Help: "Duration of processing the finalizers",
|
||||
Buckets: []float64{0.5, 1.0, 2.0, 5.0, 10.0, 30.0},
|
||||
},
|
||||
[]string{"finalizer_type", "resource_count_bucket"},
|
||||
)
|
||||
registry.MustRegister(finalizerDuration)
|
||||
|
||||
return finalizerMetrics{
|
||||
registry: registry,
|
||||
finalizerProcessedTotal: finalizerProcessedTotal,
|
||||
finalizerDuration: finalizerDuration,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *finalizerMetrics) RecordFinalizer(finalizerType string, outcome string, resourceCountChanged int, duration float64) {
|
||||
m.finalizerProcessedTotal.WithLabelValues(finalizerType, outcome).Inc()
|
||||
if outcome == utils.SuccessOutcome {
|
||||
m.finalizerDuration.WithLabelValues(finalizerType, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration)
|
||||
}
|
||||
}
|
||||
|
||||
type healthMetrics struct {
|
||||
registry prometheus.Registerer
|
||||
healthCheckedTotal *prometheus.CounterVec
|
||||
healthCheckedDuration *prometheus.HistogramVec
|
||||
}
|
||||
|
||||
func registerHealthMetrics(registry prometheus.Registerer) healthMetrics {
|
||||
healthCheckedTotal := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "grafana_provisioning_health_checked_total",
|
||||
Help: "Total number of health checks performed",
|
||||
},
|
||||
[]string{"outcome"},
|
||||
)
|
||||
registry.MustRegister(healthCheckedTotal)
|
||||
|
||||
healthCheckedDuration := prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "grafana_provisioning_health_checked_duration_seconds",
|
||||
Help: "Duration of health checks",
|
||||
Buckets: []float64{0.1, 0.2, 0.5, 1.0, 2.0, 5.0},
|
||||
},
|
||||
[]string{},
|
||||
)
|
||||
registry.MustRegister(healthCheckedDuration)
|
||||
|
||||
return healthMetrics{
|
||||
registry: registry,
|
||||
healthCheckedTotal: healthCheckedTotal,
|
||||
healthCheckedDuration: healthCheckedDuration,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *healthMetrics) RecordHealthCheck(outcome string, duration float64) {
|
||||
m.healthCheckedTotal.WithLabelValues(outcome).Inc()
|
||||
m.healthCheckedDuration.WithLabelValues().Observe(duration)
|
||||
}
|
||||
@@ -77,6 +77,8 @@ func NewRepositoryController(
|
||||
statusPatcher StatusPatcher,
|
||||
registry prometheus.Registerer,
|
||||
) (*RepositoryController, error) {
|
||||
finalizerMetrics := registerFinalizerMetrics(registry)
|
||||
|
||||
rc := &RepositoryController{
|
||||
client: provisioningClient,
|
||||
repoLister: repoInformer.Lister(),
|
||||
@@ -93,6 +95,7 @@ func NewRepositoryController(
|
||||
finalizer: &finalizer{
|
||||
lister: resourceLister,
|
||||
clientFactory: clients,
|
||||
metrics: &finalizerMetrics,
|
||||
},
|
||||
jobs: jobs,
|
||||
logger: logging.DefaultLogger.With("logger", loggerName),
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
@@ -44,7 +45,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
|
||||
opts := *job.Spec.Delete
|
||||
paths := opts.Paths
|
||||
start := time.Now()
|
||||
outcome := jobs.ErrorOutcome
|
||||
outcome := utils.ErrorOutcome
|
||||
resourcesDeleted := 0
|
||||
defer func() {
|
||||
w.metrics.RecordJob(string(provisioning.JobActionDelete), outcome, resourcesDeleted, time.Since(start).Seconds())
|
||||
@@ -119,7 +120,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
|
||||
}
|
||||
}
|
||||
|
||||
outcome = jobs.SuccessOutcome
|
||||
outcome = utils.SuccessOutcome
|
||||
jobStatus := progress.Complete(ctx, nil)
|
||||
for _, summary := range jobStatus.Summary {
|
||||
resourcesDeleted += int(summary.Delete)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
)
|
||||
|
||||
//go:generate mockery --name ExportFn --structname MockExportFn --inpackage --filename mock_export_fn.go --with-expecter
|
||||
@@ -56,7 +57,7 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository,
|
||||
|
||||
logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace())
|
||||
start := time.Now()
|
||||
outcome := jobs.ErrorOutcome
|
||||
outcome := utils.ErrorOutcome
|
||||
resourcesExported := 0
|
||||
defer func() {
|
||||
r.metrics.RecordJob(string(provisioning.JobActionPush), outcome, resourcesExported, time.Since(start).Seconds())
|
||||
@@ -118,7 +119,7 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository,
|
||||
return err
|
||||
}
|
||||
|
||||
outcome = jobs.SuccessOutcome
|
||||
outcome = utils.SuccessOutcome
|
||||
jobStatus := progress.Complete(ctx, nil)
|
||||
for _, summary := range jobStatus.Summary {
|
||||
resourcesExported += int(summary.Write)
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package jobs
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
const (
|
||||
SuccessOutcome = "success"
|
||||
ErrorOutcome = "error"
|
||||
import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type JobMetrics struct {
|
||||
@@ -87,27 +85,8 @@ func (m *JobMetrics) RecordJob(jobAction string, outcome string, resourceCountCh
|
||||
m.processedTotal.WithLabelValues(jobAction, outcome).Inc()
|
||||
|
||||
// only record duration when the job was successful. otherwise resource count will be incorrect
|
||||
if outcome == SuccessOutcome {
|
||||
m.durationHist.WithLabelValues(jobAction, getResourceCountBucket(resourceCountChanged)).Observe(duration)
|
||||
}
|
||||
}
|
||||
|
||||
func getResourceCountBucket(count int) string {
|
||||
switch {
|
||||
case count == 0:
|
||||
return "0"
|
||||
case count <= 10:
|
||||
return "1-10"
|
||||
case count <= 50:
|
||||
return "11-50"
|
||||
case count <= 100:
|
||||
return "51-100"
|
||||
case count <= 500:
|
||||
return "101-500"
|
||||
case count <= 1000:
|
||||
return "501-1000"
|
||||
default:
|
||||
return "1000+"
|
||||
if outcome == utils.SuccessOutcome {
|
||||
m.durationHist.WithLabelValues(jobAction, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/safepath"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
@@ -43,7 +44,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
|
||||
}
|
||||
opts := *job.Spec.Move
|
||||
logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace())
|
||||
outcome := jobs.ErrorOutcome
|
||||
outcome := utils.ErrorOutcome
|
||||
start := time.Now()
|
||||
resourcesMoved := 0
|
||||
defer func() {
|
||||
@@ -129,7 +130,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr
|
||||
}
|
||||
}
|
||||
|
||||
outcome = jobs.SuccessOutcome
|
||||
outcome = utils.SuccessOutcome
|
||||
jobStatus := progress.Complete(ctx, nil)
|
||||
for _, summary := range jobStatus.Summary {
|
||||
// FileActionRenamed increments both delete & create, use create here
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
|
||||
)
|
||||
|
||||
@@ -64,7 +65,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
|
||||
logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace())
|
||||
|
||||
start := time.Now()
|
||||
outcome := jobs.ErrorOutcome
|
||||
outcome := utils.ErrorOutcome
|
||||
totalChangesMade := 0
|
||||
defer func() {
|
||||
r.metrics.RecordJob(string(provisioning.JobActionPull), outcome, totalChangesMade, time.Since(start).Seconds())
|
||||
@@ -123,7 +124,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
|
||||
if syncError != nil {
|
||||
logger.Debug("failed to sync the repository", "error", syncError)
|
||||
} else {
|
||||
outcome = jobs.SuccessOutcome
|
||||
outcome = utils.SuccessOutcome
|
||||
for _, summary := range jobStatus.Summary {
|
||||
totalChangesMade += int(summary.Create + summary.Update + summary.Delete)
|
||||
}
|
||||
|
||||
@@ -716,7 +716,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
|
||||
return fmt.Errorf("create API client job store: %w", err)
|
||||
}
|
||||
|
||||
b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient(), b.registry)
|
||||
b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient())
|
||||
b.healthChecker = controller.NewHealthChecker(b.statusPatcher, b.registry)
|
||||
|
||||
// if running solely CRUD, skip the rest of the setup
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package utils
|
||||
|
||||
const (
|
||||
SuccessOutcome = "success"
|
||||
ErrorOutcome = "error"
|
||||
)
|
||||
|
||||
func GetResourceCountBucket(count int) string {
|
||||
switch {
|
||||
case count == 0:
|
||||
return "0"
|
||||
case count <= 10:
|
||||
return "1-10"
|
||||
case count <= 50:
|
||||
return "11-50"
|
||||
case count <= 100:
|
||||
return "51-100"
|
||||
case count <= 500:
|
||||
return "101-500"
|
||||
case count <= 1000:
|
||||
return "501-1000"
|
||||
default:
|
||||
return "1000+"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user