From 478f9bf59745b9b62080312867b0bf4daf9ede19 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Tue, 24 Jun 2025 18:27:52 +0200 Subject: [PATCH] Alerting: Emit metrics from prometheus state history backend (#107121) Alerting: Emit metrics from prometheus historian backend --- pkg/services/ngalert/ngalert.go | 2 +- .../ngalert/state/historian/prometheus.go | 12 +- .../state/historian/prometheus_test.go | 104 +++++++++++++++++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 066d7c5672a..9939daaa653 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -708,7 +708,7 @@ func configureHistorianBackend( if w == nil { return nil, fmt.Errorf("failed to create alert state metrics writer") } - backend := historian.NewRemotePrometheusBackend(pcfg, w, prometheusBackendLogger) + backend := historian.NewRemotePrometheusBackend(pcfg, w, prometheusBackendLogger, met) return backend, nil } diff --git a/pkg/services/ngalert/state/historian/prometheus.go b/pkg/services/ngalert/state/historian/prometheus.go index 338112f5bc6..58c22588524 100644 --- a/pkg/services/ngalert/state/historian/prometheus.go +++ b/pkg/services/ngalert/state/historian/prometheus.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/ngalert/eval" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model" @@ -89,15 +90,17 @@ type RemotePrometheusBackend struct { cfg PrometheusConfig promWriter seriesWriter logger log.Logger + metrics *metrics.Historian } -func NewRemotePrometheusBackend(cfg PrometheusConfig, promWriter seriesWriter, logger log.Logger) *RemotePrometheusBackend { +func NewRemotePrometheusBackend(cfg PrometheusConfig, promWriter seriesWriter, logger log.Logger, metrics *metrics.Historian) *RemotePrometheusBackend { logger.Info("Initializing remote Prometheus backend", "datasourceUID", cfg.DatasourceUID) return &RemotePrometheusBackend{ cfg: cfg, promWriter: promWriter, logger: logger, + metrics: metrics, } } @@ -141,9 +144,16 @@ func (b *RemotePrometheusBackend) Record(ctx context.Context, rule history_model close(errCh) }() + logger.Debug("Saving state history batch", "samples", len(frames)) + org := fmt.Sprint(st.OrgID) + b.metrics.WritesTotal.WithLabelValues(org, "prometheus").Inc() + b.metrics.TransitionsTotal.WithLabelValues(org).Add(float64(len(frames))) + var sendErr error if err := b.promWriter.WriteDatasource(ctx, b.cfg.DatasourceUID, b.cfg.MetricName, st.LastEvaluationTime, frames, st.OrgID, nil); err != nil { logger.Error("Failed to write alert state metrics batch", "error", err) + b.metrics.WritesFailed.WithLabelValues(org, "prometheus").Inc() + b.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(len(frames))) sendErr = err } errCh <- sendErr diff --git a/pkg/services/ngalert/state/historian/prometheus_test.go b/pkg/services/ngalert/state/historian/prometheus_test.go index 73cac7a5a3c..48cd9fcb322 100644 --- a/pkg/services/ngalert/state/historian/prometheus_test.go +++ b/pkg/services/ngalert/state/historian/prometheus_test.go @@ -1,6 +1,7 @@ package historian import ( + "bytes" "context" "errors" "math" @@ -9,12 +10,15 @@ import ( "github.com/grafana/dataplane/sdata/numeric" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" promValue "github.com/prometheus/prometheus/model/value" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/ngalert/eval" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model" @@ -53,13 +57,15 @@ func TestNewRemotePrometheusBackend(t *testing.T) { fakeWriter := new(fakeRemoteWriter) logger := log.NewNopLogger() + met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test") - backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger) + backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met) require.NotNil(t, backend) require.Equal(t, cfg.DatasourceUID, backend.cfg.DatasourceUID) require.Equal(t, fakeWriter, backend.promWriter) require.Equal(t, logger, backend.logger) + require.Equal(t, met, backend.metrics) } func createExpectedFrame(t *testing.T, ruleUID, ruleName, promState, grafanaState string, instanceLabels data.Labels, value float64) *data.Frame { @@ -262,7 +268,8 @@ func TestPrometheusBackend_Record(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeWriter := new(fakeRemoteWriter) - backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger) + met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test") + backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met) if tc.expectedFrames != nil { var extraLabels map[string]string @@ -298,8 +305,9 @@ func TestPrometheusBackend_Query(t *testing.T) { cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName} logger := log.NewNopLogger() fakeWriter := new(fakeRemoteWriter) + met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test") - backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger) + backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met) frame, err := backend.Query(context.Background(), ngmodels.HistoryQuery{}) require.Error(t, err) @@ -307,6 +315,93 @@ func TestPrometheusBackend_Query(t *testing.T) { require.Contains(t, err.Error(), "prometheus historian backend does not support querying") } +func TestPrometheusBackend_Record_Metrics(t *testing.T) { + cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName} + logger := log.NewNopLogger() + ctx := context.Background() + orgID := int64(1) + now := time.Now() + ruleMeta := history_model.RuleMeta{Title: "test rule"} + + t.Run("success metrics", func(t *testing.T) { + fakeWriter := new(fakeRemoteWriter) + fakeWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Return(nil).Once() + + registry := prometheus.NewRegistry() + met := metrics.NewHistorianMetrics(registry, "test") + backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met) + + states := []state.StateTransition{ + {State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}}, + {State: &state.State{AlertRuleUID: "rule-uid-2", OrgID: orgID, Labels: data.Labels{}, State: eval.Pending, LastEvaluationTime: now}}, + {State: &state.State{AlertRuleUID: "rule-uid-3", OrgID: orgID, Labels: data.Labels{}, State: eval.Normal, LastEvaluationTime: now}}, + } + + errCh := backend.Record(ctx, ruleMeta, states) + err, ok := <-errCh + require.True(t, ok) + require.NoError(t, err) + + // Only 2 frames generated (Alerting + Pending), Normal state doesn't generate frames + expectedMetrics := ` + # HELP grafana_test_state_history_writes_total The total number of state history batches that were attempted to be written. + # TYPE grafana_test_state_history_writes_total counter + grafana_test_state_history_writes_total{backend="prometheus",org="1"} 1 + # HELP grafana_test_state_history_transitions_total The total number of state transitions processed. + # TYPE grafana_test_state_history_transitions_total counter + grafana_test_state_history_transitions_total{org="1"} 2 + ` + + err = testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetrics), + "grafana_test_state_history_writes_total", + "grafana_test_state_history_transitions_total") + require.NoError(t, err) + fakeWriter.AssertExpectations(t) + }) + + t.Run("failure metrics", func(t *testing.T) { + fakeWriter := new(fakeRemoteWriter) + expectedErr := errors.New("write failed") + fakeWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Return(expectedErr).Once() + + registry := prometheus.NewRegistry() + met := metrics.NewHistorianMetrics(registry, "test") + backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met) + + states := []state.StateTransition{ + {State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}}, + } + + errCh := backend.Record(ctx, ruleMeta, states) + err, ok := <-errCh + require.True(t, ok) + require.Error(t, err) + + expectedMetrics := ` + # HELP grafana_test_state_history_writes_total The total number of state history batches that were attempted to be written. + # TYPE grafana_test_state_history_writes_total counter + grafana_test_state_history_writes_total{backend="prometheus",org="1"} 1 + # HELP grafana_test_state_history_writes_failed_total The total number of failed writes of state history batches. + # TYPE grafana_test_state_history_writes_failed_total counter + grafana_test_state_history_writes_failed_total{backend="prometheus",org="1"} 1 + # HELP grafana_test_state_history_transitions_total The total number of state transitions processed. + # TYPE grafana_test_state_history_transitions_total counter + grafana_test_state_history_transitions_total{org="1"} 1 + # HELP grafana_test_state_history_transitions_failed_total The total number of state transitions that failed to be written - they are not retried. + # TYPE grafana_test_state_history_transitions_failed_total counter + grafana_test_state_history_transitions_failed_total{org="1"} 1 + ` + + err = testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetrics), + "grafana_test_state_history_writes_total", + "grafana_test_state_history_writes_failed_total", + "grafana_test_state_history_transitions_total", + "grafana_test_state_history_transitions_failed_total") + require.NoError(t, err) + fakeWriter.AssertExpectations(t) + }) +} + func TestPrometheusBackend_Record_PanicRecovery(t *testing.T) { cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName} logger := log.NewNopLogger() @@ -320,7 +415,8 @@ func TestPrometheusBackend_Record_PanicRecovery(t *testing.T) { panicWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Once() - backend := NewRemotePrometheusBackend(cfg, panicWriter, logger) + met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test") + backend := NewRemotePrometheusBackend(cfg, panicWriter, logger, met) states := []state.StateTransition{ {State: &state.State{