public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Emit metrics from prometheus state history backend (#107121)

Browse Source 
Alerting: Emit metrics from prometheus historian backend
This commit is contained in:
Alexander Akhmetov
2025-06-24 18:27:52 +02:00
committed by GitHub
parent 2578c7cdc7
commit 478f9bf597
3 changed files with 112 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/ngalert.go
+1 -1
View File
@@ -708,7 +708,7 @@ func configureHistorianBackend(
if w == nil {
return nil, fmt.Errorf("failed to create alert state metrics writer")
}
backend := historian.NewRemotePrometheusBackend(pcfg, w, prometheusBackendLogger)
backend := historian.NewRemotePrometheusBackend(pcfg, w, prometheusBackendLogger, met)
return backend, nil
}
pkg/services/ngalert/state/historian/prometheus.go
+11 -1
View File
@@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
@@ -89,15 +90,17 @@ type RemotePrometheusBackend struct {
cfg PrometheusConfig
promWriter seriesWriter
logger log.Logger
metrics *metrics.Historian
}
func NewRemotePrometheusBackend(cfg PrometheusConfig, promWriter seriesWriter, logger log.Logger) *RemotePrometheusBackend {
func NewRemotePrometheusBackend(cfg PrometheusConfig, promWriter seriesWriter, logger log.Logger, metrics *metrics.Historian) *RemotePrometheusBackend {
logger.Info("Initializing remote Prometheus backend", "datasourceUID", cfg.DatasourceUID)
return &RemotePrometheusBackend{
cfg: cfg,
promWriter: promWriter,
logger: logger,
metrics: metrics,
}
}
@@ -141,9 +144,16 @@ func (b *RemotePrometheusBackend) Record(ctx context.Context, rule history_model
close(errCh)
}()
logger.Debug("Saving state history batch", "samples", len(frames))
org := fmt.Sprint(st.OrgID)
b.metrics.WritesTotal.WithLabelValues(org, "prometheus").Inc()
b.metrics.TransitionsTotal.WithLabelValues(org).Add(float64(len(frames)))
var sendErr error
if err := b.promWriter.WriteDatasource(ctx, b.cfg.DatasourceUID, b.cfg.MetricName, st.LastEvaluationTime, frames, st.OrgID, nil); err != nil {
logger.Error("Failed to write alert state metrics batch", "error", err)
b.metrics.WritesFailed.WithLabelValues(org, "prometheus").Inc()
b.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(len(frames)))
sendErr = err
}
errCh <- sendErr
pkg/services/ngalert/state/historian/prometheus_test.go
+100 -4
View File
@@ -1,6 +1,7 @@
package historian
import (
"bytes"
"context"
"errors"
"math"
@@ -9,12 +10,15 @@ import (
"github.com/grafana/dataplane/sdata/numeric"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
promValue "github.com/prometheus/prometheus/model/value"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
@@ -53,13 +57,15 @@ func TestNewRemotePrometheusBackend(t *testing.T) {
fakeWriter := new(fakeRemoteWriter)
logger := log.NewNopLogger()
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger)
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
require.NotNil(t, backend)
require.Equal(t, cfg.DatasourceUID, backend.cfg.DatasourceUID)
require.Equal(t, fakeWriter, backend.promWriter)
require.Equal(t, logger, backend.logger)
require.Equal(t, met, backend.metrics)
}
func createExpectedFrame(t *testing.T, ruleUID, ruleName, promState, grafanaState string, instanceLabels data.Labels, value float64) *data.Frame {
@@ -262,7 +268,8 @@ func TestPrometheusBackend_Record(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fakeWriter := new(fakeRemoteWriter)
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger)
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
if tc.expectedFrames != nil {
var extraLabels map[string]string
@@ -298,8 +305,9 @@ func TestPrometheusBackend_Query(t *testing.T) {
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
logger := log.NewNopLogger()
fakeWriter := new(fakeRemoteWriter)
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger)
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
frame, err := backend.Query(context.Background(), ngmodels.HistoryQuery{})
require.Error(t, err)
@@ -307,6 +315,93 @@ func TestPrometheusBackend_Query(t *testing.T) {
require.Contains(t, err.Error(), "prometheus historian backend does not support querying")
}
func TestPrometheusBackend_Record_Metrics(t *testing.T) {
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
logger := log.NewNopLogger()
ctx := context.Background()
orgID := int64(1)
now := time.Now()
ruleMeta := history_model.RuleMeta{Title: "test rule"}
t.Run("success metrics", func(t *testing.T) {
fakeWriter := new(fakeRemoteWriter)
fakeWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Return(nil).Once()
registry := prometheus.NewRegistry()
met := metrics.NewHistorianMetrics(registry, "test")
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
states := []state.StateTransition{
{State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}},
{State: &state.State{AlertRuleUID: "rule-uid-2", OrgID: orgID, Labels: data.Labels{}, State: eval.Pending, LastEvaluationTime: now}},
{State: &state.State{AlertRuleUID: "rule-uid-3", OrgID: orgID, Labels: data.Labels{}, State: eval.Normal, LastEvaluationTime: now}},
}
errCh := backend.Record(ctx, ruleMeta, states)
err, ok := <-errCh
require.True(t, ok)
require.NoError(t, err)
// Only 2 frames generated (Alerting + Pending), Normal state doesn't generate frames
expectedMetrics := `
# HELP grafana_test_state_history_writes_total The total number of state history batches that were attempted to be written.
# TYPE grafana_test_state_history_writes_total counter
grafana_test_state_history_writes_total{backend="prometheus",org="1"} 1
# HELP grafana_test_state_history_transitions_total The total number of state transitions processed.
# TYPE grafana_test_state_history_transitions_total counter
grafana_test_state_history_transitions_total{org="1"} 2
`
err = testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetrics),
"grafana_test_state_history_writes_total",
"grafana_test_state_history_transitions_total")
require.NoError(t, err)
fakeWriter.AssertExpectations(t)
})
t.Run("failure metrics", func(t *testing.T) {
fakeWriter := new(fakeRemoteWriter)
expectedErr := errors.New("write failed")
fakeWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Return(expectedErr).Once()
registry := prometheus.NewRegistry()
met := metrics.NewHistorianMetrics(registry, "test")
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
states := []state.StateTransition{
{State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}},
}
errCh := backend.Record(ctx, ruleMeta, states)
err, ok := <-errCh
require.True(t, ok)
require.Error(t, err)
expectedMetrics := `
# HELP grafana_test_state_history_writes_total The total number of state history batches that were attempted to be written.
# TYPE grafana_test_state_history_writes_total counter
grafana_test_state_history_writes_total{backend="prometheus",org="1"} 1
# HELP grafana_test_state_history_writes_failed_total The total number of failed writes of state history batches.
# TYPE grafana_test_state_history_writes_failed_total counter
grafana_test_state_history_writes_failed_total{backend="prometheus",org="1"} 1
# HELP grafana_test_state_history_transitions_total The total number of state transitions processed.
# TYPE grafana_test_state_history_transitions_total counter
grafana_test_state_history_transitions_total{org="1"} 1
# HELP grafana_test_state_history_transitions_failed_total The total number of state transitions that failed to be written - they are not retried.
# TYPE grafana_test_state_history_transitions_failed_total counter
grafana_test_state_history_transitions_failed_total{org="1"} 1
`
err = testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetrics),
"grafana_test_state_history_writes_total",
"grafana_test_state_history_writes_failed_total",
"grafana_test_state_history_transitions_total",
"grafana_test_state_history_transitions_failed_total")
require.NoError(t, err)
fakeWriter.AssertExpectations(t)
})
}
func TestPrometheusBackend_Record_PanicRecovery(t *testing.T) {
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
logger := log.NewNopLogger()
@@ -320,7 +415,8 @@ func TestPrometheusBackend_Record_PanicRecovery(t *testing.T) {
panicWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Once()
backend := NewRemotePrometheusBackend(cfg, panicWriter, logger)
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
backend := NewRemotePrometheusBackend(cfg, panicWriter, logger, met)
states := []state.StateTransition{
{State: &state.State{