459 lines
18 KiB
Go
459 lines
18 KiB
Go
package historian
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/grafana/dataplane/sdata/numeric"
|
|
"github.com/grafana/grafana-plugin-sdk-go/data"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
promValue "github.com/prometheus/prometheus/model/value"
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
|
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
|
history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
|
|
"github.com/grafana/grafana/pkg/setting"
|
|
)
|
|
|
|
const (
|
|
testMetricName = "test_metric_name"
|
|
)
|
|
|
|
type fakeRemoteWriter struct {
|
|
mock.Mock
|
|
}
|
|
|
|
func (f *fakeRemoteWriter) WriteDatasource(ctx context.Context, dsUID string, name string, t time.Time, frames data.Frames, orgID int64, extraLabels map[string]string) error {
|
|
args := f.Called(ctx, dsUID, name, t, frames, orgID, extraLabels)
|
|
return args.Error(0)
|
|
}
|
|
|
|
type panicRemoteWriter struct {
|
|
mock.Mock
|
|
panicMessage string
|
|
}
|
|
|
|
func (p *panicRemoteWriter) WriteDatasource(ctx context.Context, dsUID string, name string, t time.Time, frames data.Frames, orgID int64, extraLabels map[string]string) error {
|
|
p.Called(ctx, dsUID, name, t, frames, orgID, extraLabels)
|
|
panic(p.panicMessage)
|
|
}
|
|
|
|
func TestNewRemotePrometheusBackend(t *testing.T) {
|
|
cfg, err := NewPrometheusConfig(setting.UnifiedAlertingStateHistorySettings{
|
|
PrometheusTargetDatasourceUID: "test-ds-uid",
|
|
PrometheusMetricName: testMetricName,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
fakeWriter := new(fakeRemoteWriter)
|
|
logger := log.NewNopLogger()
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
|
|
|
|
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
|
|
|
|
require.NotNil(t, backend)
|
|
require.Equal(t, cfg.DatasourceUID, backend.cfg.DatasourceUID)
|
|
require.Equal(t, fakeWriter, backend.promWriter)
|
|
require.Equal(t, logger, backend.logger)
|
|
require.Equal(t, met, backend.metrics)
|
|
}
|
|
|
|
func createExpectedFrame(t *testing.T, ruleUID, ruleName, promState, grafanaState string, instanceLabels data.Labels, value float64) *data.Frame {
|
|
t.Helper()
|
|
|
|
labels := instanceLabels.Copy()
|
|
labels[alertRuleUIDLabel] = ruleUID
|
|
labels[alertNameLabel] = ruleName
|
|
labels[alertStateLabel] = promState
|
|
labels[grafanaAlertStateLabel] = grafanaState
|
|
|
|
valueField := data.NewField("", labels, []float64{value})
|
|
|
|
frame := data.NewFrame(testMetricName, valueField)
|
|
frame.SetMeta(&data.FrameMeta{
|
|
Type: data.FrameTypeNumericMulti,
|
|
TypeVersion: numeric.MultiFrameVersionLatest,
|
|
})
|
|
return frame
|
|
}
|
|
|
|
func createTransition(from, to eval.State, orgID int64, now time.Time) state.StateTransition {
|
|
return state.StateTransition{
|
|
State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{"instance": "server1"}, State: to, LastEvaluationTime: now},
|
|
PreviousState: from,
|
|
}
|
|
}
|
|
|
|
func assertFramesEqual(t *testing.T, actualFrames data.Frames, expectedFrames data.Frames) {
|
|
t.Helper()
|
|
|
|
require.Len(t, actualFrames, len(expectedFrames))
|
|
|
|
for i, expectedFrame := range expectedFrames {
|
|
actualFrame := actualFrames[i]
|
|
require.Equal(t, expectedFrame.Name, actualFrame.Name)
|
|
require.Len(t, actualFrame.Fields, 1)
|
|
|
|
expectedField := expectedFrame.Fields[0]
|
|
actualField := actualFrame.Fields[0]
|
|
|
|
// Check labels
|
|
require.Equal(t, expectedField.Labels, actualField.Labels)
|
|
|
|
// Check values with NaN handling
|
|
expectedValue := expectedField.At(0).(float64)
|
|
actualValue := actualField.At(0).(float64)
|
|
if math.IsNaN(expectedValue) {
|
|
require.True(t, math.IsNaN(actualValue))
|
|
} else {
|
|
require.Equal(t, expectedValue, actualValue)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPrometheusBackend_Record(t *testing.T) {
|
|
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
|
|
logger := log.NewNopLogger()
|
|
ctx := context.Background()
|
|
orgID := int64(1)
|
|
now := time.Now()
|
|
ruleMeta := history_model.RuleMeta{Title: "test rule"}
|
|
|
|
testCases := []struct {
|
|
name string
|
|
ruleMeta history_model.RuleMeta
|
|
states []state.StateTransition
|
|
expectedErr error
|
|
expectedFrames data.Frames
|
|
}{
|
|
{
|
|
name: "No states",
|
|
ruleMeta: history_model.RuleMeta{Title: "Test Rule No States"},
|
|
states: []state.StateTransition{},
|
|
},
|
|
{
|
|
name: "normal state only (no metrics emitted)",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{
|
|
{State: &state.State{AlertRuleUID: "rule-uid-normal", OrgID: orgID, Labels: data.Labels{"label1": "value1"}, State: eval.Normal, LastEvaluationTime: now}},
|
|
},
|
|
},
|
|
{
|
|
name: "remote writer error",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{
|
|
{State: &state.State{AlertRuleUID: "rule-uid-err", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}},
|
|
},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid-err", "test rule", "firing", "alerting", data.Labels{}, 1),
|
|
},
|
|
expectedErr: errors.New("remote write failed"),
|
|
},
|
|
{
|
|
name: "internal labels are skipped",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{
|
|
{
|
|
State: &state.State{
|
|
AlertRuleUID: "rule-uid-internal",
|
|
OrgID: orgID,
|
|
Labels: data.Labels{ngmodels.AutogeneratedRouteLabel: "ignored", "label1": "value1", "__label2": "value2"},
|
|
State: eval.Alerting,
|
|
LastEvaluationTime: now,
|
|
},
|
|
},
|
|
},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid-internal", "test rule", "firing", "alerting", data.Labels{"label1": "value1"}, 1.0),
|
|
},
|
|
},
|
|
{
|
|
name: "mixed states (normal, pending, recovering, error, nodata)",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{
|
|
{State: &state.State{AlertRuleUID: "rule-uid-normal", OrgID: orgID, Labels: data.Labels{"state": "normal"}, State: eval.Normal, LastEvaluationTime: now}},
|
|
{State: &state.State{AlertRuleUID: "rule-uid-pending", OrgID: orgID, Labels: data.Labels{"state": "pending"}, State: eval.Pending, LastEvaluationTime: now}},
|
|
{State: &state.State{AlertRuleUID: "rule-uid-recovering", OrgID: orgID, Labels: data.Labels{"state": "recovering"}, State: eval.Recovering, LastEvaluationTime: now}},
|
|
{State: &state.State{AlertRuleUID: "rule-uid-error", OrgID: orgID, Labels: data.Labels{"state": "error"}, State: eval.Error, LastEvaluationTime: now}},
|
|
{State: &state.State{AlertRuleUID: "rule-uid-nodata", OrgID: orgID, Labels: data.Labels{"state": "nodata"}, State: eval.NoData, LastEvaluationTime: now}},
|
|
},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid-pending", "test rule", "pending", "pending", data.Labels{"state": "pending"}, 1.0),
|
|
createExpectedFrame(t, "rule-uid-recovering", "test rule", "firing", "recovering", data.Labels{"state": "recovering"}, 1.0),
|
|
createExpectedFrame(t, "rule-uid-error", "test rule", "firing", "error", data.Labels{"state": "error"}, 1.0),
|
|
createExpectedFrame(t, "rule-uid-nodata", "test rule", "firing", "nodata", data.Labels{"state": "nodata"}, 1.0),
|
|
},
|
|
},
|
|
|
|
// State transitions - Normal to other states (single active frame)
|
|
{
|
|
name: "normal to alerting transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Normal, eval.Alerting, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "alerting", data.Labels{"instance": "server1"}, 1.0),
|
|
},
|
|
},
|
|
{
|
|
name: "normal to pending transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Normal, eval.Pending, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "pending", "pending", data.Labels{"instance": "server1"}, 1.0),
|
|
},
|
|
},
|
|
{
|
|
name: "normal to error transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Normal, eval.Error, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "error", data.Labels{"instance": "server1"}, 1.0),
|
|
},
|
|
},
|
|
|
|
// Transitions to Normal (StaleNaN only)
|
|
{
|
|
name: "alerting to normal transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Alerting, eval.Normal, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "alerting", data.Labels{"instance": "server1"}, math.Float64frombits(promValue.StaleNaN)),
|
|
},
|
|
},
|
|
{
|
|
name: "error to normal transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Error, eval.Normal, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "error", data.Labels{"instance": "server1"}, math.Float64frombits(promValue.StaleNaN)),
|
|
},
|
|
},
|
|
{
|
|
name: "pending to alerting transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Pending, eval.Alerting, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "pending", "pending", data.Labels{"instance": "server1"}, math.Float64frombits(promValue.StaleNaN)),
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "alerting", data.Labels{"instance": "server1"}, 1.0),
|
|
},
|
|
},
|
|
{
|
|
name: "alerting to recovering transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Alerting, eval.Recovering, orgID, now)},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "alerting", data.Labels{"instance": "server1"}, math.Float64frombits(promValue.StaleNaN)),
|
|
createExpectedFrame(t, "rule-uid", "test rule", "firing", "recovering", data.Labels{"instance": "server1"}, 1.0),
|
|
},
|
|
},
|
|
|
|
// No metric should be written
|
|
{
|
|
name: "Normal to Normal transition",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{createTransition(eval.Normal, eval.Normal, orgID, now)},
|
|
},
|
|
{
|
|
name: "labels with invalid characters are sanitized",
|
|
ruleMeta: ruleMeta,
|
|
states: []state.StateTransition{
|
|
{
|
|
State: &state.State{
|
|
AlertRuleUID: "rule-uid-sanitize",
|
|
OrgID: orgID,
|
|
Labels: data.Labels{"valid-label": "value1", "invalid.label": "value2", "label-with-dash": "value3", "123starts-with-number": "value4", "has spaces": "value5"},
|
|
State: eval.Alerting,
|
|
LastEvaluationTime: now,
|
|
},
|
|
},
|
|
},
|
|
expectedFrames: data.Frames{
|
|
createExpectedFrame(t, "rule-uid-sanitize", "test rule", "firing", "alerting", data.Labels{"valid_label": "value1", "invalid_label": "value2", "label_with_dash": "value3", "_23starts_with_number": "value4", "has_spaces": "value5"}, 1.0),
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
fakeWriter := new(fakeRemoteWriter)
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
|
|
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
|
|
|
|
if tc.expectedFrames != nil {
|
|
var extraLabels map[string]string
|
|
fakeWriter.On(
|
|
"WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, extraLabels,
|
|
).Return(tc.expectedErr).Once().Run(func(args mock.Arguments) {
|
|
if tc.expectedErr == nil {
|
|
actualFrames := args.Get(4).(data.Frames)
|
|
assertFramesEqual(t, actualFrames, tc.expectedFrames)
|
|
}
|
|
})
|
|
}
|
|
|
|
errCh := backend.Record(ctx, tc.ruleMeta, tc.states)
|
|
err, ok := <-errCh
|
|
require.True(t, ok)
|
|
|
|
if tc.expectedErr == nil {
|
|
require.Nil(t, err)
|
|
} else {
|
|
require.ErrorIs(t, err, tc.expectedErr)
|
|
}
|
|
|
|
fakeWriter.AssertExpectations(t)
|
|
if tc.expectedFrames == nil {
|
|
fakeWriter.AssertNotCalled(t, "WriteDatasource", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPrometheusBackend_Query(t *testing.T) {
|
|
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
|
|
logger := log.NewNopLogger()
|
|
fakeWriter := new(fakeRemoteWriter)
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
|
|
|
|
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
|
|
|
|
frame, err := backend.Query(context.Background(), ngmodels.HistoryQuery{})
|
|
require.Error(t, err)
|
|
require.Nil(t, frame)
|
|
require.Contains(t, err.Error(), "prometheus historian backend does not support querying")
|
|
}
|
|
|
|
func TestPrometheusBackend_Record_Metrics(t *testing.T) {
|
|
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
|
|
logger := log.NewNopLogger()
|
|
ctx := context.Background()
|
|
orgID := int64(1)
|
|
now := time.Now()
|
|
ruleMeta := history_model.RuleMeta{Title: "test rule"}
|
|
|
|
t.Run("success metrics", func(t *testing.T) {
|
|
fakeWriter := new(fakeRemoteWriter)
|
|
fakeWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Return(nil).Once()
|
|
|
|
registry := prometheus.NewRegistry()
|
|
met := metrics.NewHistorianMetrics(registry, "test")
|
|
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
|
|
|
|
states := []state.StateTransition{
|
|
{State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}},
|
|
{State: &state.State{AlertRuleUID: "rule-uid-2", OrgID: orgID, Labels: data.Labels{}, State: eval.Pending, LastEvaluationTime: now}},
|
|
{State: &state.State{AlertRuleUID: "rule-uid-3", OrgID: orgID, Labels: data.Labels{}, State: eval.Normal, LastEvaluationTime: now}},
|
|
}
|
|
|
|
errCh := backend.Record(ctx, ruleMeta, states)
|
|
err, ok := <-errCh
|
|
require.True(t, ok)
|
|
require.NoError(t, err)
|
|
|
|
// Only 2 frames generated (Alerting + Pending), Normal state doesn't generate frames
|
|
expectedMetrics := `
|
|
# HELP grafana_test_state_history_writes_total The total number of state history batches that were attempted to be written.
|
|
# TYPE grafana_test_state_history_writes_total counter
|
|
grafana_test_state_history_writes_total{backend="prometheus",org="1"} 1
|
|
# HELP grafana_test_state_history_transitions_total The total number of state transitions processed.
|
|
# TYPE grafana_test_state_history_transitions_total counter
|
|
grafana_test_state_history_transitions_total{org="1"} 2
|
|
`
|
|
|
|
err = testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetrics),
|
|
"grafana_test_state_history_writes_total",
|
|
"grafana_test_state_history_transitions_total")
|
|
require.NoError(t, err)
|
|
fakeWriter.AssertExpectations(t)
|
|
})
|
|
|
|
t.Run("failure metrics", func(t *testing.T) {
|
|
fakeWriter := new(fakeRemoteWriter)
|
|
expectedErr := errors.New("write failed")
|
|
fakeWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Return(expectedErr).Once()
|
|
|
|
registry := prometheus.NewRegistry()
|
|
met := metrics.NewHistorianMetrics(registry, "test")
|
|
backend := NewRemotePrometheusBackend(cfg, fakeWriter, logger, met)
|
|
|
|
states := []state.StateTransition{
|
|
{State: &state.State{AlertRuleUID: "rule-uid", OrgID: orgID, Labels: data.Labels{}, State: eval.Alerting, LastEvaluationTime: now}},
|
|
}
|
|
|
|
errCh := backend.Record(ctx, ruleMeta, states)
|
|
err, ok := <-errCh
|
|
require.True(t, ok)
|
|
require.Error(t, err)
|
|
|
|
expectedMetrics := `
|
|
# HELP grafana_test_state_history_writes_total The total number of state history batches that were attempted to be written.
|
|
# TYPE grafana_test_state_history_writes_total counter
|
|
grafana_test_state_history_writes_total{backend="prometheus",org="1"} 1
|
|
# HELP grafana_test_state_history_writes_failed_total The total number of failed writes of state history batches.
|
|
# TYPE grafana_test_state_history_writes_failed_total counter
|
|
grafana_test_state_history_writes_failed_total{backend="prometheus",org="1"} 1
|
|
# HELP grafana_test_state_history_transitions_total The total number of state transitions processed.
|
|
# TYPE grafana_test_state_history_transitions_total counter
|
|
grafana_test_state_history_transitions_total{org="1"} 1
|
|
# HELP grafana_test_state_history_transitions_failed_total The total number of state transitions that failed to be written - they are not retried.
|
|
# TYPE grafana_test_state_history_transitions_failed_total counter
|
|
grafana_test_state_history_transitions_failed_total{org="1"} 1
|
|
`
|
|
|
|
err = testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetrics),
|
|
"grafana_test_state_history_writes_total",
|
|
"grafana_test_state_history_writes_failed_total",
|
|
"grafana_test_state_history_transitions_total",
|
|
"grafana_test_state_history_transitions_failed_total")
|
|
require.NoError(t, err)
|
|
fakeWriter.AssertExpectations(t)
|
|
})
|
|
}
|
|
|
|
func TestPrometheusBackend_Record_PanicRecovery(t *testing.T) {
|
|
cfg := PrometheusConfig{DatasourceUID: "test-ds-uid", MetricName: testMetricName}
|
|
logger := log.NewNopLogger()
|
|
ctx := context.Background()
|
|
orgID := int64(1)
|
|
now := time.Now()
|
|
ruleMeta := history_model.RuleMeta{Title: "test rule"}
|
|
|
|
panicMessage := "panic in WriteDatasource"
|
|
panicWriter := &panicRemoteWriter{panicMessage: panicMessage}
|
|
|
|
panicWriter.On("WriteDatasource", ctx, cfg.DatasourceUID, testMetricName, now, mock.Anything, orgID, mock.Anything).Once()
|
|
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), "test")
|
|
backend := NewRemotePrometheusBackend(cfg, panicWriter, logger, met)
|
|
|
|
states := []state.StateTransition{
|
|
{State: &state.State{
|
|
AlertRuleUID: "rule-uid-panic",
|
|
OrgID: orgID,
|
|
Labels: data.Labels{"test": "panic"},
|
|
State: eval.Alerting,
|
|
LastEvaluationTime: now,
|
|
}},
|
|
}
|
|
|
|
errCh := backend.Record(ctx, ruleMeta, states)
|
|
|
|
err, ok := <-errCh
|
|
require.True(t, ok)
|
|
require.Error(t, err)
|
|
require.ErrorContains(t, err, "prometheus historian panic")
|
|
require.ErrorContains(t, err, panicMessage)
|
|
|
|
panicWriter.AssertExpectations(t)
|
|
}
|