diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 04fa3a081cd..745b0476c19 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -35,7 +35,7 @@ func TestSendingToExternalAlertmanager(t *testing.T) { fakeAM := NewFakeExternalAlertmanager(t) defer fakeAM.Close() fakeRuleStore := newFakeRuleStore(t) - fakeInstanceStore := &fakeInstanceStore{} + fakeInstanceStore := &FakeInstanceStore{} fakeAdminConfigStore := newFakeAdminConfigStore(t) // create alert rule with one second interval @@ -100,7 +100,7 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) { fakeAM := NewFakeExternalAlertmanager(t) defer fakeAM.Close() fakeRuleStore := newFakeRuleStore(t) - fakeInstanceStore := &fakeInstanceStore{} + fakeInstanceStore := &FakeInstanceStore{} fakeAdminConfigStore := newFakeAdminConfigStore(t) // First, let's create an admin configuration that holds an alertmanager. @@ -240,9 +240,9 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) { func TestSchedule_ruleRoutine(t *testing.T) { createSchedule := func( evalAppliedChan chan time.Time, - ) (*schedule, *fakeRuleStore, *fakeInstanceStore, *fakeAdminConfigStore, prometheus.Gatherer) { + ) (*schedule, *fakeRuleStore, *FakeInstanceStore, *fakeAdminConfigStore, prometheus.Gatherer) { ruleStore := newFakeRuleStore(t) - instanceStore := &fakeInstanceStore{} + instanceStore := &FakeInstanceStore{} adminConfigStore := newFakeAdminConfigStore(t) registry := prometheus.NewPedanticRegistry() diff --git a/pkg/services/ngalert/schedule/testing.go b/pkg/services/ngalert/schedule/testing.go index e5c508ea73d..8ae16425ac9 100644 --- a/pkg/services/ngalert/schedule/testing.go +++ b/pkg/services/ngalert/schedule/testing.go @@ -239,32 +239,32 @@ func (f *fakeRuleStore) UpdateRuleGroup(cmd store.UpdateRuleGroupCmd) error { return nil } -type fakeInstanceStore struct { +type FakeInstanceStore struct { mtx sync.Mutex recordedOps []interface{} } -func (f *fakeInstanceStore) GetAlertInstance(q *models.GetAlertInstanceQuery) error { +func (f *FakeInstanceStore) GetAlertInstance(q *models.GetAlertInstanceQuery) error { f.mtx.Lock() defer f.mtx.Unlock() f.recordedOps = append(f.recordedOps, *q) return nil } -func (f *fakeInstanceStore) ListAlertInstances(q *models.ListAlertInstancesQuery) error { +func (f *FakeInstanceStore) ListAlertInstances(q *models.ListAlertInstancesQuery) error { f.mtx.Lock() defer f.mtx.Unlock() f.recordedOps = append(f.recordedOps, *q) return nil } -func (f *fakeInstanceStore) SaveAlertInstance(q *models.SaveAlertInstanceCommand) error { +func (f *FakeInstanceStore) SaveAlertInstance(q *models.SaveAlertInstanceCommand) error { f.mtx.Lock() defer f.mtx.Unlock() f.recordedOps = append(f.recordedOps, *q) return nil } -func (f *fakeInstanceStore) FetchOrgIds() ([]int64, error) { return []int64{}, nil } -func (f *fakeInstanceStore) DeleteAlertInstance(_ int64, _, _ string) error { return nil } +func (f *FakeInstanceStore) FetchOrgIds() ([]int64, error) { return []int64{}, nil } +func (f *FakeInstanceStore) DeleteAlertInstance(_ int64, _, _ string) error { return nil } func newFakeAdminConfigStore(t *testing.T) *fakeAdminConfigStore { t.Helper() diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index c12f90a3a99..8d87c9b1cc1 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -6,10 +6,12 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/schedule" "github.com/grafana/grafana/pkg/services/ngalert/state" "github.com/grafana/grafana/pkg/services/ngalert/tests" @@ -356,6 +358,194 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, + { + desc: "normal -> alerting -> noData -> alerting when For is set", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 20 * time.Second, + NoDataState: models.NoData, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(20 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(30 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(40 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + }, + expectedStates: map[string]*state.State{ + `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Pending, + Results: []state.Evaluation{ + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), + }, + { + EvaluationTime: evaluationTime.Add(20 * time.Second), + EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), + }, + { + EvaluationTime: evaluationTime.Add(30 * time.Second), + EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), + }, + { + EvaluationTime: evaluationTime.Add(40 * time.Second), + EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), + }, + }, + StartsAt: evaluationTime.Add(30 * time.Second), + EndsAt: evaluationTime.Add(30 * time.Second).Add(state.ResendDelay * 3), + LastEvaluationTime: evaluationTime.Add(40 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "pending -> alerting -> noData when For is set and NoDataState is NoData", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 20 * time.Second, + NoDataState: models.NoData, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime, + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(20 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(30 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + }, + expectedStates: map[string]*state.State{ + `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.NoData, + Results: []state.Evaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), + }, + { + EvaluationTime: evaluationTime.Add(20 * time.Second), + EvaluationState: eval.Alerting, + Values: make(map[string]state.EvaluationValue), + }, + { + EvaluationTime: evaluationTime.Add(30 * time.Second), + EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), + }, + }, + StartsAt: evaluationTime, + EndsAt: evaluationTime.Add(30 * time.Second).Add(state.ResendDelay * 3), + LastEvaluationTime: evaluationTime.Add(30 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, { desc: "normal -> pending when For is set but not exceeded and first result is normal", alertRule: &models.AlertRule{ @@ -608,6 +798,192 @@ func TestProcessEvalResults(t *testing.T) { }, }, }, + { + desc: "normal -> nodata no labels when result is NoData and NoDataState is nodata", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + NoDataState: models.NoData, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + }, + expectedStates: map[string]*state.State{ + `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + }, + State: eval.NoData, + Results: []state.Evaluation{ + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal (multi-dimensional) -> nodata no labels when result is NoData and NoDataState is nodata", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + NoDataState: models.NoData, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test-1"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + EvaluationDuration: evaluationDuration, + }, + eval.Result{ + Instance: data.Labels{"instance_label": "test-2"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + }, + expectedStates: map[string]*state.State{ + `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + }, + State: eval.NoData, + Results: []state.Evaluation{ + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + Values: make(map[string]state.EvaluationValue), + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> nodata no labels -> normal when result is NoData and NoDataState is nodata", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + NoDataState: models.NoData, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime.Add(20 * time.Second), + EvaluationDuration: evaluationDuration, + }, + }, + }, + expectedStates: map[string]*state.State{ + `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Normal, + Results: []state.Evaluation{ + { + EvaluationTime: evaluationTime.Add(20 * time.Second), + EvaluationState: eval.Normal, + Values: make(map[string]state.EvaluationValue), + }, + }, + StartsAt: time.Time{}, + EndsAt: time.Time{}, + LastEvaluationTime: evaluationTime.Add(20 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, { desc: "normal -> normal when result is NoData and NoDataState is ok", alertRule: &models.AlertRule{ @@ -852,11 +1228,15 @@ func TestProcessEvalResults(t *testing.T) { } for _, tc := range testCases { - st := state.NewManager(log.New("test_state_manager"), testMetrics.GetStateMetrics(), nil, nil, nil) + st := state.NewManager(log.New("test_state_manager"), testMetrics.GetStateMetrics(), nil, nil, &schedule.FakeInstanceStore{}) t.Run(tc.desc, func(t *testing.T) { for _, res := range tc.evalResults { _ = st.ProcessEvalResults(context.Background(), tc.alertRule, res) } + + states := st.GetStatesForRuleUID(tc.alertRule.OrgID, tc.alertRule.UID) + assert.Len(t, states, len(tc.expectedStates)) + for _, s := range tc.expectedStates { cachedState, err := st.Get(s.OrgID, s.AlertRuleUID, s.CacheId) require.NoError(t, err)