diff --git a/pkg/services/ngalert/api/api_prometheus.go b/pkg/services/ngalert/api/api_prometheus.go index 8bcd531aced..7568e11dfe4 100644 --- a/pkg/services/ngalert/api/api_prometheus.go +++ b/pkg/services/ngalert/api/api_prometheus.go @@ -5,6 +5,7 @@ import ( "net/http" "time" + "github.com/grafana/grafana/pkg/services/ngalert/eval" apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -84,18 +85,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res LastEvaluation: time.Time{}, EvaluationTime: 0, // TODO: see if we are able to pass this along with evaluation results } - for _, rule := range alertRuleQuery.Result { - instanceQuery := ngmodels.ListAlertInstancesQuery{ - DefinitionOrgID: c.SignedInUser.OrgId, - DefinitionUID: rule.UID, - } - if err := srv.store.ListAlertInstances(&instanceQuery); err != nil { - ruleResponse.DiscoveryBase.Status = "error" - ruleResponse.DiscoveryBase.Error = fmt.Sprintf("failure getting alerts for rule %s: %s", rule.UID, err.Error()) - ruleResponse.DiscoveryBase.ErrorType = apiv1.ErrServer - return response.JSON(http.StatusInternalServerError, ruleResponse) - } + stateMap := srv.stateTracker.GetStatesByRuleUID() + for _, rule := range alertRuleQuery.Result { alertingRule := apimodels.AlertingRule{ State: "inactive", Name: rule.Title, @@ -106,36 +98,46 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res newRule := apimodels.Rule{ Name: rule.Title, - Labels: nil, // TODO: NG AlertRule does not have labels but does have annotations + Labels: rule.Labels, Health: "ok", // TODO: update this in the future when error and noData states are being evaluated and set Type: apiv1.RuleTypeAlerting, - LastEvaluation: time.Time{}, // TODO: set this to be rule evaluation time once it is being set - EvaluationTime: 0, // TODO: set this once we are saving it or adding it to evaluation results + LastEvaluation: time.Time{}, } - for _, instance := range instanceQuery.Result { - activeAt := instance.CurrentStateSince + + for _, alertState := range stateMap[rule.UID] { + activeAt := alertState.StartsAt alert := &apimodels.Alert{ - Labels: map[string]string(instance.Labels), - Annotations: nil, // TODO: set these once they are added to evaluation results - State: translateInstanceState(instance.CurrentState), + Labels: map[string]string(alertState.Labels), + Annotations: alertState.Annotations, + State: alertState.State.String(), ActiveAt: &activeAt, Value: "", // TODO: set this once it is added to the evaluation results } - if instance.LastEvalTime.After(newRule.LastEvaluation) { - newRule.LastEvaluation = instance.LastEvalTime - newGroup.LastEvaluation = instance.LastEvalTime + + if alertState.LastEvaluationTime.After(newRule.LastEvaluation) { + newRule.LastEvaluation = alertState.LastEvaluationTime + newGroup.LastEvaluation = alertState.LastEvaluationTime } - switch alert.State { - case "pending": + + alertingRule.Duration = alertState.EvaluationDuration.Seconds() + newRule.EvaluationTime = alertState.EvaluationDuration.Seconds() + + switch alertState.State { + case eval.Normal: + case eval.Pending: if alertingRule.State == "inactive" { alertingRule.State = "pending" } - case "firing": + case eval.Alerting: alertingRule.State = "firing" + case eval.Error: + // handle Error case based on configuration in alertRule + case eval.NoData: + // handle NoData case based on configuration in alertRule } - alertingRule.Alerts = append(alertingRule.Alerts, alert) } + alertingRule.Rule = newRule newGroup.Rules = append(newGroup.Rules, alertingRule) newGroup.Interval = float64(rule.IntervalSeconds) @@ -144,16 +146,3 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res } return response.JSON(http.StatusOK, ruleResponse) } - -func translateInstanceState(state ngmodels.InstanceStateType) string { - switch { - case state == ngmodels.InstanceStateFiring: - return "firing" - case state == ngmodels.InstanceStateNormal: - return "inactive" - case state == ngmodels.InstanceStatePending: - return "pending" - default: - return "inactive" - } -} diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index 84f4001119a..b9a5a8e512f 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -76,6 +76,11 @@ const ( // that evaluated to true (Alerting). Alerting + // Pending is the eval state for an alert instance condition + // that evaluated to true (Alerting) but has not yet met + // the For duration defined in AlertRule + Pending + // NoData is the eval state for an alert rule condition // that evaluated to NoData. NoData @@ -86,7 +91,7 @@ const ( ) func (s State) String() string { - return [...]string{"Normal", "Alerting", "NoData", "Error"}[s] + return [...]string{"Normal", "Alerting", "Pending", "NoData", "Error"}[s] } // AlertExecCtx is the context provided for executing an alert condition. diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index f4832bbef52..f003807d5d5 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -78,7 +78,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key models.AlertRul return err } - processedStates := stateTracker.ProcessEvalResults(alertRule, results) + processedStates := stateTracker.ProcessEvalResults(alertRule, results, end.Sub(start)) sch.saveAlertStates(processedStates) alerts := FromAlertStateToPostableAlerts(processedStates) sch.log.Debug("sending alerts to notifier", "count", len(alerts.PostableAlerts)) @@ -321,7 +321,7 @@ func (sch *schedule) saveAlertStates(states []state.AlertState) { for _, s := range states { cmd := models.SaveAlertInstanceCommand{ DefinitionOrgID: s.OrgID, - DefinitionUID: s.UID, + DefinitionUID: s.AlertRuleUID, Labels: models.InstanceLabels(s.Labels), State: models.InstanceStateType(s.State.String()), LastEvalTime: s.LastEvaluationTime, @@ -330,7 +330,7 @@ func (sch *schedule) saveAlertStates(states []state.AlertState) { } err := sch.store.SaveAlertInstance(&cmd) if err != nil { - sch.log.Error("failed to save alert state", "uid", s.UID, "orgId", s.OrgID, "labels", s.Labels.String(), "state", s.State.String(), "msg", err.Error()) + sch.log.Error("failed to save alert state", "uid", s.AlertRuleUID, "orgId", s.OrgID, "labels", s.Labels.String(), "state", s.State.String(), "msg", err.Error()) } } } @@ -355,7 +355,7 @@ func (sch *schedule) WarmStateCache(st *state.StateTracker) { for _, entry := range cmd.Result { lbs := map[string]string(entry.Labels) stateForEntry := state.AlertState{ - UID: entry.DefinitionUID, + AlertRuleUID: entry.DefinitionUID, OrgID: entry.DefinitionOrgID, CacheId: fmt.Sprintf("%s %s", entry.DefinitionUID, lbs), Labels: lbs, diff --git a/pkg/services/ngalert/state/state_tracker.go b/pkg/services/ngalert/state/state_tracker.go index 8d2576fd6a0..4f62ef5d32a 100644 --- a/pkg/services/ngalert/state/state_tracker.go +++ b/pkg/services/ngalert/state/state_tracker.go @@ -5,6 +5,8 @@ import ( "sync" "time" + prometheusModel "github.com/prometheus/common/model" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana-plugin-sdk-go/data" @@ -13,7 +15,7 @@ import ( ) type AlertState struct { - UID string + AlertRuleUID string OrgID int64 CacheId string Labels data.Labels @@ -22,6 +24,7 @@ type AlertState struct { StartsAt time.Time EndsAt time.Time LastEvaluationTime time.Time + EvaluationDuration time.Duration Annotations map[string]string } @@ -31,21 +34,20 @@ type StateEvaluation struct { } type cache struct { - cacheMap map[string]AlertState - mu sync.Mutex + states map[string]AlertState + mtxStates sync.Mutex } type StateTracker struct { - stateCache cache - quit chan struct{} - Log log.Logger + cache cache + quit chan struct{} + Log log.Logger } func NewStateTracker(logger log.Logger) *StateTracker { tracker := &StateTracker{ - stateCache: cache{ - cacheMap: make(map[string]AlertState), - mu: sync.Mutex{}, + cache: cache{ + states: make(map[string]AlertState), }, quit: make(chan struct{}), Log: logger, @@ -54,135 +56,158 @@ func NewStateTracker(logger log.Logger) *StateTracker { return tracker } -func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result) AlertState { - st.stateCache.mu.Lock() - defer st.stateCache.mu.Unlock() - lbs := data.Labels{} - if len(result.Instance) > 0 { - lbs = result.Instance - } +func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState { + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + + // if duplicate labels exist, alertRule label will take precedence + lbs := mergeLabels(alertRule.Labels, result.Instance) lbs["__alert_rule_uid__"] = alertRule.UID lbs["__alert_rule_namespace_uid__"] = alertRule.NamespaceUID - lbs["__alert_rule_title__"] = alertRule.Title + lbs[prometheusModel.AlertNameLabel] = alertRule.Title + + id := fmt.Sprintf("%s", map[string]string(lbs)) + if state, ok := st.cache.states[id]; ok { + return state + } annotations := map[string]string{} if len(alertRule.Annotations) > 0 { annotations = alertRule.Annotations } - idString := fmt.Sprintf("%s", map[string]string(lbs)) - if state, ok := st.stateCache.cacheMap[idString]; ok { - return state + newResults := []StateEvaluation{ + { + EvaluationTime: result.EvaluatedAt, + EvaluationState: result.State, + }, } - st.Log.Debug("adding new alert state cache entry", "cacheId", idString, "state", result.State.String(), "evaluatedAt", result.EvaluatedAt.String()) + st.Log.Debug("adding new alert state cache entry", "cacheId", id, "state", result.State.String(), "evaluatedAt", result.EvaluatedAt.String()) newState := AlertState{ - UID: alertRule.UID, - OrgID: alertRule.OrgID, - CacheId: idString, - Labels: lbs, - State: result.State, - Results: []StateEvaluation{}, - Annotations: annotations, + AlertRuleUID: alertRule.UID, + OrgID: alertRule.OrgID, + CacheId: id, + Labels: lbs, + State: result.State, + Results: newResults, + Annotations: annotations, + EvaluationDuration: evaluationDuration, } if result.State == eval.Alerting { newState.StartsAt = result.EvaluatedAt } - st.stateCache.cacheMap[idString] = newState + st.cache.states[id] = newState return newState } -func (st *StateTracker) set(stateEntry AlertState) { - st.stateCache.mu.Lock() - defer st.stateCache.mu.Unlock() - st.stateCache.cacheMap[stateEntry.CacheId] = stateEntry +func (st *StateTracker) set(entry AlertState) { + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + st.cache.states[entry.CacheId] = entry } -func (st *StateTracker) Get(stateId string) AlertState { - st.stateCache.mu.Lock() - defer st.stateCache.mu.Unlock() - return st.stateCache.cacheMap[stateId] +func (st *StateTracker) Get(id string) AlertState { + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + return st.cache.states[id] } //Used to ensure a clean cache on startup func (st *StateTracker) ResetCache() { - st.stateCache.mu.Lock() - defer st.stateCache.mu.Unlock() - st.stateCache.cacheMap = make(map[string]AlertState) + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + st.cache.states = make(map[string]AlertState) } -func (st *StateTracker) ProcessEvalResults(alertRule *ngModels.AlertRule, results eval.Results) []AlertState { +func (st *StateTracker) ProcessEvalResults(alertRule *ngModels.AlertRule, results eval.Results, evaluationDuration time.Duration) []AlertState { st.Log.Info("state tracker processing evaluation results", "uid", alertRule.UID, "resultCount", len(results)) - var changedStates []AlertState + var states []AlertState for _, result := range results { - s, _ := st.setNextState(alertRule, result) - changedStates = append(changedStates, s) + s := st.setNextState(alertRule, result, evaluationDuration) + states = append(states, s) } - st.Log.Debug("returning changed states to scheduler", "count", len(changedStates)) - return changedStates + st.Log.Debug("returning changed states to scheduler", "count", len(states)) + return states } //TODO: When calculating if an alert should not be firing anymore, we should take three things into account: // 1. The re-send the delay if any, we don't want to send every firing alert every time, we should have a fixed delay across all alerts to avoid saturating the notification system -// 2. The evaluation interval defined for this particular alert - we don't support that yet but will eventually allow you to define how often do you want this alert to be evaluted -// 3. The base interval defined by the scheduler - in the case where #2 is not yet an option we can use the base interval at which every alert runs. //Set the current state based on evaluation results -//return the state and a bool indicating whether a state transition occurred -func (st *StateTracker) setNextState(alertRule *ngModels.AlertRule, result eval.Result) (AlertState, bool) { - currentState := st.getOrCreate(alertRule, result) +func (st *StateTracker) setNextState(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState { + currentState := st.getOrCreate(alertRule, result, evaluationDuration) st.Log.Debug("setting alert state", "uid", alertRule.UID) switch { case currentState.State == result.State: st.Log.Debug("no state transition", "cacheId", currentState.CacheId, "state", currentState.State.String()) currentState.LastEvaluationTime = result.EvaluatedAt + currentState.EvaluationDuration = evaluationDuration currentState.Results = append(currentState.Results, StateEvaluation{ EvaluationTime: result.EvaluatedAt, EvaluationState: result.State, }) if currentState.State == eval.Alerting { - currentState.EndsAt = result.EvaluatedAt.Add(40 * time.Second) + currentState.EndsAt = result.EvaluatedAt.Add(alertRule.For * time.Second) } st.set(currentState) - return currentState, false + return currentState case currentState.State == eval.Normal && result.State == eval.Alerting: st.Log.Debug("state transition from normal to alerting", "cacheId", currentState.CacheId) currentState.State = eval.Alerting currentState.LastEvaluationTime = result.EvaluatedAt currentState.StartsAt = result.EvaluatedAt - currentState.EndsAt = result.EvaluatedAt.Add(40 * time.Second) + currentState.EndsAt = result.EvaluatedAt.Add(alertRule.For * time.Second) + currentState.EvaluationDuration = evaluationDuration currentState.Results = append(currentState.Results, StateEvaluation{ EvaluationTime: result.EvaluatedAt, EvaluationState: result.State, }) currentState.Annotations["alerting"] = result.EvaluatedAt.String() st.set(currentState) - return currentState, true + return currentState case currentState.State == eval.Alerting && result.State == eval.Normal: st.Log.Debug("state transition from alerting to normal", "cacheId", currentState.CacheId) currentState.State = eval.Normal currentState.LastEvaluationTime = result.EvaluatedAt currentState.EndsAt = result.EvaluatedAt + currentState.EvaluationDuration = evaluationDuration currentState.Results = append(currentState.Results, StateEvaluation{ EvaluationTime: result.EvaluatedAt, EvaluationState: result.State, }) st.set(currentState) - return currentState, true + return currentState default: - return currentState, false + return currentState } } func (st *StateTracker) GetAll() []AlertState { var states []AlertState - st.stateCache.mu.Lock() - defer st.stateCache.mu.Unlock() - for _, v := range st.stateCache.cacheMap { + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + for _, v := range st.cache.states { states = append(states, v) } return states } +func (st *StateTracker) GetStatesByRuleUID() map[string][]AlertState { + ruleMap := make(map[string][]AlertState) + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + for _, state := range st.cache.states { + if ruleStates, ok := ruleMap[state.AlertRuleUID]; ok { + ruleStates = append(ruleStates, state) + ruleMap[state.AlertRuleUID] = ruleStates + } else { + ruleStates := []AlertState{state} + ruleMap[state.AlertRuleUID] = ruleStates + } + } + return ruleMap +} + func (st *StateTracker) cleanUp() { ticker := time.NewTicker(time.Duration(60) * time.Minute) st.Log.Debug("starting cleanup process", "intervalMinutes", 60) @@ -200,9 +225,9 @@ func (st *StateTracker) cleanUp() { func (st *StateTracker) trim() { st.Log.Info("trimming alert state cache", "now", time.Now()) - st.stateCache.mu.Lock() - defer st.stateCache.mu.Unlock() - for _, v := range st.stateCache.cacheMap { + st.cache.mtxStates.Lock() + defer st.cache.mtxStates.Unlock() + for _, v := range st.cache.states { if len(v.Results) > 100 { st.Log.Debug("trimming result set", "cacheId", v.CacheId, "count", len(v.Results)-100) newResults := make([]StateEvaluation, 100) @@ -214,7 +239,7 @@ func (st *StateTracker) trim() { } func (a AlertState) Equals(b AlertState) bool { - return a.UID == b.UID && + return a.AlertRuleUID == b.AlertRuleUID && a.OrgID == b.OrgID && a.CacheId == b.CacheId && a.Labels.String() == b.Labels.String() && @@ -229,3 +254,17 @@ func (st *StateTracker) Put(states []AlertState) { st.set(s) } } + +// if duplicate labels exist, keep the value from the first set +func mergeLabels(a, b data.Labels) data.Labels { + newLbs := data.Labels{} + for k, v := range a { + newLbs[k] = v + } + for k, v := range b { + if _, ok := newLbs[k]; !ok { + newLbs[k] = v + } + } + return newLbs +} diff --git a/pkg/services/ngalert/tests/schedule_test.go b/pkg/services/ngalert/tests/schedule_test.go index 39f21da25de..7df22367cb7 100644 --- a/pkg/services/ngalert/tests/schedule_test.go +++ b/pkg/services/ngalert/tests/schedule_test.go @@ -35,11 +35,11 @@ func TestWarmStateCache(t *testing.T) { expectedEntries := []state.AlertState{ { - UID: "test_uid", - OrgID: 123, - CacheId: "test_uid map[test1:testValue1]", - Labels: data.Labels{"test1": "testValue1"}, - State: eval.Normal, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: "test_uid map[test1:testValue1]", + Labels: data.Labels{"test1": "testValue1"}, + State: eval.Normal, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, }, @@ -47,11 +47,11 @@ func TestWarmStateCache(t *testing.T) { EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime, }, { - UID: "test_uid", - OrgID: 123, - CacheId: "test_uid map[test2:testValue2]", - Labels: data.Labels{"test2": "testValue2"}, - State: eval.Alerting, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: "test_uid map[test2:testValue2]", + Labels: data.Labels{"test2": "testValue2"}, + State: eval.Alerting, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, }, diff --git a/pkg/services/ngalert/tests/state_tracker_test.go b/pkg/services/ngalert/tests/state_tracker_test.go index bf1ed6283de..3eec301076e 100644 --- a/pkg/services/ngalert/tests/state_tracker_test.go +++ b/pkg/services/ngalert/tests/state_tracker_test.go @@ -20,7 +20,11 @@ func TestProcessEvalResults(t *testing.T) { if err != nil { t.Fatalf("error parsing date format: %s", err.Error()) } - cacheId := "map[__alert_rule_namespace_uid__:test_namespace __alert_rule_title__:test_title __alert_rule_uid__:test_uid label1:value1 label2:value2]" + cacheId := "map[__alert_rule_namespace_uid__:test_namespace __alert_rule_uid__:test_uid alertname:test_title label1:value1 label2:value2 rule_label:rule_value]" + + ruleLabels := map[string]string{ + "rule_label": "rule_value", + } alertRule := models.AlertRule{ ID: 1, OrgID: 123, @@ -28,13 +32,17 @@ func TestProcessEvalResults(t *testing.T) { Condition: "A", UID: "test_uid", NamespaceUID: "test_namespace", + For: 10 * time.Second, + Labels: ruleLabels, } + processingTime := 10 * time.Millisecond expectedLabels := data.Labels{ "label1": "value1", "label2": "value2", + "rule_label": "rule_value", "__alert_rule_uid__": "test_uid", "__alert_rule_namespace_uid__": "test_namespace", - "__alert_rule_title__": "test_title", + "alertname": "test_title", } testCases := []struct { desc string @@ -60,11 +68,11 @@ func TestProcessEvalResults(t *testing.T) { expectedResultCount: 1, expectedCacheEntries: []state.AlertState{ { - UID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Normal, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: cacheId, + Labels: expectedLabels, + State: eval.Normal, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, }, @@ -94,17 +102,17 @@ func TestProcessEvalResults(t *testing.T) { expectedResultCount: 2, expectedCacheEntries: []state.AlertState{ { - UID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Alerting, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: cacheId, + Labels: expectedLabels, + State: eval.Alerting, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting}, }, StartsAt: evaluationTime.Add(1 * time.Minute), - EndsAt: evaluationTime.Add(100 * time.Second), + EndsAt: evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), }, }, @@ -129,11 +137,11 @@ func TestProcessEvalResults(t *testing.T) { expectedResultCount: 2, expectedCacheEntries: []state.AlertState{ { - UID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Normal, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: cacheId, + Labels: expectedLabels, + State: eval.Normal, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal}, @@ -164,17 +172,17 @@ func TestProcessEvalResults(t *testing.T) { expectedResultCount: 2, expectedCacheEntries: []state.AlertState{ { - UID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Alerting, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: cacheId, + Labels: expectedLabels, + State: eval.Alerting, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting}, }, StartsAt: evaluationTime, - EndsAt: evaluationTime.Add(100 * time.Second), + EndsAt: evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), }, }, @@ -199,11 +207,11 @@ func TestProcessEvalResults(t *testing.T) { expectedResultCount: 2, expectedCacheEntries: []state.AlertState{ { - UID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Normal, + AlertRuleUID: "test_uid", + OrgID: 123, + CacheId: cacheId, + Labels: expectedLabels, + State: eval.Normal, Results: []state.StateEvaluation{ {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal}, @@ -219,7 +227,7 @@ func TestProcessEvalResults(t *testing.T) { for _, tc := range testCases { t.Run("all fields for a cache entry are set correctly", func(t *testing.T) { st := state.NewStateTracker(log.New("test_state_tracker")) - _ = st.ProcessEvalResults(&alertRule, tc.evalResults) + _ = st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime) for _, entry := range tc.expectedCacheEntries { if !entry.Equals(st.Get(entry.CacheId)) { t.Log(tc.desc) @@ -231,7 +239,7 @@ func TestProcessEvalResults(t *testing.T) { t.Run("the expected number of entries are added to the cache", func(t *testing.T) { st := state.NewStateTracker(log.New("test_state_tracker")) - st.ProcessEvalResults(&alertRule, tc.evalResults) + st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime) assert.Equal(t, len(tc.expectedCacheEntries), len(st.GetAll())) }) @@ -240,15 +248,15 @@ func TestProcessEvalResults(t *testing.T) { //for a unique set of labels. t.Run("the expected number of states are returned to the caller", func(t *testing.T) { st := state.NewStateTracker(log.New("test_state_tracker")) - results := st.ProcessEvalResults(&alertRule, tc.evalResults) + results := st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime) assert.Equal(t, len(tc.evalResults), len(results)) }) } } func printEntryDiff(a, b state.AlertState, t *testing.T) { - if a.UID != b.UID { - t.Log(fmt.Sprintf("%v \t %v\n", a.UID, b.UID)) + if a.AlertRuleUID != b.AlertRuleUID { + t.Log(fmt.Sprintf("%v \t %v\n", a.AlertRuleUID, b.AlertRuleUID)) } if a.OrgID != b.OrgID { t.Log(fmt.Sprintf("%v \t %v\n", a.OrgID, b.OrgID))