From 52965de36989e640aa486467986998e62e109c5b Mon Sep 17 00:00:00 2001 From: George Robinson Date: Tue, 11 Oct 2022 09:30:33 +0100 Subject: [PATCH] Alerting: Add doc comments to state struct and normalize fields (#56647) --- .../ngalert/schedule/schedule_test.go | 6 +- .../ngalert/schedule/schedule_unit_test.go | 2 +- pkg/services/ngalert/state/cache.go | 4 +- pkg/services/ngalert/state/manager.go | 14 ++-- pkg/services/ngalert/state/manager_test.go | 74 +++++++++---------- pkg/services/ngalert/state/state.go | 56 ++++++++++---- 6 files changed, 91 insertions(+), 65 deletions(-) diff --git a/pkg/services/ngalert/schedule/schedule_test.go b/pkg/services/ngalert/schedule/schedule_test.go index b32bd4c7b16..d9dbb546d20 100644 --- a/pkg/services/ngalert/schedule/schedule_test.go +++ b/pkg/services/ngalert/schedule/schedule_test.go @@ -49,7 +49,7 @@ func TestWarmStateCache(t *testing.T) { { AlertRuleUID: rule.UID, OrgID: rule.OrgID, - CacheId: `[["test1","testValue1"]]`, + CacheID: `[["test1","testValue1"]]`, Labels: data.Labels{"test1": "testValue1"}, State: eval.Normal, Results: []state.Evaluation{ @@ -62,7 +62,7 @@ func TestWarmStateCache(t *testing.T) { }, { AlertRuleUID: rule.UID, OrgID: rule.OrgID, - CacheId: `[["test2","testValue2"]]`, + CacheID: `[["test2","testValue2"]]`, Labels: data.Labels{"test2": "testValue2"}, State: eval.Alerting, Results: []state.Evaluation{ @@ -125,7 +125,7 @@ func TestWarmStateCache(t *testing.T) { t.Run("instance cache has expected entries", func(t *testing.T) { for _, entry := range expectedEntries { - cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheId) + cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID) if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "Results")); diff != "" { t.Errorf("Result mismatch (-want +got):\n%s", diff) diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 086dbe4e2a1..9922ee9dbad 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -240,7 +240,7 @@ func TestSchedule_ruleRoutine(t *testing.T) { for i := 0; i < 2; i++ { states = append(states, &state.State{ AlertRuleUID: rule.UID, - CacheId: util.GenerateShortUID(), + CacheID: util.GenerateShortUID(), OrgID: rule.OrgID, State: s, StartsAt: sch.clock.Now(), diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go index 1371c1a8262..cc51c0547d9 100644 --- a/pkg/services/ngalert/state/cache.go +++ b/pkg/services/ngalert/state/cache.go @@ -122,7 +122,7 @@ func (rs *ruleStates) getOrCreate(ctx context.Context, log log.Logger, alertRule newState := &State{ AlertRuleUID: alertRule.UID, OrgID: alertRule.OrgID, - CacheId: id, + CacheID: id, Labels: lbs, Annotations: annotations, EvaluationDuration: result.EvaluationDuration, @@ -171,7 +171,7 @@ func (c *cache) set(entry *State) { if _, ok := c.states[entry.OrgID][entry.AlertRuleUID]; !ok { c.states[entry.OrgID][entry.AlertRuleUID] = &ruleStates{states: make(map[string]*State)} } - c.states[entry.OrgID][entry.AlertRuleUID].states[entry.CacheId] = entry + c.states[entry.OrgID][entry.AlertRuleUID].states[entry.CacheID] = entry } func (c *cache) get(orgID int64, alertRuleUID, stateId string) *State { diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index e479dfddb8a..f4d8a62e25a 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -116,14 +116,14 @@ func (st *Manager) Warm(ctx context.Context) { } lbs := map[string]string(entry.Labels) - cacheId, err := entry.Labels.StringKey() + cacheID, err := entry.Labels.StringKey() if err != nil { st.log.Error("error getting cacheId for entry", "msg", err.Error()) } - rulesStates.states[cacheId] = &State{ + rulesStates.states[cacheID] = &State{ AlertRuleUID: entry.RuleUID, OrgID: entry.RuleOrgID, - CacheId: cacheId, + CacheID: cacheID, Labels: lbs, State: translateInstanceState(entry.CurrentState), StateReason: entry.CurrentReason, @@ -169,7 +169,7 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time for _, result := range results { s := st.setNextState(ctx, alertRule, result, extraLabels) states = append(states, s) - processedResults[s.CacheId] = s + processedResults[s.CacheID] = s } resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults) if len(states) > 0 { @@ -384,9 +384,9 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim for _, s := range allStates { // Is the cached state in our recently processed results? If not, is it stale? - if _, ok := states[s.CacheId]; !ok && stateIsStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds) { - st.log.Debug("removing stale state entry", "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheId) - st.cache.deleteEntry(s.OrgID, s.AlertRuleUID, s.CacheId) + if _, ok := states[s.CacheID]; !ok && stateIsStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds) { + st.log.Debug("removing stale state entry", "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheID) + st.cache.deleteEntry(s.OrgID, s.AlertRuleUID, s.CacheID) ilbs := ngModels.InstanceLabels(s.Labels) _, labelsHash, err := ilbs.StringAndHash() if err != nil { diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 53d572b4a8f..7e8892b6572 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -117,7 +117,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -172,7 +172,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -196,7 +196,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -254,7 +254,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_1", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_1", @@ -316,7 +316,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -389,7 +389,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -484,7 +484,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -566,7 +566,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -631,7 +631,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -696,7 +696,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -761,7 +761,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -827,7 +827,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -892,7 +892,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -918,7 +918,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -983,7 +983,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-1"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-1"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-1"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1009,7 +1009,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-2"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-2"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-2"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1035,7 +1035,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1102,7 +1102,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1133,7 +1133,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1192,7 +1192,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1259,7 +1259,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1327,7 +1327,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1418,7 +1418,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1498,7 +1498,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1578,7 +1578,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1652,7 +1652,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1752,7 +1752,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1834,7 +1834,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1922,7 +1922,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1982,7 +1982,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["cluster","us-central-1"],["job","prod/grafana"],["label","test"],["namespace","prod"],["pod","grafana"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["cluster","us-central-1"],["job","prod/grafana"],["label","test"],["namespace","prod"],["pod","grafana"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["cluster","us-central-1"],["job","prod/grafana"],["label","test"],["namespace","prod"],["pod","grafana"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -2027,7 +2027,7 @@ func TestProcessEvalResults(t *testing.T) { assert.Len(t, states, len(tc.expectedStates)) for _, s := range tc.expectedStates { - cachedState := st.Get(s.OrgID, s.AlertRuleUID, s.CacheId) + cachedState := st.Get(s.OrgID, s.AlertRuleUID, s.CacheID) assert.Equal(t, s, cachedState) } @@ -2059,7 +2059,7 @@ func TestProcessEvalResults(t *testing.T) { } require.Len(t, savedStates, len(states)) for _, s := range states { - require.Contains(t, savedStates, s.CacheId) + require.Contains(t, savedStates, s.CacheID) } }) } @@ -2140,7 +2140,7 @@ func TestStaleResultsHandler(t *testing.T) { `[["__alert_rule_namespace_uid__","namespace"],["__alert_rule_uid__","` + rule.UID + `"],["alertname","` + rule.Title + `"],["test1","testValue1"]]`: { AlertRuleUID: rule.UID, OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","namespace"],["__alert_rule_uid__","` + rule.UID + `"],["alertname","` + rule.Title + `"],["test1","testValue1"]]`, + CacheID: `[["__alert_rule_namespace_uid__","namespace"],["__alert_rule_uid__","` + rule.UID + `"],["alertname","` + rule.Title + `"],["test1","testValue1"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "namespace", "__alert_rule_uid__": rule.UID, @@ -2188,7 +2188,7 @@ func TestStaleResultsHandler(t *testing.T) { "__alert_rule_uid__": rule.UID, }) for _, s := range tc.expectedStates { - cachedState := st.Get(s.OrgID, s.AlertRuleUID, s.CacheId) + cachedState := st.Get(s.OrgID, s.AlertRuleUID, s.CacheID) assert.Equal(t, s, cachedState) } } @@ -2219,8 +2219,8 @@ func TestStaleResults(t *testing.T) { t.Helper() require.Len(t, actual, len(expected)) for _, currentState := range actual { - _, ok := expected[currentState.CacheId] - require.Truef(t, ok, "State %s is not expected. States: %v", currentState.CacheId, expected) + _, ok := expected[currentState.CacheID] + require.Truef(t, ok, "State %s is not expected. States: %v", currentState.CacheID, expected) } } @@ -2283,7 +2283,7 @@ func TestStaleResults(t *testing.T) { expectedStaleReturned: {}, }) for _, s := range processed { - if s.CacheId == expectedStaleReturned { + if s.CacheID == expectedStaleReturned { assert.Truef(t, s.Resolved, "Returned stale state should have Resolved set to true") assert.Equal(t, eval.Normal, s.State) assert.Equal(t, models.StateReasonMissingSeries, s.StateReason) diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 994945b55de..cf54593fb77 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -15,26 +15,52 @@ import ( ) type State struct { - AlertRuleUID string OrgID int64 - CacheId string + AlertRuleUID string - StartsAt time.Time - EndsAt time.Time - LastSentAt time.Time + // CacheID is a unique, opaque identifier for the state, and is used to find the state + // in the state cache. It tends to be derived from the state's labels. + CacheID string - State eval.State - StateReason string + // State represents the current state. + State eval.State + + // StateReason is a textual description to explain why the state has its current state. + StateReason string + + // Results contains the result of the current and previous evaluations. + Results []Evaluation + + // Error is set if the current evaluation returned an error. If error is non-nil results + // can still contain the results of previous evaluations. + Error error + + // Resolved is set to true if this state is the transitional state between Firing and Normal. + // All subsequent states will be false until the next transition from Firing to Normal. + Resolved bool + + // Image contains an optional image for the state. It tends to be included in notifications + // as a visualization to show why the alert fired. + Image *models.Image + + // Annotations contains the annotations from the alert rule. If an annotation is templated + // then the template is first evaluated to derive the final annotation. + Annotations map[string]string + + // Labels contain the labels from the query and any custom labels from the alert rule. + // If a label is templated then the template is first evaluated to derive the final label. + Labels data.Labels + + // Values contains the values of any instant vectors, reduce and math expressions, or classic + // conditions. + Values map[string]float64 + + StartsAt time.Time + EndsAt time.Time + LastSentAt time.Time LastEvaluationString string LastEvaluationTime time.Time EvaluationDuration time.Duration - Results []Evaluation - Resolved bool - Annotations map[string]string - Labels data.Labels - Values map[string]float64 - Image *models.Image - Error error } func (a *State) GetRuleKey() models.AlertRuleKey { @@ -192,7 +218,7 @@ func (a *State) NeedsSending(resendDelay time.Duration) bool { func (a *State) Equals(b *State) bool { return a.AlertRuleUID == b.AlertRuleUID && a.OrgID == b.OrgID && - a.CacheId == b.CacheId && + a.CacheID == b.CacheID && a.Labels.String() == b.Labels.String() && a.State.String() == b.State.String() && a.StartsAt == b.StartsAt &&