From 4585d70f22804ee0b025d50c0a75d1bed4efc0c3 Mon Sep 17 00:00:00 2001 From: George Robinson Date: Tue, 11 Oct 2022 15:20:18 +0100 Subject: [PATCH] Alerting: Add doc comments to state struct and normalize fields (#56647) (#56676) (cherry picked from commit 52965de36989e640aa486467986998e62e109c5b) --- .../ngalert/schedule/schedule_test.go | 6 +- .../ngalert/schedule/schedule_unit_test.go | 2 +- pkg/services/ngalert/state/cache.go | 6 +- pkg/services/ngalert/state/manager.go | 16 ++-- pkg/services/ngalert/state/manager_test.go | 78 +++++++++---------- pkg/services/ngalert/state/state.go | 55 +++++++++---- 6 files changed, 95 insertions(+), 68 deletions(-) diff --git a/pkg/services/ngalert/schedule/schedule_test.go b/pkg/services/ngalert/schedule/schedule_test.go index 0c1dab2f329..52ee7660473 100644 --- a/pkg/services/ngalert/schedule/schedule_test.go +++ b/pkg/services/ngalert/schedule/schedule_test.go @@ -51,7 +51,7 @@ func TestWarmStateCache(t *testing.T) { { AlertRuleUID: rule.UID, OrgID: rule.OrgID, - CacheId: `[["test1","testValue1"]]`, + CacheID: `[["test1","testValue1"]]`, Labels: data.Labels{"test1": "testValue1"}, State: eval.Normal, Results: []state.Evaluation{ @@ -64,7 +64,7 @@ func TestWarmStateCache(t *testing.T) { }, { AlertRuleUID: rule.UID, OrgID: rule.OrgID, - CacheId: `[["test2","testValue2"]]`, + CacheID: `[["test2","testValue2"]]`, Labels: data.Labels{"test2": "testValue2"}, State: eval.Alerting, Results: []state.Evaluation{ @@ -118,7 +118,7 @@ func TestWarmStateCache(t *testing.T) { t.Run("instance cache has expected entries", func(t *testing.T) { for _, entry := range expectedEntries { - cacheEntry, err := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheId) + cacheEntry, err := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID) require.NoError(t, err) if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "Results")); diff != "" { diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index f063b3548fa..ea626f3bdac 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -243,7 +243,7 @@ func TestSchedule_ruleRoutine(t *testing.T) { for i := 0; i < 2; i++ { states = append(states, &state.State{ AlertRuleUID: rule.UID, - CacheId: util.GenerateShortUID(), + CacheID: util.GenerateShortUID(), OrgID: rule.OrgID, State: s, StartsAt: sch.clock.Now(), diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go index 58f53e2e25a..c59ca53f89c 100644 --- a/pkg/services/ngalert/state/cache.go +++ b/pkg/services/ngalert/state/cache.go @@ -74,7 +74,7 @@ func (c *cache) getOrCreate(ctx context.Context, alertRule *ngModels.AlertRule, il := ngModels.InstanceLabels(lbs) id, err := il.StringKey() if err != nil { - c.log.Error("error getting cacheId for entry", "err", err.Error()) + c.log.Error("error getting cacheID for entry", "err", err.Error()) } if _, ok := c.states[alertRule.OrgID]; !ok { @@ -106,7 +106,7 @@ func (c *cache) getOrCreate(ctx context.Context, alertRule *ngModels.AlertRule, newState := &State{ AlertRuleUID: alertRule.UID, OrgID: alertRule.OrgID, - CacheId: id, + CacheID: id, Labels: lbs, Annotations: annotations, EvaluationDuration: result.EvaluationDuration, @@ -148,7 +148,7 @@ func (c *cache) set(entry *State) { if _, ok := c.states[entry.OrgID][entry.AlertRuleUID]; !ok { c.states[entry.OrgID][entry.AlertRuleUID] = make(map[string]*State) } - c.states[entry.OrgID][entry.AlertRuleUID][entry.CacheId] = entry + c.states[entry.OrgID][entry.AlertRuleUID][entry.CacheID] = entry } func (c *cache) get(orgID int64, alertRuleUID, stateId string) (*State, error) { diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index b36ba1cc4b0..6d9e9cf224b 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -112,14 +112,14 @@ func (st *Manager) Warm(ctx context.Context) { } lbs := map[string]string(entry.Labels) - cacheId, err := entry.Labels.StringKey() + cacheID, err := entry.Labels.StringKey() if err != nil { - st.log.Error("error getting cacheId for entry", "msg", err.Error()) + st.log.Error("error getting cacheID for entry", "msg", err.Error()) } stateForEntry := &State{ AlertRuleUID: entry.RuleUID, OrgID: entry.RuleOrgID, - CacheId: cacheId, + CacheID: cacheID, Labels: lbs, State: translateInstanceState(entry.CurrentState), StateReason: entry.CurrentReason, @@ -180,7 +180,7 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time for _, result := range results { s := st.setNextState(ctx, alertRule, result, extraLabels) states = append(states, s) - processedResults[s.CacheId] = s + processedResults[s.CacheID] = s } resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults) if len(states) > 0 { @@ -408,10 +408,10 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim var resolvedStates []*State allStates := st.GetStatesForRuleUID(alertRule.OrgID, alertRule.UID) for _, s := range allStates { - _, ok := states[s.CacheId] + _, ok := states[s.CacheID] if !ok && isItStale(evaluatedAt, s.LastEvaluationTime, alertRule.IntervalSeconds) { - st.log.Debug("removing stale state entry", "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheId) - st.cache.deleteEntry(s.OrgID, s.AlertRuleUID, s.CacheId) + st.log.Debug("removing stale state entry", "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheID) + st.cache.deleteEntry(s.OrgID, s.AlertRuleUID, s.CacheID) ilbs := ngModels.InstanceLabels(s.Labels) _, labelsHash, err := ilbs.StringAndHash() if err != nil { @@ -419,7 +419,7 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim } if err = st.instanceStore.DeleteAlertInstance(ctx, s.OrgID, s.AlertRuleUID, labelsHash); err != nil { - st.log.Error("unable to delete stale instance from database", "err", err.Error(), "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheId) + st.log.Error("unable to delete stale instance from database", "err", err.Error(), "orgID", s.OrgID, "alertRuleUID", s.AlertRuleUID, "cacheID", s.CacheID) } if s.State == eval.Alerting { diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index ef5e612ce10..88dfef38b93 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -116,7 +116,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -170,7 +170,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -193,7 +193,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -250,7 +250,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_1", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_1", @@ -311,7 +311,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -383,7 +383,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -477,7 +477,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -558,7 +558,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -622,7 +622,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -686,7 +686,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -750,7 +750,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -815,7 +815,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -879,7 +879,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -904,7 +904,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -968,7 +968,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-1"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-1"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-1"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -993,7 +993,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-2"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-2"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test-2"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1018,7 +1018,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1084,7 +1084,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1114,7 +1114,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1172,7 +1172,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1238,7 +1238,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1305,7 +1305,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1395,7 +1395,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1474,7 +1474,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1553,7 +1553,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1626,7 +1626,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1725,7 +1725,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1806,7 +1806,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1893,7 +1893,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: { AlertRuleUID: "test_alert_rule_uid_2", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid_2", @@ -1952,7 +1952,7 @@ func TestProcessEvalResults(t *testing.T) { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["cluster","us-central-1"],["job","prod/grafana"],["label","test"],["namespace","prod"],["pod","grafana"]]`: { AlertRuleUID: "test_alert_rule_uid", OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["cluster","us-central-1"],["job","prod/grafana"],["label","test"],["namespace","prod"],["pod","grafana"]]`, + CacheID: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["cluster","us-central-1"],["job","prod/grafana"],["label","test"],["namespace","prod"],["pod","grafana"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_uid__": "test_alert_rule_uid", @@ -1995,7 +1995,7 @@ func TestProcessEvalResults(t *testing.T) { assert.Len(t, states, len(tc.expectedStates)) for _, s := range tc.expectedStates { - cachedState, err := st.Get(s.OrgID, s.AlertRuleUID, s.CacheId) + cachedState, err := st.Get(s.OrgID, s.AlertRuleUID, s.CacheID) require.NoError(t, err) assert.Equal(t, s, cachedState) } @@ -2021,14 +2021,14 @@ func TestProcessEvalResults(t *testing.T) { for _, op := range instanceStore.RecordedOps { switch q := op.(type) { case models.SaveAlertInstanceCommand: - cacheId, err := q.Labels.StringKey() + cacheID, err := q.Labels.StringKey() require.NoError(t, err) - savedStates[cacheId] = q + savedStates[cacheID] = q } } require.Len(t, savedStates, len(states)) for _, s := range states { - require.Contains(t, savedStates, s.CacheId) + require.Contains(t, savedStates, s.CacheID) } }) } @@ -2098,7 +2098,7 @@ func TestStaleResultsHandler(t *testing.T) { `[["__alert_rule_namespace_uid__","namespace"],["__alert_rule_uid__","` + rule.UID + `"],["alertname","` + rule.Title + `"],["test1","testValue1"]]`: { AlertRuleUID: rule.UID, OrgID: 1, - CacheId: `[["__alert_rule_namespace_uid__","namespace"],["__alert_rule_uid__","` + rule.UID + `"],["alertname","` + rule.Title + `"],["test1","testValue1"]]`, + CacheID: `[["__alert_rule_namespace_uid__","namespace"],["__alert_rule_uid__","` + rule.UID + `"],["alertname","` + rule.Title + `"],["test1","testValue1"]]`, Labels: data.Labels{ "__alert_rule_namespace_uid__": "namespace", "__alert_rule_uid__": rule.UID, @@ -2145,7 +2145,7 @@ func TestStaleResultsHandler(t *testing.T) { "__alert_rule_uid__": rule.UID, }) for _, s := range tc.expectedStates { - cachedState, err := st.Get(s.OrgID, s.AlertRuleUID, s.CacheId) + cachedState, err := st.Get(s.OrgID, s.AlertRuleUID, s.CacheID) require.NoError(t, err) assert.Equal(t, s, cachedState) } @@ -2177,8 +2177,8 @@ func TestStaleResults(t *testing.T) { t.Helper() require.Len(t, actual, len(expected)) for _, currentState := range actual { - _, ok := expected[currentState.CacheId] - require.Truef(t, ok, "State %s is not expected. States: %v", currentState.CacheId, expected) + _, ok := expected[currentState.CacheID] + require.Truef(t, ok, "State %s is not expected. States: %v", currentState.CacheID, expected) } } @@ -2241,7 +2241,7 @@ func TestStaleResults(t *testing.T) { expectedStaleReturned: {}, }) for _, s := range processed { - if s.CacheId == expectedStaleReturned { + if s.CacheID == expectedStaleReturned { assert.Truef(t, s.Resolved, "Returned stale state should have Resolved set to true") assert.Equal(t, eval.Normal, s.State) assert.Equal(t, models.StateReasonMissingSeries, s.StateReason) diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index b8a084eba3b..cf54593fb77 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -15,25 +15,52 @@ import ( ) type State struct { - AlertRuleUID string OrgID int64 - CacheId string + AlertRuleUID string - StartsAt time.Time - EndsAt time.Time - LastSentAt time.Time + // CacheID is a unique, opaque identifier for the state, and is used to find the state + // in the state cache. It tends to be derived from the state's labels. + CacheID string - State eval.State - StateReason string + // State represents the current state. + State eval.State + + // StateReason is a textual description to explain why the state has its current state. + StateReason string + + // Results contains the result of the current and previous evaluations. + Results []Evaluation + + // Error is set if the current evaluation returned an error. If error is non-nil results + // can still contain the results of previous evaluations. + Error error + + // Resolved is set to true if this state is the transitional state between Firing and Normal. + // All subsequent states will be false until the next transition from Firing to Normal. + Resolved bool + + // Image contains an optional image for the state. It tends to be included in notifications + // as a visualization to show why the alert fired. + Image *models.Image + + // Annotations contains the annotations from the alert rule. If an annotation is templated + // then the template is first evaluated to derive the final annotation. + Annotations map[string]string + + // Labels contain the labels from the query and any custom labels from the alert rule. + // If a label is templated then the template is first evaluated to derive the final label. + Labels data.Labels + + // Values contains the values of any instant vectors, reduce and math expressions, or classic + // conditions. + Values map[string]float64 + + StartsAt time.Time + EndsAt time.Time + LastSentAt time.Time LastEvaluationString string LastEvaluationTime time.Time EvaluationDuration time.Duration - Results []Evaluation - Resolved bool - Annotations map[string]string - Labels data.Labels - Image *models.Image - Error error } func (a *State) GetRuleKey() models.AlertRuleKey { @@ -191,7 +218,7 @@ func (a *State) NeedsSending(resendDelay time.Duration) bool { func (a *State) Equals(b *State) bool { return a.AlertRuleUID == b.AlertRuleUID && a.OrgID == b.OrgID && - a.CacheId == b.CacheId && + a.CacheID == b.CacheID && a.Labels.String() == b.Labels.String() && a.State.String() == b.State.String() && a.StartsAt == b.StartsAt &&