public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Persist AlertInstance ResolvedAt & LastSentAt (#89135)

Browse Source 
* Alerting: Persist AlertInstance ResolvedAt & LastSentAt

* Fix test

* Modify existing tests

* Fix merge conflicts from nullable LastSentAt & ResolvedAt
This commit is contained in:
Matthew Jacobson
2024-07-12 12:26:58 -04:00
committed by GitHub
parent e1f030592f
commit ba800692c6
10 changed files with 121 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/state/cache.go
+2
View File
@@ -365,6 +365,8 @@ func (c *cache) asInstances(skipNormalState bool) []ngModels.AlertInstance {
LastEvalTime: v2.LastEvaluationTime,
CurrentStateSince: v2.StartsAt,
CurrentStateEnd: v2.EndsAt,
ResolvedAt: v2.ResolvedAt,
LastSentAt: v2.LastSentAt,
ResultFingerprint: v2.ResultFingerprint.String(),
})
}
pkg/services/ngalert/state/manager.go
+2
View File
@@ -215,6 +215,8 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
LastEvaluationTime: entry.LastEvalTime,
Annotations: ruleForEntry.Annotations,
ResultFingerprint: resultFp,
ResolvedAt: entry.ResolvedAt,
LastSentAt: entry.LastSentAt,
}
statesCount++
}
pkg/services/ngalert/state/manager_test.go
+46 -13
View File
@@ -62,6 +62,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime),
ResolvedAt: util.Pointer(evaluationTime),
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64),
}, {
@@ -73,6 +75,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1),
},
@@ -85,6 +89,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(0),
},
@@ -97,6 +103,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(1),
},
@@ -109,6 +117,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: nil,
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(2),
},
@@ -128,6 +138,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: &evaluationTime,
ResolvedAt: &evaluationTime,
Labels: labels,
ResultFingerprint: data.Fingerprint(math.MaxUint64).String(),
})
@@ -144,6 +156,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1).String(),
})
@@ -160,6 +174,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(0).String(),
})
@@ -176,6 +192,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(1).String(),
})
@@ -192,6 +210,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: nil,
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(2).String(),
})
@@ -1656,7 +1676,7 @@ func printAllAnnotations(annos map[int64]annotations.Item) string {
}
func TestStaleResultsHandler(t *testing.T) {
evaluationTime := time.Now()
evaluationTime := time.Now().Truncate(time.Second).UTC() // Truncate to the second since we don't store sub-second precision.
interval := time.Minute
ctx := context.Background()
@@ -1666,9 +1686,19 @@ func TestStaleResultsHandler(t *testing.T) {
rule := tests.CreateTestAlertRule(t, ctx, dbstore, int64(interval.Seconds()), mainOrgID)
lastEval := evaluationTime.Add(-2 * interval)
labels1 := models.InstanceLabels{"test1": "testValue1"}
labels1 := models.InstanceLabels{
"__alert_rule_namespace_uid__": "namespace",
"__alert_rule_uid__": rule.UID,
"alertname": rule.Title,
"test1": "testValue1",
}
_, hash1, _ := labels1.StringAndHash()
labels2 := models.InstanceLabels{"test2": "testValue2"}
labels2 := models.InstanceLabels{
"__alert_rule_namespace_uid__": "namespace",
"__alert_rule_uid__": rule.UID,
"alertname": rule.Title,
"test2": "testValue2",
}
_, hash2, _ := labels2.StringAndHash()
instances := []models.AlertInstance{
{
@@ -1682,6 +1712,9 @@ func TestStaleResultsHandler(t *testing.T) {
LastEvalTime: lastEval,
CurrentStateSince: lastEval,
CurrentStateEnd: lastEval.Add(3 * interval),
LastSentAt: &lastEval,
ResolvedAt: &lastEval,
ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint().String(),
},
{
AlertInstanceKey: models.AlertInstanceKey{
@@ -1694,6 +1727,9 @@ func TestStaleResultsHandler(t *testing.T) {
LastEvalTime: lastEval,
CurrentStateSince: lastEval,
CurrentStateEnd: lastEval.Add(3 * interval),
LastSentAt: &lastEval,
ResolvedAt: nil,
ResultFingerprint: data.Labels{"test2": "testValue2"}.Fingerprint().String(),
},
}
@@ -1723,23 +1759,20 @@ func TestStaleResultsHandler(t *testing.T) {
{
AlertRuleUID: rule.UID,
OrgID: 1,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "namespace",
"__alert_rule_uid__": rule.UID,
"alertname": rule.Title,
"test1": "testValue1",
},
Values: make(map[string]float64),
State: eval.Normal,
Labels: data.Labels(labels1),
Values: make(map[string]float64),
State: eval.Normal,
LatestResult: &state.Evaluation{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]float64),
Condition: "A",
},
StartsAt: evaluationTime,
EndsAt: evaluationTime,
StartsAt: lastEval,
EndsAt: lastEval.Add(3 * interval),
LastEvaluationTime: evaluationTime,
LastSentAt: &lastEval,
ResolvedAt: &lastEval,
EvaluationDuration: 0,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint(),
pkg/services/ngalert/state/persister_sync.go
+2
View File
@@ -102,6 +102,8 @@ func (a *SyncStatePersister) saveAlertStates(ctx context.Context, states ...Stat
LastEvalTime: s.LastEvaluationTime,
CurrentStateSince: s.StartsAt,
CurrentStateEnd: s.EndsAt,
ResolvedAt: s.ResolvedAt,
LastSentAt: s.LastSentAt,
}
err = a.store.SaveAlertInstance(ctx, instance)