public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Move rule evaluation status logic out of prometheus API and into scheduler (#89141)

Browse Source 
* Add health fields to rules and an aggregator method to the scheduler

* Move health, last error, and last eval time in together to minimize state processing

* Wire up a readonly scheduler to prom api

* Extract to exported function

* Use health in api_prometheus and fix up tests

* Rename health struct to status

* Fix tests one more time

* Several new tests

* Handle inactive rules

* Push state mapping into state manager

* rename to StatusReader

* Rectify cyclo complexity rebase

* Convert existing package local status implementation to models one

* fix tests

* undo RuleDefs rename
This commit is contained in:
Alexander Weaver
2024-09-30 16:52:49 -05:00
committed by GitHub
parent 6a3eb276ef
commit 393faa8732
13 changed files with 213 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/schedule/alert_rule.go
+6
View File
@@ -40,6 +40,8 @@ type Rule interface {
Update(lastVersion RuleVersionAndPauseStatus) bool
// Type gives the type of the rule.
Type() ngmodels.RuleType
// Status indicates the status of the evaluating rule.
Status() ngmodels.RuleStatus
}
type ruleFactoryFunc func(context.Context, *ngmodels.AlertRule) Rule
@@ -180,6 +182,10 @@ func (a *alertRule) Type() ngmodels.RuleType {
return ngmodels.RuleTypeAlerting
}
func (a *alertRule) Status() ngmodels.RuleStatus {
return a.stateManager.GetStatusForRuleUID(a.key.OrgID, a.key.UID)
}
// eval signals the rule evaluation routine to perform the evaluation of the rule. Does nothing if the loop is stopped.
// Before sending a message into the channel, it does non-blocking read to make sure that there is no concurrent send operation.
// Returns a tuple where first element is
pkg/services/ngalert/schedule/alert_rule_test.go
+20
View File
@@ -369,6 +369,17 @@ func TestRuleRoutine(t *testing.T) {
require.Equal(t, s.Labels, data.Labels(cmd.Labels))
})
t.Run("status should accurately reflect latest evaluation", func(t *testing.T) {
states := sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)
require.NotEmpty(t, states)
status := ruleInfo.Status()
require.Equal(t, "ok", status.Health)
require.Nil(t, status.LastError)
require.Equal(t, states[0].LastEvaluationTime, status.EvaluationTimestamp)
require.Equal(t, states[0].EvaluationDuration, status.EvaluationDuration)
})
t.Run("it reports metrics", func(t *testing.T) {
// duration metric has 0 values because of mocked clock that do not advance
expectedMetric := fmt.Sprintf(
@@ -700,6 +711,15 @@ func TestRuleRoutine(t *testing.T) {
assert.Len(t, args.PostableAlerts, 1)
assert.Equal(t, state.ErrorAlertName, args.PostableAlerts[0].Labels[prometheusModel.AlertNameLabel])
})
t.Run("status should reflect unhealthy rule", func(t *testing.T) {
status := ruleInfo.Status()
require.Equal(t, "error", status.Health)
require.NotNil(t, status.LastError, "expected status to carry the latest evaluation error")
require.Contains(t, status.LastError.Error(), "cannot reference itself")
require.Equal(t, int64(0), status.EvaluationTimestamp.UTC().Unix())
require.Equal(t, time.Duration(0), status.EvaluationDuration)
})
})
t.Run("when there are alerts that should be firing", func(t *testing.T) {
pkg/services/ngalert/schedule/recording_rule.go
+2 -2
View File
@@ -84,8 +84,8 @@ func (r *recordingRule) Type() ngmodels.RuleType {
return ngmodels.RuleTypeRecording
}
func (r *recordingRule) Status() RuleStatus {
return RuleStatus{
func (r *recordingRule) Status() ngmodels.RuleStatus {
return ngmodels.RuleStatus{
Health: r.health.Load(),
LastError: r.lastError.Load(),
EvaluationTimestamp: r.evaluationTimestamp.Load(),
pkg/services/ngalert/schedule/registry.go
+8
View File
@@ -56,6 +56,14 @@ func (r *ruleRegistry) exists(key models.AlertRuleKey) bool {
return ok
}
// get fetches a rule from the registry by key. It returns (rule, ok) where ok is false if the rule did not exist.
func (r *ruleRegistry) get(key models.AlertRuleKey) (Rule, bool) {
r.mu.Lock()
defer r.mu.Unlock()
ru, ok := r.rules[key]
return ru, ok
}
// del removes pair that has specific key from the registry.
// Returns 2-tuple where the first element is value of the removed pair
// and the second element indicates whether element with the specified key existed.
pkg/services/ngalert/schedule/schedule.go
+8
View File
@@ -171,6 +171,14 @@ func (sch *schedule) Rules() ([]*ngmodels.AlertRule, map[ngmodels.FolderKey]stri
return sch.schedulableAlertRules.all()
}
// Status fetches the health of a given scheduled rule, by key.
func (sch *schedule) Status(key ngmodels.AlertRuleKey) (ngmodels.RuleStatus, bool) {
if rule, ok := sch.registry.get(key); ok {
return rule.Status(), true
}
return ngmodels.RuleStatus{}, false
}
// deleteAlertRule stops evaluation of the rule, deletes it from active rules, and cleans up state cache.
func (sch *schedule) deleteAlertRule(keys ...ngmodels.AlertRuleKey) {
for _, key := range keys {
pkg/services/ngalert/schedule/schedule_unit_test.go
+61
View File
@@ -113,6 +113,11 @@ func TestProcessTicks(t *testing.T) {
folderWithRuleGroup1 := fmt.Sprintf("%s;%s", ruleStore.getNamespaceTitle(alertRule1.NamespaceUID), alertRule1.RuleGroup)
t.Run("before 1st tick status should not be available", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.False(t, ok, "status for a rule should not be present before the scheduler has created it")
})
t.Run("on 1st tick alert rule should be evaluated", func(t *testing.T) {
tick = tick.Add(cfg.BaseInterval)
@@ -137,12 +142,25 @@ func TestProcessTicks(t *testing.T) {
require.NoError(t, err)
})
t.Run("after 1st tick status for rule should be available", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
// Interestingly, the rules in this test are randomised, and are sometimes invalid.
// Therefore, we can't reliably assert anything about the actual health. It might be error, it might not, depending on randomness.
// We are only testing that things were scheduled, not that the rule routine worked internally.
})
// add alert rule under main org with three base intervals
alertRule2 := gen.With(gen.WithOrgID(mainOrgID), gen.WithInterval(3*cfg.BaseInterval), gen.WithTitle("rule-2")).GenerateRef()
ruleStore.PutRule(ctx, alertRule2)
folderWithRuleGroup2 := fmt.Sprintf("%s;%s", ruleStore.getNamespaceTitle(alertRule2.NamespaceUID), alertRule2.RuleGroup)
t.Run("before 2nd tick status for rule should not be available", func(t *testing.T) {
_, ok := sched.Status(alertRule2.GetKey())
require.False(t, ok, "status for a rule should not be present before the scheduler has created it")
})
t.Run("on 2nd tick first alert rule should be evaluated", func(t *testing.T) {
tick = tick.Add(cfg.BaseInterval)
scheduled, stopped, updated := sched.processTick(ctx, dispatcherGroup, tick)
@@ -184,6 +202,16 @@ func TestProcessTicks(t *testing.T) {
assertEvalRun(t, evalAppliedCh, tick, keys...)
})
t.Run("after 3rd tick status for both rules should be available", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
_, ok = sched.Status(alertRule2.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
// Interestingly, the rules in this test are randomised, and are sometimes invalid.
// Therefore, we can't reliably assert anything about the actual health. It might be error, it might not, depending on randomness.
// We are only testing that things were scheduled, not that the rule routine worked internally.
})
t.Run("on 4th tick only one alert rule should be evaluated", func(t *testing.T) {
tick = tick.Add(cfg.BaseInterval)
scheduled, stopped, updated := sched.processTick(ctx, dispatcherGroup, tick)
@@ -223,6 +251,16 @@ func TestProcessTicks(t *testing.T) {
require.NoError(t, err)
})
t.Run("after 5th tick status for both rules should be available regardless of pause state", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
_, ok = sched.Status(alertRule2.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
// Interestingly, the rules in this test are randomised, and are sometimes invalid.
// Therefore, we can't reliably assert anything about the actual health. It might be error, it might not, depending on randomness.
// We are only testing that things were scheduled, not that the rule routine worked internally.
})
t.Run("on 6th tick all alert rule are paused (it still enters evaluation but it is early skipped)", func(t *testing.T) {
tick = tick.Add(cfg.BaseInterval)
@@ -309,6 +347,13 @@ func TestProcessTicks(t *testing.T) {
require.NoError(t, err)
})
t.Run("after 8th tick status for deleted rule should not be available", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.False(t, ok, "status for a rule that was deleted should not be available")
_, ok = sched.Status(alertRule2.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
})
t.Run("on 9th tick one alert rule should be evaluated", func(t *testing.T) {
tick = tick.Add(cfg.BaseInterval)
@@ -338,6 +383,14 @@ func TestProcessTicks(t *testing.T) {
require.Emptyf(t, updated, "None rules are expected to be updated")
assertEvalRun(t, evalAppliedCh, tick, alertRule3.GetKey())
})
t.Run("after 10th tick status for remaining rules should be available", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.False(t, ok, "status for a rule that was deleted should not be available")
_, ok = sched.Status(alertRule2.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
_, ok = sched.Status(alertRule3.GetKey())
require.True(t, ok, "status for a rule that just evaluated was not available")
})
t.Run("on 11th tick rule2 should be updated", func(t *testing.T) {
newRule2 := models.CopyRule(alertRule2)
newRule2.Version++
@@ -465,6 +518,14 @@ func TestProcessTicks(t *testing.T) {
require.Emptyf(t, updated, "No rules should be updated")
})
t.Run("after 12th tick no status should be available", func(t *testing.T) {
_, ok := sched.Status(alertRule1.GetKey())
require.False(t, ok, "status for a rule that was deleted should not be available")
_, ok = sched.Status(alertRule2.GetKey())
require.False(t, ok, "status for a rule that just evaluated was not available")
_, ok = sched.Status(alertRule3.GetKey())
require.False(t, ok, "status for a rule that just evaluated was not available")
})
t.Run("scheduled rules should be sorted", func(t *testing.T) {
rules := gen.With(gen.WithOrgID(mainOrgID), gen.WithInterval(cfg.BaseInterval)).GenerateManyRef(10, 20)