public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Alerting]: Update scheduler to evaluate rules created by the unified API (#32589)

Browse Source 
* Update scheduler

* Fix tests

* Fixes after code review feedback

* lint - add uncommitted modifications

Co-authored-by: kyle <kyle@grafana.com>
This commit is contained in:
Sofia Papagiannaki
2021-04-03 20:13:29 +03:00
committed by GitHub
co-authored by kyle
parent 455fbce020
commit daabf64aa1
11 changed files with 275 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/tests/database_test.go
+30
View File
@@ -5,6 +5,8 @@ package tests
import (
"encoding/json"
"errors"
"fmt"
"math/rand"
"testing"
"time"
@@ -418,3 +420,31 @@ func getLongString(n int) string {
}
return string(b)
}
// createTestAlertDefinition creates a dummy alert definition to be used by the tests.
func createTestAlertDefinition(t *testing.T, dbstore *store.DBstore, intervalSeconds int64) *models.AlertDefinition {
cmd := models.SaveAlertDefinitionCommand{
OrgID: 1,
Title: fmt.Sprintf("an alert definition %d", rand.Intn(1000)),
Condition: "A",
Data: []models.AlertQuery{
{
Model: json.RawMessage(`{
"datasource": "__expr__",
"type":"math",
"expression":"2 + 2 > 1"
}`),
RelativeTimeRange: models.RelativeTimeRange{
From: models.Duration(5 * time.Hour),
To: models.Duration(3 * time.Hour),
},
RefID: "A",
},
},
IntervalSeconds: &intervalSeconds,
}
err := dbstore.SaveAlertDefinition(&cmd)
require.NoError(t, err)
t.Logf("alert definition: %v with interval: %d created", cmd.Result.GetKey(), intervalSeconds)
return cmd.Result
}
pkg/services/ngalert/tests/schedule_test.go
+61 -92
View File
@@ -26,7 +26,7 @@ import (
)
type evalAppliedInfo struct {
alertDefKey models.AlertDefinitionKey
alertDefKey models.AlertRuleKey
now time.Time
}
@@ -109,15 +109,15 @@ func TestAlertingTicker(t *testing.T) {
dbstore := setupTestEnv(t, 1)
t.Cleanup(registry.ClearOverrides)
alerts := make([]*models.AlertDefinition, 0)
// create alert definition with zero interval (should never run)
alerts = append(alerts, createTestAlertDefinition(t, dbstore, 0))
alerts := make([]*models.AlertRule, 0)
// create alert rule with zero interval (should never run)
alerts = append(alerts, createTestAlertRule(t, dbstore, 0))
// create alert definition with one second interval
alerts = append(alerts, createTestAlertDefinition(t, dbstore, 1))
// create alert rule with one second interval
alerts = append(alerts, createTestAlertRule(t, dbstore, 1))
evalAppliedCh := make(chan evalAppliedInfo, len(alerts))
stopAppliedCh := make(chan models.AlertDefinitionKey, len(alerts))
stopAppliedCh := make(chan models.AlertRuleKey, len(alerts))
mockedClock := clock.NewMock()
baseInterval := time.Second
@@ -125,14 +125,15 @@ func TestAlertingTicker(t *testing.T) {
schedCfg := schedule.SchedulerCfg{
C: mockedClock,
BaseInterval: baseInterval,
EvalAppliedFunc: func(alertDefKey models.AlertDefinitionKey, now time.Time) {
EvalAppliedFunc: func(alertDefKey models.AlertRuleKey, now time.Time) {
evalAppliedCh <- evalAppliedInfo{alertDefKey: alertDefKey, now: now}
},
StopAppliedFunc: func(alertDefKey models.AlertDefinitionKey) {
StopAppliedFunc: func(alertDefKey models.AlertRuleKey) {
stopAppliedCh <- alertDefKey
},
Store: dbstore,
Logger: log.New("ngalert schedule test"),
Store: dbstore,
RuleStore: dbstore,
Logger: log.New("ngalert schedule test"),
}
sched := schedule.NewScheduler(schedCfg, nil)
@@ -145,101 +146,69 @@ func TestAlertingTicker(t *testing.T) {
}()
runtime.Gosched()
expectedAlertDefinitionsEvaluated := []models.AlertDefinitionKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 1st tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
expectedAlertRulesEvaluated := []models.AlertRuleKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 1st tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
// change alert definition interval to three seconds
// change alert rule interval to three seconds
var threeSecInterval int64 = 3
err := dbstore.UpdateAlertDefinition(&models.UpdateAlertDefinitionCommand{
UID: alerts[0].UID,
IntervalSeconds: &threeSecInterval,
OrgID: alerts[0].OrgID,
alerts[0] = updateTestAlertRuleIntervalSeconds(t, dbstore, alerts[0], threeSecInterval)
t.Logf("alert rule: %v interval reset to: %d", alerts[0].GetKey(), threeSecInterval)
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 2nd tick alert rule: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[1].GetKey(), alerts[0].GetKey()}
t.Run(fmt.Sprintf("on 3rd tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 4th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
err := dbstore.DeleteAlertRuleByUID(alerts[1].OrgID, alerts[1].UID)
require.NoError(t, err)
t.Logf("alert definition: %v interval reset to: %d", alerts[0].GetKey(), threeSecInterval)
t.Logf("alert rule: %v deleted", alerts[1].GetKey())
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 2nd tick alert definition: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
expectedAlertRulesEvaluated = []models.AlertRuleKey{}
t.Run(fmt.Sprintf("on 5th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
expectedAlertRulesStopped := []models.AlertRuleKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 5th tick alert rules: %s should be stopped", concatenate(expectedAlertRulesStopped)), func(t *testing.T) {
assertStopRun(t, stopAppliedCh, expectedAlertRulesStopped...)
})
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{alerts[1].GetKey(), alerts[0].GetKey()}
t.Run(fmt.Sprintf("on 3rd tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[0].GetKey()}
t.Run(fmt.Sprintf("on 6th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 4th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
// create alert rule with one second interval
alerts = append(alerts, createTestAlertRule(t, dbstore, 1))
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[2].GetKey()}
t.Run(fmt.Sprintf("on 7th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
err = dbstore.DeleteAlertDefinitionByUID(&models.DeleteAlertDefinitionByUIDCommand{UID: alerts[1].UID, OrgID: alerts[1].OrgID})
require.NoError(t, err)
t.Logf("alert definition: %v deleted", alerts[1].GetKey())
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{}
t.Run(fmt.Sprintf("on 5th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
expectedAlertDefinitionsStopped := []models.AlertDefinitionKey{alerts[1].GetKey()}
t.Run(fmt.Sprintf("on 5th tick alert definitions: %s should be stopped", concatenate(expectedAlertDefinitionsStopped)), func(t *testing.T) {
assertStopRun(t, stopAppliedCh, expectedAlertDefinitionsStopped...)
})
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{alerts[0].GetKey()}
t.Run(fmt.Sprintf("on 6th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
// create alert definition with one second interval
alerts = append(alerts, createTestAlertDefinition(t, dbstore, 1))
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{alerts[2].GetKey()}
t.Run(fmt.Sprintf("on 7th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
// pause alert definition
err = dbstore.UpdateAlertDefinitionPaused(&models.UpdateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: true})
require.NoError(t, err)
t.Logf("alert definition: %v paused", alerts[2].GetKey())
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{}
t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
expectedAlertDefinitionsStopped = []models.AlertDefinitionKey{alerts[2].GetKey()}
t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be stopped", concatenate(expectedAlertDefinitionsStopped)), func(t *testing.T) {
assertStopRun(t, stopAppliedCh, expectedAlertDefinitionsStopped...)
})
// unpause alert definition
err = dbstore.UpdateAlertDefinitionPaused(&models.UpdateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: false})
require.NoError(t, err)
t.Logf("alert definition: %v unpaused", alerts[2].GetKey())
expectedAlertDefinitionsEvaluated = []models.AlertDefinitionKey{alerts[0].GetKey(), alerts[2].GetKey()}
t.Run(fmt.Sprintf("on 9th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...)
})
}
func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys ...models.AlertDefinitionKey) {
func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys ...models.AlertRuleKey) {
timeout := time.After(time.Second)
expected := make(map[models.AlertDefinitionKey]struct{}, len(keys))
expected := make(map[models.AlertRuleKey]struct{}, len(keys))
for _, k := range keys {
expected[k] = struct{}{}
}
@@ -248,7 +217,7 @@ func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys
select {
case info := <-ch:
_, ok := expected[info.alertDefKey]
t.Logf("alert definition: %v evaluated at: %v", info.alertDefKey, info.now)
t.Logf("alert rule: %v evaluated at: %v", info.alertDefKey, info.now)
assert.True(t, ok)
assert.Equal(t, tick, info.now)
delete(expected, info.alertDefKey)
@@ -264,10 +233,10 @@ func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys
}
}
func assertStopRun(t *testing.T, ch <-chan models.AlertDefinitionKey, keys ...models.AlertDefinitionKey) {
func assertStopRun(t *testing.T, ch <-chan models.AlertRuleKey, keys ...models.AlertRuleKey) {
timeout := time.After(time.Second)
expected := make(map[models.AlertDefinitionKey]struct{}, len(keys))
expected := make(map[models.AlertRuleKey]struct{}, len(keys))
for _, k := range keys {
expected[k] = struct{}{}
}
@@ -276,7 +245,7 @@ func assertStopRun(t *testing.T, ch <-chan models.AlertDefinitionKey, keys ...mo
select {
case alertDefKey := <-ch:
_, ok := expected[alertDefKey]
t.Logf("alert definition: %v stopped", alertDefKey)
t.Logf("alert rule: %v stopped", alertDefKey)
assert.True(t, ok)
delete(expected, alertDefKey)
if len(expected) == 0 {
@@ -297,7 +266,7 @@ func advanceClock(t *testing.T, mockedClock *clock.Mock) time.Time {
// t.Logf("Tick: %v", mockedClock.Now())
}
func concatenate(keys []models.AlertDefinitionKey) string {
func concatenate(keys []models.AlertRuleKey) string {
s := make([]string, len(keys))
for _, k := range keys {
s = append(s, k.String())
pkg/services/ngalert/tests/util.go
+84 -22
View File
@@ -7,6 +7,9 @@ import (
"testing"
"time"
apimodels "github.com/grafana/alerting-api/pkg/api"
"github.com/prometheus/common/model"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert"
@@ -61,30 +64,89 @@ func overrideAlertNGInRegistry(t *testing.T, cfg *setting.Cfg) ngalert.AlertNG {
return ng
}
// createTestAlertDefinition creates a dummy alert definition to be used by the tests.
func createTestAlertDefinition(t *testing.T, store *store.DBstore, intervalSeconds int64) *models.AlertDefinition {
cmd := models.SaveAlertDefinitionCommand{
OrgID: 1,
Title: fmt.Sprintf("an alert definition %d", rand.Intn(1000)),
Condition: "A",
Data: []models.AlertQuery{
{
Model: json.RawMessage(`{
"datasource": "__expr__",
"type":"math",
"expression":"2 + 2 > 1"
}`),
RelativeTimeRange: models.RelativeTimeRange{
From: models.Duration(5 * time.Hour),
To: models.Duration(3 * time.Hour),
// createTestAlertRule creates a dummy alert definition to be used by the tests.
func createTestAlertRule(t *testing.T, dbstore *store.DBstore, intervalSeconds int64) *models.AlertRule {
d := rand.Intn(1000)
ruleGroup := fmt.Sprintf("ruleGroup-%d", d)
err := dbstore.UpdateRuleGroup(store.UpdateRuleGroupCmd{
OrgID: 1,
NamespaceUID: "namespace",
RuleGroupConfig: apimodels.PostableRuleGroupConfig{
Name: ruleGroup,
Interval: model.Duration(time.Duration(intervalSeconds) * time.Second),
Rules: []apimodels.PostableExtendedRuleNode{
{
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
OrgID: 1,
Title: fmt.Sprintf("an alert definition %d", d),
Condition: "A",
Data: []models.AlertQuery{
{
Model: json.RawMessage(`{
"datasource": "__expr__",
"type":"math",
"expression":"2 + 2 > 1"
}`),
RelativeTimeRange: models.RelativeTimeRange{
From: models.Duration(5 * time.Hour),
To: models.Duration(3 * time.Hour),
},
RefID: "A",
},
},
},
},
RefID: "A",
},
},
IntervalSeconds: &intervalSeconds,
}
err := store.SaveAlertDefinition(&cmd)
})
require.NoError(t, err)
t.Logf("alert definition: %v with interval: %d created", cmd.Result.GetKey(), intervalSeconds)
return cmd.Result
q := models.ListRuleGroupAlertRulesQuery{
OrgID: 1,
NamespaceUID: "namespace",
RuleGroup: ruleGroup,
}
err = dbstore.GetRuleGroupAlertRules(&q)
require.NoError(t, err)
require.NotEmpty(t, q.Result)
rule := q.Result[0]
t.Logf("alert definition: %v with interval: %d created", rule.GetKey(), rule.IntervalSeconds)
return rule
}
// updateTestAlertRule update a dummy alert definition to be used by the tests.
func updateTestAlertRuleIntervalSeconds(t *testing.T, dbstore *store.DBstore, existingRule *models.AlertRule, intervalSeconds int64) *models.AlertRule {
cmd := store.UpdateRuleGroupCmd{
OrgID: 1,
NamespaceUID: "namespace",
RuleGroupConfig: apimodels.PostableRuleGroupConfig{
Name: existingRule.RuleGroup,
Interval: model.Duration(time.Duration(intervalSeconds) * time.Second),
Rules: []apimodels.PostableExtendedRuleNode{
{
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
OrgID: 1,
UID: existingRule.UID,
},
},
},
},
}
err := dbstore.UpdateRuleGroup(cmd)
require.NoError(t, err)
q := models.ListRuleGroupAlertRulesQuery{
OrgID: 1,
NamespaceUID: "namespace",
RuleGroup: existingRule.RuleGroup,
}
err = dbstore.GetRuleGroupAlertRules(&q)
require.NoError(t, err)
require.NotEmpty(t, q.Result)
rule := q.Result[0]
t.Logf("alert definition: %v with interval: %d created", rule.GetKey(), rule.IntervalSeconds)
return rule
}