public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Move rule evaluation status logic out of prometheus API and into scheduler (#89141)

Browse Source 
* Add health fields to rules and an aggregator method to the scheduler

* Move health, last error, and last eval time in together to minimize state processing

* Wire up a readonly scheduler to prom api

* Extract to exported function

* Use health in api_prometheus and fix up tests

* Rename health struct to status

* Fix tests one more time

* Several new tests

* Handle inactive rules

* Push state mapping into state manager

* rename to StatusReader

* Rectify cyclo complexity rebase

* Convert existing package local status implementation to models one

* fix tests

* undo RuleDefs rename
This commit is contained in:
Alexander Weaver
2024-09-30 16:52:49 -05:00
committed by GitHub
parent 6a3eb276ef
commit 393faa8732
13 changed files with 213 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api.go
+2 -1
View File
@@ -63,6 +63,7 @@ type API struct {
DataProxy *datasourceproxy.DataSourceProxyService
MultiOrgAlertmanager *notifier.MultiOrgAlertmanager
StateManager *state.Manager
Scheduler StatusReader
AccessControl ac.AccessControl
Policies *provisioning.NotificationPolicyService
ReceiverService *notifier.ReceiverService
@@ -115,7 +116,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
api.RegisterPrometheusApiEndpoints(NewForkingProm(
api.DatasourceCache,
NewLotexProm(proxy, logger),
&PrometheusSrv{log: logger, manager: api.StateManager, store: api.RuleStore, authz: ruleAuthzService},
&PrometheusSrv{log: logger, manager: api.StateManager, status: api.Scheduler, store: api.RuleStore, authz: ruleAuthzService},
), m)
// Register endpoints for proxying to Cortex Ruler-compatible backends.
api.RegisterRulerApiEndpoints(NewForkingRuler(
pkg/services/ngalert/api/api_prometheus.go
+29 -20
View File
@@ -9,7 +9,6 @@ import (
"sort"
"strconv"
"strings"
"time"
"github.com/prometheus/alertmanager/pkg/labels"
apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
@@ -24,9 +23,14 @@ import (
"github.com/grafana/grafana/pkg/util"
)
type StatusReader interface {
Status(key ngmodels.AlertRuleKey) (ngmodels.RuleStatus, bool)
}
type PrometheusSrv struct {
log log.Logger
manager state.AlertInstanceManager
status StatusReader
store RuleStore
authz RuleAccessControlService
}
@@ -222,7 +226,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
namespaces[namespaceUID] = folder.Fullpath
}
ruleResponse = PrepareRuleGroupStatuses(srv.log, srv.manager, srv.store, RuleGroupStatusesOptions{
ruleResponse = PrepareRuleGroupStatuses(srv.log, srv.manager, srv.status, srv.store, RuleGroupStatusesOptions{
Ctx: c.Req.Context(),
OrgID: c.OrgID,
Query: c.Req.Form,
@@ -235,7 +239,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
return response.JSON(ruleResponse.HTTPStatusCode(), ruleResponse)
}
func PrepareRuleGroupStatuses(log log.Logger, manager state.AlertInstanceManager, store ListAlertRulesStore, opts RuleGroupStatusesOptions) apimodels.RuleResponse {
func PrepareRuleGroupStatuses(log log.Logger, manager state.AlertInstanceManager, status StatusReader, store ListAlertRulesStore, opts RuleGroupStatusesOptions) apimodels.RuleResponse {
ruleResponse := apimodels.RuleResponse{
DiscoveryBase: apimodels.DiscoveryBase{
Status: "success",
@@ -346,7 +350,7 @@ func PrepareRuleGroupStatuses(log log.Logger, manager state.AlertInstanceManager
continue
}
ruleGroup, totals := toRuleGroup(log, manager, groupKey, folder, rules, limitAlertsPerRule, withStatesFast, matchers, labelOptions)
ruleGroup, totals := toRuleGroup(log, manager, status, groupKey, folder, rules, limitAlertsPerRule, withStatesFast, matchers, labelOptions)
ruleGroup.Totals = totals
for k, v := range totals {
rulesTotals[k] += v
@@ -432,7 +436,7 @@ func matchersMatch(matchers []*labels.Matcher, labels map[string]string) bool {
return true
}
func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, groupKey ngmodels.AlertRuleGroupKey, folderFullPath string, rules []*ngmodels.AlertRule, limitAlerts int64, withStates map[eval.State]struct{}, matchers labels.Matchers, labelOptions []ngmodels.LabelOption) (*apimodels.RuleGroup, map[string]int64) {
func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusReader, groupKey ngmodels.AlertRuleGroupKey, folderFullPath string, rules []*ngmodels.AlertRule, limitAlerts int64, withStates map[eval.State]struct{}, matchers labels.Matchers, labelOptions []ngmodels.LabelOption) (*apimodels.RuleGroup, map[string]int64) {
newGroup := &apimodels.RuleGroup{
Name: groupKey.RuleGroup,
// file is what Prometheus uses for provisioning, we replace it with namespace which is the folder in Grafana.
@@ -443,6 +447,15 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, groupKey ng
ngmodels.RulesGroup(rules).SortByGroupIndex()
for _, rule := range rules {
status, ok := sr.Status(rule.GetKey())
// Grafana by design return "ok" health and default other fields for unscheduled rules.
// This differs from Prometheus.
if !ok {
status = ngmodels.RuleStatus{
Health: "ok",
}
}
alertingRule := apimodels.AlertingRule{
State: "inactive",
Name: rule.Title,
@@ -454,9 +467,11 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, groupKey ng
newRule := apimodels.Rule{
Name: rule.Title,
Labels: apimodels.LabelsFromMap(rule.GetLabels(labelOptions...)),
Health: "ok",
Health: status.Health,
LastError: errorOrEmpty(status.LastError),
Type: rule.Type().String(),
LastEvaluation: time.Time{},
LastEvaluation: status.EvaluationTimestamp,
EvaluationTime: status.EvaluationDuration.Seconds(),
}
states := manager.GetStatesForRuleUID(rule.OrgID, rule.UID)
@@ -485,12 +500,6 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, groupKey ng
Value: valString,
}
if alertState.LastEvaluationTime.After(newRule.LastEvaluation) {
newRule.LastEvaluation = alertState.LastEvaluationTime
}
newRule.EvaluationTime = alertState.EvaluationDuration.Seconds()
switch alertState.State {
case eval.Normal:
case eval.Pending:
@@ -503,14 +512,7 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, groupKey ng
}
alertingRule.State = "firing"
case eval.Error:
newRule.Health = "error"
case eval.NoData:
newRule.Health = "nodata"
}
if alertState.Error != nil {
newRule.LastError = alertState.Error.Error()
newRule.Health = "error"
}
if len(withStates) > 0 {
@@ -604,3 +606,10 @@ func encodedQueriesOrError(rules []ngmodels.AlertQuery) string {
return err.Error()
}
func errorOrEmpty(err error) string {
if err != nil {
return err.Error()
}
return ""
}
pkg/services/ngalert/api/api_prometheus_test.go
+6
View File
@@ -489,6 +489,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
t.Run("should return sorted", func(t *testing.T) {
ruleStore := fakes.NewRuleStore(t)
fakeAIM := NewFakeAlertInstanceManager(t)
fakeSch := newFakeSchedulerReader(t).setupStates(fakeAIM)
groupKey := ngmodels.GenerateGroupKey(orgID)
gen := ngmodels.RuleGen
rules := gen.With(gen.WithGroupKey(groupKey), gen.WithUniqueGroupIndex()).GenerateManyRef(5, 10)
@@ -497,6 +498,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
api := PrometheusSrv{
log: log.NewNopLogger(),
manager: fakeAIM,
status: fakeSch,
store: ruleStore,
authz: &fakeRuleAccessControlService{},
}
@@ -558,6 +560,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
api := PrometheusSrv{
log: log.NewNopLogger(),
manager: fakeAIM,
status: newFakeSchedulerReader(t).setupStates(fakeAIM),
store: ruleStore,
authz: accesscontrol.NewRuleService(acimpl.ProvideAccessControl(featuremgmt.WithFeatures(), zanzana.NewNoopClient())),
}
@@ -673,6 +676,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
api := PrometheusSrv{
log: log.NewNopLogger(),
manager: fakeAIM,
status: newFakeSchedulerReader(t).setupStates(fakeAIM),
store: ruleStore,
authz: accesscontrol.NewRuleService(acimpl.ProvideAccessControl(featuremgmt.WithFeatures(), zanzana.NewNoopClient())),
}
@@ -1389,11 +1393,13 @@ func TestRouteGetRuleStatuses(t *testing.T) {
func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, PrometheusSrv) {
fakeStore := fakes.NewRuleStore(t)
fakeAIM := NewFakeAlertInstanceManager(t)
fakeSch := newFakeSchedulerReader(t).setupStates(fakeAIM)
fakeAuthz := &fakeRuleAccessControlService{}
api := PrometheusSrv{
log: log.NewNopLogger(),
manager: fakeAIM,
status: fakeSch,
store: fakeStore,
authz: fakeAuthz,
}
pkg/services/ngalert/api/testing.go
+26
View File
@@ -166,3 +166,29 @@ func (f fakeRuleAccessControlService) AuthorizeDatasourceAccessForRule(ctx conte
func (f fakeRuleAccessControlService) AuthorizeDatasourceAccessForRuleGroup(ctx context.Context, user identity.Requester, rules models.RulesGroup) error {
return nil
}
type statesReader interface {
GetStatesForRuleUID(orgID int64, alertRuleUID string) []*state.State
}
type fakeSchedulerReader struct {
states statesReader
}
func newFakeSchedulerReader(t *testing.T) *fakeSchedulerReader {
return &fakeSchedulerReader{}
}
// setupStates allows the fake scheduler to return data consistent with states defined elsewhere.
// This can be combined with fakeAlertInstanceManager, for instance.
func (f *fakeSchedulerReader) setupStates(reader statesReader) *fakeSchedulerReader {
f.states = reader
return f
}
func (f *fakeSchedulerReader) Status(key models.AlertRuleKey) (models.RuleStatus, bool) {
if f.states == nil {
return models.RuleStatus{}, false
}
return state.StatesToRuleStatus(f.states.GetStatesForRuleUID(key.OrgID, key.UID)), true
}