From 567a6a09bdacdaa2663f38418273359a08b78dc4 Mon Sep 17 00:00:00 2001 From: David Parrott Date: Tue, 13 Apr 2021 14:38:09 -0700 Subject: [PATCH] Alerting: Return RuleResponse for api/prometheus/grafana/api/v1/rules (#32919) * Return RuleResponse for api/prometheus/grafana/api/v1/rules * change TODO to note Co-authored-by: gotjosh * pr feedback * test fixup Co-authored-by: gotjosh --- pkg/services/ngalert/api/api.go | 2 +- pkg/services/ngalert/api/api_prometheus.go | 115 +++++++++++++++++- pkg/services/ngalert/models/alert_rule.go | 7 ++ pkg/services/ngalert/models/instance.go | 2 + pkg/services/ngalert/schedule/schedule.go | 12 +- pkg/services/ngalert/state/state_tracker.go | 2 +- .../ngalert/state/state_tracker_test.go | 11 +- pkg/services/ngalert/store/alert_rule.go | 15 +++ pkg/services/ngalert/tests/schedule_test.go | 4 +- 9 files changed, 147 insertions(+), 23 deletions(-) diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index 1f745d1bc69..888874a4bbb 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -69,7 +69,7 @@ func (api *API) RegisterAPIEndpoints() { api.RegisterPrometheusApiEndpoints(NewForkedProm( api.DatasourceCache, NewLotexProm(proxy, logger), - PrometheusSrv{log: logger, stateTracker: api.StateTracker}, + PrometheusSrv{log: logger, stateTracker: api.StateTracker, store: api.RuleStore}, )) // Register endpoints for proxing to Cortex Ruler-compatible backends. api.RegisterRulerApiEndpoints(NewForkedRuler( diff --git a/pkg/services/ngalert/api/api_prometheus.go b/pkg/services/ngalert/api/api_prometheus.go index 3f6a545698f..79f7099bdbc 100644 --- a/pkg/services/ngalert/api/api_prometheus.go +++ b/pkg/services/ngalert/api/api_prometheus.go @@ -1,7 +1,14 @@ package api import ( + "fmt" "net/http" + "time" + + apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/store" apimodels "github.com/grafana/alerting-api/pkg/api" "github.com/grafana/grafana/pkg/api/response" @@ -13,6 +20,7 @@ import ( type PrometheusSrv struct { log log.Logger stateTracker *state.StateTracker + store store.RuleStore } func (srv PrometheusSrv) RouteGetAlertStatuses(c *models.ReqContext) response.Response { @@ -38,7 +46,108 @@ func (srv PrometheusSrv) RouteGetAlertStatuses(c *models.ReqContext) response.Re } func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Response { - recipient := c.Params(":Recipient") - srv.log.Info("RouteGetRuleStatuses: ", "Recipient", recipient) - return response.Error(http.StatusNotImplemented, "", nil) + ruleResponse := apimodels.RuleResponse{ + DiscoveryBase: apimodels.DiscoveryBase{ + Status: "success", + }, + Data: apimodels.RuleDiscovery{}, + } + + ruleGroupQuery := ngmodels.ListOrgRuleGroupsQuery{ + OrgID: c.SignedInUser.OrgId, + } + if err := srv.store.GetOrgRuleGroups(&ruleGroupQuery); err != nil { + ruleResponse.DiscoveryBase.Status = "error" + ruleResponse.DiscoveryBase.Error = fmt.Sprintf("failure getting rule groups: %s", err.Error()) + ruleResponse.DiscoveryBase.ErrorType = apiv1.ErrServer + return response.JSON(http.StatusInternalServerError, ruleResponse) + } + + for _, groupId := range ruleGroupQuery.Result { + alertRuleQuery := ngmodels.ListRuleGroupAlertRulesQuery{OrgID: c.SignedInUser.OrgId, RuleGroup: groupId} + if err := srv.store.GetRuleGroupAlertRules(&alertRuleQuery); err != nil { + ruleResponse.DiscoveryBase.Status = "error" + ruleResponse.DiscoveryBase.Error = fmt.Sprintf("failure getting rules for group %s: %s", groupId, err.Error()) + ruleResponse.DiscoveryBase.ErrorType = apiv1.ErrServer + return response.JSON(http.StatusInternalServerError, ruleResponse) + } + + newGroup := &apimodels.RuleGroup{ + Name: groupId, + File: "", // This doesn't make sense in our architecture but would be a good use case for provisioned alerts. + LastEvaluation: time.Time{}, + EvaluationTime: 0, // TODO: see if we are able to pass this along with evaluation results + } + for _, rule := range alertRuleQuery.Result { + instanceQuery := ngmodels.ListAlertInstancesQuery{ + DefinitionOrgID: c.SignedInUser.OrgId, + DefinitionUID: rule.UID, + } + if err := srv.store.ListAlertInstances(&instanceQuery); err != nil { + ruleResponse.DiscoveryBase.Status = "error" + ruleResponse.DiscoveryBase.Error = fmt.Sprintf("failure getting alerts for rule %s: %s", rule.UID, err.Error()) + ruleResponse.DiscoveryBase.ErrorType = apiv1.ErrServer + return response.JSON(http.StatusInternalServerError, ruleResponse) + } + + alertingRule := apimodels.AlertingRule{ + State: "inactive", + Name: rule.Title, + Query: "", // TODO: get this from parsing AlertRule.Data + Duration: time.Duration(rule.For).Seconds(), + Annotations: rule.Annotations, + } + + newRule := apimodels.Rule{ + Name: rule.Title, + Labels: nil, // TODO: NG AlertRule does not have labels but does have annotations + Health: "ok", // TODO: update this in the future when error and noData states are being evaluated and set + Type: apiv1.RuleTypeAlerting, + LastEvaluation: time.Time{}, // TODO: set this to be rule evaluation time once it is being set + EvaluationTime: 0, // TODO: set this once we are saving it or adding it to evaluation results + } + for _, instance := range instanceQuery.Result { + activeAt := instance.CurrentStateSince + alert := &apimodels.Alert{ + Labels: map[string]string(instance.Labels), + Annotations: nil, // TODO: set these once they are added to evaluation results + State: translateInstanceState(instance.CurrentState), + ActiveAt: &activeAt, + Value: "", // TODO: set this once it is added to the evaluation results + } + if instance.LastEvalTime.After(newRule.LastEvaluation) { + newRule.LastEvaluation = instance.LastEvalTime + newGroup.LastEvaluation = instance.LastEvalTime + } + switch alert.State { + case "pending": + if alertingRule.State == "inactive" { + alertingRule.State = "pending" + } + case "firing": + alertingRule.State = "firing" + } + + alertingRule.Alerts = append(alertingRule.Alerts, alert) + } + alertingRule.Rule = newRule + newGroup.Rules = append(newGroup.Rules, alertingRule) + newGroup.Interval = float64(rule.IntervalSeconds) + } + ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, newGroup) + } + return response.JSON(http.StatusOK, ruleResponse) +} + +func translateInstanceState(state ngmodels.InstanceStateType) string { + switch { + case state == ngmodels.InstanceStateFiring: + return "firing" + case state == ngmodels.InstanceStateNormal: + return "inactive" + case state == ngmodels.InstanceStatePending: + return "pending" + default: + return "inactive" + } } diff --git a/pkg/services/ngalert/models/alert_rule.go b/pkg/services/ngalert/models/alert_rule.go index c9ad6b2aaef..135c1ceb6d3 100644 --- a/pkg/services/ngalert/models/alert_rule.go +++ b/pkg/services/ngalert/models/alert_rule.go @@ -140,6 +140,13 @@ type ListRuleGroupAlertRulesQuery struct { Result []*AlertRule } +// ListOrgRuleGroupsQuery is the query for listing unique rule groups +type ListOrgRuleGroupsQuery struct { + OrgID int64 + + Result []string +} + // Condition contains backend expressions and queries and the RefID // of the query or expression that will be evaluated. type Condition struct { diff --git a/pkg/services/ngalert/models/instance.go b/pkg/services/ngalert/models/instance.go index 6e23ee5be83..1a9eb69164c 100644 --- a/pkg/services/ngalert/models/instance.go +++ b/pkg/services/ngalert/models/instance.go @@ -25,6 +25,8 @@ const ( InstanceStateFiring InstanceStateType = "Alerting" // InstanceStateNormal is for a normal alert. InstanceStateNormal InstanceStateType = "Normal" + // InstanceStatePending is for an alert that is firing but has not met the duration + InstanceStatePending InstanceStateType = "Pending" // InstanceStateNoData is for an alert with no data. InstanceStateNoData InstanceStateType = "NoData" // InstanceStateError is for a erroring alert. diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index adbaa8d85ec..a91903c336a 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -6,8 +6,6 @@ import ( "sync" "time" - "github.com/grafana/grafana-plugin-sdk-go/data" - "golang.org/x/sync/errgroup" "github.com/benbjohnson/clock" @@ -339,14 +337,6 @@ func (sch *schedule) saveAlertStates(states []state.AlertState) { } } -func dataLabelsFromInstanceLabels(il models.InstanceLabels) data.Labels { - lbs := data.Labels{} - for k, v := range il { - lbs[k] = v - } - return lbs -} - func (sch *schedule) WarmStateCache(st *state.StateTracker) { sch.log.Info("warming cache for startup") st.ResetCache() @@ -365,7 +355,7 @@ func (sch *schedule) WarmStateCache(st *state.StateTracker) { sch.log.Error("unable to fetch previous state", "msg", err.Error()) } for _, entry := range cmd.Result { - lbs := dataLabelsFromInstanceLabels(entry.Labels) + lbs := map[string]string(entry.Labels) stateForEntry := state.AlertState{ UID: entry.DefinitionUID, OrgID: entry.DefinitionOrgID, diff --git a/pkg/services/ngalert/state/state_tracker.go b/pkg/services/ngalert/state/state_tracker.go index f136778a910..9d9f0d27100 100644 --- a/pkg/services/ngalert/state/state_tracker.go +++ b/pkg/services/ngalert/state/state_tracker.go @@ -57,7 +57,7 @@ func (st *StateTracker) getOrCreate(uid string, orgId int64, result eval.Result) st.stateCache.mu.Lock() defer st.stateCache.mu.Unlock() - idString := fmt.Sprintf("%s %s", uid, result.Instance.String()) + idString := fmt.Sprintf("%s %s", uid, map[string]string(result.Instance)) if state, ok := st.stateCache.cacheMap[idString]; ok { return state } diff --git a/pkg/services/ngalert/state/state_tracker_test.go b/pkg/services/ngalert/state/state_tracker_test.go index d89bce52dc7..6632b4da2c2 100644 --- a/pkg/services/ngalert/state/state_tracker_test.go +++ b/pkg/services/ngalert/state/state_tracker_test.go @@ -18,6 +18,7 @@ func TestProcessEvalResults(t *testing.T) { if err != nil { t.Fatalf("error parsing date format: %s", err.Error()) } + cacheId := "test_uid map[label1:value1 label2:value2]" testCases := []struct { desc string uid string @@ -49,7 +50,7 @@ func TestProcessEvalResults(t *testing.T) { { UID: "test_uid", OrgID: 123, - CacheId: "test_uid label1=value1, label2=value2", + CacheId: cacheId, Labels: data.Labels{"label1": "value1", "label2": "value2"}, State: eval.Normal, Results: []StateEvaluation{ @@ -87,7 +88,7 @@ func TestProcessEvalResults(t *testing.T) { { UID: "test_uid", OrgID: 123, - CacheId: "test_uid label1=value1, label2=value2", + CacheId: cacheId, Labels: data.Labels{"label1": "value1", "label2": "value2"}, State: eval.Alerting, Results: []StateEvaluation{ @@ -126,7 +127,7 @@ func TestProcessEvalResults(t *testing.T) { { UID: "test_uid", OrgID: 123, - CacheId: "test_uid label1=value1, label2=value2", + CacheId: cacheId, Labels: data.Labels{"label1": "value1", "label2": "value2"}, State: eval.Normal, Results: []StateEvaluation{ @@ -165,7 +166,7 @@ func TestProcessEvalResults(t *testing.T) { { UID: "test_uid", OrgID: 123, - CacheId: "test_uid label1=value1, label2=value2", + CacheId: cacheId, Labels: data.Labels{"label1": "value1", "label2": "value2"}, State: eval.Alerting, Results: []StateEvaluation{ @@ -204,7 +205,7 @@ func TestProcessEvalResults(t *testing.T) { { UID: "test_uid", OrgID: 123, - CacheId: "test_uid label1=value1, label2=value2", + CacheId: cacheId, Labels: data.Labels{"label1": "value1", "label2": "value2"}, State: eval.Normal, Results: []StateEvaluation{ diff --git a/pkg/services/ngalert/store/alert_rule.go b/pkg/services/ngalert/store/alert_rule.go index 9bb80936afd..13043a83cfa 100644 --- a/pkg/services/ngalert/store/alert_rule.go +++ b/pkg/services/ngalert/store/alert_rule.go @@ -45,6 +45,7 @@ type RuleStore interface { GetRuleGroupAlertRules(query *ngmodels.ListRuleGroupAlertRulesQuery) error GetNamespaceUIDBySlug(string, int64, *models.SignedInUser) (string, error) GetNamespaceByUID(string, int64, *models.SignedInUser) (string, error) + GetOrgRuleGroups(query *ngmodels.ListOrgRuleGroupsQuery) error UpsertAlertRules([]UpsertRule) error UpdateRuleGroup(UpdateRuleGroupCmd) error GetAlertInstance(*ngmodels.GetAlertInstanceQuery) error @@ -298,6 +299,7 @@ func (st DBstore) GetNamespaceAlertRules(query *ngmodels.ListNamespaceAlertRules func (st DBstore) GetRuleGroupAlertRules(query *ngmodels.ListRuleGroupAlertRulesQuery) error { return st.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { alertRules := make([]*ngmodels.AlertRule, 0) + q := "SELECT * FROM alert_rule WHERE org_id = ? and namespace_uid = ? and rule_group = ?" if err := sess.SQL(q, query.OrgID, query.NamespaceUID, query.RuleGroup).Find(&alertRules); err != nil { return err @@ -455,3 +457,16 @@ func (st DBstore) UpdateRuleGroup(cmd UpdateRuleGroupCmd) error { return nil }) } + +func (st DBstore) GetOrgRuleGroups(query *ngmodels.ListOrgRuleGroupsQuery) error { + return st.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { + var ruleGroups []string + q := "SELECT DISTINCT rule_group FROM alert_rule WHERE org_id = ?" + if err := sess.SQL(q, query.OrgID).Find(&ruleGroups); err != nil { + return err + } + + query.Result = ruleGroups + return nil + }) +} diff --git a/pkg/services/ngalert/tests/schedule_test.go b/pkg/services/ngalert/tests/schedule_test.go index df8cacdda1e..39f21da25de 100644 --- a/pkg/services/ngalert/tests/schedule_test.go +++ b/pkg/services/ngalert/tests/schedule_test.go @@ -37,7 +37,7 @@ func TestWarmStateCache(t *testing.T) { { UID: "test_uid", OrgID: 123, - CacheId: "test_uid test1=testValue1", + CacheId: "test_uid map[test1:testValue1]", Labels: data.Labels{"test1": "testValue1"}, State: eval.Normal, Results: []state.StateEvaluation{ @@ -49,7 +49,7 @@ func TestWarmStateCache(t *testing.T) { }, { UID: "test_uid", OrgID: 123, - CacheId: "test_uid test2=testValue2", + CacheId: "test_uid map[test2:testValue2]", Labels: data.Labels{"test2": "testValue2"}, State: eval.Alerting, Results: []state.StateEvaluation{