Alerting: Add limits and filters to Prometheus Rules API (#66627)

This commit adds support for limits and filters to the Prometheus Rules
API.

Limits:

It adds a number of limits to the Grafana flavour of the Prometheus Rules
API:

- `limit` limits the maximum number of Rule Groups returned
- `limit_rules` limits the maximum number of rules per Rule Group
- `limit_alerts` limits the maximum number of alerts per rule

It sorts Rule Groups and rules within Rule Groups such that data in the
response is stable across requests. It also returns summaries (totals)
for all Rule Groups, individual Rule Groups and rules.

Filters:

Alerts can be filtered by state with the `state` query string. An example
of an HTTP request asking for just firing alerts might be
`/api/prometheus/grafana/api/v1/rules?state=alerting`.

A request can filter by two or more states by adding additional `state`
query strings to the URL. For example `?state=alerting&state=normal`.

Like the alert list panel, the `firing`, `pending` and `normal` state are
first compared against the state of each alert rule. All other states are
ignored. If the alert rule matches then its alert instances are filtered
against states once more.

Alerts can also be filtered by labels using the `matcher` query string.
Like `state`, multiple matchers can be provided by adding additional
`matcher` query strings to the URL.

The match expression should be parsed using existing regular expression
and sent to the API as URL-encoded JSON in the format:

{
    "name": "test",
    "value": "value1",
    "isRegex": false,
    "isEqual": true
}

The `isRegex` and `isEqual` options work as follows:

| IsEqual | IsRegex  | Operator |
| ------- | -------- | -------- |
| true    | false    |    =     |
| true    | true     |    =~    |
| false   | true     |    !~    |
| false   | false    |    !=    |

pkg/services/ngalert/api/api_prometheus.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/prometheus/alertmanager/pkg/labels"
 	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 
 	"github.com/grafana/grafana/pkg/api/response"
@@ -21,6 +22,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
+	"github.com/grafana/grafana/pkg/util"
 )
 
 type PrometheusSrv struct {
@@ -105,6 +107,44 @@ func getPanelIDFromRequest(r *http.Request) (int64, error) {
 	return 0, nil
 }
 
+func getMatchersFromRequest(r *http.Request) (labels.Matchers, error) {
+	var matchers labels.Matchers
+	for _, s := range r.URL.Query()["matcher"] {
+		var m labels.Matcher
+		if err := json.Unmarshal([]byte(s), &m); err != nil {
+			return nil, err
+		}
+		if len(m.Name) == 0 {
+			return nil, errors.New("bad matcher: the name cannot be blank")
+		}
+		matchers = append(matchers, &m)
+	}
+	return matchers, nil
+}
+
+func getStatesFromRequest(r *http.Request) ([]eval.State, error) {
+	var states []eval.State
+	for _, s := range r.URL.Query()["state"] {
+		s = strings.ToLower(s)
+		switch s {
+		case "normal", "inactive":
+			states = append(states, eval.Normal)
+		case "alerting", "firing":
+			states = append(states, eval.Alerting)
+		case "pending":
+			states = append(states, eval.Pending)
+		case "nodata":
+			states = append(states, eval.NoData)
+		// nolint:goconst
+		case "error":
+			states = append(states, eval.Error)
+		default:
+			return states, fmt.Errorf("unknown state '%s'", s)
+		}
+	}
+	return states, nil
+}
+
 func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) response.Response {
 	dashboardUID := c.Query("dashboard_uid")
 	panelID, err := getPanelIDFromRequest(c.Req)
@@ -115,12 +155,28 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
 		return ErrResp(http.StatusBadRequest, errors.New("panel_id must be set with dashboard_uid"), "")
 	}
 
+	limitGroups := c.QueryInt64WithDefault("limit", -1)
+	limitRulesPerGroup := c.QueryInt64WithDefault("limit_rules", -1)
+	limitAlertsPerRule := c.QueryInt64WithDefault("limit_alerts", -1)
+	matchers, err := getMatchersFromRequest(c.Req)
+	if err != nil {
+		return ErrResp(http.StatusBadRequest, err, "")
+	}
+	withStates, err := getStatesFromRequest(c.Req)
+	if err != nil {
+		return ErrResp(http.StatusBadRequest, err, "")
+	}
+	withStatesFast := make(map[eval.State]struct{})
+	for _, state := range withStates {
+		withStatesFast[state] = struct{}{}
+	}
+
 	ruleResponse := apimodels.RuleResponse{
 		DiscoveryBase: apimodels.DiscoveryBase{
 			Status: "success",
 		},
 		Data: apimodels.RuleDiscovery{
-			RuleGroups: []*apimodels.RuleGroup{},
+			RuleGroups: []apimodels.RuleGroup{},
 		},
 	}
 
@@ -161,14 +217,22 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
 		return accesscontrol.HasAccess(srv.ac, c)(accesscontrol.ReqViewer, evaluator)
 	}
 
+	// Group rules together by Namespace and Rule Group. Rules are also grouped by Org ID,
+	// but in this API all rules belong to the same organization.
 	groupedRules := make(map[ngmodels.AlertRuleGroupKey][]*ngmodels.AlertRule)
 	for _, rule := range ruleList {
-		key := rule.GetGroupKey()
-		rulesInGroup := groupedRules[key]
-		rulesInGroup = append(rulesInGroup, rule)
-		groupedRules[key] = rulesInGroup
+		groupKey := rule.GetGroupKey()
+		ruleGroup := groupedRules[groupKey]
+		ruleGroup = append(ruleGroup, rule)
+		groupedRules[groupKey] = ruleGroup
+	}
+	// Sort the rules in each rule group by index. We do this at the end instead of
+	// after each append to avoid having to sort each group multiple times.
+	for _, groupRules := range groupedRules {
+		ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(groupRules)
 	}
 
+	rulesTotals := make(map[string]int64, len(groupedRules))
 	for groupKey, rules := range groupedRules {
 		folder := namespaceMap[groupKey.NamespaceUID]
 		if folder == nil {
@@ -178,16 +242,73 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *contextmodel.ReqContext) respon
 		if !authorizeAccessToRuleGroup(rules, hasAccess) {
 			continue
 		}
-		ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, srv.toRuleGroup(groupKey.RuleGroup, folder, rules, labelOptions))
+		ruleGroup, totals := srv.toRuleGroup(groupKey, folder, rules, limitAlertsPerRule, withStatesFast, matchers, labelOptions)
+		ruleGroup.Totals = totals
+		for k, v := range totals {
+			rulesTotals[k] += v
+		}
+
+		if len(withStates) > 0 {
+			// Filtering is weird but firing, pending, and normal filters also need to be
+			// applied to the rule. Others such as nodata and error should have no effect.
+			// This is to match the current behavior in the UI.
+			filteredRules := make([]apimodels.AlertingRule, 0, len(ruleGroup.Rules))
+			for _, rule := range ruleGroup.Rules {
+				var state *eval.State
+				switch rule.State {
+				case "normal", "inactive":
+					state = util.Pointer(eval.Normal)
+				case "alerting", "firing":
+					state = util.Pointer(eval.Alerting)
+				case "pending":
+					state = util.Pointer(eval.Pending)
+				}
+				if state != nil {
+					if _, ok := withStatesFast[*state]; ok {
+						filteredRules = append(filteredRules, rule)
+					}
+				}
+			}
+			ruleGroup.Rules = filteredRules
+		}
+
+		if limitRulesPerGroup > -1 && int64(len(ruleGroup.Rules)) > limitRulesPerGroup {
+			ruleGroup.Rules = ruleGroup.Rules[0:limitRulesPerGroup]
+		}
+
+		ruleResponse.Data.RuleGroups = append(ruleResponse.Data.RuleGroups, *ruleGroup)
 	}
+
+	ruleResponse.Data.Totals = rulesTotals
+
+	// Sort Rule Groups before checking limits
+	apimodels.RuleGroupsBy(apimodels.RuleGroupsByFileAndName).Sort(ruleResponse.Data.RuleGroups)
+	if limitGroups > -1 && int64(len(ruleResponse.Data.RuleGroups)) >= limitGroups {
+		ruleResponse.Data.RuleGroups = ruleResponse.Data.RuleGroups[0:limitGroups]
+	}
+
 	return response.JSON(http.StatusOK, ruleResponse)
 }
 
-func (srv PrometheusSrv) toRuleGroup(groupName string, folder *folder.Folder, rules []*ngmodels.AlertRule, labelOptions []ngmodels.LabelOption) *apimodels.RuleGroup {
-	newGroup := &apimodels.RuleGroup{
-		Name: groupName,
-		File: folder.Title, // file is what Prometheus uses for provisioning, we replace it with namespace.
+// This is the same as matchers.Matches but avoids the need to create a LabelSet
+func matchersMatch(matchers []*labels.Matcher, labels map[string]string) bool {
+	for _, m := range matchers {
+		if !m.Matches(labels[m.Name]) {
+			return false
+		}
 	}
+	return true
+}
+
+func (srv PrometheusSrv) toRuleGroup(groupKey ngmodels.AlertRuleGroupKey, folder *folder.Folder, rules []*ngmodels.AlertRule, limitAlerts int64, withStates map[eval.State]struct{}, matchers labels.Matchers, labelOptions []ngmodels.LabelOption) (*apimodels.RuleGroup, map[string]int64) {
+	newGroup := &apimodels.RuleGroup{
+		Name: groupKey.RuleGroup,
+		// file is what Prometheus uses for provisioning, we replace it with namespace which is the folder in Grafana.
+		File: folder.Title,
+	}
+
+	rulesTotals := make(map[string]int64, len(rules))
+
 	ngmodels.RulesGroup(rules).SortByGroupIndex()
 	for _, rule := range rules {
 		alertingRule := apimodels.AlertingRule{
@@ -206,14 +327,20 @@ func (srv PrometheusSrv) toRuleGroup(groupName string, folder *folder.Folder, ru
 			LastEvaluation: time.Time{},
 		}
 
-		for _, alertState := range srv.manager.GetStatesForRuleUID(rule.OrgID, rule.UID) {
+		states := srv.manager.GetStatesForRuleUID(rule.OrgID, rule.UID)
+		totals := make(map[string]int64)
+		for _, alertState := range states {
 			activeAt := alertState.StartsAt
 			valString := ""
 			if alertState.State == eval.Alerting || alertState.State == eval.Pending {
 				valString = formatValues(alertState)
 			}
-
-			alert := &apimodels.Alert{
+			totals[strings.ToLower(alertState.State.String())] += 1
+			// Do not add error twice when execution error state is Error
+			if alertState.Error != nil && rule.ExecErrState != ngmodels.ErrorErrState {
+				totals["error"] += 1
+			}
+			alert := apimodels.Alert{
 				Labels:      alertState.GetLabels(labelOptions...),
 				Annotations: alertState.Annotations,
 
@@ -237,6 +364,9 @@ func (srv PrometheusSrv) toRuleGroup(groupName string, folder *folder.Folder, ru
 					alertingRule.State = "pending"
 				}
 			case eval.Alerting:
+				if alertingRule.ActiveAt == nil || alertingRule.ActiveAt.After(activeAt) {
+					alertingRule.ActiveAt = &activeAt
+				}
 				alertingRule.State = "firing"
 			case eval.Error:
 				newRule.Health = "error"
@@ -249,17 +379,43 @@ func (srv PrometheusSrv) toRuleGroup(groupName string, folder *folder.Folder, ru
 				newRule.Health = "error"
 			}
 
+			if len(withStates) > 0 {
+				if _, ok := withStates[alertState.State]; !ok {
+					continue
+				}
+			}
+
+			if !matchersMatch(matchers, alertState.Labels) {
+				continue
+			}
+
 			alertingRule.Alerts = append(alertingRule.Alerts, alert)
 		}
 
+		if alertingRule.State != "" {
+			rulesTotals[alertingRule.State] += 1
+		}
+
+		if newRule.Health == "error" || newRule.Health == "nodata" {
+			rulesTotals[newRule.Health] += 1
+		}
+
+		apimodels.AlertsBy(apimodels.AlertsByImportance).Sort(alertingRule.Alerts)
+
+		if limitAlerts > -1 && int64(len(alertingRule.Alerts)) > limitAlerts {
+			alertingRule.Alerts = alertingRule.Alerts[0:limitAlerts]
+		}
+
 		alertingRule.Rule = newRule
+		alertingRule.Totals = totals
 		newGroup.Rules = append(newGroup.Rules, alertingRule)
 		newGroup.Interval = float64(rule.IntervalSeconds)
 		// TODO yuri. Change that when scheduler will process alerts in groups
 		newGroup.EvaluationTime = newRule.EvaluationTime
 		newGroup.LastEvaluation = newRule.LastEvaluation
 	}
-	return newGroup
+
+	return newGroup, rulesTotals
 }
 
 // ruleToQuery attempts to extract the datasource queries from the alert query model.