Alerting: Add limits and filters to Prometheus Rules API (#66627)
This commit adds support for limits and filters to the Prometheus Rules
API.
Limits:
It adds a number of limits to the Grafana flavour of the Prometheus Rules
API:
- `limit` limits the maximum number of Rule Groups returned
- `limit_rules` limits the maximum number of rules per Rule Group
- `limit_alerts` limits the maximum number of alerts per rule
It sorts Rule Groups and rules within Rule Groups such that data in the
response is stable across requests. It also returns summaries (totals)
for all Rule Groups, individual Rule Groups and rules.
Filters:
Alerts can be filtered by state with the `state` query string. An example
of an HTTP request asking for just firing alerts might be
`/api/prometheus/grafana/api/v1/rules?state=alerting`.
A request can filter by two or more states by adding additional `state`
query strings to the URL. For example `?state=alerting&state=normal`.
Like the alert list panel, the `firing`, `pending` and `normal` state are
first compared against the state of each alert rule. All other states are
ignored. If the alert rule matches then its alert instances are filtered
against states once more.
Alerts can also be filtered by labels using the `matcher` query string.
Like `state`, multiple matchers can be provided by adding additional
`matcher` query strings to the URL.
The match expression should be parsed using existing regular expression
and sent to the API as URL-encoded JSON in the format:
{
"name": "test",
"value": "value1",
"isRegex": false,
"isEqual": true
}
The `isRegex` and `isEqual` options work as follows:
| IsEqual | IsRegex | Operator |
| ------- | -------- | -------- |
| true | false | = |
| true | true | =~ |
| false | true | !~ |
| false | false | != |
This commit is contained in:
George Robinson
@@ -3,6 +3,7 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
|
||||
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
|
||||
"github.com/grafana/grafana/pkg/services/folder"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
||||
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
@@ -254,6 +256,30 @@ func withAlertingState() forEachState {
|
||||
}
|
||||
}
|
||||
|
||||
func withAlertingErrorState() forEachState {
|
||||
return func(s *state.State) *state.State {
|
||||
s.SetAlerting("", timeNow(), timeNow().Add(5*time.Minute))
|
||||
s.Error = errors.New("this is an error")
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
func withErrorState() forEachState {
|
||||
return func(s *state.State) *state.State {
|
||||
s.SetError(errors.New("this is an error"), timeNow(), timeNow().Add(5*time.Minute))
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
func withLabels(labels data.Labels) forEachState {
|
||||
return func(s *state.State) *state.State {
|
||||
for k, v := range labels {
|
||||
s.Labels[k] = v
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
timeNow = func() time.Time { return time.Date(2022, 3, 10, 14, 0, 0, 0, time.UTC) }
|
||||
orgID := int64(1)
|
||||
@@ -305,6 +331,9 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
"activeAt": "0001-01-01T00:00:00Z",
|
||||
"value": ""
|
||||
}],
|
||||
"totals": {
|
||||
"normal": 1
|
||||
},
|
||||
"labels": {
|
||||
"__a_private_label_on_the_rule__": "a_value"
|
||||
},
|
||||
@@ -314,10 +343,16 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
"duration": 180,
|
||||
"evaluationTime": 60
|
||||
}],
|
||||
"totals": {
|
||||
"inactive": 1
|
||||
},
|
||||
"interval": 60,
|
||||
"lastEvaluation": "2022-03-10T14:01:00Z",
|
||||
"evaluationTime": 60
|
||||
}]
|
||||
}],
|
||||
"totals": {
|
||||
"inactive": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
`, folder.Title), string(r.Body()))
|
||||
@@ -358,6 +393,9 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
"activeAt": "0001-01-01T00:00:00Z",
|
||||
"value": ""
|
||||
}],
|
||||
"totals": {
|
||||
"normal": 1
|
||||
},
|
||||
"labels": {
|
||||
"__a_private_label_on_the_rule__": "a_value",
|
||||
"__alert_rule_uid__": "RuleUID"
|
||||
@@ -368,10 +406,16 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
"duration": 180,
|
||||
"evaluationTime": 60
|
||||
}],
|
||||
"totals": {
|
||||
"inactive": 1
|
||||
},
|
||||
"interval": 60,
|
||||
"lastEvaluation": "2022-03-10T14:01:00Z",
|
||||
"evaluationTime": 60
|
||||
}]
|
||||
}],
|
||||
"totals": {
|
||||
"inactive": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
`, folder.Title), string(r.Body()))
|
||||
@@ -406,6 +450,9 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
"activeAt": "0001-01-01T00:00:00Z",
|
||||
"value": ""
|
||||
}],
|
||||
"totals": {
|
||||
"normal": 1
|
||||
},
|
||||
"labels": {
|
||||
"__a_private_label_on_the_rule__": "a_value"
|
||||
},
|
||||
@@ -415,10 +462,16 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
"duration": 180,
|
||||
"evaluationTime": 60
|
||||
}],
|
||||
"totals": {
|
||||
"inactive": 1
|
||||
},
|
||||
"interval": 60,
|
||||
"lastEvaluation": "2022-03-10T14:01:00Z",
|
||||
"evaluationTime": 60
|
||||
}]
|
||||
}],
|
||||
"totals": {
|
||||
"inactive": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
`, folder.Title), string(r.Body()))
|
||||
@@ -504,6 +557,637 @@ func TestRouteGetRuleStatuses(t *testing.T) {
|
||||
assert.Emptyf(t, rules, "not all expected rules were returned")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("test totals are expected", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, _, api := setupAPI(t)
|
||||
// Create rules in the same Rule Group to keep assertions simple
|
||||
rules := ngmodels.GenerateAlertRules(3, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1"), withNamespace(&folder.Folder{
|
||||
Title: "Folder-1",
|
||||
})))
|
||||
// Need to sort these so we add alerts to the rules as ordered in the response
|
||||
ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(rules)
|
||||
// The last two rules will have errors, however the first will be alerting
|
||||
// while the second one will have a DatasourceError alert.
|
||||
rules[1].ExecErrState = ngmodels.AlertingErrState
|
||||
rules[2].ExecErrState = ngmodels.ErrorErrState
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
|
||||
// create a normal and alerting state for the first rule
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1)
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1, withAlertingState())
|
||||
// create an error state for the last two rules
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[1].UID, 1, withAlertingErrorState())
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[2].UID, 1, withErrorState())
|
||||
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// Even though there are just 3 rules, the totals should show two firing rules,
|
||||
// one inactive rules and two errors
|
||||
require.Equal(t, map[string]int64{"firing": 2, "inactive": 1, "error": 2}, res.Data.Totals)
|
||||
// There should be 1 Rule Group that contains all rules
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 3)
|
||||
|
||||
// The first rule should have an alerting and normal alert
|
||||
r1 := rg.Rules[0]
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, r1.Totals)
|
||||
require.Len(t, r1.Alerts, 2)
|
||||
// The second rule should have an alerting alert
|
||||
r2 := rg.Rules[1]
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "error": 1}, r2.Totals)
|
||||
require.Len(t, r2.Alerts, 1)
|
||||
// The last rule should have an error alert
|
||||
r3 := rg.Rules[2]
|
||||
require.Equal(t, map[string]int64{"error": 1}, r3.Totals)
|
||||
require.Len(t, r3.Alerts, 1)
|
||||
})
|
||||
|
||||
t.Run("test time of first firing alert", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, _, api := setupAPI(t)
|
||||
// Create rules in the same Rule Group to keep assertions simple
|
||||
rules := ngmodels.GenerateAlertRules(1, ngmodels.AlertRuleGen(withOrgID(orgID)))
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
|
||||
getRuleResponse := func() apimodels.RuleResponse {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
return res
|
||||
}
|
||||
|
||||
// no alerts so timestamp should be nil
|
||||
res := getRuleResponse()
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Nil(t, rg.Rules[0].ActiveAt)
|
||||
|
||||
// create a normal alert, the timestamp should still be nil
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1)
|
||||
res = getRuleResponse()
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg = res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Nil(t, rg.Rules[0].ActiveAt)
|
||||
|
||||
// create a firing alert, the timestamp should be non-nil
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1, withAlertingState())
|
||||
res = getRuleResponse()
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg = res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.NotNil(t, rg.Rules[0].ActiveAt)
|
||||
|
||||
lastActiveAt := rg.Rules[0].ActiveAt
|
||||
// create a second firing alert, the timestamp of first firing alert should be the same
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1, withAlertingState())
|
||||
res = getRuleResponse()
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg = res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Equal(t, lastActiveAt, rg.Rules[0].ActiveAt)
|
||||
})
|
||||
|
||||
t.Run("test with limit on Rule Groups", func(t *testing.T) {
|
||||
fakeStore, _, _, api := setupAPI(t)
|
||||
|
||||
rules := ngmodels.GenerateAlertRules(2, ngmodels.AlertRuleGen(withOrgID(orgID)))
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
|
||||
t.Run("first without limit", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 inactive rules across all Rule Groups
|
||||
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
|
||||
require.Len(t, res.Data.RuleGroups, 2)
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
// Each Rule Group should have 1 inactive rule
|
||||
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
|
||||
require.Len(t, rg.Rules, 1)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("then with limit", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 inactive rules across all Rule Groups
|
||||
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
// The Rule Group within the limit should have 1 inactive rule
|
||||
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
|
||||
require.Len(t, rg.Rules, 1)
|
||||
})
|
||||
|
||||
t.Run("then with limit larger than number of rule groups", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("test with limit rules", func(t *testing.T) {
|
||||
fakeStore, _, _, api := setupAPI(t)
|
||||
rules := ngmodels.GenerateAlertRules(2, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1")))
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
|
||||
t.Run("first without limit", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 inactive rules across all Rule Groups
|
||||
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
|
||||
require.Len(t, res.Data.RuleGroups, 2)
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
// Each Rule Group should have 1 inactive rule
|
||||
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
|
||||
require.Len(t, rg.Rules, 1)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("then with limit", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1&limit_rules=1", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 inactive rules
|
||||
require.Equal(t, map[string]int64{"inactive": 2}, res.Data.Totals)
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
// The Rule Group within the limit should have 1 inactive rule because of the limit
|
||||
require.Equal(t, map[string]int64{"inactive": 1}, rg.Totals)
|
||||
require.Len(t, rg.Rules, 1)
|
||||
})
|
||||
|
||||
t.Run("then with limit larger than number of rules", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1&limit_rules=2", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
require.Len(t, res.Data.RuleGroups[0].Rules, 1)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("test with limit alerts", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, _, api := setupAPI(t)
|
||||
rules := ngmodels.GenerateAlertRules(2, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1")))
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
// create a normal and firing alert for each rule
|
||||
for _, r := range rules {
|
||||
fakeAIM.GenerateAlertInstances(orgID, r.UID, 1)
|
||||
fakeAIM.GenerateAlertInstances(orgID, r.UID, 1, withAlertingState())
|
||||
}
|
||||
|
||||
t.Run("first without limit", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 firing rules across all Rule Groups
|
||||
require.Equal(t, map[string]int64{"firing": 2}, res.Data.Totals)
|
||||
require.Len(t, res.Data.RuleGroups, 2)
|
||||
for _, rg := range res.Data.RuleGroups {
|
||||
// Each Rule Group should have 1 firing rule
|
||||
require.Equal(t, map[string]int64{"firing": 1}, rg.Totals)
|
||||
require.Len(t, rg.Rules, 1)
|
||||
// Each rule should have two alerts
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rg.Rules[0].Totals)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("then with limits", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1&limit_rules=1&limit_alerts=1", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 firing rules across all Rule Groups
|
||||
require.Equal(t, map[string]int64{"firing": 2}, res.Data.Totals)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
// The Rule Group within the limit should have 1 inactive rule because of the limit
|
||||
require.Equal(t, map[string]int64{"firing": 1}, rg.Totals)
|
||||
require.Len(t, rg.Rules, 1)
|
||||
rule := rg.Rules[0]
|
||||
// The rule should have two alerts, but just one should be returned
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rule.Totals)
|
||||
require.Len(t, rule.Alerts, 1)
|
||||
// Firing alerts should have precedence over normal alerts
|
||||
require.Equal(t, "Alerting", rule.Alerts[0].State)
|
||||
})
|
||||
|
||||
t.Run("then with limit larger than number of alerts", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?limit=1&limit_rules=1&limit_alerts=3", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
require.Len(t, res.Data.RuleGroups[0].Rules, 1)
|
||||
require.Len(t, res.Data.RuleGroups[0].Rules[0].Alerts, 2)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("test with filters on state", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, _, api := setupAPI(t)
|
||||
// create two rules in the same Rule Group to keep assertions simple
|
||||
rules := ngmodels.GenerateAlertRules(3, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1"), withNamespace(&folder.Folder{
|
||||
Title: "Folder-1",
|
||||
})))
|
||||
// Need to sort these so we add alerts to the rules as ordered in the response
|
||||
ngmodels.AlertRulesBy(ngmodels.AlertRulesByIndex).Sort(rules)
|
||||
// The last two rules will have errors, however the first will be alerting
|
||||
// while the second one will have a DatasourceError alert.
|
||||
rules[1].ExecErrState = ngmodels.AlertingErrState
|
||||
rules[2].ExecErrState = ngmodels.ErrorErrState
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
|
||||
// create a normal and alerting state for the first rule
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1)
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1, withAlertingState())
|
||||
// create an error state for the last two rules
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[1].UID, 1, withAlertingErrorState())
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[2].UID, 1, withErrorState())
|
||||
|
||||
t.Run("invalid state returns 400 Bad Request", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?state=unknown", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusBadRequest, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
require.Equal(t, "unknown state 'unknown'", res.Error)
|
||||
})
|
||||
|
||||
t.Run("first without filters", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be 2 firing rules, 1 inactive rule, and 2 with errors
|
||||
require.Equal(t, map[string]int64{"firing": 2, "inactive": 1, "error": 2}, res.Data.Totals)
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 3)
|
||||
|
||||
// The first two rules should be firing and the last should be inactive
|
||||
require.Equal(t, "firing", rg.Rules[0].State)
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rg.Rules[0].Totals)
|
||||
require.Len(t, rg.Rules[0].Alerts, 2)
|
||||
require.Equal(t, "firing", rg.Rules[1].State)
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "error": 1}, rg.Rules[1].Totals)
|
||||
require.Len(t, rg.Rules[1].Alerts, 1)
|
||||
require.Equal(t, "inactive", rg.Rules[2].State)
|
||||
require.Equal(t, map[string]int64{"error": 1}, rg.Rules[2].Totals)
|
||||
require.Len(t, rg.Rules[2].Alerts, 1)
|
||||
})
|
||||
|
||||
t.Run("then with filter for firing alerts", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?state=firing", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// The totals should be the same
|
||||
require.Equal(t, map[string]int64{"firing": 2, "inactive": 1, "error": 2}, res.Data.Totals)
|
||||
|
||||
// The inactive rules should be filtered out of the result
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 2)
|
||||
|
||||
// Both firing rules should be returned with their totals unchanged
|
||||
require.Equal(t, "firing", rg.Rules[0].State)
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rg.Rules[0].Totals)
|
||||
// The first rule should have just 1 firing alert as the inactive alert
|
||||
// has been removed by the filter for firing alerts
|
||||
require.Len(t, rg.Rules[0].Alerts, 1)
|
||||
|
||||
require.Equal(t, "firing", rg.Rules[1].State)
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "error": 1}, rg.Rules[1].Totals)
|
||||
require.Len(t, rg.Rules[1].Alerts, 1)
|
||||
})
|
||||
|
||||
t.Run("then with filters for both inactive and firing alerts", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?state=inactive&state=firing", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// The totals should be the same
|
||||
require.Equal(t, map[string]int64{"firing": 2, "inactive": 1, "error": 2}, res.Data.Totals)
|
||||
|
||||
// The number of rules returned should also be the same
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 3)
|
||||
|
||||
// The first two rules should be firing and the last should be inactive
|
||||
require.Equal(t, "firing", rg.Rules[0].State)
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "normal": 1}, rg.Rules[0].Totals)
|
||||
require.Len(t, rg.Rules[0].Alerts, 2)
|
||||
require.Equal(t, "firing", rg.Rules[1].State)
|
||||
require.Equal(t, map[string]int64{"alerting": 1, "error": 1}, rg.Rules[1].Totals)
|
||||
require.Len(t, rg.Rules[1].Alerts, 1)
|
||||
|
||||
// The last rule should have 1 alert as the filter includes errors too
|
||||
require.Equal(t, "inactive", rg.Rules[2].State)
|
||||
require.Equal(t, map[string]int64{"error": 1}, rg.Rules[2].Totals)
|
||||
// The error alert has been removed as the filters are inactive and firing
|
||||
require.Len(t, rg.Rules[2].Alerts, 0)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("test with matcher on labels", func(t *testing.T) {
|
||||
fakeStore, fakeAIM, _, api := setupAPI(t)
|
||||
// create two rules in the same Rule Group to keep assertions simple
|
||||
rules := ngmodels.GenerateAlertRules(1, ngmodels.AlertRuleGen(withOrgID(orgID), withGroup("Rule-Group-1"), withNamespace(&folder.Folder{
|
||||
Title: "Folder-1",
|
||||
})))
|
||||
fakeStore.PutRule(context.Background(), rules...)
|
||||
|
||||
// create a normal and alerting state for each rule
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1,
|
||||
withLabels(data.Labels{"test": "value1"}))
|
||||
fakeAIM.GenerateAlertInstances(orgID, rules[0].UID, 1,
|
||||
withLabels(data.Labels{"test": "value2"}), withAlertingState())
|
||||
|
||||
t.Run("invalid matchers returns 400 Bad Request", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?matcher={\"name\":\"\"}", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusBadRequest, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
require.Equal(t, "bad matcher: the name cannot be blank", res.Error)
|
||||
})
|
||||
|
||||
t.Run("first without matchers", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Len(t, rg.Rules[0].Alerts, 2)
|
||||
})
|
||||
|
||||
t.Run("then with single matcher", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?matcher={\"name\":\"test\",\"isEqual\":true,\"value\":\"value1\"}", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be just the alert with the label test=value1
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Len(t, rg.Rules[0].Alerts, 1)
|
||||
})
|
||||
|
||||
t.Run("then with URL encoded regex matcher", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?matcher=%7B%22name%22:%22test%22%2C%22isEqual%22:true%2C%22isRegex%22:true%2C%22value%22:%22value%5B0-9%5D%2B%22%7D%0A", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be just the alert with the label test=value1
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Len(t, rg.Rules[0].Alerts, 2)
|
||||
})
|
||||
|
||||
t.Run("then with multiple matchers", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?matcher={\"name\":\"alertname\",\"isEqual\":true,\"value\":\"test_title_0\"}&matcher={\"name\":\"test\",\"isEqual\":true,\"value\":\"value1\"}", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should be just the alert with the label test=value1
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Len(t, rg.Rules[0].Alerts, 1)
|
||||
})
|
||||
|
||||
t.Run("then with multiple matchers that don't match", func(t *testing.T) {
|
||||
r, err := http.NewRequest("GET", "/api/v1/rules?matcher={\"name\":\"alertname\",\"isEqual\":true,\"value\":\"test_title_0\"}&matcher={\"name\":\"test\",\"isEqual\":true,\"value\":\"value3\"}", nil)
|
||||
require.NoError(t, err)
|
||||
c := &contextmodel.ReqContext{
|
||||
Context: &web.Context{Req: r},
|
||||
SignedInUser: &user.SignedInUser{
|
||||
OrgID: orgID,
|
||||
OrgRole: org.RoleViewer,
|
||||
},
|
||||
}
|
||||
resp := api.RouteGetRuleStatuses(c)
|
||||
require.Equal(t, http.StatusOK, resp.Status())
|
||||
var res apimodels.RuleResponse
|
||||
require.NoError(t, json.Unmarshal(resp.Body(), &res))
|
||||
|
||||
// There should no alerts
|
||||
require.Len(t, res.Data.RuleGroups, 1)
|
||||
rg := res.Data.RuleGroups[0]
|
||||
require.Len(t, rg.Rules, 1)
|
||||
require.Len(t, rg.Rules[0].Alerts, 0)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func setupAPI(t *testing.T) (*fakes.RuleStore, *fakeAlertInstanceManager, *acmock.Mock, PrometheusSrv) {
|
||||
|
||||
Reference in New Issue
Block a user