Alerting: API to create rule groups using mimirtool (#100558)
What is this feature?
Adds an API endpoint to create alert rules with mimirtool:
- POST /convert/prometheus/config/v1/rules/{NamespaceTitle} - Accepts a single rule group in a Prometheus YAML format and creates or updates a Grafana rule group from it.
The endpoint uses the conversion package from #100224.
Key parts
The API works similarly to the provisioning API. If the rule does not exist, it will be created, otherwise updated. Any rules not present in the new group will be deleted, ensuring the group is fully synchronized with the provided configuration.
Since the API works with namespace titles (folders), the handler automatically creates a folder in the root based on the provided title if it does not exist. It also requires a special header, X-Grafana-Alerting-Datasource-UID. This header specifies which datasource to use for the new rules.
If the rule group's evaluation interval is not specified, it uses the DefaultRuleEvaluationInterval from settings.
This commit is contained in:
Alexander Akhmetov
@@ -0,0 +1,196 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
prommodel "github.com/prometheus/common/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/datasources"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/services/org"
|
||||
"github.com/grafana/grafana/pkg/services/user"
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
)
|
||||
|
||||
func TestIntegrationConvertPrometheusEndpoints(t *testing.T) {
|
||||
testinfra.SQLiteIntegrationTest(t)
|
||||
|
||||
// Setup Grafana and its Database
|
||||
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
||||
DisableLegacyAlerting: true,
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"alertingConversionAPI"},
|
||||
})
|
||||
|
||||
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
|
||||
|
||||
// Create a user to make authenticated requests
|
||||
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
|
||||
DefaultOrgRole: string(org.RoleAdmin),
|
||||
Password: "password",
|
||||
Login: "admin",
|
||||
})
|
||||
|
||||
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
|
||||
namespace := "test-namespace"
|
||||
|
||||
promGroup1 := apimodels.PrometheusRuleGroup{
|
||||
Name: "test-group-1",
|
||||
Interval: prommodel.Duration(60 * time.Second),
|
||||
Rules: []apimodels.PrometheusRule{
|
||||
// Recording rule
|
||||
{
|
||||
Record: "test:requests:rate5m",
|
||||
Expr: "sum(rate(test_requests_total[5m])) by (job)",
|
||||
Labels: map[string]string{
|
||||
"env": "prod",
|
||||
"team": "infra",
|
||||
},
|
||||
},
|
||||
// Two alerting rules
|
||||
{
|
||||
Alert: "HighMemoryUsage",
|
||||
Expr: "process_memory_usage > 80",
|
||||
For: util.Pointer(prommodel.Duration(5 * time.Minute)),
|
||||
Labels: map[string]string{
|
||||
"severity": "warning",
|
||||
"team": "alerting",
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"annotation-1": "value-1",
|
||||
"annotation-2": "value-2",
|
||||
},
|
||||
},
|
||||
{
|
||||
Alert: "ServiceDown",
|
||||
Expr: "up == 0",
|
||||
For: util.Pointer(prommodel.Duration(2 * time.Minute)),
|
||||
Labels: map[string]string{
|
||||
"severity": "critical",
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"annotation-1": "value-1",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
promGroup2 := apimodels.PrometheusRuleGroup{
|
||||
Name: "test-group-2",
|
||||
Interval: prommodel.Duration(60 * time.Second),
|
||||
Rules: []apimodels.PrometheusRule{
|
||||
{
|
||||
Alert: "HighDiskUsage",
|
||||
Expr: "disk_usage > 80",
|
||||
For: util.Pointer(prommodel.Duration(1 * time.Minute)),
|
||||
Labels: map[string]string{
|
||||
"severity": "low",
|
||||
"team": "alerting",
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"annotation-5": "value-5",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
|
||||
|
||||
t.Run("create two rule groups and get them back", func(t *testing.T) {
|
||||
apiClient.ConvertPrometheusPostRuleGroup(t, namespace, ds.Body.Datasource.UID, promGroup1, nil)
|
||||
apiClient.ConvertPrometheusPostRuleGroup(t, namespace, ds.Body.Datasource.UID, promGroup2, nil)
|
||||
|
||||
ns, _, _ := apiClient.GetAllRulesWithStatus(t)
|
||||
|
||||
require.Len(t, ns[namespace], 2)
|
||||
|
||||
rulesByGroupName := map[string][]apimodels.GettableExtendedRuleNode{}
|
||||
for _, group := range ns[namespace] {
|
||||
rulesByGroupName[group.Name] = append(rulesByGroupName[group.Name], group.Rules...)
|
||||
}
|
||||
|
||||
require.Len(t, rulesByGroupName[promGroup1.Name], 3)
|
||||
require.Len(t, rulesByGroupName[promGroup2.Name], 1)
|
||||
})
|
||||
|
||||
t.Run("when pausing header is set, rules should be paused", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
recordingPaused bool
|
||||
alertPaused bool
|
||||
}{
|
||||
{
|
||||
name: "do not pause rules",
|
||||
recordingPaused: false,
|
||||
alertPaused: false,
|
||||
},
|
||||
{
|
||||
name: "pause recording rules",
|
||||
recordingPaused: true,
|
||||
alertPaused: false,
|
||||
},
|
||||
{
|
||||
name: "pause alert rules",
|
||||
recordingPaused: false,
|
||||
alertPaused: true,
|
||||
},
|
||||
{
|
||||
name: "pause both recording and alert rules",
|
||||
recordingPaused: true,
|
||||
alertPaused: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
headers := map[string]string{}
|
||||
if tc.recordingPaused {
|
||||
headers["X-Grafana-Alerting-Recording-Rules-Paused"] = "true"
|
||||
}
|
||||
if tc.alertPaused {
|
||||
headers["X-Grafana-Alerting-Alert-Rules-Paused"] = "true"
|
||||
}
|
||||
apiClient.ConvertPrometheusPostRuleGroup(t, namespace, ds.Body.Datasource.UID, promGroup1, headers)
|
||||
|
||||
ns, _, _ := apiClient.GetAllRulesWithStatus(t)
|
||||
|
||||
rulesByGroupName := map[string][]apimodels.GettableExtendedRuleNode{}
|
||||
for _, group := range ns[namespace] {
|
||||
rulesByGroupName[group.Name] = append(rulesByGroupName[group.Name], group.Rules...)
|
||||
}
|
||||
|
||||
require.Len(t, rulesByGroupName[promGroup1.Name], 3)
|
||||
|
||||
pausedRecordingRules := 0
|
||||
pausedAlertRules := 0
|
||||
|
||||
for _, rule := range rulesByGroupName[promGroup1.Name] {
|
||||
if rule.GrafanaManagedAlert.IsPaused {
|
||||
if rule.GrafanaManagedAlert.Record != nil {
|
||||
pausedRecordingRules++
|
||||
} else {
|
||||
pausedAlertRules++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if tc.recordingPaused {
|
||||
require.Equal(t, 1, pausedRecordingRules)
|
||||
} else {
|
||||
require.Equal(t, 0, pausedRecordingRules)
|
||||
}
|
||||
|
||||
if tc.alertPaused {
|
||||
require.Equal(t, 2, pausedAlertRules)
|
||||
} else {
|
||||
require.Equal(t, 0, pausedAlertRules)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api"
|
||||
"github.com/grafana/grafana/pkg/expr"
|
||||
@@ -752,7 +753,13 @@ func (a apiClient) SubmitRuleForTesting(t *testing.T, config apimodels.PostableE
|
||||
func (a apiClient) CreateTestDatasource(t *testing.T) (result api.CreateOrUpdateDatasourceResponse) {
|
||||
t.Helper()
|
||||
|
||||
payload := fmt.Sprintf(`{"name":"TestData-%s","type":"testdata","access":"proxy","isDefault":false}`, uuid.NewString())
|
||||
return a.CreateDatasource(t, "testdata")
|
||||
}
|
||||
|
||||
func (a apiClient) CreateDatasource(t *testing.T, dsType string) (result api.CreateOrUpdateDatasourceResponse) {
|
||||
t.Helper()
|
||||
|
||||
payload := fmt.Sprintf(`{"name":"TestDatasource-%s","type":"%s","access":"proxy","isDefault":false}`, uuid.NewString(), dsType)
|
||||
buf := bytes.Buffer{}
|
||||
buf.Write([]byte(payload))
|
||||
|
||||
@@ -1094,6 +1101,25 @@ func (a apiClient) GetRuleByUID(t *testing.T, ruleUID string) apimodels.Gettable
|
||||
return rule
|
||||
}
|
||||
|
||||
func (a apiClient) ConvertPrometheusPostRuleGroup(t *testing.T, namespaceTitle, datasourceUID string, promGroup apimodels.PrometheusRuleGroup, headers map[string]string) {
|
||||
t.Helper()
|
||||
|
||||
data, err := yaml.Marshal(promGroup)
|
||||
require.NoError(t, err)
|
||||
buf := bytes.NewReader(data)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/convert/prometheus/config/v1/rules/%s", a.url, namespaceTitle), buf)
|
||||
require.NoError(t, err)
|
||||
req.Header.Add("X-Grafana-Alerting-Datasource-UID", datasourceUID)
|
||||
|
||||
for key, value := range headers {
|
||||
req.Header.Add(key, value)
|
||||
}
|
||||
|
||||
_, status, raw := sendRequest[apimodels.ConvertPrometheusResponse](t, req, http.StatusAccepted)
|
||||
requireStatusCode(t, http.StatusAccepted, status, raw)
|
||||
}
|
||||
|
||||
func sendRequest[T any](t *testing.T, req *http.Request, successStatusCode int) (T, int, string) {
|
||||
t.Helper()
|
||||
client := &http.Client{}
|
||||
|
||||
Reference in New Issue
Block a user