public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support simplified routing receivers in Prometheus conversion API (#105135)

Browse Source 
Adds ability to set notifications settings using the Prometheus conversion API.

The API now supports a new optional header: X-Grafana-Alerting-Notification-Settings which can be used to specify notification settings.

The value of the header is the AlertRuleNotificationSettings structure in JSON:
mimirtool rules load alerts.yaml --extra-headers 'X-Grafana-Alerting-Notification-Settings: {"receiver": "my-webhook", "group_by": ["cluster", "pod"]}'
This commit is contained in:
Alexander Akhmetov
2025-05-12 22:07:02 +02:00
committed by GitHub
parent e965b85e19
commit c17b019ab1
12 changed files with 559 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_convert_prometheus.go
+50 -5
View File
@@ -1,6 +1,7 @@
package api
import (
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -21,6 +22,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/api/validation"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/prom"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
@@ -41,6 +43,10 @@ const (
// These headers control the paused state of newly created rules. By default, rules are not paused.
recordingRulesPausedHeader = "X-Grafana-Alerting-Recording-Rules-Paused"
alertRulesPausedHeader = "X-Grafana-Alerting-Alert-Rules-Paused"
// notificationSettingsHeader is the header that specifies the notification settings to be used for the rules.
// The value should be a JSON-encoded AlertRuleNotificationSettings object.
notificationSettingsHeader = "X-Grafana-Alerting-Notification-Settings"
)
var (
@@ -49,7 +55,7 @@ var (
errutil.WithPublicMessage(fmt.Sprintf("Missing datasource UID header: %s", datasourceUIDHeader)),
).Errorf("missing datasource UID header")
errInvalidHeaderValueMsg = "Invalid value for header {{.Public.Header}}: must be 'true' or 'false'"
errInvalidHeaderValueMsg = "Invalid value for header {{.Public.Header}}: {{.Public.Error}}"
errInvalidHeaderValueBase = errutil.ValidationFailed("alerting.invalidHeaderValue").MustTemplate(errInvalidHeaderValueMsg, errutil.WithPublic(errInvalidHeaderValueMsg))
errRecordingRulesNotEnabled = errutil.ValidationFailed(
@@ -63,8 +69,8 @@ var (
).Errorf("recording rules target datasources configuration not enabled")
)
func errInvalidHeaderValue(header string) error {
return errInvalidHeaderValueBase.Build(errutil.TemplateData{Public: map[string]any{"Header": header}})
func errInvalidHeaderValue(header string, err error) error {
return errInvalidHeaderValueBase.Build(errutil.TemplateData{Public: map[string]any{"Header": header, "Error": err}})
}
// ConvertPrometheusSrv converts Prometheus rules to Grafana rules
@@ -369,6 +375,12 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroups(c *context
// to ensure we can return them in this API in Prometheus format.
keepOriginalRuleDefinition := provenance == models.ProvenanceConvertedPrometheus
notificationSettings, err := parseNotificationSettingsHeader(c)
if err != nil {
logger.Error("Failed to parse notification settings header", "error", err)
return errorToResponse(err)
}
// 2. Convert Prometheus Rules to GMA
grafanaGroups := make([]*models.AlertRuleGroup, 0, len(promNamespaces))
for ns, rgs := range promNamespaces {
@@ -394,7 +406,18 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroups(c *context
}
}
grafanaGroup, err := srv.convertToGrafanaRuleGroup(c, ds, tds, namespace.UID, rg, pauseRecordingRules, pauseAlertRules, keepOriginalRuleDefinition, logger)
grafanaGroup, err := srv.convertToGrafanaRuleGroup(
c,
ds,
tds,
namespace.UID,
rg,
pauseRecordingRules,
pauseAlertRules,
keepOriginalRuleDefinition,
notificationSettings,
logger,
)
if err != nil {
logger.Error("Failed to convert Prometheus rules to Grafana rules", "error", err)
return errorToResponse(err)
@@ -442,6 +465,7 @@ func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup(
pauseRecordingRules bool,
pauseAlertRules bool,
keepOriginalRuleDefinition bool,
notificationSettings []models.NotificationSettings,
logger log.Logger,
) (*models.AlertRuleGroup, error) {
logger.Info("Converting Prometheus rules to Grafana rules", "rules", len(promGroup.Rules), "folder_uid", namespaceUID, "datasource_uid", ds.UID, "datasource_type", ds.Type)
@@ -479,6 +503,7 @@ func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup(
},
KeepOriginalRuleDefinition: util.Pointer(keepOriginalRuleDefinition),
EvaluationOffset: &srv.cfg.PrometheusConversion.RuleQueryOffset,
NotificationSettings: notificationSettings,
},
)
if err != nil {
@@ -503,7 +528,7 @@ func parseBooleanHeader(header string, headerName string) (bool, error) {
}
val, err := strconv.ParseBool(header)
if err != nil {
return false, errInvalidHeaderValue(headerName)
return false, errInvalidHeaderValue(headerName, errors.New("must be 'true' or 'false'"))
}
return val, nil
}
@@ -597,3 +622,23 @@ func getProvenance(ctx *contextmodel.ReqContext) models.Provenance {
}
return models.ProvenanceConvertedPrometheus
}
func parseNotificationSettingsHeader(ctx *contextmodel.ReqContext) ([]models.NotificationSettings, error) {
var notificationSettings []models.NotificationSettings
notificationSettingsJSON := ctx.Req.Header.Get(notificationSettingsHeader)
if notificationSettingsJSON != "" {
var settings apimodels.AlertRuleNotificationSettings
var err error
if err := json.Unmarshal([]byte(notificationSettingsJSON), &settings); err != nil {
return nil, errInvalidHeaderValue(notificationSettingsHeader, errors.New("invalid JSON"))
}
notificationSettings, err = validation.ValidateNotificationSettings(&settings)
if err != nil {
return nil, errInvalidHeaderValue(notificationSettingsHeader, err)
}
}
return notificationSettings, nil
}
pkg/services/ngalert/api/api_convert_prometheus_test.go
+92
View File
@@ -2,6 +2,7 @@ package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
@@ -425,6 +426,97 @@ func TestRouteConvertPrometheusPostRuleGroup(t *testing.T) {
require.NotNil(t, remaining[0].Record)
require.Equal(t, targetDSUID, remaining[0].Record.TargetDatasourceUID)
})
t.Run("sets notification settings for rules if specified", func(t *testing.T) {
srv, _, ruleStore, _ := createConvertPrometheusSrv(t)
rc := createRequestCtx()
receiver := "test-receiver"
groupBy := []string{"cluster", "pod"}
settings := apimodels.AlertRuleNotificationSettings{
Receiver: receiver,
GroupBy: groupBy,
}
settingsJSON, err := json.Marshal(settings)
require.NoError(t, err)
rc.Req.Header.Set(notificationSettingsHeader, string(settingsJSON))
simpleGroup := apimodels.PrometheusRuleGroup{
Name: "Test Group",
Interval: prommodel.Duration(1 * time.Minute),
Rules: []apimodels.PrometheusRule{
{
Alert: "TestAlert",
Expr: "up == 0",
For: util.Pointer(prommodel.Duration(5 * time.Minute)),
Labels: map[string]string{
"severity": "critical",
},
},
},
}
response := srv.RouteConvertPrometheusPostRuleGroup(rc, "test", simpleGroup)
require.Equal(t, http.StatusAccepted, response.Status())
createdRules, err := ruleStore.ListAlertRules(context.Background(), &models.ListAlertRulesQuery{
OrgID: 1,
})
require.NoError(t, err)
require.Len(t, createdRules, 1)
require.Len(t, createdRules[0].NotificationSettings, 1)
require.Equal(t, receiver, createdRules[0].NotificationSettings[0].Receiver)
require.Equal(t, groupBy, createdRules[0].NotificationSettings[0].GroupBy)
})
t.Run("returns error when notification settings header contains invalid JSON", func(t *testing.T) {
srv, _, _, _ := createConvertPrometheusSrv(t)
rc := createRequestCtx()
rc.Req.Header.Set(notificationSettingsHeader, "{invalid json")
simpleGroup := apimodels.PrometheusRuleGroup{
Name: "Test Group",
Interval: prommodel.Duration(1 * time.Minute),
Rules: []apimodels.PrometheusRule{
{
Alert: "TestAlert",
Expr: "up == 0",
},
},
}
response := srv.RouteConvertPrometheusPostRuleGroup(rc, "test", simpleGroup)
require.Equal(t, http.StatusBadRequest, response.Status())
require.Contains(t, string(response.Body()), "Invalid value for header X-Grafana-Alerting-Notification-Settings")
})
t.Run("returns error when notification settings contain invalid values", func(t *testing.T) {
srv, _, _, _ := createConvertPrometheusSrv(t)
rc := createRequestCtx()
settings := apimodels.AlertRuleNotificationSettings{
Receiver: "", // empty receiver is invalid
}
settingsJSON, err := json.Marshal(settings)
require.NoError(t, err)
rc.Req.Header.Set(notificationSettingsHeader, string(settingsJSON))
simpleGroup := apimodels.PrometheusRuleGroup{
Name: "Test Group",
Interval: prommodel.Duration(1 * time.Minute),
Rules: []apimodels.PrometheusRule{
{
Alert: "TestAlert",
Expr: "up == 0",
},
},
}
response := srv.RouteConvertPrometheusPostRuleGroup(rc, "test", simpleGroup)
require.Equal(t, http.StatusBadRequest, response.Status())
require.Contains(t, string(response.Body()), "Invalid value for header X-Grafana-Alerting-Notification-Settings")
})
}
func TestRouteConvertPrometheusGetRuleGroup(t *testing.T) {
pkg/services/ngalert/api/tooling/api.json
+30 -4
View File
@@ -314,6 +314,16 @@
},
"AlertRuleNotificationSettings": {
"properties": {
"active_time_intervals": {
"description": "Override the times when notifications should not be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration time_intervals section. All notifications will be suppressed unless they are sent\nat the time that matches any interval.",
"example": [
"maintenance"
],
"items": {
"type": "string"
},
"type": "array"
},
"group_by": {
"default": [
"alertname",
@@ -341,7 +351,7 @@
"type": "string"
},
"mute_time_intervals": {
"description": "Override the times when notifications should be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration mute_time_intervals section. When muted it will not send any notifications, but\notherwise acts normally.",
"description": "Override the times when notifications should be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration time_intervals section. When muted it will not send any notifications, but\notherwise acts normally.",
"example": [
"maintenance"
],
@@ -368,6 +378,12 @@
},
"AlertRuleNotificationSettingsExport": {
"properties": {
"active_time_intervals": {
"items": {
"type": "string"
},
"type": "array"
},
"group_by": {
"items": {
"type": "string"
@@ -2325,6 +2341,12 @@
},
"NotificationPolicyExport": {
"properties": {
"active_time_intervals": {
"items": {
"type": "string"
},
"type": "array"
},
"continue": {
"type": "boolean"
},
@@ -3710,6 +3732,12 @@
"RouteExport": {
"description": "RouteExport is the provisioned file export of definitions.Route. This is needed to hide fields that aren't useable in\nprovisioning file format. An alternative would be to define a custom MarshalJSON and MarshalYAML that excludes them.",
"properties": {
"active_time_intervals": {
"items": {
"type": "string"
},
"type": "array"
},
"continue": {
"type": "boolean"
},
@@ -4889,7 +4917,6 @@
"type": "object"
},
"alertGroups": {
"description": "AlertGroups alert groups",
"items": {
"$ref": "#/definitions/alertGroup",
"type": "object"
@@ -5176,7 +5203,6 @@
"type": "object"
},
"gettableSilences": {
"description": "GettableSilences gettable silences",
"items": {
"$ref": "#/definitions/gettableSilence",
"type": "object"
@@ -6661,4 +6687,4 @@
}
},
"swagger": "2.0"
}
}
pkg/services/ngalert/api/tooling/definitions/convert_prometheus_api.go
+6
View File
@@ -210,6 +210,12 @@ type RouteConvertPrometheusPostRuleGroupParams struct {
RecordingRulesPaused bool `json:"x-grafana-alerting-recording-rules-paused"`
// in: header
AlertRulesPaused bool `json:"x-grafana-alerting-alert-rules-paused"`
// in: header
TargetDatasourceUID string `json:"x-grafana-alerting-target-datasource-uid"`
// in: header
FolderUID string `json:"x-grafana-alerting-folder-uid"`
// in: header
NotificationReceiver string `json:"x-grafana-alerting-notification-receiver"`
// in:body
Body PrometheusRuleGroup
}
pkg/services/ngalert/api/tooling/post.json
+61 -4
View File
@@ -314,6 +314,16 @@
},
"AlertRuleNotificationSettings": {
"properties": {
"active_time_intervals": {
"description": "Override the times when notifications should not be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration time_intervals section. All notifications will be suppressed unless they are sent\nat the time that matches any interval.",
"example": [
"maintenance"
],
"items": {
"type": "string"
},
"type": "array"
},
"group_by": {
"default": [
"alertname",
@@ -341,7 +351,7 @@
"type": "string"
},
"mute_time_intervals": {
"description": "Override the times when notifications should be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration mute_time_intervals section. When muted it will not send any notifications, but\notherwise acts normally.",
"description": "Override the times when notifications should be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration time_intervals section. When muted it will not send any notifications, but\notherwise acts normally.",
"example": [
"maintenance"
],
@@ -368,6 +378,12 @@
},
"AlertRuleNotificationSettingsExport": {
"properties": {
"active_time_intervals": {
"items": {
"type": "string"
},
"type": "array"
},
"group_by": {
"items": {
"type": "string"
@@ -2325,6 +2341,12 @@
},
"NotificationPolicyExport": {
"properties": {
"active_time_intervals": {
"items": {
"type": "string"
},
"type": "array"
},
"continue": {
"type": "boolean"
},
@@ -3710,6 +3732,12 @@
"RouteExport": {
"description": "RouteExport is the provisioned file export of definitions.Route. This is needed to hide fields that aren't useable in\nprovisioning file format. An alternative would be to define a custom MarshalJSON and MarshalYAML that excludes them.",
"properties": {
"active_time_intervals": {
"items": {
"type": "string"
},
"type": "array"
},
"continue": {
"type": "boolean"
},
@@ -4888,7 +4916,6 @@
"type": "object"
},
"alertGroups": {
"description": "AlertGroups alert groups",
"items": {
"$ref": "#/definitions/alertGroup",
"type": "object"
@@ -5050,7 +5077,6 @@
"type": "object"
},
"gettableAlerts": {
"description": "GettableAlerts gettable alerts",
"items": {
"$ref": "#/definitions/gettableAlert",
"type": "object"
@@ -5175,6 +5201,7 @@
"type": "object"
},
"gettableSilences": {
"description": "GettableSilences gettable silences",
"items": {
"$ref": "#/definitions/gettableSilence",
"type": "object"
@@ -6644,6 +6671,21 @@
"name": "x-grafana-alerting-alert-rules-paused",
"type": "boolean"
},
{
"in": "header",
"name": "x-grafana-alerting-target-datasource-uid",
"type": "string"
},
{
"in": "header",
"name": "x-grafana-alerting-folder-uid",
"type": "string"
},
{
"in": "header",
"name": "x-grafana-alerting-notification-receiver",
"type": "string"
},
{
"in": "body",
"name": "Body",
@@ -6919,6 +6961,21 @@
"name": "x-grafana-alerting-alert-rules-paused",
"type": "boolean"
},
{
"in": "header",
"name": "x-grafana-alerting-target-datasource-uid",
"type": "string"
},
{
"in": "header",
"name": "x-grafana-alerting-folder-uid",
"type": "string"
},
{
"in": "header",
"name": "x-grafana-alerting-notification-receiver",
"type": "string"
},
{
"in": "body",
"name": "Body",
@@ -9533,4 +9590,4 @@
}
},
"swagger": "2.0"
}
}
pkg/services/ngalert/api/tooling/spec.json
+61 -4
View File
@@ -1204,6 +1204,21 @@
"name": "x-grafana-alerting-alert-rules-paused",
"in": "header"
},
{
"type": "string",
"name": "x-grafana-alerting-target-datasource-uid",
"in": "header"
},
{
"type": "string",
"name": "x-grafana-alerting-folder-uid",
"in": "header"
},
{
"type": "string",
"name": "x-grafana-alerting-notification-receiver",
"in": "header"
},
{
"name": "Body",
"in": "body",
@@ -1479,6 +1494,21 @@
"name": "x-grafana-alerting-alert-rules-paused",
"in": "header"
},
{
"type": "string",
"name": "x-grafana-alerting-target-datasource-uid",
"in": "header"
},
{
"type": "string",
"name": "x-grafana-alerting-folder-uid",
"in": "header"
},
{
"type": "string",
"name": "x-grafana-alerting-notification-receiver",
"in": "header"
},
{
"name": "Body",
"in": "body",
@@ -4416,6 +4446,16 @@
"receiver"
],
"properties": {
"active_time_intervals": {
"description": "Override the times when notifications should not be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration time_intervals section. All notifications will be suppressed unless they are sent\nat the time that matches any interval.",
"type": "array",
"items": {
"type": "string"
},
"example": [
"maintenance"
]
},
"group_by": {
"description": "Override the labels by which incoming alerts are grouped together. For example, multiple alerts coming in for\ncluster=A and alertname=LatencyHigh would be batched into a single group. To aggregate by all possible labels\nuse the special value '...' as the sole label name.\nThis effectively disables aggregation entirely, passing through all alerts as-is. This is unlikely to be what\nyou want, unless you have a very low alert volume or your upstream notification system performs its own grouping.\nMust include 'alertname' and 'grafana_folder' if not using '...'.",
"type": "array",
@@ -4443,7 +4483,7 @@
"example": "30s"
},
"mute_time_intervals": {
"description": "Override the times when notifications should be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration mute_time_intervals section. When muted it will not send any notifications, but\notherwise acts normally.",
"description": "Override the times when notifications should be muted. These must match the name of a mute time interval defined\nin the alertmanager configuration time_intervals section. When muted it will not send any notifications, but\notherwise acts normally.",
"type": "array",
"items": {
"type": "string"
@@ -4468,6 +4508,12 @@
"type": "object",
"title": "AlertRuleNotificationSettingsExport is the provisioned export of models.NotificationSettings.",
"properties": {
"active_time_intervals": {
"type": "array",
"items": {
"type": "string"
}
},
"group_by": {
"type": "array",
"items": {
@@ -6427,6 +6473,12 @@
"type": "object",
"title": "NotificationPolicyExport is the provisioned file export of alerting.NotificiationPolicyV1.",
"properties": {
"active_time_intervals": {
"type": "array",
"items": {
"type": "string"
}
},
"continue": {
"type": "boolean"
},
@@ -7811,6 +7863,12 @@
"description": "RouteExport is the provisioned file export of definitions.Route. This is needed to hide fields that aren't useable in\nprovisioning file format. An alternative would be to define a custom MarshalJSON and MarshalYAML that excludes them.",
"type": "object",
"properties": {
"active_time_intervals": {
"type": "array",
"items": {
"type": "string"
}
},
"continue": {
"type": "boolean"
},
@@ -8988,7 +9046,6 @@
}
},
"alertGroups": {
"description": "AlertGroups alert groups",
"type": "array",
"items": {
"type": "object",
@@ -9150,7 +9207,6 @@
}
},
"gettableAlerts": {
"description": "GettableAlerts gettable alerts",
"type": "array",
"items": {
"type": "object",
@@ -9275,6 +9331,7 @@
}
},
"gettableSilences": {
"description": "GettableSilences gettable silences",
"type": "array",
"items": {
"type": "object",
@@ -9566,4 +9623,4 @@
"type": "basic"
}
}
}
}
pkg/services/ngalert/api/validation/api_ruler_validation.go
+2 -2
View File
@@ -142,7 +142,7 @@ func validateAlertingRuleFields(in *apimodels.PostableExtendedRuleNode, newRule
}
if in.GrafanaManagedAlert.NotificationSettings != nil {
newRule.NotificationSettings, err = validateNotificationSettings(in.GrafanaManagedAlert.NotificationSettings)
newRule.NotificationSettings, err = ValidateNotificationSettings(in.GrafanaManagedAlert.NotificationSettings)
if err != nil {
return ngmodels.AlertRule{}, err
}
@@ -391,7 +391,7 @@ func ValidateRuleGroup(
return result, nil
}
func validateNotificationSettings(n *apimodels.AlertRuleNotificationSettings) ([]ngmodels.NotificationSettings, error) {
func ValidateNotificationSettings(n *apimodels.AlertRuleNotificationSettings) ([]ngmodels.NotificationSettings, error) {
s := ngmodels.NotificationSettings{
Receiver: n.Receiver,
GroupBy: n.GroupBy,