public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Expose info about notification delivery errors in a new /receivers endpoint (#55429)

Browse Source 
* (WIP) switch to fork AM, first implementation of the API, generate spec

* get receivers avoiding race conditions

* use latest version of our forked AM, tests

* make linter happy, delete TODO comment

* update number of expected paths to += 2

* delete unused endpoint code, code review comments, tests

* Update pkg/services/ngalert/notifier/alertmanager.go

Co-authored-by: Matthew Jacobson <matthew.jacobson@grafana.com>

* remove call to fmt.Println

* clear naming for fields

* shorter variable names in GetReceivers

Co-authored-by: Matthew Jacobson <matthew.jacobson@grafana.com>
This commit is contained in:
Santiago
2022-10-03 10:58:41 -03:00
committed by GitHub
co-authored by Matthew Jacobson
parent 1e16dd5b7c
commit 09f8e026a1
17 changed files with 667 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api.go
+2 -1
View File
@@ -49,7 +49,8 @@ type Alertmanager interface {
GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error)
// Testing
// Receivers
GetReceivers(ctx context.Context) apimodels.Receivers
TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*notifier.TestReceiversResult, error)
}
pkg/services/ngalert/api/api_alertmanager.go
+10
View File
@@ -248,6 +248,16 @@ func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.P
return NotImplementedResp
}
func (srv AlertmanagerSrv) RouteGetReceivers(c *models.ReqContext) response.Response {
am, errResp := srv.AlertmanagerFor(c.OrgID)
if errResp != nil {
return errResp
}
rcvs := am.GetReceivers(c.Req.Context())
return response.JSON(http.StatusOK, rcvs)
}
func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigBodyParams) response.Response {
if err := srv.crypto.LoadSecureSettings(c.Req.Context(), c.OrgID, body.Receivers); err != nil {
var unknownReceiverError UnknownReceiverError
pkg/services/ngalert/api/authorization.go
+2
View File
@@ -156,6 +156,8 @@ func (api *API) authorize(method, path string) web.Handler {
case http.MethodPost + "/api/alertmanager/grafana/config/api/v1/alerts":
// additional authorization is done in the request handler
eval = ac.EvalAny(ac.EvalPermission(ac.ActionAlertingNotificationsWrite))
case http.MethodGet + "/api/alertmanager/grafana/config/api/v1/receivers":
eval = ac.EvalPermission(ac.ActionAlertingNotificationsRead)
case http.MethodPost + "/api/alertmanager/grafana/config/api/v1/receivers/test":
fallback = middleware.ReqEditorRole
eval = ac.EvalPermission(ac.ActionAlertingNotificationsRead)
pkg/services/ngalert/api/authorization_test.go
+1 -1
View File
@@ -49,7 +49,7 @@ func TestAuthorize(t *testing.T) {
}
paths[p] = methods
}
require.Len(t, paths, 40)
require.Len(t, paths, 41)
ac := acmock.New()
api := &API{AccessControl: ac}
pkg/services/ngalert/api/forking_alertmanager.go
+4
View File
@@ -187,6 +187,10 @@ func (f *AlertmanagerApiHandler) handleRoutePostGrafanaAlertingConfig(ctx *model
return f.GrafanaSvc.RoutePostAlertingConfig(ctx, conf)
}
func (f *AlertmanagerApiHandler) handleRouteGetGrafanaReceivers(ctx *models.ReqContext) response.Response {
return f.GrafanaSvc.RouteGetReceivers(ctx)
}
func (f *AlertmanagerApiHandler) handleRoutePostTestGrafanaReceivers(ctx *models.ReqContext, conf apimodels.TestReceiversConfigBodyParams) response.Response {
return f.GrafanaSvc.RoutePostTestReceivers(ctx, conf)
}
pkg/services/ngalert/api/generated_base_api_alertmanager.go
+14
View File
@@ -33,6 +33,7 @@ type AlertmanagerApi interface {
RouteGetGrafanaAMAlerts(*models.ReqContext) response.Response
RouteGetGrafanaAMStatus(*models.ReqContext) response.Response
RouteGetGrafanaAlertingConfig(*models.ReqContext) response.Response
RouteGetGrafanaReceivers(*models.ReqContext) response.Response
RouteGetGrafanaSilence(*models.ReqContext) response.Response
RouteGetGrafanaSilences(*models.ReqContext) response.Response
RouteGetSilence(*models.ReqContext) response.Response
@@ -114,6 +115,9 @@ func (f *AlertmanagerApiHandler) RouteGetGrafanaAMStatus(ctx *models.ReqContext)
func (f *AlertmanagerApiHandler) RouteGetGrafanaAlertingConfig(ctx *models.ReqContext) response.Response {
return f.handleRouteGetGrafanaAlertingConfig(ctx)
}
func (f *AlertmanagerApiHandler) RouteGetGrafanaReceivers(ctx *models.ReqContext) response.Response {
return f.handleRouteGetGrafanaReceivers(ctx)
}
func (f *AlertmanagerApiHandler) RouteGetGrafanaSilence(ctx *models.ReqContext) response.Response {
// Parse Path Parameters
silenceIdParam := web.Params(ctx.Req)[":SilenceId"]
@@ -330,6 +334,16 @@ func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApi, m *metrics
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/grafana/config/api/v1/receivers"),
api.authorize(http.MethodGet, "/api/alertmanager/grafana/config/api/v1/receivers"),
metrics.Instrument(
http.MethodGet,
"/api/alertmanager/grafana/config/api/v1/receivers",
srv.RouteGetGrafanaReceivers,
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/grafana/api/v2/silence/{SilenceId}"),
api.authorize(http.MethodGet, "/api/alertmanager/grafana/api/v2/silence/{SilenceId}"),
pkg/services/ngalert/api/tooling/api.json
+83 -107
View File
@@ -260,7 +260,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"labels": {
"additionalProperties": {
@@ -378,25 +378,6 @@
"title": "DataTopic is used to identify which topic the frame should be assigned to.",
"type": "string"
},
"DateTime": {
"description": "DateTime is a time but it serializes to ISO8601 format with millis\nIt knows how to read 3 different variations of a RFC3339 date time.\nMost APIs we encounter want either millisecond or second precision times.\nThis just tries to make it worry-free.",
"format": "date-time",
"type": "string"
},
"DayOfMonthRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A DayOfMonthRange is an inclusive range that may have negative Beginning/End values that represent distance from the End of the month Beginning at -1.",
"type": "object"
},
"DiscoveryBase": {
"properties": {
"error": {
@@ -625,16 +606,12 @@
"FieldConfig": {
"properties": {
"color": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Map values to a display color\nNOTE: this interface is under development in the frontend... so simple map for now",
"type": "object"
},
"custom": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Panel Specific Values",
"type": "object"
},
@@ -742,8 +719,7 @@
"type": "string"
},
"custom": {
"description": "Custom datasource specific values.",
"type": "object"
"description": "Custom datasource specific values."
},
"dataTopic": {
"$ref": "#/definitions/DataTopic"
@@ -939,7 +915,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"grafana_alert": {
"$ref": "#/definitions/GettableGrafanaRule"
@@ -1239,6 +1215,10 @@
"description": "The bearer token file for the targets. Deprecated in favour of\nAuthorization.CredentialsFile.",
"type": "string"
},
"enable_http2": {
"description": "EnableHTTP2 specifies whether the client should configure HTTP2.\nThe omitempty flag is not set, because it would be hidden from the\nmarshalled configuration when set to false.",
"type": "boolean"
},
"follow_redirects": {
"description": "FollowRedirects specifies whether the client should follow HTTP 3xx redirects.\nThe omitempty flag is not set, because it would be hidden from the\nmarshalled configuration when set to false.",
"type": "boolean"
@@ -1268,20 +1248,6 @@
"title": "HostPort represents a \"host:port\" network address.",
"type": "object"
},
"InclusiveRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "InclusiveRange is used to hold the Beginning and End values of many time interval components.",
"type": "object"
},
"InhibitRule": {
"description": "InhibitRule defines an inhibition rule that mutes alerts that match the\ntarget labels if an alert matching the source labels exists.\nBoth alerts have to have a set of labels being equal.",
"properties": {
@@ -1501,20 +1467,6 @@
},
"type": "array"
},
"MonthRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A MonthRange is an inclusive range between [1, 12] where 1 = January.",
"type": "object"
},
"MultiStatus": {
"type": "object"
},
@@ -1605,6 +1557,9 @@
},
"type": "object"
},
"proxy_url": {
"$ref": "#/definitions/URL"
},
"scopes": {
"items": {
"type": "string"
@@ -1931,7 +1886,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"grafana_alert": {
"$ref": "#/definitions/PostableGrafanaRule"
@@ -2220,7 +2175,7 @@
"PushoverConfig": {
"properties": {
"expire": {
"$ref": "#/definitions/duration"
"type": "string"
},
"html": {
"type": "boolean"
@@ -2235,7 +2190,7 @@
"type": "string"
},
"retry": {
"$ref": "#/definitions/duration"
"type": "string"
},
"send_resolved": {
"type": "boolean"
@@ -2265,16 +2220,12 @@
"description": "The embedded FieldConfig's display name must be set.\nIt corresponds to the QueryResultMetaStat on the frontend (https://github.com/grafana/grafana/blob/master/packages/grafana-data/src/types/data.ts#L53).",
"properties": {
"color": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Map values to a display color\nNOTE: this interface is under development in the frontend... so simple map for now",
"type": "object"
},
"custom": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Panel Specific Values",
"type": "object"
},
@@ -2462,10 +2413,10 @@
"type": "array"
},
"group_interval": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"group_wait": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"match": {
"additionalProperties": {
@@ -2496,7 +2447,7 @@
"type": "string"
},
"repeat_interval": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"routes": {
"items": {
@@ -2897,6 +2848,9 @@
"description": "The client key file for the targets.",
"type": "string"
},
"min_version": {
"$ref": "#/definitions/TLSVersion"
},
"server_name": {
"description": "Used to verify the hostname for the targets.",
"type": "string"
@@ -2905,6 +2859,10 @@
"title": "TLSConfig configures the options for TLS connections.",
"type": "object"
},
"TLSVersion": {
"format": "uint16",
"type": "integer"
},
"TelegramConfig": {
"properties": {
"api_url": {
@@ -3073,13 +3031,13 @@
"properties": {
"days_of_month": {
"items": {
"$ref": "#/definitions/DayOfMonthRange"
"type": "string"
},
"type": "array"
},
"months": {
"items": {
"$ref": "#/definitions/MonthRange"
"type": "string"
},
"type": "array"
},
@@ -3091,13 +3049,13 @@
},
"weekdays": {
"items": {
"$ref": "#/definitions/WeekdayRange"
"type": "string"
},
"type": "array"
},
"years": {
"items": {
"$ref": "#/definitions/YearRange"
"type": "string"
},
"type": "array"
}
@@ -3120,7 +3078,6 @@
"type": "object"
},
"URL": {
"description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -3131,6 +3088,9 @@
"Host": {
"type": "string"
},
"OmitHost": {
"type": "boolean"
},
"Opaque": {
"type": "string"
},
@@ -3153,7 +3113,7 @@
"$ref": "#/definitions/Userinfo"
}
},
"title": "A URL represents a parsed URL (technically, a URI reference).",
"title": "URL is a custom URL type that allows validation at configuration load time.",
"type": "object"
},
"Userinfo": {
@@ -3291,34 +3251,6 @@
"title": "WechatConfig configures notifications via Wechat.",
"type": "object"
},
"WeekdayRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A WeekdayRange is an inclusive range between [0, 6] where 0 = Sunday.",
"type": "object"
},
"YearRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A YearRange is a positive inclusive range.",
"type": "object"
},
"alert": {
"description": "Alert alert",
"properties": {
@@ -3361,6 +3293,7 @@
"type": "object"
},
"alertGroups": {
"description": "AlertGroups alert groups",
"items": {
"$ref": "#/definitions/alertGroup"
},
@@ -3464,10 +3397,8 @@
],
"type": "object"
},
"duration": {
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
"description": "GettableAlert gettable alert",
"properties": {
"annotations": {
"$ref": "#/definitions/labelSet"
@@ -3530,6 +3461,7 @@
"type": "array"
},
"gettableSilence": {
"description": "GettableSilence gettable silence",
"properties": {
"comment": {
"description": "comment",
@@ -3578,11 +3510,42 @@
"type": "object"
},
"gettableSilences": {
"description": "GettableSilences gettable silences",
"items": {
"$ref": "#/definitions/gettableSilence"
},
"type": "array"
},
"integration": {
"properties": {
"lastNotifyAttempt": {
"description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time",
"format": "date-time",
"type": "string"
},
"lastNotifyAttemptDuration": {
"description": "Duration of the last attempt to deliver a notification in humanized format (`1s` or `15ms`, etc).",
"type": "string"
},
"lastNotifyAttemptError": {
"description": "Error string for the last attempt to deliver a notification. Empty if the last attempt was successful.",
"type": "string"
},
"name": {
"description": "name",
"type": "string"
},
"sendResolved": {
"description": "send resolved",
"type": "boolean"
}
},
"required": [
"name",
"sendResolved"
],
"type": "object"
},
"labelSet": {
"additionalProperties": {
"type": "string"
@@ -3688,6 +3651,7 @@
"type": "array"
},
"postableSilence": {
"description": "PostableSilence postable silence",
"properties": {
"comment": {
"description": "comment",
@@ -3725,14 +3689,26 @@
"type": "object"
},
"receiver": {
"description": "Receiver receiver",
"properties": {
"active": {
"description": "active",
"type": "boolean"
},
"integrations": {
"description": "integrations",
"items": {
"$ref": "#/definitions/integration"
},
"type": "array"
},
"name": {
"description": "name",
"type": "string"
}
},
"required": [
"active",
"integrations",
"name"
],
"type": "object"
pkg/services/ngalert/api/tooling/definitions/alertmanager.go
+13
View File
@@ -142,6 +142,13 @@ import (
// 400: ValidationError
// 404: NotFound
// swagger:route GET /api/alertmanager/grafana/config/api/v1/receivers alertmanager RouteGetGrafanaReceivers
//
// Get a list of all receivers.
//
// Responses:
// 200: receivers
// swagger:route POST /api/alertmanager/grafana/config/api/v1/receivers/test alertmanager RoutePostTestGrafanaReceivers
//
// Test Grafana managed receivers without saving them.
@@ -403,6 +410,12 @@ type AlertGroup = amv2.AlertGroup
// swagger:model receiver
type Receiver = amv2.Receiver
// swagger:model receivers
type Receivers = []amv2.Receiver
// swagger:model integration
type Integration = amv2.Integration
// swagger:parameters RouteGetAMAlerts RouteGetAMAlertGroups RouteGetGrafanaAMAlerts RouteGetGrafanaAMAlertGroups
type AlertsParams struct {
pkg/services/ngalert/api/tooling/post.json
+116 -128
View File
@@ -260,7 +260,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"labels": {
"additionalProperties": {
@@ -378,25 +378,6 @@
"title": "DataTopic is used to identify which topic the frame should be assigned to.",
"type": "string"
},
"DateTime": {
"description": "DateTime is a time but it serializes to ISO8601 format with millis\nIt knows how to read 3 different variations of a RFC3339 date time.\nMost APIs we encounter want either millisecond or second precision times.\nThis just tries to make it worry-free.",
"format": "date-time",
"type": "string"
},
"DayOfMonthRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A DayOfMonthRange is an inclusive range that may have negative Beginning/End values that represent distance from the End of the month Beginning at -1.",
"type": "object"
},
"DiscoveryBase": {
"properties": {
"error": {
@@ -625,16 +606,12 @@
"FieldConfig": {
"properties": {
"color": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Map values to a display color\nNOTE: this interface is under development in the frontend... so simple map for now",
"type": "object"
},
"custom": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Panel Specific Values",
"type": "object"
},
@@ -742,8 +719,7 @@
"type": "string"
},
"custom": {
"description": "Custom datasource specific values.",
"type": "object"
"description": "Custom datasource specific values."
},
"dataTopic": {
"$ref": "#/definitions/DataTopic"
@@ -939,7 +915,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"grafana_alert": {
"$ref": "#/definitions/GettableGrafanaRule"
@@ -1239,6 +1215,10 @@
"description": "The bearer token file for the targets. Deprecated in favour of\nAuthorization.CredentialsFile.",
"type": "string"
},
"enable_http2": {
"description": "EnableHTTP2 specifies whether the client should configure HTTP2.\nThe omitempty flag is not set, because it would be hidden from the\nmarshalled configuration when set to false.",
"type": "boolean"
},
"follow_redirects": {
"description": "FollowRedirects specifies whether the client should follow HTTP 3xx redirects.\nThe omitempty flag is not set, because it would be hidden from the\nmarshalled configuration when set to false.",
"type": "boolean"
@@ -1268,20 +1248,6 @@
"title": "HostPort represents a \"host:port\" network address.",
"type": "object"
},
"InclusiveRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "InclusiveRange is used to hold the Beginning and End values of many time interval components.",
"type": "object"
},
"InhibitRule": {
"description": "InhibitRule defines an inhibition rule that mutes alerts that match the\ntarget labels if an alert matching the source labels exists.\nBoth alerts have to have a set of labels being equal.",
"properties": {
@@ -1501,20 +1467,6 @@
},
"type": "array"
},
"MonthRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A MonthRange is an inclusive range between [1, 12] where 1 = January.",
"type": "object"
},
"MultiStatus": {
"type": "object"
},
@@ -1605,6 +1557,9 @@
},
"type": "object"
},
"proxy_url": {
"$ref": "#/definitions/URL"
},
"scopes": {
"items": {
"type": "string"
@@ -1931,7 +1886,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"grafana_alert": {
"$ref": "#/definitions/PostableGrafanaRule"
@@ -2220,7 +2175,7 @@
"PushoverConfig": {
"properties": {
"expire": {
"$ref": "#/definitions/duration"
"type": "string"
},
"html": {
"type": "boolean"
@@ -2235,7 +2190,7 @@
"type": "string"
},
"retry": {
"$ref": "#/definitions/duration"
"type": "string"
},
"send_resolved": {
"type": "boolean"
@@ -2265,16 +2220,12 @@
"description": "The embedded FieldConfig's display name must be set.\nIt corresponds to the QueryResultMetaStat on the frontend (https://github.com/grafana/grafana/blob/master/packages/grafana-data/src/types/data.ts#L53).",
"properties": {
"color": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Map values to a display color\nNOTE: this interface is under development in the frontend... so simple map for now",
"type": "object"
},
"custom": {
"additionalProperties": {
"type": "object"
},
"additionalProperties": {},
"description": "Panel Specific Values",
"type": "object"
},
@@ -2462,10 +2413,10 @@
"type": "array"
},
"group_interval": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"group_wait": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"match": {
"additionalProperties": {
@@ -2496,7 +2447,7 @@
"type": "string"
},
"repeat_interval": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"routes": {
"items": {
@@ -2897,6 +2848,9 @@
"description": "The client key file for the targets.",
"type": "string"
},
"min_version": {
"$ref": "#/definitions/TLSVersion"
},
"server_name": {
"description": "Used to verify the hostname for the targets.",
"type": "string"
@@ -2905,6 +2859,10 @@
"title": "TLSConfig configures the options for TLS connections.",
"type": "object"
},
"TLSVersion": {
"format": "uint16",
"type": "integer"
},
"TelegramConfig": {
"properties": {
"api_url": {
@@ -3073,13 +3031,13 @@
"properties": {
"days_of_month": {
"items": {
"$ref": "#/definitions/DayOfMonthRange"
"type": "string"
},
"type": "array"
},
"months": {
"items": {
"$ref": "#/definitions/MonthRange"
"type": "string"
},
"type": "array"
},
@@ -3091,13 +3049,13 @@
},
"weekdays": {
"items": {
"$ref": "#/definitions/WeekdayRange"
"type": "string"
},
"type": "array"
},
"years": {
"items": {
"$ref": "#/definitions/YearRange"
"type": "string"
},
"type": "array"
}
@@ -3120,6 +3078,7 @@
"type": "object"
},
"URL": {
"description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -3130,6 +3089,9 @@
"Host": {
"type": "string"
},
"OmitHost": {
"type": "boolean"
},
"Opaque": {
"type": "string"
},
@@ -3152,7 +3114,7 @@
"$ref": "#/definitions/Userinfo"
}
},
"title": "URL is a custom URL type that allows validation at configuration load time.",
"title": "A URL represents a parsed URL (technically, a URI reference).",
"type": "object"
},
"Userinfo": {
@@ -3290,34 +3252,6 @@
"title": "WechatConfig configures notifications via Wechat.",
"type": "object"
},
"WeekdayRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A WeekdayRange is an inclusive range between [0, 6] where 0 = Sunday.",
"type": "object"
},
"YearRange": {
"properties": {
"Begin": {
"format": "int64",
"type": "integer"
},
"End": {
"format": "int64",
"type": "integer"
}
},
"title": "A YearRange is a positive inclusive range.",
"type": "object"
},
"alert": {
"description": "Alert alert",
"properties": {
@@ -3462,11 +3396,7 @@
],
"type": "object"
},
"duration": {
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
"description": "GettableAlert gettable alert",
"properties": {
"annotations": {
"$ref": "#/definitions/labelSet"
@@ -3522,6 +3452,7 @@
"type": "object"
},
"gettableAlerts": {
"description": "GettableAlerts gettable alerts",
"items": {
"$ref": "#/definitions/gettableAlert"
},
@@ -3576,12 +3507,42 @@
"type": "object"
},
"gettableSilences": {
"description": "GettableSilences gettable silences",
"items": {
"$ref": "#/definitions/gettableSilence"
},
"type": "array"
},
"integration": {
"description": "Integration integration",
"properties": {
"lastNotifyAttempt": {
"description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time",
"format": "date-time",
"type": "string"
},
"lastNotifyAttemptDuration": {
"description": "Duration of the last attempt to deliver a notification in humanized format (`1s` or `15ms`, etc).",
"type": "string"
},
"lastNotifyAttemptError": {
"description": "Error string for the last attempt to deliver a notification. Empty if the last attempt was successful.",
"type": "string"
},
"name": {
"description": "name",
"type": "string"
},
"sendResolved": {
"description": "send resolved",
"type": "boolean"
}
},
"required": [
"name",
"sendResolved"
],
"type": "object"
},
"labelSet": {
"additionalProperties": {
"type": "string"
@@ -3687,7 +3648,6 @@
"type": "array"
},
"postableSilence": {
"description": "PostableSilence postable silence",
"properties": {
"comment": {
"description": "comment",
@@ -3725,13 +3685,27 @@
"type": "object"
},
"receiver": {
"description": "Receiver receiver",
"properties": {
"active": {
"description": "active",
"type": "boolean"
},
"integrations": {
"description": "integrations",
"items": {
"$ref": "#/definitions/integration"
},
"type": "array"
},
"name": {
"description": "name",
"type": "string"
}
},
"required": [
"active",
"integrations",
"name"
],
"type": "object"
@@ -4206,6 +4180,20 @@
]
}
},
"/api/alertmanager/grafana/config/api/v1/receivers": {
"get": {
"operationId": "RouteGetGrafanaReceivers",
"responses": {
"200": {
"$ref": "#/responses/receivers"
}
},
"summary": "Get a list of all receivers.",
"tags": [
"alertmanager"
]
}
},
"/api/alertmanager/grafana/config/api/v1/receivers/test": {
"post": {
"operationId": "RoutePostTestGrafanaReceivers",
@@ -5462,6 +5450,26 @@
]
}
},
"/api/v1/ngalert": {
"get": {
"description": "Get the status of the alerting engine",
"operationId": "RouteGetStatus",
"produces": [
"application/json"
],
"responses": {
"200": {
"description": "AlertingStatus",
"schema": {
"$ref": "#/definitions/AlertingStatus"
}
}
},
"tags": [
"configuration"
]
}
},
"/api/v1/ngalert/admin_config": {
"delete": {
"consumes": [
@@ -5571,26 +5579,6 @@
]
}
},
"/api/v1/ngalert": {
"get": {
"description": "Get the status of the alerting engine",
"operationId": "RouteGetStatus",
"produces": [
"application/json"
],
"responses": {
"200": {
"description": "AlertingStatus",
"schema": {
"$ref": "#/definitions/AlertingStatus"
}
}
},
"tags": [
"configuration"
]
}
},
"/api/v1/provisioning/alert-rules": {
"post": {
"consumes": [
pkg/services/ngalert/api/tooling/spec.json
+117 -128
View File
@@ -392,6 +392,20 @@
}
}
},
"/api/alertmanager/grafana/config/api/v1/receivers": {
"get": {
"tags": [
"alertmanager"
],
"summary": "Get a list of all receivers.",
"operationId": "RouteGetGrafanaReceivers",
"responses": {
"200": {
"$ref": "#/responses/receivers"
}
}
}
},
"/api/alertmanager/grafana/config/api/v1/receivers/test": {
"post": {
"tags": [
@@ -1648,6 +1662,26 @@
}
}
},
"/api/v1/ngalert": {
"get": {
"description": "Get the status of the alerting engine",
"produces": [
"application/json"
],
"tags": [
"configuration"
],
"operationId": "RouteGetStatus",
"responses": {
"200": {
"description": "AlertingStatus",
"schema": {
"$ref": "#/definitions/AlertingStatus"
}
}
}
}
},
"/api/v1/ngalert/admin_config": {
"get": {
"produces": [
@@ -1757,26 +1791,6 @@
}
}
},
"/api/v1/ngalert": {
"get": {
"description": "Get the status of the alerting engine",
"produces": [
"application/json"
],
"tags": [
"configuration"
],
"operationId": "RouteGetStatus",
"responses": {
"200": {
"description": "AlertingStatus",
"schema": {
"$ref": "#/definitions/AlertingStatus"
}
}
}
}
},
"/api/v1/provisioning/alert-rules": {
"post": {
"consumes": [
@@ -2782,7 +2796,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"labels": {
"type": "object",
@@ -2899,25 +2913,6 @@
"type": "string",
"title": "DataTopic is used to identify which topic the frame should be assigned to."
},
"DateTime": {
"description": "DateTime is a time but it serializes to ISO8601 format with millis\nIt knows how to read 3 different variations of a RFC3339 date time.\nMost APIs we encounter want either millisecond or second precision times.\nThis just tries to make it worry-free.",
"type": "string",
"format": "date-time"
},
"DayOfMonthRange": {
"type": "object",
"title": "A DayOfMonthRange is an inclusive range that may have negative Beginning/End values that represent distance from the End of the month Beginning at -1.",
"properties": {
"Begin": {
"type": "integer",
"format": "int64"
},
"End": {
"type": "integer",
"format": "int64"
}
}
},
"DiscoveryBase": {
"type": "object",
"required": [
@@ -3153,16 +3148,12 @@
"color": {
"description": "Map values to a display color\nNOTE: this interface is under development in the frontend... so simple map for now",
"type": "object",
"additionalProperties": {
"type": "object"
}
"additionalProperties": {}
},
"custom": {
"description": "Panel Specific Values",
"type": "object",
"additionalProperties": {
"type": "object"
}
"additionalProperties": {}
},
"decimals": {
"type": "integer",
@@ -3268,8 +3259,7 @@
"type": "string"
},
"custom": {
"description": "Custom datasource specific values.",
"type": "object"
"description": "Custom datasource specific values."
},
"dataTopic": {
"$ref": "#/definitions/DataTopic"
@@ -3464,7 +3454,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"grafana_alert": {
"$ref": "#/definitions/GettableGrafanaRule"
@@ -3765,6 +3755,10 @@
"description": "The bearer token file for the targets. Deprecated in favour of\nAuthorization.CredentialsFile.",
"type": "string"
},
"enable_http2": {
"description": "EnableHTTP2 specifies whether the client should configure HTTP2.\nThe omitempty flag is not set, because it would be hidden from the\nmarshalled configuration when set to false.",
"type": "boolean"
},
"follow_redirects": {
"description": "FollowRedirects specifies whether the client should follow HTTP 3xx redirects.\nThe omitempty flag is not set, because it would be hidden from the\nmarshalled configuration when set to false.",
"type": "boolean"
@@ -3792,20 +3786,6 @@
}
}
},
"InclusiveRange": {
"type": "object",
"title": "InclusiveRange is used to hold the Beginning and End values of many time interval components.",
"properties": {
"Begin": {
"type": "integer",
"format": "int64"
},
"End": {
"type": "integer",
"format": "int64"
}
}
},
"InhibitRule": {
"description": "InhibitRule defines an inhibition rule that mutes alerts that match the\ntarget labels if an alert matching the source labels exists.\nBoth alerts have to have a set of labels being equal.",
"type": "object",
@@ -4026,20 +4006,6 @@
"$ref": "#/definitions/MessageTemplate"
}
},
"MonthRange": {
"type": "object",
"title": "A MonthRange is an inclusive range between [1, 12] where 1 = January.",
"properties": {
"Begin": {
"type": "integer",
"format": "int64"
},
"End": {
"type": "integer",
"format": "int64"
}
}
},
"MultiStatus": {
"type": "object"
},
@@ -4132,6 +4098,9 @@
"type": "string"
}
},
"proxy_url": {
"$ref": "#/definitions/URL"
},
"scopes": {
"type": "array",
"items": {
@@ -4457,7 +4426,7 @@
"type": "string"
},
"for": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"grafana_alert": {
"$ref": "#/definitions/PostableGrafanaRule"
@@ -4746,7 +4715,7 @@
"type": "object",
"properties": {
"expire": {
"$ref": "#/definitions/duration"
"type": "string"
},
"html": {
"type": "boolean"
@@ -4761,7 +4730,7 @@
"type": "string"
},
"retry": {
"$ref": "#/definitions/duration"
"type": "string"
},
"send_resolved": {
"type": "boolean"
@@ -4794,16 +4763,12 @@
"color": {
"description": "Map values to a display color\nNOTE: this interface is under development in the frontend... so simple map for now",
"type": "object",
"additionalProperties": {
"type": "object"
}
"additionalProperties": {}
},
"custom": {
"description": "Panel Specific Values",
"type": "object",
"additionalProperties": {
"type": "object"
}
"additionalProperties": {}
},
"decimals": {
"type": "integer",
@@ -4988,10 +4953,10 @@
}
},
"group_interval": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"group_wait": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"match": {
"description": "Deprecated. Remove before v1.0 release.",
@@ -5022,7 +4987,7 @@
"type": "string"
},
"repeat_interval": {
"$ref": "#/definitions/Duration"
"type": "string"
},
"routes": {
"type": "array",
@@ -5424,12 +5389,19 @@
"description": "The client key file for the targets.",
"type": "string"
},
"min_version": {
"$ref": "#/definitions/TLSVersion"
},
"server_name": {
"description": "Used to verify the hostname for the targets.",
"type": "string"
}
}
},
"TLSVersion": {
"type": "integer",
"format": "uint16"
},
"TelegramConfig": {
"type": "object",
"title": "TelegramConfig configures notifications via Telegram.",
@@ -5600,13 +5572,13 @@
"days_of_month": {
"type": "array",
"items": {
"$ref": "#/definitions/DayOfMonthRange"
"type": "string"
}
},
"months": {
"type": "array",
"items": {
"$ref": "#/definitions/MonthRange"
"type": "string"
}
},
"times": {
@@ -5618,13 +5590,13 @@
"weekdays": {
"type": "array",
"items": {
"$ref": "#/definitions/WeekdayRange"
"type": "string"
}
},
"years": {
"type": "array",
"items": {
"$ref": "#/definitions/YearRange"
"type": "string"
}
}
}
@@ -5645,8 +5617,9 @@
}
},
"URL": {
"description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"type": "object",
"title": "URL is a custom URL type that allows validation at configuration load time.",
"title": "A URL represents a parsed URL (technically, a URI reference).",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -5657,6 +5630,9 @@
"Host": {
"type": "string"
},
"OmitHost": {
"type": "boolean"
},
"Opaque": {
"type": "string"
},
@@ -5815,34 +5791,6 @@
}
}
},
"WeekdayRange": {
"type": "object",
"title": "A WeekdayRange is an inclusive range between [0, 6] where 0 = Sunday.",
"properties": {
"Begin": {
"type": "integer",
"format": "int64"
},
"End": {
"type": "integer",
"format": "int64"
}
}
},
"YearRange": {
"type": "object",
"title": "A YearRange is a positive inclusive range.",
"properties": {
"Begin": {
"type": "integer",
"format": "int64"
},
"End": {
"type": "integer",
"format": "int64"
}
}
},
"alert": {
"description": "Alert alert",
"type": "object",
@@ -5989,11 +5937,7 @@
}
}
},
"duration": {
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
"description": "GettableAlert gettable alert",
"type": "object",
"required": [
"labels",
@@ -6050,6 +5994,7 @@
"$ref": "#/definitions/gettableAlert"
},
"gettableAlerts": {
"description": "GettableAlerts gettable alerts",
"type": "array",
"items": {
"$ref": "#/definitions/gettableAlert"
@@ -6106,13 +6051,44 @@
"$ref": "#/definitions/gettableSilence"
},
"gettableSilences": {
"description": "GettableSilences gettable silences",
"type": "array",
"items": {
"$ref": "#/definitions/gettableSilence"
},
"$ref": "#/definitions/gettableSilences"
},
"integration": {
"description": "Integration integration",
"type": "object",
"required": [
"name",
"sendResolved"
],
"properties": {
"lastNotifyAttempt": {
"description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time",
"type": "string",
"format": "date-time"
},
"lastNotifyAttemptDuration": {
"description": "Duration of the last attempt to deliver a notification in humanized format (`1s` or `15ms`, etc).",
"type": "string"
},
"lastNotifyAttemptError": {
"description": "Error string for the last attempt to deliver a notification. Empty if the last attempt was successful.",
"type": "string"
},
"name": {
"description": "name",
"type": "string"
},
"sendResolved": {
"description": "send resolved",
"type": "boolean"
}
},
"$ref": "#/definitions/integration"
},
"labelSet": {
"description": "LabelSet label set",
"type": "object",
@@ -6218,7 +6194,6 @@
}
},
"postableSilence": {
"description": "PostableSilence postable silence",
"type": "object",
"required": [
"comment",
@@ -6257,11 +6232,25 @@
"$ref": "#/definitions/postableSilence"
},
"receiver": {
"description": "Receiver receiver",
"type": "object",
"required": [
"active",
"integrations",
"name"
],
"properties": {
"active": {
"description": "active",
"type": "boolean"
},
"integrations": {
"description": "integrations",
"type": "array",
"items": {
"$ref": "#/definitions/integration"
}
},
"name": {
"description": "name",
"type": "string"
pkg/services/ngalert/notifier/alertmanager.go
+34 -15
View File
@@ -118,6 +118,8 @@ type Alertmanager struct {
silencer *silence.Silencer
silences *silence.Silences
receivers []*notify.Receiver
// muteTimes is a map where the key is the name of the mute_time_interval
// and the value represents all configured time_interval(s)
muteTimes map[string][]timeinterval.TimeInterval
@@ -206,7 +208,7 @@ func newAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
}()
// Initialize in-memory alerts
am.alerts, err = mem.NewAlerts(context.Background(), am.marker, memoryAlertsGCInterval, nil, am.logger)
am.alerts, err = mem.NewAlerts(context.Background(), am.marker, memoryAlertsGCInterval, nil, am.logger, m.Registerer)
if err != nil {
return nil, fmt.Errorf("unable to initialize the alert provider component of alerting: %w", err)
}
@@ -421,14 +423,20 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
inhibitionStage := notify.NewMuteStage(am.inhibitor)
timeMuteStage := notify.NewTimeMuteStage(am.muteTimes)
silencingStage := notify.NewMuteStage(am.silencer)
for name := range integrationsMap {
stage := am.createReceiverStage(name, integrationsMap[name], am.waitFunc, am.notificationLog)
routingStage[name] = notify.MultiStage{meshStage, silencingStage, timeMuteStage, inhibitionStage, stage}
}
am.route = dispatch.NewRoute(cfg.AlertmanagerConfig.Route.AsAMRoute(), nil)
am.dispatcher = dispatch.NewDispatcher(am.alerts, am.route, routingStage, am.marker, am.timeoutFunc, &nilLimits{}, am.logger, am.dispatcherMetrics)
// Check which receivers are active and create the receiver stage.
activeReceivers := am.getActiveReceiversMap(am.route)
for name := range integrationsMap {
stage := am.createReceiverStage(name, integrationsMap[name], am.waitFunc, am.notificationLog)
routingStage[name] = notify.MultiStage{meshStage, silencingStage, timeMuteStage, inhibitionStage, stage}
_, isActive := activeReceivers[name]
am.receivers = append(am.receivers, notify.NewReceiver(name, isActive, integrationsMap[name]))
}
am.wg.Add(1)
go func() {
defer am.wg.Done()
@@ -452,8 +460,8 @@ func (am *Alertmanager) WorkingDirPath() string {
}
// buildIntegrationsMap builds a map of name to the list of Grafana integration notifiers off of a list of receiver config.
func (am *Alertmanager) buildIntegrationsMap(receivers []*apimodels.PostableApiReceiver, templates *template.Template) (map[string][]notify.Integration, error) {
integrationsMap := make(map[string][]notify.Integration, len(receivers))
func (am *Alertmanager) buildIntegrationsMap(receivers []*apimodels.PostableApiReceiver, templates *template.Template) (map[string][]*notify.Integration, error) {
integrationsMap := make(map[string][]*notify.Integration, len(receivers))
for _, receiver := range receivers {
integrations, err := am.buildReceiverIntegrations(receiver, templates)
if err != nil {
@@ -466,8 +474,8 @@ func (am *Alertmanager) buildIntegrationsMap(receivers []*apimodels.PostableApiR
}
// buildReceiverIntegrations builds a list of integration notifiers off of a receiver config.
func (am *Alertmanager) buildReceiverIntegrations(receiver *apimodels.PostableApiReceiver, tmpl *template.Template) ([]notify.Integration, error) {
var integrations []notify.Integration
func (am *Alertmanager) buildReceiverIntegrations(receiver *apimodels.PostableApiReceiver, tmpl *template.Template) ([]*notify.Integration, error) {
var integrations []*notify.Integration
for i, r := range receiver.GrafanaManagedReceivers {
n, err := am.buildReceiverIntegration(r, tmpl)
if err != nil {
@@ -667,18 +675,18 @@ func (e AlertValidationError) Error() string {
}
// createReceiverStage creates a pipeline of stages for a receiver.
func (am *Alertmanager) createReceiverStage(name string, integrations []notify.Integration, wait func() time.Duration, notificationLog notify.NotificationLog) notify.Stage {
func (am *Alertmanager) createReceiverStage(name string, integrations []*notify.Integration, wait func() time.Duration, notificationLog notify.NotificationLog) notify.Stage {
var fs notify.FanoutStage
for i := range integrations {
for _, integration := range integrations {
recv := &nflogpb.Receiver{
GroupName: name,
Integration: integrations[i].Name(),
Idx: uint32(integrations[i].Index()),
Integration: integration.Name(),
Idx: uint32(integration.Index()),
}
var s notify.MultiStage
s = append(s, notify.NewWaitStage(wait))
s = append(s, notify.NewDedupStage(&integrations[i], notificationLog, recv))
s = append(s, notify.NewRetryStage(integrations[i], name, am.stageMetrics))
s = append(s, notify.NewDedupStage(integration, notificationLog, recv))
s = append(s, notify.NewRetryStage(integration, name, am.stageMetrics))
s = append(s, notify.NewSetNotifiesStage(notificationLog, recv))
fs = append(fs, s)
@@ -686,6 +694,17 @@ func (am *Alertmanager) createReceiverStage(name string, integrations []notify.I
return fs
}
// getActiveReceiversMap returns all receivers that are in use by a route.
func (am *Alertmanager) getActiveReceiversMap(r *dispatch.Route) map[string]struct{} {
receiversMap := make(map[string]struct{})
visitFunc := func(r *dispatch.Route) {
receiversMap[r.RouteOpts.Receiver] = struct{}{}
}
r.Walk(visitFunc)
return receiversMap
}
func (am *Alertmanager) waitFunc() time.Duration {
return time.Duration(am.peer.Position()) * am.peerTimeout
}
pkg/services/ngalert/notifier/alertmanager_test.go
+1 -1
View File
@@ -314,7 +314,7 @@ func TestPutAlert(t *testing.T) {
t.Run(c.title, func(t *testing.T) {
r := prometheus.NewRegistry()
am.marker = types.NewMarker(r)
am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 15*time.Minute, nil, am.logger)
am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 15*time.Minute, nil, am.logger, r)
require.NoError(t, err)
alerts := []*types.Alert{}
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+1 -1
View File
@@ -85,7 +85,7 @@ func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore AlertingStore, orgSto
true,
cfg.UnifiedAlerting.HAPushPullInterval,
cfg.UnifiedAlerting.HAGossipInterval,
cluster.DefaultTcpTimeout,
cluster.DefaultTCPTimeout,
cluster.DefaultProbeTimeout,
cluster.DefaultProbeInterval,
nil,
pkg/services/ngalert/notifier/receivers.go
+40
View File
@@ -8,7 +8,9 @@ import (
"sort"
"time"
"github.com/go-openapi/strfmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/prometheus/alertmanager/api/v2/models"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/common/model"
@@ -197,6 +199,44 @@ func (am *Alertmanager) TestReceivers(ctx context.Context, c apimodels.TestRecei
return newTestReceiversResult(testAlert, append(invalid, results...), now), nil
}
func (am *Alertmanager) GetReceivers(ctx context.Context) apimodels.Receivers {
am.reloadConfigMtx.RLock()
defer am.reloadConfigMtx.RUnlock()
var apiReceivers apimodels.Receivers
for _, rcv := range am.receivers {
// Build integrations slice for each receiver.
var integrations []*models.Integration
for _, integration := range rcv.Integrations() {
name := integration.Name()
sendResolved := integration.SendResolved()
ts, d, err := integration.GetReport()
integrations = append(integrations, &apimodels.Integration{
Name: &name,
SendResolved: &sendResolved,
LastNotifyAttempt: strfmt.DateTime(ts),
LastNotifyAttemptDuration: d.String(),
LastNotifyAttemptError: func() string {
if err != nil {
return err.Error()
}
return ""
}(),
})
}
active := rcv.Active()
name := rcv.Name()
apiReceivers = append(apiReceivers, apimodels.Receiver{
Active: &active,
Integrations: integrations,
Name: &name,
})
}
return apiReceivers
}
func newTestAlert(c apimodels.TestReceiversConfigBodyParams, startsAt, updatedAt time.Time) types.Alert {
var (
defaultAnnotations = model.LabelSet{
pkg/tests/api/alerting/api_notification_channel_test.go
+211 -10
View File
@@ -728,7 +728,7 @@ func TestNotificationChannels(t *testing.T) {
channels.GetBoundary = func() string { return "abcd" }
env.NotificationService.EmailHandlerSync = mockEmail.sendEmailCommandHandlerSync
// As we are using a NotificationService mock here, but he test expects real NotificationService -
// As we are using a NotificationService mock here, but the test expects real NotificationService -
// we try to issue a real POST request here
env.NotificationService.WebhookHandler = func(_ context.Context, cmd *models.SendWebhookSync) error {
if res, err := http.Post(cmd.Url, "", strings.NewReader(cmd.Body)); err == nil {
@@ -770,11 +770,31 @@ func TestNotificationChannels(t *testing.T) {
re := regexp.MustCompile(`"uid":"([\w|-]*)"`)
e := getExpAlertmanagerConfigFromAPI(mockChannel.server.Addr)
require.JSONEq(t, e, string(re.ReplaceAll([]byte(b), []byte(`"uid":""`))))
// Check the receivers API. No errors nor attempts to notify should be registered.
receiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers", grafanaListedAddr)
resp = getRequest(t, receiversURL, http.StatusOK) // nolint
b = getBody(t, resp.Body)
var receivers apimodels.Receivers
err := json.Unmarshal([]byte(b), &receivers)
require.NoError(t, err)
for _, rcv := range receivers {
require.NotNil(t, rcv.Name)
require.NotNil(t, rcv.Active)
require.NotEmpty(t, rcv.Integrations)
for _, integration := range rcv.Integrations {
require.NotNil(t, integration.Name)
require.NotNil(t, integration.SendResolved)
require.Equal(t, "", integration.LastNotifyAttemptError)
require.Zero(t, integration.LastNotifyAttempt)
require.Equal(t, "0s", integration.LastNotifyAttemptDuration)
}
}
}
{
// Create rules that will fire as quickly as possible
originalFunction := store.GenerateNewAlertRuleUID
t.Cleanup(func() {
store.GenerateNewAlertRuleUID = originalFunction
@@ -791,6 +811,7 @@ func TestNotificationChannels(t *testing.T) {
// Eventually, we'll get all the desired alerts.
// nolint:gosec
require.Eventually(t, func() bool {
// TODO: not waiting for the failed notifications, flaky test?
return mockChannel.totalNotifications() >= len(nonEmailAlertNames) && len(mockEmail.emails) >= 1
}, 30*time.Second, 1*time.Second)
@@ -798,6 +819,60 @@ func TestNotificationChannels(t *testing.T) {
require.Equal(t, expEmailNotifications, mockEmail.emails)
require.NoError(t, mockChannel.Close())
// Check the receivers API. Errors and inactive receivers are expected, attempts to deliver notifications should be registered.
receiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers", grafanaListedAddr)
resp := getRequest(t, receiversURL, http.StatusOK) // nolint
b := getBody(t, resp.Body)
var receivers apimodels.Receivers
err := json.Unmarshal([]byte(b), &receivers)
require.NoError(t, err)
for _, rcv := range receivers {
var expActive bool
if _, ok := expInactiveReceivers[*rcv.Name]; !ok {
expActive = true
}
var expErr bool
if _, ok := expNotificationErrors[*rcv.Name]; ok {
expErr = true
}
require.NotNil(t, rcv.Name)
require.NotNil(t, rcv.Active)
require.NotEmpty(t, rcv.Integrations)
if expActive {
require.True(t, *rcv.Active)
}
// We don't have test alerts for the default notifier, continue iterating.
if *rcv.Name == "grafana-default-email" {
continue
}
for _, integration := range rcv.Integrations {
require.NotNil(t, integration.Name)
require.NotNil(t, integration.SendResolved)
// If the receiver is not active, no attempts to send notifications should be registered.
if expActive {
require.NotZero(t, integration.LastNotifyAttempt)
require.NotEqual(t, "0s", integration.LastNotifyAttemptDuration)
} else {
require.Zero(t, integration.LastNotifyAttempt)
require.Equal(t, "0s", integration.LastNotifyAttemptDuration)
}
// Check whether we're expecting an error on this integration.
if expErr {
for _, integration := range rcv.Integrations {
require.Equal(t, expNotificationErrors[*rcv.Name], integration.LastNotifyAttemptError)
}
} else {
require.Equal(t, "", integration.LastNotifyAttemptError)
}
}
}
{
// Delete the configuration; so it returns the default configuration.
u := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
@@ -859,6 +934,10 @@ var emailAlertNames = []string{
"EmailAlert",
}
var failedAlertNames = []string{
"SlackFailedAlert",
}
func getRulesConfig(t *testing.T) string {
t.Helper()
interval, err := model.ParseDuration("10s")
@@ -869,7 +948,10 @@ func getRulesConfig(t *testing.T) string {
}
// Create rules that will fire as quickly as possible for all the routes.
for _, alertName := range append(nonEmailAlertNames, emailAlertNames...) {
rulesToCreate := append(nonEmailAlertNames, emailAlertNames...)
rulesToCreate = append(rulesToCreate, failedAlertNames...)
for _, alertName := range rulesToCreate {
rules.Rules = append(rules.Rules, apimodels.PostableExtendedRuleNode{
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
Title: alertName,
@@ -1124,6 +1206,26 @@ const alertmanagerConfig = `
"alertname=\"SlackAlert2\""
]
},
{
"receiver": "slack_failed_recv",
"group_wait": "0s",
"group_by": [
"alertname"
],
"matchers": [
"alertname=\"SlackFailedAlert\""
]
},
{
"receiver": "slack_inactive_recv",
"group_wait": "0s",
"group_by": [
"alertname"
],
"matchers": [
"alertname=\"Inactive\""
]
},
{
"receiver": "pagerduty_recv",
"group_wait": "0s",
@@ -1273,7 +1375,7 @@ const alertmanagerConfig = `
"matchers": [
"alertname=\"TelegramAlert\""
]
}
}
]
},
"receivers": [
@@ -1483,7 +1585,7 @@ const alertmanagerConfig = `
}
]
},
{
{
"name": "slack_recv1",
"grafana_managed_receiver_configs": [
{
@@ -1506,8 +1608,8 @@ const alertmanagerConfig = `
}
}
]
},
{
},
{
"name": "slack_recv2",
"grafana_managed_receiver_configs": [
{
@@ -1523,6 +1625,39 @@ const alertmanagerConfig = `
}
}
]
},
{
"name": "slack_failed_recv",
"grafana_managed_receiver_configs": [
{
"name": "slack_failed_test",
"type": "slack",
"settings": {
"recipient": "#test-channel",
"username": "test",
"text": "Integration Test"
},
"secureSettings": {
"url": "htt://127.0.0.1:8080/slack_failed_recv/slack_failed_test"
}
}
]
},
{
"name": "slack_inactive_recv",
"grafana_managed_receiver_configs": [
{
"name": "inactive",
"type": "slack",
"settings": {
"recipient": "#inactive-channel",
"username": "Integration Test"
},
"secureSettings": {
"token": "myfullysecrettoken"
}
}
]
},
{
"name": "pagerduty_recv",
@@ -1589,7 +1724,26 @@ var expAlertmanagerConfigFromAPI = `
"alertname=\"SlackAlert2\""
]
},
{
{
"receiver": "slack_failed_recv",
"group_wait": "0s",
"group_by": [
"alertname"
],
"matchers": [
"alertname=\"SlackFailedAlert\""
]
},
{
"receiver": "slack_inactive_recv",
"group_wait": "0s",
"group_by": [
"alertname"
],
"matchers": [
"alertname=\"Inactive\""
]
}, {
"receiver": "pagerduty_recv",
"group_wait": "0s",
"group_by": [
@@ -1738,7 +1892,7 @@ var expAlertmanagerConfigFromAPI = `
"matchers": [
"alertname=\"TelegramAlert\""
]
}
}
]
},
"templates": null,
@@ -1986,7 +2140,7 @@ var expAlertmanagerConfigFromAPI = `
}
]
},
{
{
"name": "slack_recv1",
"grafana_managed_receiver_configs": [
{
@@ -2031,6 +2185,43 @@ var expAlertmanagerConfigFromAPI = `
}
]
},
{
"name": "slack_failed_recv",
"grafana_managed_receiver_configs": [
{
"uid": "",
"name": "slack_failed_test",
"type": "slack",
"disableResolveMessage": false,
"settings": {
"recipient": "#test-channel",
"username": "test",
"text": "Integration Test"
},
"secureFields": {
"url": true
}
}
]
},
{
"name": "slack_inactive_recv",
"grafana_managed_receiver_configs": [
{
"uid": "",
"name": "inactive",
"type": "slack",
"disableResolveMessage": false,
"settings": {
"recipient": "#inactive-channel",
"username": "Integration Test"
},
"secureFields": {
"token": true
}
}
]
},
{
"name": "pagerduty_recv",
"grafana_managed_receiver_configs": [
@@ -2435,3 +2626,13 @@ var expNonEmailNotifications = map[string][]string{
]`,
},
}
// expNotificationErrors maps a receiver name with its expected error string.
var expNotificationErrors = map[string]string{
"slack_failed_recv": `Post "htt://127.0.0.1:8080/slack_failed_recv/slack_failed_test": unsupported protocol scheme "htt"`,
}
// expNotificationErrors maps a receiver name with its expected error string.
var expInactiveReceivers = map[string]struct{}{
"slack_inactive_recv": {},
}