Alerting: Send alerts to external Alertmanager(s) (#37298)
* Alerting: Send alerts to external Alertmanager(s) Within this PR we're adding support for registering or unregistering sending to a set of external alertmanagers. A few of the things that are going are: - Introduce a new table to hold "admin" (either org or global) configuration we can change at runtime. - A new periodic check that polls for this configuration and adjusts the "senders" accordingly. - Introduces a new concept of "senders" that are responsible for shipping the alerts to the external Alertmanager(s). In a nutshell, this is the Prometheus notifier (the one in charge of sending the alert) mapped to a multi-tenant map. There are a few code movements here and there but those are minor, I tried to keep things intact as much as possible so that we could have an easier diff.
This commit is contained in:
gotjosh
@@ -0,0 +1,51 @@
|
||||
package definitions
|
||||
|
||||
// swagger:route GET /api/v1/ngalert/admin_config configuration RouteGetNGalertConfig
|
||||
//
|
||||
// Get the NGalert configuration of the user's organization, returns 404 if no configuration is present.
|
||||
//
|
||||
// Produces:
|
||||
// - application/json
|
||||
//
|
||||
// Responses:
|
||||
// 200: GettableNGalertConfig
|
||||
// 404: Failure
|
||||
// 500: Failure
|
||||
|
||||
// swagger:route POST /api/v1/ngalert/admin_config configuration RoutePostNGalertConfig
|
||||
//
|
||||
// Creates or updates the NGalert configuration of the user's organization.
|
||||
//
|
||||
// Consumes:
|
||||
// - application/json
|
||||
//
|
||||
// Responses:
|
||||
// 201: Ack
|
||||
// 400: ValidationError
|
||||
|
||||
// swagger:route DELETE /api/v1/ngalert/admin_config configuration RouteDeleteNGalertConfig
|
||||
//
|
||||
// Deletes the NGalert configuration of the user's organization.
|
||||
//
|
||||
// Consumes:
|
||||
// - application/json
|
||||
//
|
||||
// Responses:
|
||||
// 200: Ack
|
||||
// 500: Failure
|
||||
|
||||
// swagger:parameters RoutePostNGalertConfig
|
||||
type NGalertConfig struct {
|
||||
// in:body
|
||||
Body PostableNGalertConfig
|
||||
}
|
||||
|
||||
// swagger:model
|
||||
type PostableNGalertConfig struct {
|
||||
Alertmanagers []string `json:"alertmanagers"`
|
||||
}
|
||||
|
||||
// swagger:model
|
||||
type GettableNGalertConfig struct {
|
||||
Alertmanagers []string `json:"alertmanagers"`
|
||||
}
|
||||
@@ -57,7 +57,9 @@
|
||||
"type": "object",
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"AlertGroup": {},
|
||||
"AlertGroup": {
|
||||
"$ref": "#/definitions/alertGroup"
|
||||
},
|
||||
"AlertGroups": {
|
||||
"$ref": "#/definitions/alertGroups"
|
||||
},
|
||||
@@ -751,6 +753,19 @@
|
||||
"type": "object",
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"GettableNGalertConfig": {
|
||||
"properties": {
|
||||
"alertmanagers": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"x-go-name": "Alertmanagers"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"GettableRuleGroupConfig": {
|
||||
"properties": {
|
||||
"interval": {
|
||||
@@ -771,12 +786,8 @@
|
||||
"type": "object",
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"GettableSilence": {
|
||||
"$ref": "#/definitions/gettableSilence"
|
||||
},
|
||||
"GettableSilences": {
|
||||
"$ref": "#/definitions/gettableSilences"
|
||||
},
|
||||
"GettableSilence": {},
|
||||
"GettableSilences": {},
|
||||
"GettableStatus": {
|
||||
"properties": {
|
||||
"cluster": {
|
||||
@@ -1545,6 +1556,19 @@
|
||||
"type": "object",
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"PostableNGalertConfig": {
|
||||
"properties": {
|
||||
"alertmanagers": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"x-go-name": "Alertmanagers"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"PostableRuleGroupConfig": {
|
||||
"properties": {
|
||||
"interval": {
|
||||
@@ -2425,7 +2449,7 @@
|
||||
"alerts": {
|
||||
"description": "alerts",
|
||||
"items": {
|
||||
"$ref": "#/definitions/GettableAlert"
|
||||
"$ref": "#/definitions/gettableAlert"
|
||||
},
|
||||
"type": "array",
|
||||
"x-go-name": "Alerts"
|
||||
@@ -2434,7 +2458,7 @@
|
||||
"$ref": "#/definitions/labelSet"
|
||||
},
|
||||
"receiver": {
|
||||
"$ref": "#/definitions/receiver"
|
||||
"$ref": "#/definitions/Receiver"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -2601,7 +2625,7 @@
|
||||
"receivers": {
|
||||
"description": "receivers",
|
||||
"items": {
|
||||
"$ref": "#/definitions/receiver"
|
||||
"$ref": "#/definitions/Receiver"
|
||||
},
|
||||
"type": "array",
|
||||
"x-go-name": "Receivers"
|
||||
@@ -2639,7 +2663,7 @@
|
||||
"gettableAlerts": {
|
||||
"description": "GettableAlerts gettable alerts",
|
||||
"items": {
|
||||
"$ref": "#/definitions/GettableAlert"
|
||||
"$ref": "#/definitions/gettableAlert"
|
||||
},
|
||||
"type": "array",
|
||||
"x-go-name": "GettableAlerts",
|
||||
@@ -2705,7 +2729,7 @@
|
||||
"gettableSilences": {
|
||||
"description": "GettableSilences gettable silences",
|
||||
"items": {
|
||||
"$ref": "#/definitions/gettableSilence"
|
||||
"$ref": "#/definitions/GettableSilence"
|
||||
},
|
||||
"type": "array",
|
||||
"x-go-name": "GettableSilences",
|
||||
@@ -3775,6 +3799,95 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/ngalert/admin_config": {
|
||||
"delete": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"operationId": "RouteDeleteNGalertConfig",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Ack",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Ack"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failure",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Failure"
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": "Deletes the NGalert configuration of the user's organization.",
|
||||
"tags": [
|
||||
"configuration"
|
||||
]
|
||||
},
|
||||
"get": {
|
||||
"operationId": "RouteGetNGalertConfig",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "GettableNGalertConfig",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/GettableNGalertConfig"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Failure",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Failure"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failure",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Failure"
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": "Get the NGalert configuration of the user's organization, returns 404 if no configuration is present.",
|
||||
"tags": [
|
||||
"configuration"
|
||||
]
|
||||
},
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"operationId": "RoutePostNGalertConfig",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "body",
|
||||
"name": "Body",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/PostableNGalertConfig"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Ack",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Ack"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "ValidationError",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/ValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": "Creates or updates the NGalert configuration of the user's organization.",
|
||||
"tags": [
|
||||
"configuration"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/receiver/test/{Recipient}": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
|
||||
@@ -781,6 +781,95 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/ngalert/admin_config": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"configuration"
|
||||
],
|
||||
"summary": "Get the NGalert configuration of the user's organization, returns 404 if no configuration is present.",
|
||||
"operationId": "RouteGetNGalertConfig",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "GettableNGalertConfig",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/GettableNGalertConfig"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Failure",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Failure"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failure",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Failure"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"configuration"
|
||||
],
|
||||
"summary": "Creates or updates the NGalert configuration of the user's organization.",
|
||||
"operationId": "RoutePostNGalertConfig",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "Body",
|
||||
"in": "body",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/PostableNGalertConfig"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Ack",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Ack"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "ValidationError",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/ValidationError"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"configuration"
|
||||
],
|
||||
"summary": "Deletes the NGalert configuration of the user's organization.",
|
||||
"operationId": "RouteDeleteNGalertConfig",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Ack",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Ack"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failure",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Failure"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/receiver/test/{Recipient}": {
|
||||
"post": {
|
||||
"description": "Test receiver",
|
||||
@@ -927,7 +1016,7 @@
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"AlertGroup": {
|
||||
"$ref": "#/definitions/AlertGroup"
|
||||
"$ref": "#/definitions/alertGroup"
|
||||
},
|
||||
"AlertGroups": {
|
||||
"$ref": "#/definitions/alertGroups"
|
||||
@@ -1625,6 +1714,19 @@
|
||||
},
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"GettableNGalertConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"alertmanagers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"x-go-name": "Alertmanagers"
|
||||
}
|
||||
},
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"GettableRuleGroupConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1646,10 +1748,10 @@
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"GettableSilence": {
|
||||
"$ref": "#/definitions/gettableSilence"
|
||||
"$ref": "#/definitions/GettableSilence"
|
||||
},
|
||||
"GettableSilences": {
|
||||
"$ref": "#/definitions/gettableSilences"
|
||||
"$ref": "#/definitions/GettableSilences"
|
||||
},
|
||||
"GettableStatus": {
|
||||
"type": "object",
|
||||
@@ -2420,6 +2522,19 @@
|
||||
},
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"PostableNGalertConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"alertmanagers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"x-go-name": "Alertmanagers"
|
||||
}
|
||||
},
|
||||
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
},
|
||||
"PostableRuleGroupConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -3310,7 +3425,7 @@
|
||||
"description": "alerts",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/GettableAlert"
|
||||
"$ref": "#/definitions/gettableAlert"
|
||||
},
|
||||
"x-go-name": "Alerts"
|
||||
},
|
||||
@@ -3318,7 +3433,7 @@
|
||||
"$ref": "#/definitions/labelSet"
|
||||
},
|
||||
"receiver": {
|
||||
"$ref": "#/definitions/receiver"
|
||||
"$ref": "#/definitions/Receiver"
|
||||
}
|
||||
},
|
||||
"x-go-name": "AlertGroup",
|
||||
@@ -3491,7 +3606,7 @@
|
||||
"description": "receivers",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/receiver"
|
||||
"$ref": "#/definitions/Receiver"
|
||||
},
|
||||
"x-go-name": "Receivers"
|
||||
},
|
||||
@@ -3518,7 +3633,7 @@
|
||||
"description": "GettableAlerts gettable alerts",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/GettableAlert"
|
||||
"$ref": "#/definitions/gettableAlert"
|
||||
},
|
||||
"x-go-name": "GettableAlerts",
|
||||
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
|
||||
@@ -3584,7 +3699,7 @@
|
||||
"description": "GettableSilences gettable silences",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/gettableSilence"
|
||||
"$ref": "#/definitions/GettableSilence"
|
||||
},
|
||||
"x-go-name": "GettableSilences",
|
||||
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
|
||||
|
||||
Reference in New Issue
Block a user