public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Contact point testing (#37308)

Browse Source 
This commit adds contact point testing to ngalerts via a new API
endpoint. This endpoint accepts JSON containing a list of
receiver configurations which are validated and then tested
with a notification for a test alert. The endpoint returns JSON
for each receiver with a status and error message. It accepts
a configurable timeout via the Request-Timeout header (in seconds)
up to a maximum of 30 seconds.
This commit is contained in:
George Robinson
2021-08-17 13:49:05 +01:00
committed by GitHub
parent afabc617ed
commit 3ca00f90b5
15 changed files with 1348 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tests/api/alerting/api_alertmanager_configuration_test.go
+1 -1
View File
@@ -85,7 +85,7 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) {
}
`
resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
require.JSONEq(t, `{"message":"failed to save and apply Alertmanager configuration: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body))
require.JSONEq(t, `{"message":"failed to save and apply Alertmanager configuration: the receiver is invalid: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body))
resp = getRequest(t, alertConfigURL, http.StatusOK) // nolint
require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))
pkg/tests/api/alerting/api_notification_channel_test.go
+337
View File
@@ -30,6 +30,328 @@ import (
"github.com/grafana/grafana/pkg/tests/testinfra"
)
func TestTestReceivers(t *testing.T) {
t.Run("assert no receivers returns 400 Bad Request", func(t *testing.T) {
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
})
store := testinfra.SetUpDatabase(t, dir)
store.Bus = bus.GetBus()
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
createUser(t, store, models.CreateUserCommand{
DefaultOrgRole: string(models.ROLE_EDITOR),
Login: "grafana",
Password: "password",
})
testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
// nolint
resp := postRequest(t, testReceiversURL, `{
"receivers": []
}`, http.StatusBadRequest)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, `{"error":"no receivers"}`, string(b))
})
t.Run("assert working receiver returns OK", func(t *testing.T) {
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
})
store := testinfra.SetUpDatabase(t, dir)
store.Bus = bus.GetBus()
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
createUser(t, store, models.CreateUserCommand{
DefaultOrgRole: string(models.ROLE_EDITOR),
Login: "grafana",
Password: "password",
})
oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
mockEmails := &mockEmailHandler{}
bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
t.Cleanup(func() {
bus.AddHandlerCtx("", oldEmailBus)
})
testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
// nolint
resp := postRequest(t, testReceiversURL, `{
"receivers": [{
"name":"receiver-1",
"grafana_managed_receiver_configs": [
{
"uid":"",
"name":"receiver-1",
"type":"email",
"disableResolveMessage":false,
"settings":{
"addresses":"example@email.com"
},
"secureFields":{}
}
]
}]
}`, http.StatusOK)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
var result apimodels.TestReceiversResult
require.NoError(t, json.NewDecoder(resp.Body).Decode(&result))
require.Len(t, result.Receivers, 1)
require.Len(t, result.Receivers[0].Configs, 1)
require.Equal(t, apimodels.TestReceiversResult{
Receivers: []apimodels.TestReceiverResult{{
Name: "receiver-1",
Configs: []apimodels.TestReceiverConfigResult{{
Name: "receiver-1",
UID: result.Receivers[0].Configs[0].UID,
Status: "ok",
}},
}},
NotifedAt: result.NotifedAt,
}, result)
require.Len(t, mockEmails.emails, 1)
require.Equal(t, []string{"example@email.com"}, mockEmails.emails[0].To)
})
t.Run("assert invalid receiver returns 400 Bad Request", func(t *testing.T) {
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
})
store := testinfra.SetUpDatabase(t, dir)
store.Bus = bus.GetBus()
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
createUser(t, store, models.CreateUserCommand{
DefaultOrgRole: string(models.ROLE_EDITOR),
Login: "grafana",
Password: "password",
})
oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
mockEmails := &mockEmailHandler{}
bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
t.Cleanup(func() {
bus.AddHandlerCtx("", oldEmailBus)
})
testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
// nolint
resp := postRequest(t, testReceiversURL, `{
"receivers": [{
"name":"receiver-1",
"grafana_managed_receiver_configs": [
{
"uid":"",
"name":"receiver-1",
"type":"email",
"disableResolveMessage":false,
"settings":{},
"secureFields":{}
}
]
}]
}`, http.StatusBadRequest)
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, resp.Body.Close())
})
var result apimodels.TestReceiversResult
require.NoError(t, json.Unmarshal(b, &result))
require.Len(t, result.Receivers, 1)
require.Len(t, result.Receivers[0].Configs, 1)
require.Equal(t, apimodels.TestReceiversResult{
Receivers: []apimodels.TestReceiverResult{{
Name: "receiver-1",
Configs: []apimodels.TestReceiverConfigResult{{
Name: "receiver-1",
UID: result.Receivers[0].Configs[0].UID,
Status: "failed",
Error: "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings",
}},
}},
NotifedAt: result.NotifedAt,
}, result)
})
t.Run("assert timed out receiver returns 408 Request Timeout", func(t *testing.T) {
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
})
store := testinfra.SetUpDatabase(t, dir)
store.Bus = bus.GetBus()
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
createUser(t, store, models.CreateUserCommand{
DefaultOrgRole: string(models.ROLE_EDITOR),
Login: "grafana",
Password: "password",
})
oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
mockEmails := &mockEmailHandlerWithTimeout{
timeout: 5 * time.Second,
}
bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
t.Cleanup(func() {
bus.AddHandlerCtx("", oldEmailBus)
})
testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
req, err := http.NewRequest(http.MethodPost, testReceiversURL, strings.NewReader(`{
"receivers": [{
"name":"receiver-1",
"grafana_managed_receiver_configs": [
{
"uid":"",
"name":"receiver-1",
"type":"email",
"disableResolveMessage":false,
"settings":{
"addresses":"example@email.com"
},
"secureFields":{}
}
]
}]
}`))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Request-Timeout", "1")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, resp.Body.Close())
})
require.Equal(t, http.StatusRequestTimeout, resp.StatusCode)
var result apimodels.TestReceiversResult
require.NoError(t, json.NewDecoder(resp.Body).Decode(&result))
require.Len(t, result.Receivers, 1)
require.Len(t, result.Receivers[0].Configs, 1)
require.Equal(t, apimodels.TestReceiversResult{
Receivers: []apimodels.TestReceiverResult{{
Name: "receiver-1",
Configs: []apimodels.TestReceiverConfigResult{{
Name: "receiver-1",
UID: result.Receivers[0].Configs[0].UID,
Status: "failed",
Error: "the receiver timed out: context deadline exceeded",
}},
}},
NotifedAt: result.NotifedAt,
}, result)
})
t.Run("assert multiple different errors returns 207 Multi Status", func(t *testing.T) {
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
})
store := testinfra.SetUpDatabase(t, dir)
store.Bus = bus.GetBus()
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
createUser(t, store, models.CreateUserCommand{
DefaultOrgRole: string(models.ROLE_EDITOR),
Login: "grafana",
Password: "password",
})
oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
mockEmails := &mockEmailHandlerWithTimeout{
timeout: 5 * time.Second,
}
bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
t.Cleanup(func() {
bus.AddHandlerCtx("", oldEmailBus)
})
testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
req, err := http.NewRequest(http.MethodPost, testReceiversURL, strings.NewReader(`{
"receivers": [{
"name":"receiver-1",
"grafana_managed_receiver_configs": [
{
"uid":"",
"name":"receiver-1",
"type":"email",
"disableResolveMessage":false,
"settings":{},
"secureFields":{}
}
]
}, {
"name":"receiver-2",
"grafana_managed_receiver_configs": [
{
"uid":"",
"name":"receiver-2",
"type":"email",
"disableResolveMessage":false,
"settings":{
"addresses":"example@email.com"
},
"secureFields":{}
}
]
}]
}`))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Request-Timeout", "1")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, resp.Body.Close())
})
require.Equal(t, http.StatusMultiStatus, resp.StatusCode)
var result apimodels.TestReceiversResult
require.NoError(t, json.NewDecoder(resp.Body).Decode(&result))
require.Len(t, result.Receivers, 2)
require.Len(t, result.Receivers[0].Configs, 1)
require.Len(t, result.Receivers[1].Configs, 1)
require.Equal(t, apimodels.TestReceiversResult{
Receivers: []apimodels.TestReceiverResult{{
Name: "receiver-1",
Configs: []apimodels.TestReceiverConfigResult{{
Name: "receiver-1",
UID: result.Receivers[0].Configs[0].UID,
Status: "failed",
Error: "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings",
}},
}, {
Name: "receiver-2",
Configs: []apimodels.TestReceiverConfigResult{{
Name: "receiver-2",
UID: result.Receivers[1].Configs[0].UID,
Status: "failed",
Error: "the receiver timed out: context deadline exceeded",
}},
}},
NotifedAt: result.NotifedAt,
}, result)
})
}
func TestNotificationChannels(t *testing.T) {
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
@@ -391,6 +713,21 @@ func (e *mockEmailHandler) sendEmailCommandHandlerSync(_ context.Context, cmd *m
return nil
}
// mockEmailHandlerWithTimeout blocks until the timeout has expired.
type mockEmailHandlerWithTimeout struct {
mockEmailHandler
timeout time.Duration
}
func (e *mockEmailHandlerWithTimeout) sendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error {
select {
case <-time.After(e.timeout):
return e.mockEmailHandler.sendEmailCommandHandlerSync(ctx, cmd)
case <-ctx.Done():
return ctx.Err()
}
}
// alertmanagerConfig has the config for all the notification channels
// that we want to test. It is recommended to use different URL for each
// channel and have 1 route per channel.