c8ccc4649c
This pull request updates our fork of Alertmanager to commit 65bdab0, which is based on commit 5658f8c in Prometheus Alertmanager. It applies the changes from grafana/alerting#155 which removes the overrides for validation of alerts, labels and silences that we had put in place to allow alerts and silences to work for non-Prometheus datasources. However, as this is now supported in Alertmanager with the UTF-8 work, we can use the new upstream functions and remove these overrides. The compat package is a package in Alertmanager that takes care of backwards compatibility when parsing matchers, validating alerts, labels and silences. It has three modes: classic mode, UTF-8 strict mode, fallback mode. These modes are controlled via compat.InitFromFlags. Grafana initializes the compat package without any feature flags, which is the equivalent of fallback mode. Classic and UTF-8 strict mode are used in Mimir. While Grafana Managed Alerts have no need for fallback mode, Grafana can still be used as an interface to manage the configurations of Mimir Alertmanagers and view configurations of Prometheus Alertmanager, and those installations might not have migrated or being running on older versions. Such installations behave as if in classic mode, and Grafana must be able to parse their configurations to interact with them for some period of time. As such, Grafana uses fallback mode until we are ready to drop support for outdated installations of Mimir and the Prometheus Alertmanager.
494 lines
13 KiB
Go
494 lines
13 KiB
Go
package alerting
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/alertmanager/config"
|
|
"github.com/prometheus/alertmanager/pkg/labels"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
|
"github.com/grafana/grafana/pkg/services/org"
|
|
"github.com/grafana/grafana/pkg/services/org/orgimpl"
|
|
"github.com/grafana/grafana/pkg/services/quota/quotatest"
|
|
"github.com/grafana/grafana/pkg/services/user"
|
|
"github.com/grafana/grafana/pkg/tests/testinfra"
|
|
)
|
|
|
|
func TestIntegrationAlertmanagerConfiguration(t *testing.T) {
|
|
testinfra.SQLiteIntegrationTest(t)
|
|
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
|
DisableLegacyAlerting: true,
|
|
EnableUnifiedAlerting: true,
|
|
AppModeProduction: true,
|
|
})
|
|
grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
|
|
createUser(t, store, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleAdmin),
|
|
Password: "admin",
|
|
Login: "admin",
|
|
})
|
|
client := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
|
|
|
|
cases := []struct {
|
|
name string
|
|
cfg apimodels.PostableUserConfig
|
|
expErr string
|
|
}{{
|
|
name: "configuration with default route",
|
|
cfg: apimodels.PostableUserConfig{
|
|
AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
|
|
Config: apimodels.Config{
|
|
Route: &apimodels.Route{
|
|
Receiver: "test",
|
|
},
|
|
},
|
|
Receivers: []*apimodels.PostableApiReceiver{{
|
|
Receiver: config.Receiver{
|
|
Name: "test",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}, {
|
|
name: "configuration with UTF-8 matchers",
|
|
cfg: apimodels.PostableUserConfig{
|
|
AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
|
|
Config: apimodels.Config{
|
|
Route: &apimodels.Route{
|
|
Receiver: "test",
|
|
Routes: []*apimodels.Route{{
|
|
GroupBy: []model.LabelName{"foo🙂"},
|
|
Matchers: config.Matchers{{
|
|
Type: labels.MatchEqual,
|
|
Name: "foo🙂",
|
|
Value: "bar",
|
|
}, {
|
|
Type: labels.MatchNotEqual,
|
|
Name: "_bar1",
|
|
Value: "baz🙂",
|
|
}, {
|
|
Type: labels.MatchRegexp,
|
|
Name: "0baz",
|
|
Value: "[a-zA-Z0-9]+,?",
|
|
}, {
|
|
Type: labels.MatchNotRegexp,
|
|
Name: "corge",
|
|
Value: "^[0-9]+((,[0-9]{3})*(,[0-9]{0,3})?)?$",
|
|
}, {
|
|
Type: labels.MatchEqual,
|
|
Name: "Προμηθέας", // Prometheus in Greek
|
|
Value: "Prom",
|
|
}, {
|
|
Type: labels.MatchNotEqual,
|
|
Name: "犬", // Dog in Japanese
|
|
Value: "Shiba Inu",
|
|
}},
|
|
}},
|
|
},
|
|
},
|
|
Receivers: []*apimodels.PostableApiReceiver{{
|
|
Receiver: config.Receiver{
|
|
Name: "test",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}, {
|
|
name: "configuration with UTF-8 object matchers",
|
|
cfg: apimodels.PostableUserConfig{
|
|
AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
|
|
Config: apimodels.Config{
|
|
Route: &apimodels.Route{
|
|
Receiver: "test",
|
|
Routes: []*apimodels.Route{{
|
|
GroupBy: []model.LabelName{"foo🙂"},
|
|
ObjectMatchers: apimodels.ObjectMatchers{{
|
|
Type: labels.MatchEqual,
|
|
Name: "foo🙂",
|
|
Value: "bar",
|
|
}, {
|
|
Type: labels.MatchNotEqual,
|
|
Name: "_bar1",
|
|
Value: "baz🙂",
|
|
}, {
|
|
Type: labels.MatchRegexp,
|
|
Name: "0baz",
|
|
Value: "[a-zA-Z0-9]+,?",
|
|
}, {
|
|
Type: labels.MatchNotRegexp,
|
|
Name: "corge",
|
|
Value: "^[0-9]+((,[0-9]{3})*(,[0-9]{0,3})?)?$",
|
|
}, {
|
|
Type: labels.MatchEqual,
|
|
Name: "Προμηθέας", // Prometheus in Greek
|
|
Value: "Prom",
|
|
}, {
|
|
Type: labels.MatchNotEqual,
|
|
Name: "犬", // Dog in Japanese
|
|
Value: "Shiba Inu",
|
|
}},
|
|
}},
|
|
},
|
|
},
|
|
Receivers: []*apimodels.PostableApiReceiver{{
|
|
Receiver: config.Receiver{
|
|
Name: "test",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}, {
|
|
name: "configuration with UTF-8 in both matchers and object matchers",
|
|
cfg: apimodels.PostableUserConfig{
|
|
AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
|
|
Config: apimodels.Config{
|
|
Route: &apimodels.Route{
|
|
Receiver: "test",
|
|
Routes: []*apimodels.Route{{
|
|
GroupBy: []model.LabelName{"foo🙂"},
|
|
Matchers: config.Matchers{{
|
|
Type: labels.MatchEqual,
|
|
Name: "foo🙂",
|
|
Value: "bar",
|
|
}, {
|
|
Type: labels.MatchNotEqual,
|
|
Name: "_bar1",
|
|
Value: "baz🙂",
|
|
}, {
|
|
Type: labels.MatchRegexp,
|
|
Name: "0baz",
|
|
Value: "[a-zA-Z0-9]+,?",
|
|
}, {
|
|
Type: labels.MatchNotRegexp,
|
|
Name: "corge",
|
|
Value: "^[0-9]+((,[0-9]{3})*(,[0-9]{0,3})?)?$",
|
|
}},
|
|
ObjectMatchers: apimodels.ObjectMatchers{{
|
|
Type: labels.MatchEqual,
|
|
Name: "Προμηθέας", // Prometheus in Greek
|
|
Value: "Prom",
|
|
}, {
|
|
Type: labels.MatchNotEqual,
|
|
Name: "犬", // Dog in Japanese
|
|
Value: "Shiba Inu",
|
|
}},
|
|
}},
|
|
},
|
|
},
|
|
Receivers: []*apimodels.PostableApiReceiver{{
|
|
Receiver: config.Receiver{
|
|
Name: "test",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
}}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ok, err := client.PostConfiguration(t, tc.cfg)
|
|
if tc.expErr != "" {
|
|
require.EqualError(t, err, tc.expErr)
|
|
require.False(t, ok)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.True(t, ok)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIntegrationAlertmanagerConfigurationIsTransactional(t *testing.T) {
|
|
testinfra.SQLiteIntegrationTest(t)
|
|
|
|
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
|
DisableLegacyAlerting: true,
|
|
EnableUnifiedAlerting: true,
|
|
NGAlertAlertmanagerConfigPollInterval: 2 * time.Second,
|
|
DisableAnonymous: true,
|
|
AppModeProduction: true,
|
|
})
|
|
|
|
grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
|
|
|
|
orgService, err := orgimpl.ProvideService(store, store.Cfg, quotatest.New(false, nil))
|
|
require.NoError(t, err)
|
|
|
|
// editor from main organisation requests configuration
|
|
alertConfigURL := fmt.Sprintf("http://editor:editor@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
|
|
|
// create user under main organisation
|
|
userID := createUser(t, store, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleEditor),
|
|
Password: "editor",
|
|
Login: "editor",
|
|
})
|
|
|
|
// create another organisation
|
|
newOrg, err := orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "another org", UserID: userID})
|
|
require.NoError(t, err)
|
|
orgID := newOrg.ID
|
|
|
|
// create user under different organisation
|
|
createUser(t, store, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleEditor),
|
|
Password: "editor-42",
|
|
Login: "editor-42",
|
|
OrgID: orgID,
|
|
})
|
|
|
|
// On a blank start with no configuration, it saves and delivers the default configuration.
|
|
{
|
|
resp := getRequest(t, alertConfigURL, http.StatusOK) // nolint
|
|
require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))
|
|
}
|
|
|
|
// When creating new configuration, if it fails to apply - it does not save it.
|
|
{
|
|
payload := `
|
|
{
|
|
"template_files": {},
|
|
"alertmanager_config": {
|
|
"route": {
|
|
"receiver": "slack.receiver"
|
|
},
|
|
"templates": null,
|
|
"receivers": [{
|
|
"name": "slack.receiver",
|
|
"grafana_managed_receiver_configs": [{
|
|
"settings": {
|
|
"iconEmoji": "",
|
|
"iconUrl": "",
|
|
"mentionGroups": "",
|
|
"mentionUsers": "",
|
|
"recipient": "#unified-alerting-test",
|
|
"username": ""
|
|
},
|
|
"secureSettings": {},
|
|
"type": "slack",
|
|
"name": "slack.receiver",
|
|
"disableResolveMessage": false,
|
|
"uid": ""
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
`
|
|
resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
require.NoError(t, json.Unmarshal(b, &res))
|
|
require.Regexp(t, `^failed to save and apply Alertmanager configuration: failed to validate integration "slack.receiver" \(UID [^\)]+\) of type "slack": token must be specified when using the Slack chat API`, res["message"])
|
|
resp = getRequest(t, alertConfigURL, http.StatusOK) // nolint
|
|
|
|
require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))
|
|
}
|
|
|
|
// editor42 from organisation 42 posts configuration
|
|
alertConfigURL = fmt.Sprintf("http://editor-42:editor-42@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
|
|
|
// Before we start operating, make sure we've synced this org.
|
|
require.Eventually(t, func() bool {
|
|
resp, err := http.Get(alertConfigURL) // nolint
|
|
require.NoError(t, err)
|
|
return resp.StatusCode == http.StatusOK
|
|
}, 10*time.Second, 2*time.Second)
|
|
|
|
// Post the alertmanager config.
|
|
{
|
|
mockChannel := newMockNotificationChannel(t, grafanaListedAddr)
|
|
amConfig := getAlertmanagerConfig(mockChannel.server.Addr)
|
|
postRequest(t, alertConfigURL, amConfig, http.StatusAccepted) // nolint
|
|
|
|
// Verifying that the new configuration is returned
|
|
resp := getRequest(t, alertConfigURL, http.StatusOK) // nolint
|
|
b := getBody(t, resp.Body)
|
|
re := regexp.MustCompile(`"uid":"([\w|-]*)"`)
|
|
e := getExpAlertmanagerConfigFromAPI(mockChannel.server.Addr)
|
|
require.JSONEq(t, e, string(re.ReplaceAll([]byte(b), []byte(`"uid":""`))))
|
|
}
|
|
|
|
// verify that main organisation still gets the default configuration
|
|
alertConfigURL = fmt.Sprintf("http://editor:editor@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
|
{
|
|
resp := getRequest(t, alertConfigURL, http.StatusOK) // nolint
|
|
require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))
|
|
}
|
|
}
|
|
|
|
func TestIntegrationAlertmanagerConfigurationPersistSecrets(t *testing.T) {
|
|
testinfra.SQLiteIntegrationTest(t)
|
|
|
|
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
|
DisableLegacyAlerting: true,
|
|
EnableUnifiedAlerting: true,
|
|
DisableAnonymous: true,
|
|
AppModeProduction: true,
|
|
})
|
|
|
|
grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
|
|
alertConfigURL := fmt.Sprintf("http://editor:editor@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
|
|
|
createUser(t, store, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleEditor),
|
|
Password: "editor",
|
|
Login: "editor",
|
|
})
|
|
generatedUID := ""
|
|
|
|
// create a new configuration that has a secret
|
|
{
|
|
payload := `
|
|
{
|
|
"template_files": {},
|
|
"alertmanager_config": {
|
|
"route": {
|
|
"receiver": "slack.receiver"
|
|
},
|
|
"templates": null,
|
|
"receivers": [{
|
|
"name": "slack.receiver",
|
|
"grafana_managed_receiver_configs": [{
|
|
"settings": {
|
|
"recipient": "#unified-alerting-test"
|
|
},
|
|
"secureSettings": {
|
|
"url": "http://averysecureurl.com/webhook"
|
|
},
|
|
"type": "slack",
|
|
"name": "slack.receiver",
|
|
"disableResolveMessage": false
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
`
|
|
resp := postRequest(t, alertConfigURL, payload, http.StatusAccepted) // nolint
|
|
require.JSONEq(t, `{"message":"configuration created"}`, getBody(t, resp.Body))
|
|
}
|
|
|
|
// Try to update a receiver with unknown UID
|
|
{
|
|
// Then, update the recipient
|
|
payload := `
|
|
{
|
|
"template_files": {},
|
|
"alertmanager_config": {
|
|
"route": {
|
|
"receiver": "slack.receiver"
|
|
},
|
|
"templates": null,
|
|
"receivers": [{
|
|
"name": "slack.receiver",
|
|
"grafana_managed_receiver_configs": [{
|
|
"settings": {
|
|
"recipient": "#unified-alerting-test-but-updated"
|
|
},
|
|
"secureFields": {
|
|
"url": true
|
|
},
|
|
"type": "slack",
|
|
"name": "slack.receiver",
|
|
"disableResolveMessage": false,
|
|
"uid": "invalid"
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
`
|
|
|
|
resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
|
|
s := getBody(t, resp.Body)
|
|
var res map[string]any
|
|
require.NoError(t, json.Unmarshal([]byte(s), &res))
|
|
require.Equal(t, "unknown receiver: invalid", res["message"])
|
|
}
|
|
|
|
// The secure settings must be present
|
|
{
|
|
resp := getRequest(t, alertConfigURL, http.StatusOK) // nolint
|
|
var c apimodels.GettableUserConfig
|
|
bb := getBody(t, resp.Body)
|
|
err := json.Unmarshal([]byte(bb), &c)
|
|
require.NoError(t, err)
|
|
m := c.GetGrafanaReceiverMap()
|
|
assert.Len(t, m, 1)
|
|
for k := range m {
|
|
generatedUID = m[k].UID
|
|
}
|
|
|
|
// Then, update the recipient
|
|
payload := fmt.Sprintf(`
|
|
{
|
|
"template_files": {},
|
|
"alertmanager_config": {
|
|
"route": {
|
|
"receiver": "slack.receiver"
|
|
},
|
|
"templates": null,
|
|
"receivers": [{
|
|
"name": "slack.receiver",
|
|
"grafana_managed_receiver_configs": [{
|
|
"settings": {
|
|
"recipient": "#unified-alerting-test-but-updated"
|
|
},
|
|
"secureFields": {
|
|
"url": true
|
|
},
|
|
"type": "slack",
|
|
"name": "slack.receiver",
|
|
"disableResolveMessage": false,
|
|
"uid": %q
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
`, generatedUID)
|
|
|
|
resp = postRequest(t, alertConfigURL, payload, http.StatusAccepted) // nolint
|
|
require.JSONEq(t, `{"message": "configuration created"}`, getBody(t, resp.Body))
|
|
}
|
|
|
|
// The secure settings must be present
|
|
{
|
|
resp := getRequest(t, alertConfigURL, http.StatusOK) // nolint
|
|
require.JSONEq(t, fmt.Sprintf(`
|
|
{
|
|
"template_files": {},
|
|
"alertmanager_config": {
|
|
"route": {
|
|
"receiver": "slack.receiver"
|
|
},
|
|
"templates": null,
|
|
"receivers": [{
|
|
"name": "slack.receiver",
|
|
"grafana_managed_receiver_configs": [{
|
|
"uid": %q,
|
|
"name": "slack.receiver",
|
|
"type": "slack",
|
|
"disableResolveMessage": false,
|
|
"settings": {
|
|
"recipient": "#unified-alerting-test-but-updated"
|
|
},
|
|
"secureFields": {
|
|
"url": true
|
|
}
|
|
}]
|
|
}]
|
|
}
|
|
}
|
|
`, generatedUID), getBody(t, resp.Body))
|
|
}
|
|
}
|