public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[8.2.x] Alerting: Validate contact point configuration during migration to Unified Alerting (#40717) (#40801)

Browse Source 
* Alerting: Validate contact point configuration during migration to Unified Alerting (#40717)

* Alerting: Validate contact point configuration during the migration

This minimises the chances of generating broken configuration as part of the migration. Originally, we wanted to generate it and not produce a hard stop in Grafana but this strategy has the chance to avoid delivering notifications for our users.

We now think it's better to hard stop the migration and let the user take care of resolving the configuration manually.

(cherry picked from commit 74fb491b6a)
This commit is contained in:
gotjosh
2021-10-22 12:16:36 +01:00
committed by GitHub
parent c7f488580a
commit abd1050f98
23 changed files with 299 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/migrations/ualert/channel.go
-16
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"sort"
"strings"
@@ -131,21 +130,6 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, orgID int64, channelUid
return err
}
// Grafana accepts any type of string as a URL for the Slack notification channel.
// However, the Alertmanager will fail if provided with an invalid URL we have two options at this point:
// Either we fail the migration or remove the URL, we've chosen the latter and assume that the notification
// channel was broken to begin with.
if c.Type == "slack" {
u, ok := decryptedSecureSettings["url"]
if ok {
_, err := url.Parse(u)
if err != nil {
m.mg.Logger.Warn("slack notification channel had invalid URL, removing", "name", c.Name, "uid", c.Uid, "org", c.OrgID)
delete(decryptedSecureSettings, "url")
}
}
}
portedChannels = append(portedChannels, &PostableGrafanaReceiver{
UID: uid,
Name: c.Name,
pkg/services/sqlstore/migrations/ualert/channel_test.go
-120
View File
@@ -1,120 +0,0 @@
package ualert
import (
"fmt"
"math/rand"
"testing"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/util"
)
func Test_makeReceiverAndRoute(t *testing.T) {
emptyMigration := func() *migration {
return &migration{
mg: &migrator.Migrator{
Logger: log.New("test"),
},
migratedChannelsPerOrg: make(map[int64]map[*notificationChannel]struct{}),
portedChannelGroupsPerOrg: make(map[int64]map[string]string),
seenChannelUIDs: make(map[string]struct{}),
}
}
generateChannel := func(channelType string, settings map[string]interface{}, secureSettings map[string]string) *notificationChannel {
uid := util.GenerateShortUID()
return &notificationChannel{
ID: rand.Int63(),
OrgID: rand.Int63(),
Uid: uid,
Name: fmt.Sprintf("Test-%s", uid),
Type: channelType,
DisableResolveMessage: rand.Int63()%2 == 0,
IsDefault: rand.Int63()%2 == 0,
Settings: simplejson.NewFromAny(settings),
SecureSettings: securejsondata.GetEncryptedJsonData(secureSettings),
}
}
t.Run("Slack channel is migrated", func(t *testing.T) {
t.Run("url is removed if it is invalid (secure settings)", func(t *testing.T) {
secureSettings := map[string]string{
"url": invalidUri,
"token": util.GenerateShortUID(),
}
settings := map[string]interface{}{
"test": "data",
"some_map": map[string]interface{}{
"test": rand.Int63(),
},
}
channel := generateChannel("slack", settings, secureSettings)
channelsUid := []interface{}{
channel.Uid,
}
defaultChannels := make([]*notificationChannel, 0)
allChannels := map[interface{}]*notificationChannel{
channel.Uid: channel,
}
apiReceiver, _, err := emptyMigration().makeReceiverAndRoute(util.GenerateShortUID(), channel.OrgID, channelsUid, defaultChannels, allChannels)
require.NoError(t, err)
require.Len(t, apiReceiver.GrafanaManagedReceivers, 1)
receiver := apiReceiver.GrafanaManagedReceivers[0]
require.NotContains(t, receiver.SecureSettings, "url")
require.Contains(t, receiver.SecureSettings, "token")
require.Equal(t, secureSettings["token"], receiver.SecureSettings["token"])
actualSettings, err := receiver.Settings.Map()
require.NoError(t, err)
require.Equal(t, settings, actualSettings)
})
t.Run("url is removed if it is invalid (settings)", func(t *testing.T) {
secureSettings := map[string]string{
"token": util.GenerateShortUID(),
}
settings := map[string]interface{}{
"url": invalidUri,
"test": "data",
"some_map": map[string]interface{}{
"test": rand.Int63(),
},
}
channel := generateChannel("slack", settings, secureSettings)
channelsUid := []interface{}{
channel.Uid,
}
defaultChannels := make([]*notificationChannel, 0)
allChannels := map[interface{}]*notificationChannel{
channel.Uid: channel,
}
apiReceiver, _, err := emptyMigration().makeReceiverAndRoute(util.GenerateShortUID(), channel.OrgID, channelsUid, defaultChannels, allChannels)
require.NoError(t, err)
require.Len(t, apiReceiver.GrafanaManagedReceivers, 1)
receiver := apiReceiver.GrafanaManagedReceivers[0]
require.NotContains(t, receiver.SecureSettings, "url")
require.Contains(t, receiver.SecureSettings, "token")
require.Equal(t, secureSettings["token"], receiver.SecureSettings["token"])
actualSettings, err := receiver.Settings.Map()
require.NoError(t, err)
delete(settings, "url")
require.Equal(t, settings, actualSettings)
})
})
}
const invalidUri = "�6�M��)uk譹1(�h`$�o�N>mĕ����cS2�dh![ę� ���`csB�!��OSxP�{�"
pkg/services/sqlstore/migrations/ualert/testing.go
+23
View File
@@ -0,0 +1,23 @@
package ualert
import (
"testing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)
// newTestMigration generates an empty migration to use in tests.
func newTestMigration(t *testing.T) *migration {
t.Helper()
return &migration{
mg: &migrator.Migrator{
Logger: log.New("test"),
},
migratedChannelsPerOrg: make(map[int64]map[*notificationChannel]struct{}),
portedChannelGroupsPerOrg: make(map[int64]map[string]string),
seenChannelUIDs: make(map[string]struct{}),
}
}
pkg/services/sqlstore/migrations/ualert/ualert.go
+99 -14
View File
@@ -1,6 +1,7 @@
package ualert
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
@@ -8,10 +9,11 @@ import (
"strconv"
"strings"
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
pb "github.com/prometheus/alertmanager/silence/silencepb"
"xorm.io/xorm"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)
const GENERAL_FOLDER = "General Alerting"
@@ -215,6 +217,7 @@ func (m *migration) SQL(dialect migrator.Dialect) string {
return "code migration"
}
//nolint: gocyclo
func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
m.sess = sess
m.mg = mg
@@ -375,7 +378,24 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
return err
}
if err := m.writeAlertmanagerConfig(orgID, amConfig, allChannelsPerOrg[orgID]); err != nil {
// No channels, hence don't require Alertmanager config - skip it.
if len(allChannelsPerOrg[orgID]) == 0 {
m.mg.Logger.Info("alert migration: no notification channel found, skipping Alertmanager config")
continue
}
// Encrypt the secure settings before we continue.
if err := amConfig.EncryptSecureSettings(); err != nil {
return err
}
// Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time.
// Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case.
if err := m.validateAlertmanagerConfig(orgID, amConfig); err != nil {
return err
}
if err := m.writeAlertmanagerConfig(orgID, amConfig); err != nil {
return err
}
@@ -387,22 +407,13 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
return nil
}
func (m *migration) writeAlertmanagerConfig(orgID int64, amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel) error {
if len(allChannels) == 0 {
// No channels, hence don't require Alertmanager config.
m.mg.Logger.Info("alert migration: no notification channel found, skipping Alertmanager config")
return nil
}
if err := amConfig.EncryptSecureSettings(); err != nil {
return err
}
func (m *migration) writeAlertmanagerConfig(orgID int64, amConfig *PostableUserConfig) error {
rawAmConfig, err := json.Marshal(amConfig)
if err != nil {
return err
}
// TODO: should we apply the config here? Because Alertmanager can take upto 1 min to pick it up.
// We don't need to apply the configuration, given the multi org alertmanager will do an initial sync before the server is ready.
_, err = m.sess.Insert(AlertConfiguration{
AlertmanagerConfiguration: string(rawAmConfig),
// Since we are migration for a snapshot of the code, it is always going to migrate to
@@ -417,6 +428,80 @@ func (m *migration) writeAlertmanagerConfig(orgID int64, amConfig *PostableUserC
return nil
}
// validateAlertmanagerConfig validates the alertmanager configuration produced by the migration against the receivers.
func (m *migration) validateAlertmanagerConfig(orgID int64, config *PostableUserConfig) error {
for _, r := range config.AlertmanagerConfig.Receivers {
for _, gr := range r.GrafanaManagedReceivers {
// First, let's decode the secure settings - given they're stored as base64.
secureSettings := make(map[string][]byte, len(gr.SecureSettings))
for k, v := range gr.SecureSettings {
d, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return err
}
secureSettings[k] = d
}
var (
cfg = &channels.NotificationChannelConfig{
UID: gr.UID,
Name: gr.Name,
Type: gr.Type,
DisableResolveMessage: gr.DisableResolveMessage,
Settings: gr.Settings,
SecureSettings: secureSettings,
}
err error
)
switch gr.Type {
case "email":
_, err = channels.NewEmailNotifier(cfg, nil) // Email notifier already has a default template.
case "pagerduty":
_, err = channels.NewPagerdutyNotifier(cfg, nil)
case "pushover":
_, err = channels.NewPushoverNotifier(cfg, nil)
case "slack":
_, err = channels.NewSlackNotifier(cfg, nil)
case "telegram":
_, err = channels.NewTelegramNotifier(cfg, nil)
case "victorops":
_, err = channels.NewVictoropsNotifier(cfg, nil)
case "teams":
_, err = channels.NewTeamsNotifier(cfg, nil)
case "dingding":
_, err = channels.NewDingDingNotifier(cfg, nil)
case "kafka":
_, err = channels.NewKafkaNotifier(cfg, nil)
case "webhook":
_, err = channels.NewWebHookNotifier(cfg, nil)
case "sensugo":
_, err = channels.NewSensuGoNotifier(cfg, nil)
case "discord":
_, err = channels.NewDiscordNotifier(cfg, nil)
case "googlechat":
_, err = channels.NewGoogleChatNotifier(cfg, nil)
case "LINE":
_, err = channels.NewLineNotifier(cfg, nil)
case "threema":
_, err = channels.NewThreemaNotifier(cfg, nil)
case "opsgenie":
_, err = channels.NewOpsgenieNotifier(cfg, nil)
case "prometheus-alertmanager":
_, err = channels.NewAlertmanagerNotifier(cfg, nil)
default:
return fmt.Errorf("notifier %s is not supported", gr.Type)
}
if err != nil {
return err
}
}
}
return nil
}
type AlertConfiguration struct {
ID int64 `xorm:"pk autoincr 'id'"`
OrgID int64 `xorm:"org_id"`
pkg/services/sqlstore/migrations/ualert/ualert_test.go
+90
View File
@@ -0,0 +1,90 @@
package ualert
import (
"errors"
"fmt"
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/require"
)
func Test_validateAlertmanagerConfig(t *testing.T) {
tc := []struct {
name string
receivers []*PostableGrafanaReceiver
err error
}{
{
name: "when a slack receiver does not have a valid URL - it should error",
receivers: []*PostableGrafanaReceiver{
{
UID: util.GenerateShortUID(),
Name: "SlackWithBadURL",
Type: "slack",
Settings: simplejson.NewFromAny(map[string]interface{}{}),
SecureSettings: map[string]string{"url": invalidUri},
},
},
err: fmt.Errorf("failed to validate receiver \"SlackWithBadURL\" of type \"slack\": invalid URL %q: parse %q: net/url: invalid control character in URL", invalidUri, invalidUri),
},
{
name: "when a slack receiver has an invalid recipient - it should error",
receivers: []*PostableGrafanaReceiver{
{
UID: util.GenerateShortUID(),
Name: "SlackWithBadRecipient",
Type: "slack",
Settings: simplejson.NewFromAny(map[string]interface{}{"recipient": "this-doesnt-pass"}),
SecureSettings: map[string]string{"url": "http://webhook.slack.com/myuser"},
},
},
err: errors.New("failed to validate receiver \"SlackWithBadRecipient\" of type \"slack\": recipient on invalid format: \"this-doesnt-pass\""),
},
{
name: "when the configuration is valid - it should not error",
receivers: []*PostableGrafanaReceiver{
{
UID: util.GenerateShortUID(),
Name: "SlackWithBadURL",
Type: "slack",
Settings: simplejson.NewFromAny(map[string]interface{}{"recipient": "#a-good-channel"}),
SecureSettings: map[string]string{"url": "http://webhook.slack.com/myuser"},
},
},
},
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
mg := newTestMigration(t)
orgID := int64(1)
config := configFromReceivers(t, tt.receivers)
require.NoError(t, config.EncryptSecureSettings()) // make sure we encrypt the settings
err := mg.validateAlertmanagerConfig(orgID, config)
if tt.err != nil {
require.Error(t, err)
require.EqualError(t, err, tt.err.Error())
} else {
require.NoError(t, err)
}
})
}
}
func configFromReceivers(t *testing.T, receivers []*PostableGrafanaReceiver) *PostableUserConfig {
t.Helper()
return &PostableUserConfig{
AlertmanagerConfig: PostableApiAlertingConfig{
Receivers: []*PostableApiReceiver{
{GrafanaManagedReceivers: receivers},
},
},
}
}
const invalidUri = "�6�M��)uk譹1(�h`$�o�N>mĕ����cS2�dh![ę� ���`csB�!��OSxP�{�"