From e52f718a87f9f26813bfe45cd30d823e7a1d42f7 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Thu, 20 May 2021 00:40:12 +0530 Subject: [PATCH] NGAlert: Migrate old notification channel config (#33846) * NGAlert: Migrate old notification channel config Signed-off-by: Ganesh Vernekar * Copy the required structs and functions Signed-off-by: Ganesh Vernekar * Fix lint Signed-off-by: Ganesh Vernekar * Fix review comments and error on discontinued channels Signed-off-by: Ganesh Vernekar * Use both ID and UID to map channels, attach UID to channels Signed-off-by: Ganesh Vernekar * Fix lint Signed-off-by: Ganesh Vernekar * Bug fixes Signed-off-by: Ganesh Vernekar * Use alert table instead of dashboard for channel IDs Signed-off-by: Ganesh Vernekar * Fix offline reviews: don't add empty receivers and routes, add receiver for unmigrated channels Signed-off-by: Ganesh Vernekar * Handle multiple default channels, pick default from used channels Signed-off-by: Ganesh Vernekar * Include default channels in all receivers Signed-off-by: Ganesh Vernekar --- .../sqlstore/migrations/ualert/channel.go | 383 ++++++++++++++++++ .../sqlstore/migrations/ualert/dash_alert.go | 7 +- .../sqlstore/migrations/ualert/ualert.go | 58 ++- 3 files changed, 442 insertions(+), 6 deletions(-) create mode 100644 pkg/services/sqlstore/migrations/ualert/channel.go diff --git a/pkg/services/sqlstore/migrations/ualert/channel.go b/pkg/services/sqlstore/migrations/ualert/channel.go new file mode 100644 index 00000000000..8bbce6b0ef5 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/channel.go @@ -0,0 +1,383 @@ +package ualert + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + "github.com/prometheus/alertmanager/pkg/labels" + + "github.com/grafana/grafana/pkg/components/securejsondata" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +type notificationChannel struct { + ID int `xorm:"id"` + Uid string `xorm:"uid"` + Name string `xorm:"name"` + Type string `xorm:"type"` + DisableResolveMessage bool `xorm:"disable_resolve_message"` + IsDefault bool `xorm:"is_default"` + Settings *simplejson.Json `xorm:"settings"` + SecureSettings securejsondata.SecureJsonData `xorm:"secure_settings"` +} + +func (m *migration) getNotificationChannelMap() (map[interface{}]*notificationChannel, []*notificationChannel, error) { + q := ` + SELECT id, + uid, + name, + type, + disable_resolve_message, + is_default, + settings, + secure_settings + FROM + alert_notification + ` + allChannels := []notificationChannel{} + err := m.sess.SQL(q).Find(&allChannels) + if err != nil { + return nil, nil, err + } + + if len(allChannels) == 0 { + return nil, nil, nil + } + + allChannelsMap := make(map[interface{}]*notificationChannel) + var defaultChannels []*notificationChannel + for i, c := range allChannels { + if c.Uid != "" { + allChannelsMap[c.Uid] = &allChannels[i] + } + if c.ID != 0 { + allChannelsMap[c.ID] = &allChannels[i] + } + if c.IsDefault { + // TODO: verify that there will be only 1 default channel. + defaultChannels = append(defaultChannels, &allChannels[i]) + } + } + + return allChannelsMap, defaultChannels, nil +} + +func (m *migration) updateReceiverAndRoute(allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel, da dashAlert, rule *alertRule, amConfig *PostableUserConfig) error { + rule.Labels["alertname"] = da.Name + rule.Annotations["message"] = da.Message + + // Create receiver and route for this rule. + if allChannels == nil { + return nil + } + + channelIDs := extractChannelIDs(da) + if len(channelIDs) == 0 { + // If there are no channels associated, we skip adding any routes, + // receivers or labels to rules so that it goes through the default + // route. + return nil + } + + recv, route, err := m.makeReceiverAndRoute(rule.Uid, channelIDs, defaultChannels, allChannels) + if err != nil { + return err + } + + // Attach label for routing. + n, v := getLabelForRouteMatching(rule.Uid) + rule.Labels[n] = v + + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) + amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route) + + return nil +} + +func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface{}, defaultChannels []*notificationChannel, allChannels map[interface{}]*notificationChannel) (*PostableApiReceiver, *Route, error) { + receiverName := getMigratedReceiverNameFromRuleUID(ruleUid) + + portedChannels := []*PostableGrafanaReceiver{} + receiver := &PostableApiReceiver{ + Name: receiverName, + } + + addedChannels := map[*notificationChannel]struct{}{} + + addChannel := func(c *notificationChannel) error { + if c.Type == "hipchat" || c.Type == "sensu" { + return fmt.Errorf("discontinued notification channel found: %s", c.Type) + } + + uid, ok := m.generateChannelUID() + if !ok { + return errors.New("failed to generate UID for notification channel") + } + + m.migratedChannels[c] = struct{}{} + settings, secureSettings := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) + portedChannels = append(portedChannels, &PostableGrafanaReceiver{ + UID: uid, + Name: c.Name, + Type: c.Type, + DisableResolveMessage: c.DisableResolveMessage, + Settings: settings, + SecureSettings: secureSettings, + }) + addedChannels[c] = struct{}{} + + return nil + } + + for _, n := range channelUids { + c, ok := allChannels[n] + if !ok { + continue + } + + if err := addChannel(c); err != nil { + return nil, nil, err + } + } + + for _, c := range defaultChannels { + if _, ok := addedChannels[c]; !ok { + if err := addChannel(c); err != nil { + return nil, nil, err + } + } + } + + receiver.GrafanaManagedReceivers = portedChannels + + n, v := getLabelForRouteMatching(ruleUid) + mat, err := labels.NewMatcher(labels.MatchEqual, n, v) + if err != nil { + return nil, nil, err + } + route := &Route{ + Receiver: receiverName, + Matchers: Matchers{mat}, + } + + return receiver, route, nil +} + +func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error { + // Unmigrated channels. + portedChannels := []*PostableGrafanaReceiver{} + receiver := &PostableApiReceiver{ + Name: "autogen-unlinked-channel-recv", + } + for _, c := range allChannels { + _, ok := m.migratedChannels[c] + if ok { + continue + } + if c.Type == "hipchat" || c.Type == "sensu" { + return fmt.Errorf("discontinued notification channel found: %s", c.Type) + } + + uid, ok := m.generateChannelUID() + if !ok { + return errors.New("failed to generate UID for notification channel") + } + + m.migratedChannels[c] = struct{}{} + settings, secureSettings := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) + portedChannels = append(portedChannels, &PostableGrafanaReceiver{ + UID: uid, + Name: c.Name, + Type: c.Type, + DisableResolveMessage: c.DisableResolveMessage, + Settings: settings, + SecureSettings: secureSettings, + }) + } + receiver.GrafanaManagedReceivers = portedChannels + if len(portedChannels) > 0 { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, receiver) + } + + // Default route and receiver. + if len(defaultChannels) == 0 { + // Pick one from the migrated channels. Preference to email channel. + for c := range m.migratedChannels { + if len(defaultChannels) == 0 { + defaultChannels = append(defaultChannels, c) + } + if c.Type == "email" { + defaultChannels[0] = c + break + } + } + } + + var channelUids = []interface{}{} + for _, c := range defaultChannels { + if c.Uid == "" { + channelUids = append(channelUids, c.ID) + } else { + channelUids = append(channelUids, c.Uid) + } + } + + recv, route, err := m.makeReceiverAndRoute("default_route", channelUids, defaultChannels, allChannels) + if err != nil { + return err + } + + route.Matchers = nil // Don't need matchers for root route. + route.Routes = amConfig.AlertmanagerConfig.Route.Routes + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) + amConfig.AlertmanagerConfig.Route = route + + return nil +} + +func (m *migration) generateChannelUID() (string, bool) { + for i := 0; i < 5; i++ { + gen := util.GenerateShortUID() + if _, ok := m.seenChannelUIDs[gen]; !ok { + m.seenChannelUIDs[gen] = struct{}{} + return gen, true + } + } + + return "", false +} + +// Some settings were migrated from settings to secure settings in between. +// See https://grafana.com/docs/grafana/latest/installation/upgrading/#ensure-encryption-of-existing-alert-notification-channel-secrets. +// migrateSettingsToSecureSettings takes care of that. +func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings securejsondata.SecureJsonData) (*simplejson.Json, map[string]string) { + keys := []string{} + switch chanType { + case "slack": + keys = []string{"url", "token"} + case "pagerduty": + keys = []string{"integrationKey"} + case "webhook": + keys = []string{"password"} + case "prometheus-alertmanager": + keys = []string{"basicAuthPassword"} + case "opsgenie": + keys = []string{"apiKey"} + case "telegram": + keys = []string{"bottoken"} + case "line": + keys = []string{"token"} + case "pushover": + keys = []string{"apiToken", "userKey"} + case "threema": + keys = []string{"api_secret"} + } + + ss := secureSettings.Decrypt() + for _, k := range keys { + if v, ok := ss[k]; ok && v != "" { + continue + } + + sv := settings.Get(k).MustString() + if sv != "" { + ss[k] = sv + settings.Del(k) + } + } + + return settings, ss +} + +func getMigratedReceiverNameFromRuleUID(ruleUID string) string { + return fmt.Sprintf("autogen-panel-recv-%s", ruleUID) +} + +func getLabelForRouteMatching(ruleUID string) (string, string) { + return "rule_uid", ruleUID +} + +func extractChannelIDs(d dashAlert) (channelUids []interface{}) { + // Extracting channel UID/ID. + for _, ui := range d.ParsedSettings.Notifications { + if ui.UID != "" { + channelUids = append(channelUids, ui.UID) + continue + } + // In certain circumstances, id is used instead of uid. + // We add this if there was no uid. + if ui.ID > 0 { + channelUids = append(channelUids, ui.ID) + } + } + + return channelUids +} + +// Below is a snapshot of all the config and supporting functions imported +// to avoid vendoring those packages. + +type PostableUserConfig struct { + TemplateFiles map[string]string `yaml:"template_files" json:"template_files"` + AlertmanagerConfig PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"` +} + +func (c *PostableUserConfig) EncryptSecureSettings() error { + for _, r := range c.AlertmanagerConfig.Receivers { + for _, gr := range r.GrafanaManagedReceivers { + for k, v := range gr.SecureSettings { + encryptedData, err := util.Encrypt([]byte(v), setting.SecretKey) + if err != nil { + return fmt.Errorf("failed to encrypt secure settings: %w", err) + } + gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(encryptedData) + } + } + } + return nil +} + +type PostableApiAlertingConfig struct { + Route *Route `yaml:"route,omitempty" json:"route,omitempty"` + Templates []string `yaml:"templates" json:"templates"` + Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"` +} + +type Route struct { + Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"` + Matchers Matchers `yaml:"matchers,omitempty" json:"matchers,omitempty"` + Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"` +} + +type Matchers labels.Matchers + +func (m Matchers) MarshalJSON() ([]byte, error) { + if len(m) == 0 { + return nil, nil + } + result := make([]string, len(m)) + for i, matcher := range m { + result[i] = matcher.String() + } + return json.Marshal(result) +} + +type PostableApiReceiver struct { + Name string `yaml:"name" json:"name"` + GrafanaManagedReceivers []*PostableGrafanaReceiver `yaml:"grafana_managed_receiver_configs,omitempty" json:"grafana_managed_receiver_configs,omitempty"` +} + +type PostableGrafanaReceiver CreateAlertNotificationCommand + +type CreateAlertNotificationCommand struct { + UID string `json:"uid"` + Name string `json:"name"` + Type string `json:"type"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Settings *simplejson.Json `json:"settings"` + SecureSettings map[string]string `json:"secureSettings"` +} diff --git a/pkg/services/sqlstore/migrations/ualert/dash_alert.go b/pkg/services/sqlstore/migrations/ualert/dash_alert.go index 19b433f23d8..b6338709261 100644 --- a/pkg/services/sqlstore/migrations/ualert/dash_alert.go +++ b/pkg/services/sqlstore/migrations/ualert/dash_alert.go @@ -21,8 +21,8 @@ type dashAlert struct { } var slurpDashSQL = ` -SELECT id, - org_id, +SELECT id, + org_id, dashboard_id, panel_id, org_id, @@ -70,7 +70,8 @@ type dashAlertSettings struct { // dashAlertNot is the object that represents the Notifications array in // dashAlertSettings type dashAlertNot struct { - UID string `json:"uid"` + UID string `json:"uid,omitempty"` + ID int64 `json:"id,omitempty"` } // dashAlertingConditionJSON is like classic.ClassicConditionJSON except that it diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index 05421d50b1e..ebe8ead72fb 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -1,11 +1,14 @@ package ualert import ( + "encoding/json" "fmt" "os" + "time" + + "xorm.io/xorm" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "xorm.io/xorm" ) const GENERAL_FOLDER = "General Alerting" @@ -49,7 +52,10 @@ func AddDashAlertMigration(mg *migrator.Migrator) { if err != nil { mg.Logger.Error("alert migration error: could not clear alert migration for removing data", "error", err) } - mg.AddMigration(migTitle, &migration{}) + mg.AddMigration(migTitle, &migration{ + seenChannelUIDs: make(map[string]struct{}), + migratedChannels: make(map[*notificationChannel]struct{}), + }) case !ngEnabled && migrationRun: // Remove the migration entry that creates unified alerting data. This is so when the feature // flag is enabled in the future the migration "move dashboard alerts to unified alerting" will be run again. @@ -66,6 +72,9 @@ type migration struct { // session and mg are attached for convenience. sess *xorm.Session mg *migrator.Migrator + + seenChannelUIDs map[string]struct{} + migratedChannels map[*notificationChannel]struct{} } func (m *migration) SQL(dialect migrator.Dialect) string { @@ -93,6 +102,15 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { return err } + // allChannels: channelUID -> channelConfig + allChannels, defaultChannels, err := m.getNotificationChannelMap() + if err != nil { + return err + } + + amConfig := PostableUserConfig{} + amConfig.AlertmanagerConfig.Route = &Route{} + for _, da := range dashAlerts { newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap) if err != nil { @@ -194,6 +212,10 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { return err } + if err := m.updateReceiverAndRoute(allChannels, defaultChannels, da, rule, &amConfig); err != nil { + return err + } + _, err = m.sess.Insert(rule) if err != nil { // TODO better error handling, if constraint @@ -213,7 +235,37 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { } } - return nil + // Create a separate receiver for all the unmigrated channels. + err = m.updateDefaultAndUnmigratedChannels(&amConfig, allChannels, defaultChannels) + if err != nil { + return err + } + + if err := amConfig.EncryptSecureSettings(); err != nil { + return err + } + rawAmConfig, err := json.Marshal(&amConfig) + if err != nil { + return err + } + + // TODO: should we apply the config here? Because Alertmanager can take upto 1 min to pick it up. + _, err = m.sess.Insert(AlertConfiguration{ + AlertmanagerConfiguration: string(rawAmConfig), + // Since we are migration for a snapshot of the code, it is always going to migrate to + // the v1 config. + ConfigurationVersion: "v1", + }) + + return err +} + +type AlertConfiguration struct { + ID int64 `xorm:"pk autoincr 'id'"` + + AlertmanagerConfiguration string + ConfigurationVersion string + CreatedAt time.Time `xorm:"created"` } type rmMigration struct {