From 10f8a138aee99a9e95d4af1a76f0d6b86e80d4c8 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Thu, 29 Jul 2021 19:26:19 +0530 Subject: [PATCH] Alerting: Deduplicate receivers during migration (#36812) * Alerting: Deduplicate receivers during migration Signed-off-by: Ganesh Vernekar * Fix review comments Signed-off-by: Ganesh Vernekar * Better naming of contact points Signed-off-by: Ganesh Vernekar * Handle some edge cases Signed-off-by: Ganesh Vernekar * Don't add routes which point to default contact point Signed-off-by: Ganesh Vernekar --- .../sqlstore/migrations/ualert/channel.go | 147 ++++++++++++------ .../sqlstore/migrations/ualert/ualert.go | 20 ++- 2 files changed, 113 insertions(+), 54 deletions(-) diff --git a/pkg/services/sqlstore/migrations/ualert/channel.go b/pkg/services/sqlstore/migrations/ualert/channel.go index b53cf527693..a16ef11ba1e 100644 --- a/pkg/services/sqlstore/migrations/ualert/channel.go +++ b/pkg/services/sqlstore/migrations/ualert/channel.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "sort" + "strings" "github.com/prometheus/alertmanager/pkg/labels" @@ -85,21 +87,19 @@ func (m *migration) updateReceiverAndRoute(allChannels map[interface{}]*notifica return err } - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) - amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route) + if recv != nil { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) + } + if route != nil { + amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route) + } return nil } func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface{}, defaultChannels []*notificationChannel, allChannels map[interface{}]*notificationChannel) (*PostableApiReceiver, *Route, error) { - receiverName := getMigratedReceiverNameFromRuleUID(ruleUid) - portedChannels := []*PostableGrafanaReceiver{} - receiver := &PostableApiReceiver{ - Name: receiverName, - } - - addedChannels := map[*notificationChannel]struct{}{} + var receiver *PostableApiReceiver addChannel := func(c *notificationChannel) error { if c.Type == "hipchat" || c.Type == "sensu" { @@ -122,31 +122,70 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface Settings: settings, SecureSettings: secureSettings, }) - addedChannels[c] = struct{}{} return nil } - for _, n := range channelUids { - c, ok := allChannels[n] - if !ok { - continue + // Remove obsolete notification channels. + filteredChannelUids := make(map[interface{}]struct{}) + for _, uid := range channelUids { + _, ok := allChannels[uid] + if ok { + filteredChannelUids[uid] = struct{}{} + } else { + m.mg.Logger.Warn("ignoring obsolete notification channel", "uid", uid) } - - if err := addChannel(c); err != nil { - return nil, nil, err + } + // Add default channels that are not obsolete. + for _, c := range defaultChannels { + id := interface{}(c.Uid) + if c.Uid == "" { + id = c.ID + } + _, ok := allChannels[id] + if ok { + filteredChannelUids[id] = struct{}{} } } - for _, c := range defaultChannels { - if _, ok := addedChannels[c]; !ok { - if err := addChannel(c); err != nil { + if len(filteredChannelUids) == 0 && ruleUid != "default_route" { + // We use the default route instead. No need to add additional route. + return nil, nil, nil + } + + chanKey, err := makeKeyForChannelGroup(filteredChannelUids) + if err != nil { + return nil, nil, err + } + + var receiverName string + if rn, ok := m.portedChannelGroups[chanKey]; ok { + // We have ported these exact set of channels already. Re-use it. + receiverName = rn + if receiverName == "autogen-contact-point-default" { + // We don't need to create new routes if it's the default contact point. + return nil, nil, nil + } + } else { + for n := range filteredChannelUids { + if err := addChannel(allChannels[n]); err != nil { return nil, nil, err } } - } - receiver.GrafanaManagedReceivers = portedChannels + if ruleUid == "default_route" { + receiverName = "autogen-contact-point-default" + } else { + m.lastReceiverID++ + receiverName = fmt.Sprintf("autogen-contact-point-%d", m.lastReceiverID) + } + + m.portedChannelGroups[chanKey] = receiverName + receiver = &PostableApiReceiver{ + Name: receiverName, + GrafanaManagedReceivers: portedChannels, + } + } n, v := getLabelForRouteMatching(ruleUid) mat, err := labels.NewMatcher(labels.MatchEqual, n, v) @@ -161,7 +200,45 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface return receiver, route, nil } -func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error { +// makeKeyForChannelGroup generates a unique for this group of channels UIDs. +func makeKeyForChannelGroup(channelUids map[interface{}]struct{}) (string, error) { + uids := make([]string, 0, len(channelUids)) + for u := range channelUids { + switch uid := u.(type) { + case string: + uids = append(uids, uid) + case int, int32, int64: + uids = append(uids, fmt.Sprintf("%d", uid)) + default: + // Should never happen. + return "", fmt.Errorf("unknown channel UID type: %T", u) + } + } + + sort.Strings(uids) + return strings.Join(uids, "::sep::"), nil +} + +// addDefaultChannels should be called before adding any other routes. +func (m *migration) addDefaultChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error { + // Default route and receiver. + recv, route, err := m.makeReceiverAndRoute("default_route", nil, defaultChannels, allChannels) + if err != nil { + return err + } + + if recv != nil { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) + } + if route != nil { + route.Matchers = nil // Don't need matchers for root route. + amConfig.AlertmanagerConfig.Route = route + } + + return nil +} + +func (m *migration) addUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error { // Unmigrated channels. portedChannels := []*PostableGrafanaReceiver{} receiver := &PostableApiReceiver{ @@ -198,26 +275,6 @@ func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserCon amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, receiver) } - // Default route and receiver. - var channelUids = []interface{}{} - for _, c := range defaultChannels { - if c.Uid == "" { - channelUids = append(channelUids, c.ID) - } else { - channelUids = append(channelUids, c.Uid) - } - } - - recv, route, err := m.makeReceiverAndRoute("default_route", channelUids, defaultChannels, allChannels) - if err != nil { - return err - } - - route.Matchers = nil // Don't need matchers for root route. - route.Routes = amConfig.AlertmanagerConfig.Route.Routes - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) - amConfig.AlertmanagerConfig.Route = route - return nil } @@ -275,10 +332,6 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, return settings, ss } -func getMigratedReceiverNameFromRuleUID(ruleUID string) string { - return fmt.Sprintf("autogen-panel-recv-%s", ruleUID) -} - func getLabelForRouteMatching(ruleUID string) (string, string) { return "rule_uid", ruleUID } diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index 80fc01b4762..65b103bb5b2 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -54,8 +54,9 @@ func AddDashAlertMigration(mg *migrator.Migrator) { mg.Logger.Error("alert migration error: could not clear alert migration for removing data", "error", err) } mg.AddMigration(migTitle, &migration{ - seenChannelUIDs: make(map[string]struct{}), - migratedChannels: make(map[*notificationChannel]struct{}), + seenChannelUIDs: make(map[string]struct{}), + migratedChannels: make(map[*notificationChannel]struct{}), + portedChannelGroups: make(map[string]string), }) case !ngEnabled && migrationRun: // Remove the migration entry that creates unified alerting data. This is so when the feature @@ -74,9 +75,11 @@ type migration struct { sess *xorm.Session mg *migrator.Migrator - seenChannelUIDs map[string]struct{} - migratedChannels map[*notificationChannel]struct{} - silences []*pb.MeshSilence + seenChannelUIDs map[string]struct{} + migratedChannels map[*notificationChannel]struct{} + silences []*pb.MeshSilence + portedChannelGroups map[string]string // Channel group key -> receiver name. + lastReceiverID int // For the auto generated receivers. } func (m *migration) SQL(dialect migrator.Dialect) string { @@ -111,7 +114,10 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { } amConfig := PostableUserConfig{} - amConfig.AlertmanagerConfig.Route = &Route{} + err = m.addDefaultChannels(&amConfig, allChannels, defaultChannels) + if err != nil { + return err + } for _, da := range dashAlerts { newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap) @@ -245,7 +251,7 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { } // Create a separate receiver for all the unmigrated channels. - err = m.updateDefaultAndUnmigratedChannels(&amConfig, allChannels, defaultChannels) + err = m.addUnmigratedChannels(&amConfig, allChannels, defaultChannels) if err != nil { return err }