From 087cf5a877e9c091bc44a7aada0eef19fea54299 Mon Sep 17 00:00:00 2001 From: Matthew Jacobson Date: Fri, 3 Nov 2023 15:12:52 -0400 Subject: [PATCH] [v9.4.x] Alerting: During legacy migration do not create one silence per rule (#77642) * Alerting: During legacy migration do not create one silence per rule During legacy migration every migrated rule is given a label rule_uid=. This is used to silence migrated alerts if they were: - Paused in legacy alerting. - Had Error state set to keep last state. - Had NoData state set to keep last state. This can potentially create a large amount of silences and a high cardinality label. Both of these scenarios have poor outcomes for CPU load and latency in unified alerting. Instead, this change opts to create one or more of three labels on each migrated alert rule as well as three silence rules: - migration_paused = true - migration_keep_last_state_error = true - migration_keep_last_state_nodata = true This will drastically reduce the number of created silence rules in most cases as well as not create the potentially high cardinalty label `rule_uid`. --- .../sqlstore/migrations/ualert/alert_rule.go | 22 +++---- .../migrations/ualert/alert_rule_test.go | 39 +++++++++++ .../sqlstore/migrations/ualert/silences.go | 66 +++++++++---------- .../sqlstore/migrations/ualert/ualert.go | 9 +++ 4 files changed, 90 insertions(+), 46 deletions(-) diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule.go b/pkg/services/sqlstore/migrations/ualert/alert_rule.go index c906c0150a9..58e47f302c0 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule.go @@ -156,20 +156,18 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string return nil, err } - // Label for routing and silences. - n, v := getLabelForSilenceMatching(ar.UID) - ar.Labels[n] = v - - if err := m.addSilence(da, ar); err != nil { - m.mg.Logger.Error("alert migration error: failed to create silence", "rule_name", ar.Title, "err", err) + // Label for routing silences. + if da.State == "paused" { + n, v := getLabelForPauseSilenceMatching() + ar.Labels[n] = v } - - if err := m.addErrorSilence(da, ar); err != nil { - m.mg.Logger.Error("alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err) + if da.ParsedSettings.ExecutionErrorState == "keep_state" { + n, v := getLabelForErrorSilenceMatching() + ar.Labels[n] = v } - - if err := m.addNoDataSilence(da, ar); err != nil { - m.mg.Logger.Error("alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err) + if da.ParsedSettings.NoDataState == "keep_state" { + n, v := getLabelForNoDataSilenceMatching() + ar.Labels[n] = v } return ar, nil diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go b/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go index 1e730fc8465..2132e9fa652 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go @@ -138,6 +138,45 @@ func TestMakeAlertRule(t *testing.T) { require.NoError(t, err) require.True(t, ar.IsPaused) }) + + t.Run("paused dash alert is silenced", func(t *testing.T) { + m := newTestMigration(t) + da := createTestDashAlert() + da.State = "paused" + cnd := createTestDashAlertCondition() + + ar, err := m.makeAlertRule(cnd, da, "folder") + require.NoError(t, err) + + n, v := getLabelForPauseSilenceMatching() + require.Equal(t, ar.Labels[n], v) + }) + + t.Run("keep last state error dash alert is silenced", func(t *testing.T) { + m := newTestMigration(t) + da := createTestDashAlert() + da.ParsedSettings.ExecutionErrorState = "keep_state" + cnd := createTestDashAlertCondition() + + ar, err := m.makeAlertRule(cnd, da, "folder") + require.NoError(t, err) + + n, v := getLabelForErrorSilenceMatching() + require.Equal(t, ar.Labels[n], v) + }) + + t.Run("keep last state nodata dash alert is silenced", func(t *testing.T) { + m := newTestMigration(t) + da := createTestDashAlert() + da.ParsedSettings.NoDataState = "keep_state" + cnd := createTestDashAlertCondition() + + ar, err := m.makeAlertRule(cnd, da, "folder") + require.NoError(t, err) + + n, v := getLabelForNoDataSilenceMatching() + require.Equal(t, ar.Labels[n], v) + }) } func createTestDashAlert() dashAlert { diff --git a/pkg/services/sqlstore/migrations/ualert/silences.go b/pkg/services/sqlstore/migrations/ualert/silences.go index c5fdfbf1948..e58fe90d615 100644 --- a/pkg/services/sqlstore/migrations/ualert/silences.go +++ b/pkg/services/sqlstore/migrations/ualert/silences.go @@ -26,17 +26,13 @@ const ( ErrorAlertName = "DatasourceError" ) -func (m *migration) addSilence(da dashAlert, rule *alertRule) error { - if da.State != "paused" { - return nil - } - +func (m *migration) addPauseSilence(orgId int64) error { uid, err := uuid.NewRandom() if err != nil { return errors.New("failed to create uuid for silence") } - n, v := getLabelForSilenceMatching(rule.UID) + n, v := getLabelForPauseSilenceMatching() s := &pb.MeshSilence{ Silence: &pb.Silence{ Id: uid.String(), @@ -50,29 +46,26 @@ func (m *migration) addSilence(da dashAlert, rule *alertRule) error { StartsAt: time.Now(), EndsAt: time.Now().Add(365 * 20 * time.Hour), // 1 year. CreatedBy: "Grafana Migration", - Comment: "Created during auto migration to unified alerting", + Comment: "Created during migration to unified alerting to silence paused alerts", }, ExpiresAt: time.Now().Add(365 * 20 * time.Hour), // 1 year. } - _, ok := m.silences[da.OrgId] + _, ok := m.silences[orgId] if !ok { - m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) + m.silences[orgId] = make([]*pb.MeshSilence, 0) } - m.silences[da.OrgId] = append(m.silences[da.OrgId], s) + m.silences[orgId] = append(m.silences[orgId], s) return nil } -func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error { - if da.ParsedSettings.ExecutionErrorState != "keep_state" { - return nil - } - +func (m *migration) addErrorSilence(orgId int64) error { uid, err := uuid.NewRandom() if err != nil { return errors.New("failed to create uuid for silence") } + n, v := getLabelForErrorSilenceMatching() s := &pb.MeshSilence{ Silence: &pb.Silence{ Id: uid.String(), @@ -84,34 +77,31 @@ func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error { }, { Type: pb.Matcher_EQUAL, - Name: "rule_uid", - Pattern: rule.UID, + Name: n, + Pattern: v, }, }, StartsAt: time.Now(), EndsAt: time.Now().AddDate(1, 0, 0), // 1 year CreatedBy: "Grafana Migration", - Comment: fmt.Sprintf("Created during migration to unified alerting to silence Error state for alert rule ID '%s' and Title '%s' because the option 'Keep Last State' was selected for Error state", rule.UID, rule.Title), + Comment: "Created during migration to unified alerting to silence Error state when the option 'Keep Last State' was selected for Error state", }, ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year } - if _, ok := m.silences[da.OrgId]; !ok { - m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) + if _, ok := m.silences[orgId]; !ok { + m.silences[orgId] = make([]*pb.MeshSilence, 0) } - m.silences[da.OrgId] = append(m.silences[da.OrgId], s) + m.silences[orgId] = append(m.silences[orgId], s) return nil } -func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error { - if da.ParsedSettings.NoDataState != "keep_state" { - return nil - } - +func (m *migration) addNoDataSilence(orgId int64) error { uid, err := uuid.NewRandom() if err != nil { return errors.New("failed to create uuid for silence") } + n, v := getLabelForNoDataSilenceMatching() s := &pb.MeshSilence{ Silence: &pb.Silence{ Id: uid.String(), @@ -123,22 +113,22 @@ func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error { }, { Type: pb.Matcher_EQUAL, - Name: "rule_uid", - Pattern: rule.UID, + Name: n, + Pattern: v, }, }, StartsAt: time.Now(), EndsAt: time.Now().AddDate(1, 0, 0), // 1 year. CreatedBy: "Grafana Migration", - Comment: fmt.Sprintf("Created during migration to unified alerting to silence NoData state for alert rule ID '%s' and Title '%s' because the option 'Keep Last State' was selected for NoData state", rule.UID, rule.Title), + Comment: "Created during migration to unified alerting to silence NoData state when the option 'Keep Last State' was selected for NoData state", }, ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year. } - _, ok := m.silences[da.OrgId] + _, ok := m.silences[orgId] if !ok { - m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) + m.silences[orgId] = make([]*pb.MeshSilence, 0) } - m.silences[da.OrgId] = append(m.silences[da.OrgId], s) + m.silences[orgId] = append(m.silences[orgId], s) return nil } @@ -212,6 +202,14 @@ func openReplace(filename string) (*replaceFile, error) { return rf, nil } -func getLabelForSilenceMatching(ruleUID string) (string, string) { - return "rule_uid", ruleUID +func getLabelForPauseSilenceMatching() (string, string) { + return "migration_paused", "true" +} + +func getLabelForErrorSilenceMatching() (string, string) { + return "migration_keep_last_state_error", "true" +} + +func getLabelForNoDataSilenceMatching() (string, string) { + return "migration_keep_last_state_nodata", "true" } diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index fee2a10562e..6c404357709 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -392,6 +392,15 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { } for orgID := range rulesPerOrg { + if err := m.addPauseSilence(orgID); err != nil { + m.mg.Logger.Error("alert migration error: failed to create silence for paused alerts") + } + if err := m.addErrorSilence(orgID); err != nil { + m.mg.Logger.Error("alert migration error: failed to create silence for Error keep last state") + } + if err := m.addNoDataSilence(orgID); err != nil { + m.mg.Logger.Error("alert migration error: failed to create silence for Error keep last state") + } if err := m.writeSilencesFile(orgID); err != nil { m.mg.Logger.Error("alert migration error: failed to write silence file", "err", err) }