public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[v9.4.x] Alerting: During legacy migration do not create one silence per rule (#77642)

Browse Source 
* Alerting: During legacy migration do not create one silence per rule

During legacy migration every migrated rule is given a label rule_uid=<uid>.
This is used to silence migrated alerts if they were:

- Paused in legacy alerting.
- Had Error state set to keep last state.
- Had NoData state set to keep last state.

This can potentially create a large amount of silences and a high cardinality
label. Both of these scenarios have poor outcomes for CPU load and latency in
unified alerting.

Instead, this change opts to create one or more of three labels on each
migrated alert rule as well as three silence rules:

- migration_paused = true
- migration_keep_last_state_error = true
- migration_keep_last_state_nodata = true

 This will drastically reduce the number of created silence rules in most cases
 as well as not create the potentially high cardinalty label `rule_uid`.
This commit is contained in:
Matthew Jacobson
2023-11-03 15:12:52 -04:00
committed by GitHub
parent a6610343c9
commit 087cf5a877
4 changed files with 90 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/migrations/ualert/alert_rule.go
+10 -12
View File
@@ -156,20 +156,18 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string
return nil, err
}
// Label for routing and silences.
n, v := getLabelForSilenceMatching(ar.UID)
ar.Labels[n] = v
if err := m.addSilence(da, ar); err != nil {
m.mg.Logger.Error("alert migration error: failed to create silence", "rule_name", ar.Title, "err", err)
// Label for routing silences.
if da.State == "paused" {
n, v := getLabelForPauseSilenceMatching()
ar.Labels[n] = v
}
if err := m.addErrorSilence(da, ar); err != nil {
m.mg.Logger.Error("alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err)
if da.ParsedSettings.ExecutionErrorState == "keep_state" {
n, v := getLabelForErrorSilenceMatching()
ar.Labels[n] = v
}
if err := m.addNoDataSilence(da, ar); err != nil {
m.mg.Logger.Error("alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err)
if da.ParsedSettings.NoDataState == "keep_state" {
n, v := getLabelForNoDataSilenceMatching()
ar.Labels[n] = v
}
return ar, nil
pkg/services/sqlstore/migrations/ualert/alert_rule_test.go
+39
View File
@@ -138,6 +138,45 @@ func TestMakeAlertRule(t *testing.T) {
require.NoError(t, err)
require.True(t, ar.IsPaused)
})
t.Run("paused dash alert is silenced", func(t *testing.T) {
m := newTestMigration(t)
da := createTestDashAlert()
da.State = "paused"
cnd := createTestDashAlertCondition()
ar, err := m.makeAlertRule(cnd, da, "folder")
require.NoError(t, err)
n, v := getLabelForPauseSilenceMatching()
require.Equal(t, ar.Labels[n], v)
})
t.Run("keep last state error dash alert is silenced", func(t *testing.T) {
m := newTestMigration(t)
da := createTestDashAlert()
da.ParsedSettings.ExecutionErrorState = "keep_state"
cnd := createTestDashAlertCondition()
ar, err := m.makeAlertRule(cnd, da, "folder")
require.NoError(t, err)
n, v := getLabelForErrorSilenceMatching()
require.Equal(t, ar.Labels[n], v)
})
t.Run("keep last state nodata dash alert is silenced", func(t *testing.T) {
m := newTestMigration(t)
da := createTestDashAlert()
da.ParsedSettings.NoDataState = "keep_state"
cnd := createTestDashAlertCondition()
ar, err := m.makeAlertRule(cnd, da, "folder")
require.NoError(t, err)
n, v := getLabelForNoDataSilenceMatching()
require.Equal(t, ar.Labels[n], v)
})
}
func createTestDashAlert() dashAlert {
pkg/services/sqlstore/migrations/ualert/silences.go
+32 -34
View File
@@ -26,17 +26,13 @@ const (
ErrorAlertName = "DatasourceError"
)
func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
if da.State != "paused" {
return nil
}
func (m *migration) addPauseSilence(orgId int64) error {
uid, err := uuid.NewRandom()
if err != nil {
return errors.New("failed to create uuid for silence")
}
n, v := getLabelForSilenceMatching(rule.UID)
n, v := getLabelForPauseSilenceMatching()
s := &pb.MeshSilence{
Silence: &pb.Silence{
Id: uid.String(),
@@ -50,29 +46,26 @@ func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
StartsAt: time.Now(),
EndsAt: time.Now().Add(365 * 20 * time.Hour), // 1 year.
CreatedBy: "Grafana Migration",
Comment: "Created during auto migration to unified alerting",
Comment: "Created during migration to unified alerting to silence paused alerts",
},
ExpiresAt: time.Now().Add(365 * 20 * time.Hour), // 1 year.
}
_, ok := m.silences[da.OrgId]
_, ok := m.silences[orgId]
if !ok {
m.silences[da.OrgId] = make([]*pb.MeshSilence, 0)
m.silences[orgId] = make([]*pb.MeshSilence, 0)
}
m.silences[da.OrgId] = append(m.silences[da.OrgId], s)
m.silences[orgId] = append(m.silences[orgId], s)
return nil
}
func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error {
if da.ParsedSettings.ExecutionErrorState != "keep_state" {
return nil
}
func (m *migration) addErrorSilence(orgId int64) error {
uid, err := uuid.NewRandom()
if err != nil {
return errors.New("failed to create uuid for silence")
}
n, v := getLabelForErrorSilenceMatching()
s := &pb.MeshSilence{
Silence: &pb.Silence{
Id: uid.String(),
@@ -84,34 +77,31 @@ func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error {
},
{
Type: pb.Matcher_EQUAL,
Name: "rule_uid",
Pattern: rule.UID,
Name: n,
Pattern: v,
},
},
StartsAt: time.Now(),
EndsAt: time.Now().AddDate(1, 0, 0), // 1 year
CreatedBy: "Grafana Migration",
Comment: fmt.Sprintf("Created during migration to unified alerting to silence Error state for alert rule ID '%s' and Title '%s' because the option 'Keep Last State' was selected for Error state", rule.UID, rule.Title),
Comment: "Created during migration to unified alerting to silence Error state when the option 'Keep Last State' was selected for Error state",
},
ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year
}
if _, ok := m.silences[da.OrgId]; !ok {
m.silences[da.OrgId] = make([]*pb.MeshSilence, 0)
if _, ok := m.silences[orgId]; !ok {
m.silences[orgId] = make([]*pb.MeshSilence, 0)
}
m.silences[da.OrgId] = append(m.silences[da.OrgId], s)
m.silences[orgId] = append(m.silences[orgId], s)
return nil
}
func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error {
if da.ParsedSettings.NoDataState != "keep_state" {
return nil
}
func (m *migration) addNoDataSilence(orgId int64) error {
uid, err := uuid.NewRandom()
if err != nil {
return errors.New("failed to create uuid for silence")
}
n, v := getLabelForNoDataSilenceMatching()
s := &pb.MeshSilence{
Silence: &pb.Silence{
Id: uid.String(),
@@ -123,22 +113,22 @@ func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error {
},
{
Type: pb.Matcher_EQUAL,
Name: "rule_uid",
Pattern: rule.UID,
Name: n,
Pattern: v,
},
},
StartsAt: time.Now(),
EndsAt: time.Now().AddDate(1, 0, 0), // 1 year.
CreatedBy: "Grafana Migration",
Comment: fmt.Sprintf("Created during migration to unified alerting to silence NoData state for alert rule ID '%s' and Title '%s' because the option 'Keep Last State' was selected for NoData state", rule.UID, rule.Title),
Comment: "Created during migration to unified alerting to silence NoData state when the option 'Keep Last State' was selected for NoData state",
},
ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year.
}
_, ok := m.silences[da.OrgId]
_, ok := m.silences[orgId]
if !ok {
m.silences[da.OrgId] = make([]*pb.MeshSilence, 0)
m.silences[orgId] = make([]*pb.MeshSilence, 0)
}
m.silences[da.OrgId] = append(m.silences[da.OrgId], s)
m.silences[orgId] = append(m.silences[orgId], s)
return nil
}
@@ -212,6 +202,14 @@ func openReplace(filename string) (*replaceFile, error) {
return rf, nil
}
func getLabelForSilenceMatching(ruleUID string) (string, string) {
return "rule_uid", ruleUID
func getLabelForPauseSilenceMatching() (string, string) {
return "migration_paused", "true"
}
func getLabelForErrorSilenceMatching() (string, string) {
return "migration_keep_last_state_error", "true"
}
func getLabelForNoDataSilenceMatching() (string, string) {
return "migration_keep_last_state_nodata", "true"
}
pkg/services/sqlstore/migrations/ualert/ualert.go
+9
View File
@@ -392,6 +392,15 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
}
for orgID := range rulesPerOrg {
if err := m.addPauseSilence(orgID); err != nil {
m.mg.Logger.Error("alert migration error: failed to create silence for paused alerts")
}
if err := m.addErrorSilence(orgID); err != nil {
m.mg.Logger.Error("alert migration error: failed to create silence for Error keep last state")
}
if err := m.addNoDataSilence(orgID); err != nil {
m.mg.Logger.Error("alert migration error: failed to create silence for Error keep last state")
}
if err := m.writeSilencesFile(orgID); err != nil {
m.mg.Logger.Error("alert migration error: failed to write silence file", "err", err)
}