public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Time interval Delete API to check for usages in alert rules (#90500)

Browse Source 
* Check if a time interval is used in alert rules before deleting it
* Add time interval to parameters of ListAlertRulesQuery and ListNotificationSettings of DbStore

== Refacorings == 
* refactor isMuteTimeInUse to accept a single route
* update getMuteTiming to not return err
* update delete to get the mute timing from config first
This commit is contained in:
Yuri Tseretyan
2024-07-17 10:53:54 -04:00
committed by GitHub
parent 79df1b1e10
commit 970cafa20f
15 changed files with 492 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/provisioning/errors.go
+21 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/grafana/grafana/pkg/apimachinery/errutil"
"github.com/grafana/grafana/pkg/services/ngalert/models"
)
var ErrValidation = fmt.Errorf("invalid object specification")
@@ -20,7 +21,7 @@ var (
ErrTimeIntervalNotFound = errutil.NotFound("alerting.notifications.time-intervals.notFound")
ErrTimeIntervalExists = errutil.BadRequest("alerting.notifications.time-intervals.nameExists", errutil.WithPublicMessage("Time interval with this name already exists. Use a different name or update existing one."))
ErrTimeIntervalInvalid = errutil.BadRequest("alerting.notifications.time-intervals.invalidFormat").MustTemplate("Invalid format of the submitted time interval", errutil.WithPublic("Time interval is in invalid format. Correct the payload and try again."))
ErrTimeIntervalInUse = errutil.Conflict("alerting.notifications.time-intervals.used", errutil.WithPublicMessage("Time interval is used by one or many notification policies"))
ErrTimeIntervalInUse = errutil.Conflict("alerting.notifications.time-intervals.used").MustTemplate("Time interval is used")
ErrContactPointReferenced = errutil.Conflict("alerting.notifications.contact-points.referenced", errutil.WithPublicMessage("Contact point is currently referenced by a notification policy."))
ErrContactPointUsedInRule = errutil.Conflict("alerting.notifications.contact-points.used-by-rule", errutil.WithPublicMessage("Contact point is currently used in the notification settings of one or many alert rules."))
@@ -47,3 +48,22 @@ func MakeErrTimeIntervalInvalid(err error) error {
return ErrTimeIntervalInvalid.Build(data)
}
func MakeErrTimeIntervalInUse(usedByRoutes bool, rules []models.AlertRuleKey) error {
uids := make([]string, 0, len(rules))
for _, key := range rules {
uids = append(uids, key.UID)
}
data := make(map[string]any, 2)
if len(uids) > 0 {
data["UsedByRules"] = uids
}
if usedByRoutes {
data["UsedByRoutes"] = true
}
return ErrTimeIntervalInUse.Build(errutil.TemplateData{
Public: data,
Error: nil,
})
}
pkg/services/ngalert/provisioning/mute_timings.go
+63 -59
View File
@@ -5,10 +5,12 @@ import (
"encoding/binary"
"fmt"
"hash/fnv"
"slices"
"unsafe"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/timeinterval"
"golang.org/x/exp/maps"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -17,20 +19,22 @@ import (
)
type MuteTimingService struct {
configStore alertmanagerConfigStore
provenanceStore ProvisioningStore
xact TransactionManager
log log.Logger
validator validation.ProvenanceStatusTransitionValidator
configStore alertmanagerConfigStore
provenanceStore ProvisioningStore
xact TransactionManager
log log.Logger
validator validation.ProvenanceStatusTransitionValidator
ruleNotificationsStore AlertRuleNotificationSettingsStore
}
func NewMuteTimingService(config AMConfigStore, prov ProvisioningStore, xact TransactionManager, log log.Logger) *MuteTimingService {
func NewMuteTimingService(config AMConfigStore, prov ProvisioningStore, xact TransactionManager, log log.Logger, ns AlertRuleNotificationSettingsStore) *MuteTimingService {
return &MuteTimingService{
configStore: &alertmanagerConfigStoreImpl{store: config},
provenanceStore: prov,
xact: xact,
log: log,
validator: validation.ValidateProvenanceRelaxed,
configStore: &alertmanagerConfigStoreImpl{store: config},
provenanceStore: prov,
xact: xact,
log: log,
validator: validation.ValidateProvenanceRelaxed,
ruleNotificationsStore: ns,
}
}
@@ -69,9 +73,9 @@ func (svc *MuteTimingService) GetMuteTiming(ctx context.Context, name string, or
return definitions.MuteTimeInterval{}, err
}
mt, _, err := getMuteTiming(rev, name)
if err != nil {
return definitions.MuteTimeInterval{}, err
mt, idx := getMuteTiming(rev, name)
if idx == -1 {
return definitions.MuteTimeInterval{}, ErrTimeIntervalNotFound.Errorf("")
}
result := definitions.MuteTimeInterval{
@@ -98,13 +102,9 @@ func (svc *MuteTimingService) CreateMuteTiming(ctx context.Context, mt definitio
return definitions.MuteTimeInterval{}, err
}
if revision.cfg.AlertmanagerConfig.MuteTimeIntervals == nil {
revision.cfg.AlertmanagerConfig.MuteTimeIntervals = []config.MuteTimeInterval{}
}
for _, existing := range revision.cfg.AlertmanagerConfig.MuteTimeIntervals {
if mt.Name == existing.Name {
return definitions.MuteTimeInterval{}, ErrTimeIntervalExists.Errorf("")
}
_, idx := getMuteTiming(revision, mt.Name)
if idx != -1 {
return definitions.MuteTimeInterval{}, ErrTimeIntervalExists.Errorf("")
}
revision.cfg.AlertmanagerConfig.MuteTimeIntervals = append(revision.cfg.AlertmanagerConfig.MuteTimeIntervals, mt.MuteTimeInterval)
@@ -148,9 +148,9 @@ func (svc *MuteTimingService) UpdateMuteTiming(ctx context.Context, mt definitio
return definitions.MuteTimeInterval{}, nil
}
old, idx, err := getMuteTiming(revision, mt.Name)
if err != nil {
return definitions.MuteTimeInterval{}, err
old, idx := getMuteTiming(revision, mt.Name)
if idx == -1 {
return definitions.MuteTimeInterval{}, ErrTimeIntervalNotFound.Errorf("")
}
err = svc.checkOptimisticConcurrency(old, models.Provenance(mt.Provenance), mt.Version, "update")
@@ -179,7 +179,18 @@ func (svc *MuteTimingService) UpdateMuteTiming(ctx context.Context, mt definitio
// DeleteMuteTiming deletes the mute timing with the given name in the given org. If the mute timing does not exist, no error is returned.
func (svc *MuteTimingService) DeleteMuteTiming(ctx context.Context, name string, orgID int64, provenance definitions.Provenance, version string) error {
target := definitions.MuteTimeInterval{MuteTimeInterval: config.MuteTimeInterval{Name: name}, Provenance: provenance}
revision, err := svc.configStore.Get(ctx, orgID)
if err != nil {
return err
}
existing, idx := getMuteTiming(revision, name)
if idx == -1 {
svc.log.FromContext(ctx).Debug("Time interval was not found. Skip deleting", "name", name)
return nil
}
target := definitions.MuteTimeInterval{MuteTimeInterval: existing, Provenance: provenance}
// check that provenance is not changed in an invalid way
storedProvenance, err := svc.provenanceStore.GetProvenance(ctx, &target, orgID)
if err != nil {
@@ -189,30 +200,27 @@ func (svc *MuteTimingService) DeleteMuteTiming(ctx context.Context, name string,
return err
}
revision, err := svc.configStore.Get(ctx, orgID)
if isMuteTimeInUseInRoutes(name, revision.cfg.AlertmanagerConfig.Route) {
ns, _ := svc.ruleNotificationsStore.ListNotificationSettings(ctx, models.ListNotificationSettingsQuery{OrgID: orgID, TimeIntervalName: existing.Name})
// ignore error here because it's not important
return MakeErrTimeIntervalInUse(true, maps.Keys(ns))
}
err = svc.checkOptimisticConcurrency(existing, models.Provenance(provenance), version, "delete")
if err != nil {
return err
}
revision.cfg.AlertmanagerConfig.MuteTimeIntervals = slices.Delete(revision.cfg.AlertmanagerConfig.MuteTimeIntervals, idx, idx+1)
if revision.cfg.AlertmanagerConfig.MuteTimeIntervals == nil {
return nil
}
if isMuteTimeInUse(name, []*definitions.Route{revision.cfg.AlertmanagerConfig.Route}) {
return ErrTimeIntervalInUse.Errorf("")
}
for i, existing := range revision.cfg.AlertmanagerConfig.MuteTimeIntervals {
if name != existing.Name {
continue
}
err = svc.checkOptimisticConcurrency(existing, models.Provenance(provenance), version, "delete")
return svc.xact.InTransaction(ctx, func(ctx context.Context) error {
keys, err := svc.ruleNotificationsStore.ListNotificationSettings(ctx, models.ListNotificationSettingsQuery{OrgID: orgID, TimeIntervalName: existing.Name})
if err != nil {
return err
}
intervals := revision.cfg.AlertmanagerConfig.MuteTimeIntervals
revision.cfg.AlertmanagerConfig.MuteTimeIntervals = append(intervals[:i], intervals[i+1:]...)
}
if len(keys) > 0 {
return MakeErrTimeIntervalInUse(false, maps.Keys(keys))
}
return svc.xact.InTransaction(ctx, func(ctx context.Context) error {
if err := svc.configStore.Save(ctx, revision, orgID); err != nil {
return err
}
@@ -220,33 +228,29 @@ func (svc *MuteTimingService) DeleteMuteTiming(ctx context.Context, name string,
})
}
func isMuteTimeInUse(name string, routes []*definitions.Route) bool {
if len(routes) == 0 {
func isMuteTimeInUseInRoutes(name string, route *definitions.Route) bool {
if route == nil {
return false
}
for _, route := range routes {
for _, mtName := range route.MuteTimeIntervals {
if mtName == name {
return true
}
}
if isMuteTimeInUse(name, route.Routes) {
if slices.Contains(route.MuteTimeIntervals, name) {
return true
}
for _, route := range route.Routes {
if isMuteTimeInUseInRoutes(name, route) {
return true
}
}
return false
}
func getMuteTiming(rev *cfgRevision, name string) (config.MuteTimeInterval, int, error) {
if rev.cfg.AlertmanagerConfig.MuteTimeIntervals == nil {
return config.MuteTimeInterval{}, -1, ErrTimeIntervalNotFound.Errorf("")
func getMuteTiming(rev *cfgRevision, name string) (config.MuteTimeInterval, int) {
idx := slices.IndexFunc(rev.cfg.AlertmanagerConfig.MuteTimeIntervals, func(interval config.MuteTimeInterval) bool {
return interval.Name == name
})
if idx == -1 {
return config.MuteTimeInterval{}, idx
}
for idx, mt := range rev.cfg.AlertmanagerConfig.MuteTimeIntervals {
if mt.Name == name {
return mt, idx, nil
}
}
return config.MuteTimeInterval{}, -1, ErrTimeIntervalNotFound.Errorf("")
return rev.cfg.AlertmanagerConfig.MuteTimeIntervals[idx], idx
}
func calculateMuteTimeIntervalFingerprint(interval config.MuteTimeInterval) string {
pkg/services/ngalert/provisioning/mute_timings_test.go
+35 -31
View File
@@ -9,6 +9,7 @@ import (
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/timeinterval"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@@ -635,31 +636,7 @@ func TestDeleteMuteTimings(t *testing.T) {
}
}
t.Run("re-saves config and deletes provenance if mute timing does not exist", func(t *testing.T) {
sut, store, prov := createMuteTimingSvcSut()
store.GetFn = func(ctx context.Context, orgID int64) (*cfgRevision, error) {
return &cfgRevision{cfg: initialConfig()}, nil
}
prov.EXPECT().GetProvenance(mock.Anything, mock.Anything, mock.Anything).Return(models.ProvenanceAPI, nil)
prov.EXPECT().DeleteProvenance(mock.Anything, mock.Anything, mock.Anything).Return(nil)
err := sut.DeleteMuteTiming(context.Background(), "no-timing", orgID, definitions.Provenance(models.ProvenanceAPI), "")
require.NoError(t, err)
require.Len(t, store.Calls, 2)
require.Equal(t, "Get", store.Calls[0].Method)
require.Equal(t, orgID, store.Calls[0].Args[1])
require.Equal(t, "Save", store.Calls[1].Method)
require.Equal(t, orgID, store.Calls[1].Args[2])
revision := store.Calls[1].Args[1].(*cfgRevision)
require.EqualValues(t, initialConfig().AlertmanagerConfig.MuteTimeIntervals, revision.cfg.AlertmanagerConfig.MuteTimeIntervals)
prov.AssertCalled(t, "DeleteProvenance", mock.Anything, &definitions.MuteTimeInterval{MuteTimeInterval: config.MuteTimeInterval{Name: "no-timing"}, Provenance: definitions.Provenance(models.ProvenanceAPI)}, orgID)
})
t.Run("fails if it was created with different provenance", func(t *testing.T) {
t.Run("fails if provenance check fails", func(t *testing.T) {
sut, store, prov := createMuteTimingSvcSut()
expectedErr := errors.New("test")
sut.validator = func(from, to models.Provenance) error {
@@ -669,15 +646,12 @@ func TestDeleteMuteTimings(t *testing.T) {
return &cfgRevision{cfg: initialConfig()}, nil
}
prov.EXPECT().GetProvenance(mock.Anything, mock.Anything, mock.Anything).Return(models.ProvenanceAPI, nil)
prov.EXPECT().DeleteProvenance(mock.Anything, mock.Anything, mock.Anything).Return(nil)
err := sut.DeleteMuteTiming(context.Background(), "no-timing", orgID, definitions.Provenance(models.ProvenanceNone), correctVersion)
err := sut.DeleteMuteTiming(context.Background(), timingToDelete.Name, orgID, definitions.Provenance(models.ProvenanceNone), correctVersion)
require.ErrorIs(t, err, expectedErr)
require.Len(t, store.Calls, 0)
})
t.Run("returns ErrTimeIntervalInUse if mute timing is used", func(t *testing.T) {
t.Run("returns ErrTimeIntervalInUse if mute timing is used by a route", func(t *testing.T) {
sut, store, prov := createMuteTimingSvcSut()
store.GetFn = func(ctx context.Context, orgID int64) (*cfgRevision, error) {
return &cfgRevision{cfg: initialConfig()}, nil
@@ -689,7 +663,36 @@ func TestDeleteMuteTimings(t *testing.T) {
require.Len(t, store.Calls, 1)
require.Equal(t, "Get", store.Calls[0].Method)
require.Equal(t, orgID, store.Calls[0].Args[1])
require.Truef(t, ErrTimeIntervalInUse.Is(err), "expected ErrTimeIntervalInUse but got %s", err)
require.ErrorIs(t, err, ErrTimeIntervalInUse)
})
t.Run("returns ErrTimeIntervalInUse if mute timing is used by rules", func(t *testing.T) {
sut, store, prov := createMuteTimingSvcSut()
ruleNsStore := fakeAlertRuleNotificationStore{
ListNotificationSettingsFn: func(ctx context.Context, q models.ListNotificationSettingsQuery) (map[models.AlertRuleKey][]models.NotificationSettings, error) {
assertInTransaction(t, ctx)
assert.Equal(t, orgID, q.OrgID)
assert.Equal(t, timingToDelete.Name, q.TimeIntervalName)
assert.Empty(t, q.ReceiverName)
return map[models.AlertRuleKey][]models.NotificationSettings{
models.GenerateRuleKey(orgID): nil,
}, nil
},
}
sut.ruleNotificationsStore = &ruleNsStore
store.GetFn = func(ctx context.Context, orgID int64) (*cfgRevision, error) {
return &cfgRevision{cfg: initialConfig()}, nil
}
prov.EXPECT().GetProvenance(mock.Anything, mock.Anything, mock.Anything).Return(models.ProvenanceAPI, nil)
err := sut.DeleteMuteTiming(context.Background(), timingToDelete.Name, orgID, definitions.Provenance(models.ProvenanceAPI), correctVersion)
require.Len(t, store.Calls, 1)
require.Equal(t, "Get", store.Calls[0].Method)
require.Equal(t, orgID, store.Calls[0].Args[1])
require.ErrorIs(t, err, ErrTimeIntervalInUse)
require.Len(t, ruleNsStore.Calls, 1)
require.Equal(t, "ListNotificationSettings", ruleNsStore.Calls[0].Method)
})
t.Run("returns ErrVersionConflict if provided version does not match", func(t *testing.T) {
@@ -828,5 +831,6 @@ func createMuteTimingSvcSut() (*MuteTimingService, *alertmanagerConfigStoreFake,
validator: func(from, to models.Provenance) error {
return nil
},
ruleNotificationsStore: &fakeAlertRuleNotificationStore{},
}, store, prov
}
pkg/services/ngalert/provisioning/testing.go
+37
View File
@@ -217,3 +217,40 @@ func (s *fakeRuleAccessControlService) CanWriteAllRules(ctx context.Context, use
}
return false, nil
}
type fakeAlertRuleNotificationStore struct {
Calls []call
RenameReceiverInNotificationSettingsFn func(ctx context.Context, orgID int64, oldReceiver, newReceiver string) (int, error)
ListNotificationSettingsFn func(ctx context.Context, q models.ListNotificationSettingsQuery) (map[models.AlertRuleKey][]models.NotificationSettings, error)
}
func (f *fakeAlertRuleNotificationStore) RenameReceiverInNotificationSettings(ctx context.Context, orgID int64, oldReceiver, newReceiver string) (int, error) {
call := call{
Method: "RenameReceiverInNotificationSettings",
Args: []interface{}{ctx, orgID, oldReceiver, newReceiver},
}
f.Calls = append(f.Calls, call)
if f.RenameReceiverInNotificationSettingsFn != nil {
return f.RenameReceiverInNotificationSettingsFn(ctx, orgID, oldReceiver, newReceiver)
}
// Default values when no function hook is provided
return 0, nil
}
func (f *fakeAlertRuleNotificationStore) ListNotificationSettings(ctx context.Context, q models.ListNotificationSettingsQuery) (map[models.AlertRuleKey][]models.NotificationSettings, error) {
call := call{
Method: "ListNotificationSettings",
Args: []interface{}{ctx, q},
}
f.Calls = append(f.Calls, call)
if f.ListNotificationSettingsFn != nil {
return f.ListNotificationSettingsFn(ctx, q)
}
// Default values when no function hook is provided
return nil, nil
}