public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Persist silence state immediately on Create/Delete (#84705)

Browse Source 
* Alerting: Persist silence state immediately on Create/Delete

Persists the silence state to the kvstore immediately instead of waiting for the
 next maintenance run. This is used after Create/Delete to prevent silences from
 being lost when a new Alertmanager is started before the state has persisted.
 This can happen, for example, in a rolling deployment scenario.

* Fix test that requires real data

* Don't error if silence state persist fails, maintenance will correct
This commit is contained in:
Matthew Jacobson
2024-04-09 13:39:34 -04:00
committed by GitHub
parent c7e4baff75
commit f79dd7c7f9
14 changed files with 449 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api_alertmanager.go
+15 -12
View File
@@ -65,11 +65,6 @@ func (srv AlertmanagerSrv) RouteCreateSilence(c *contextmodel.ReqContext, postab
return ErrResp(http.StatusBadRequest, err, "silence failed validation")
}
am, errResp := srv.AlertmanagerFor(c.SignedInUser.GetOrgID())
if errResp != nil {
return errResp
}
action := accesscontrol.ActionAlertingInstanceUpdate
if postableSilence.ID == "" {
action = accesscontrol.ActionAlertingInstanceCreate
@@ -83,8 +78,15 @@ func (srv AlertmanagerSrv) RouteCreateSilence(c *contextmodel.ReqContext, postab
return response.Err(authz.NewAuthorizationErrorWithPermissions(fmt.Sprintf("%s silences", errAction), evaluator))
}
silenceID, err := am.CreateSilence(c.Req.Context(), &postableSilence)
silenceID, err := srv.mam.CreateSilence(c.Req.Context(), c.SignedInUser.GetOrgID(), &postableSilence)
if err != nil {
if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
return ErrResp(http.StatusNotFound, err, "")
}
if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
return ErrResp(http.StatusConflict, err, "")
}
if errors.Is(err, alertingNotify.ErrSilenceNotFound) {
return ErrResp(http.StatusNotFound, err, "")
}
@@ -115,12 +117,13 @@ func (srv AlertmanagerSrv) RouteDeleteAlertingConfig(c *contextmodel.ReqContext)
}
func (srv AlertmanagerSrv) RouteDeleteSilence(c *contextmodel.ReqContext, silenceID string) response.Response {
am, errResp := srv.AlertmanagerFor(c.SignedInUser.GetOrgID())
if errResp != nil {
return errResp
}
if err := am.DeleteSilence(c.Req.Context(), silenceID); err != nil {
if err := srv.mam.DeleteSilence(c.Req.Context(), c.SignedInUser.GetOrgID(), silenceID); err != nil {
if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
return ErrResp(http.StatusNotFound, err, "")
}
if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
return ErrResp(http.StatusConflict, err, "")
}
if errors.Is(err, alertingNotify.ErrSilenceNotFound) {
return ErrResp(http.StatusNotFound, err, "")
}