Alerting: Introduce the silencing interface (#32517)

* Alerting: Introduce the silencing interface

The operations introduced are:

- Listing silences
- Retrieving an specific silence
- Deleting a silence
- Creating a silence

Signed-off-by: Josue Abreu <josue@grafana.com>

* Add a comment to listing silences

* Update to upstream alertmanager

* Remove copied code from the Alertmanager

pkg/services/ngalert/api/api_alertmanager_base.go

@@ -18,14 +18,15 @@ import (
 )
 
 type AlertmanagerApiService interface {
-	RouteCreateSilence(*models.ReqContext, apimodels.SilenceBody) response.Response
-	RouteDeleteAlertingConfig(*models.ReqContext) response.Response
+	RouteCreateSilence(*models.ReqContext, apimodels.CreateSilenceParams) response.Response
+	RouteGetSilences(*models.ReqContext) response.Response
+	RouteGetSilence(*models.ReqContext) response.Response
 	RouteDeleteSilence(*models.ReqContext) response.Response
+
+	RouteDeleteAlertingConfig(*models.ReqContext) response.Response
 	RouteGetAMAlertGroups(*models.ReqContext) response.Response
 	RouteGetAMAlerts(*models.ReqContext) response.Response
 	RouteGetAlertingConfig(*models.ReqContext) response.Response
-	RouteGetSilence(*models.ReqContext) response.Response
-	RouteGetSilences(*models.ReqContext) response.Response
 	RoutePostAMAlerts(*models.ReqContext, apimodels.PostableAlerts) response.Response
 	RoutePostAlertingConfig(*models.ReqContext, apimodels.PostableUserConfig) response.Response
 }
@@ -36,29 +37,44 @@ type AlertmanagerApiBase struct {
 
 func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService) {
 	api.RouteRegister.Group("", func(group routing.RouteRegister) {
-		group.Post(toMacaronPath("/alertmanager/{Recipient}/api/v2/silences"), binding.Bind(apimodels.SilenceBody{}), routing.Wrap(srv.RouteCreateSilence))
-		group.Delete(toMacaronPath("/alertmanager/{Recipient}/config/api/v1/alerts"), routing.Wrap(srv.RouteDeleteAlertingConfig))
+		// Silences
+		group.Post(toMacaronPath("/alertmanager/{Recipient}/api/v2/silences"), binding.Bind(apimodels.CreateSilenceParams{}), routing.Wrap(srv.RouteCreateSilence))
+		group.Get(toMacaronPath("/alertmanager/{Recipient}/api/v2/silences"), routing.Wrap(srv.RouteGetSilences))
+		group.Get(toMacaronPath("/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"), routing.Wrap(srv.RouteGetSilence))
 		group.Delete(toMacaronPath("/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"), routing.Wrap(srv.RouteDeleteSilence))
+
+		// Alerts
 		group.Get(toMacaronPath("/alertmanager/{Recipient}/api/v2/alerts/groups"), routing.Wrap(srv.RouteGetAMAlertGroups))
 		group.Get(toMacaronPath("/alertmanager/{Recipient}/api/v2/alerts"), routing.Wrap(srv.RouteGetAMAlerts))
-		group.Get(toMacaronPath("/alertmanager/{Recipient}/config/api/v1/alerts"), routing.Wrap(srv.RouteGetAlertingConfig))
-		group.Get(toMacaronPath("/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"), routing.Wrap(srv.RouteGetSilence))
-		group.Get(toMacaronPath("/alertmanager/{Recipient}/api/v2/silences"), routing.Wrap(srv.RouteGetSilences))
 		group.Post(toMacaronPath("/alertmanager/{Recipient}/api/v2/alerts"), binding.Bind(apimodels.PostableAlerts{}), routing.Wrap(srv.RoutePostAMAlerts))
+
+		// Configuration
+		group.Delete(toMacaronPath("/alertmanager/{Recipient}/config/api/v1/alerts"), routing.Wrap(srv.RouteDeleteAlertingConfig))
+		group.Get(toMacaronPath("/alertmanager/{Recipient}/config/api/v1/alerts"), routing.Wrap(srv.RouteGetAlertingConfig))
 		group.Post(toMacaronPath("/alertmanager/{Recipient}/config/api/v1/alerts"), binding.Bind(apimodels.PostableUserConfig{}), routing.Wrap(srv.RoutePostAlertingConfig))
 	})
 }
 
-func (base AlertmanagerApiBase) RouteCreateSilence(c *models.ReqContext, body apimodels.SilenceBody) response.Response {
+func (base AlertmanagerApiBase) RouteCreateSilence(c *models.ReqContext, params apimodels.CreateSilenceParams) response.Response {
 	recipient := c.Params(":Recipient")
 	base.log.Info("RouteCreateSilence: ", "Recipient", recipient)
-	base.log.Info("RouteCreateSilence: ", "body", body)
+	base.log.Info("RouteCreateSilence: ", "params", params)
 	return response.Error(http.StatusNotImplemented, "", nil)
 }
 
-func (base AlertmanagerApiBase) RouteDeleteAlertingConfig(c *models.ReqContext) response.Response {
+func (base AlertmanagerApiBase) RouteGetSilences(c *models.ReqContext) response.Response {
 	recipient := c.Params(":Recipient")
-	base.log.Info("RouteDeleteAlertingConfig: ", "Recipient", recipient)
+	base.log.Info("RouteGetSilences: ", "Recipient", recipient)
+	filter := c.Params(":Filter")
+	base.log.Info("RouteGetSilences: ", "params", filter)
+	return response.Error(http.StatusNotImplemented, "", nil)
+}
+
+func (base AlertmanagerApiBase) RouteGetSilence(c *models.ReqContext) response.Response {
+	silenceId := c.Params(":SilenceId")
+	base.log.Info("RouteGetSilence: ", "SilenceId", silenceId)
+	recipient := c.Params(":Recipient")
+	base.log.Info("RouteGetSilence: ", "Recipient", recipient)
 	return response.Error(http.StatusNotImplemented, "", nil)
 }
 
@@ -70,6 +86,12 @@ func (base AlertmanagerApiBase) RouteDeleteSilence(c *models.ReqContext) respons
 	return response.Error(http.StatusNotImplemented, "", nil)
 }
 
+func (base AlertmanagerApiBase) RouteDeleteAlertingConfig(c *models.ReqContext) response.Response {
+	recipient := c.Params(":Recipient")
+	base.log.Info("RouteDeleteAlertingConfig: ", "Recipient", recipient)
+	return response.Error(http.StatusNotImplemented, "", nil)
+}
+
 func (base AlertmanagerApiBase) RouteGetAMAlertGroups(c *models.ReqContext) response.Response {
 	recipient := c.Params(":Recipient")
 	base.log.Info("RouteGetAMAlertGroups: ", "Recipient", recipient)
@@ -88,20 +110,6 @@ func (base AlertmanagerApiBase) RouteGetAlertingConfig(c *models.ReqContext) res
 	return response.Error(http.StatusNotImplemented, "", nil)
 }
 
-func (base AlertmanagerApiBase) RouteGetSilence(c *models.ReqContext) response.Response {
-	silenceId := c.Params(":SilenceId")
-	base.log.Info("RouteGetSilence: ", "SilenceId", silenceId)
-	recipient := c.Params(":Recipient")
-	base.log.Info("RouteGetSilence: ", "Recipient", recipient)
-	return response.Error(http.StatusNotImplemented, "", nil)
-}
-
-func (base AlertmanagerApiBase) RouteGetSilences(c *models.ReqContext) response.Response {
-	recipient := c.Params(":Recipient")
-	base.log.Info("RouteGetSilences: ", "Recipient", recipient)
-	return response.Error(http.StatusNotImplemented, "", nil)
-}
-
 func (base AlertmanagerApiBase) RoutePostAMAlerts(c *models.ReqContext, body apimodels.PostableAlerts) response.Response {
 	recipient := c.Params(":Recipient")
 	base.log.Info("RoutePostAMAlerts: ", "Recipient", recipient)

pkg/services/ngalert/api/api_alertmanager_mock.go

@@ -523,7 +523,7 @@ type AlertmanagerApiMock struct {
 	log log.Logger
 }
 
-func (mock AlertmanagerApiMock) RouteCreateSilence(c *models.ReqContext, body apimodels.SilenceBody) response.Response {
+func (mock AlertmanagerApiMock) RouteCreateSilence(c *models.ReqContext, body apimodels.CreateSilenceParams) response.Response {
 	recipient := c.Params(":Recipient")
 	mock.log.Info("RouteCreateSilence: ", "Recipient", recipient)
 	mock.log.Info("RouteCreateSilence: ", "body", body)

pkg/services/ngalert/api/forked_am.go

@@ -38,7 +38,7 @@ func (am *ForkedAMSvc) getService(ctx *models.ReqContext) (AlertmanagerApiServic
 	}
 }
 
-func (am *ForkedAMSvc) RouteCreateSilence(ctx *models.ReqContext, body apimodels.SilenceBody) response.Response {
+func (am *ForkedAMSvc) RouteCreateSilence(ctx *models.ReqContext, body apimodels.CreateSilenceParams) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
 		return response.Error(400, err.Error(), nil)

pkg/services/ngalert/api/lotex_am.go

@@ -32,7 +32,7 @@ func NewLotexAM(proxy *AlertingProxy, log log.Logger) *LotexAM {
 	}
 }
 
-func (am *LotexAM) RouteCreateSilence(ctx *models.ReqContext, silenceBody apimodels.SilenceBody) response.Response {
+func (am *LotexAM) RouteCreateSilence(ctx *models.ReqContext, silenceBody apimodels.CreateSilenceParams) response.Response {
 	blob, err := json.Marshal(silenceBody)
 	if err != nil {
 		return response.Error(500, "Failed marshal silence", err)

pkg/services/ngalert/notifier/alertmanager.go

@@ -3,7 +3,6 @@ package notifier
 import (
 	"context"
 	"path/filepath"
-	"sort"
 	"sync"
 	"time"
 
@@ -14,9 +13,7 @@ import (
 	"github.com/prometheus/alertmanager/nflog"
 	"github.com/prometheus/alertmanager/nflog/nflogpb"
 	"github.com/prometheus/alertmanager/notify"
-	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/alertmanager/silence"
-	"github.com/prometheus/alertmanager/silence/silencepb"
 	"github.com/prometheus/alertmanager/template"
 	"github.com/prometheus/alertmanager/types"
 	"github.com/prometheus/client_golang/prometheus"
@@ -33,6 +30,8 @@ import (
 
 const (
 	workingDir = "alerting"
+	// How long should we keep silences and notification entries on-disk after they've served their purpose.
+	retentionNotificationsAndSilences = 5 * 24 * time.Hour
 )
 
 type Alertmanager struct {
@@ -74,15 +73,15 @@ func (am *Alertmanager) Init() (err error) {
 	am.Store = store.DBstore{SQLStore: am.SQLStore}
 
 	am.notificationLog, err = nflog.New(
-		nflog.WithRetention(time.Hour*24),                         //TODO: This is a setting.
-		nflog.WithSnapshot(filepath.Join("dir", "notifications")), //TODO: This should be a setting
+		nflog.WithRetention(retentionNotificationsAndSilences),
+		nflog.WithSnapshot(filepath.Join(am.WorkingDirPath(), "notifications")),
 	)
 	if err != nil {
 		return errors.Wrap(err, "unable to initialize the notification log component of alerting")
 	}
 	am.silences, err = silence.New(silence.Options{
-		SnapshotFile: filepath.Join("dir", "silences"), //TODO: This is a setting
-		Retention:    time.Hour * 24,                   //TODO: This is also a setting
+		SnapshotFile: filepath.Join(am.WorkingDirPath(), "silences"),
+		Retention:    retentionNotificationsAndSilences,
 	})
 	if err != nil {
 		return errors.Wrap(err, "unable to initialize the silencing component of alerting")
@@ -251,66 +250,6 @@ func (am *Alertmanager) PutAlerts(alerts ...*PostableAlert) error {
 	return am.alerts.PutPostableAlert(alerts...)
 }
 
-func (am *Alertmanager) ListSilences(matchers []*labels.Matcher) ([]types.Silence, error) {
-	pbsilences, _, err := am.silences.Query()
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to query for the list of silences")
-	}
-	r := []types.Silence{}
-	for _, pbs := range pbsilences {
-		s, err := silenceFromProto(pbs)
-		if err != nil {
-			return nil, errors.Wrap(err, "unable to marshal silence")
-		}
-
-		sms := make(map[string]string)
-		for _, m := range s.Matchers {
-			sms[m.Name] = m.Value
-		}
-
-		if !matchFilterLabels(matchers, sms) {
-			continue
-		}
-
-		r = append(r, *s)
-	}
-
-	var active, pending, expired []types.Silence
-	for _, s := range r {
-		switch s.Status.State {
-		case types.SilenceStateActive:
-			active = append(active, s)
-		case types.SilenceStatePending:
-			pending = append(pending, s)
-		case types.SilenceStateExpired:
-			expired = append(expired, s)
-		}
-	}
-
-	sort.Slice(active, func(i int, j int) bool {
-		return active[i].EndsAt.Before(active[j].EndsAt)
-	})
-	sort.Slice(pending, func(i int, j int) bool {
-		return pending[i].EndsAt.Before(pending[j].EndsAt)
-	})
-	sort.Slice(expired, func(i int, j int) bool {
-		return expired[i].EndsAt.After(expired[j].EndsAt)
-	})
-
-	// Initialize silences explicitly to an empty list (instead of nil)
-	// So that it does not get converted to "null" in JSON.
-	silences := []types.Silence{}
-	silences = append(silences, active...)
-	silences = append(silences, pending...)
-	silences = append(silences, expired...)
-
-	return silences, nil
-}
-
-func (am *Alertmanager) GetSilence(silence *types.Silence)    {}
-func (am *Alertmanager) CreateSilence(silence *types.Silence) {}
-func (am *Alertmanager) DeleteSilence(silence *types.Silence) {}
-
 // createReceiverStage creates a pipeline of stages for a receiver.
 func (am *Alertmanager) createReceiverStage(name string, integrations []notify.Integration, wait func() time.Duration, notificationLog notify.NotificationLog) notify.Stage {
 	var fs notify.FanoutStage
@@ -343,63 +282,3 @@ func timeoutFunc(d time.Duration) time.Duration {
 	}
 	return d + waitFunc()
 }
-
-// copied from the Alertmanager
-func silenceFromProto(s *silencepb.Silence) (*types.Silence, error) {
-	sil := &types.Silence{
-		ID:        s.Id,
-		StartsAt:  s.StartsAt,
-		EndsAt:    s.EndsAt,
-		UpdatedAt: s.UpdatedAt,
-		Status: types.SilenceStatus{
-			State: types.CalcSilenceState(s.StartsAt, s.EndsAt),
-		},
-		Comment:   s.Comment,
-		CreatedBy: s.CreatedBy,
-	}
-	for _, m := range s.Matchers {
-		var t labels.MatchType
-		switch m.Type {
-		case silencepb.Matcher_EQUAL:
-			t = labels.MatchEqual
-		case silencepb.Matcher_REGEXP:
-			t = labels.MatchRegexp
-		case silencepb.Matcher_NOT_EQUAL:
-			t = labels.MatchNotEqual
-		case silencepb.Matcher_NOT_REGEXP:
-			t = labels.MatchNotRegexp
-		}
-		matcher, err := labels.NewMatcher(t, m.Name, m.Pattern)
-		if err != nil {
-			return nil, err
-		}
-
-		sil.Matchers = append(sil.Matchers, matcher)
-	}
-
-	return sil, nil
-}
-
-func matchFilterLabels(matchers []*labels.Matcher, sms map[string]string) bool {
-	for _, m := range matchers {
-		v, prs := sms[m.Name]
-		switch m.Type {
-		case labels.MatchNotRegexp, labels.MatchNotEqual:
-			if m.Value == "" && prs {
-				continue
-			}
-			if !m.Matches(v) {
-				return false
-			}
-		default:
-			if m.Value == "" && !prs {
-				continue
-			}
-			if !m.Matches(v) {
-				return false
-			}
-		}
-	}
-
-	return true
-}

pkg/services/ngalert/notifier/silences.go

@@ -1 +1,122 @@
 package notifier
+
+import (
+	"fmt"
+	"time"
+
+	apimodels "github.com/grafana/alerting-api/pkg/api"
+	"github.com/pkg/errors"
+	v2 "github.com/prometheus/alertmanager/api/v2"
+	"github.com/prometheus/alertmanager/pkg/labels"
+	"github.com/prometheus/alertmanager/silence"
+)
+
+var (
+	ErrGetSilencesInternal     = errors.New("unable to retrieve silence(s) due to an internal error")
+	ErrDeleteSilenceInternal   = errors.New("unable to delete silence due to an internal error")
+	ErrCreateSilenceBadPayload = errors.New("unable to create silence")
+	ErrListSilencesBadPayload  = errors.New("unable to list silences")
+	ErrSilenceNotFound         = silence.ErrNotFound
+)
+
+// ListSilences retrieves a list of stored silences. It supports a set of labels as filters.
+func (am *Alertmanager) ListSilences(filters []string) (apimodels.GettableSilences, error) {
+	matchers := []*labels.Matcher{}
+	for _, matcherString := range filters {
+		matcher, err := labels.ParseMatcher(matcherString)
+		if err != nil {
+			am.logger.Error("failed to parse matcher", "err", err, "matcher", matcherString)
+			return nil, errors.Wrap(ErrListSilencesBadPayload, err.Error())
+		}
+
+		matchers = append(matchers, matcher)
+	}
+
+	psils, _, err := am.silences.Query()
+	if err != nil {
+		am.logger.Error(ErrGetSilencesInternal.Error(), "err", err)
+		return nil, errors.Wrap(ErrGetSilencesInternal, err.Error())
+	}
+
+	sils := apimodels.GettableSilences{}
+	for _, ps := range psils {
+		if !v2.CheckSilenceMatchesFilterLabels(ps, matchers) {
+			continue
+		}
+		silence, err := v2.GettableSilenceFromProto(ps)
+		if err != nil {
+			am.logger.Error("unmarshaling from protobuf failed", "err", err)
+			return apimodels.GettableSilences{}, errors.Wrap(ErrGetSilencesInternal, fmt.Sprintf("failed to convert internal silence to API silence: %v", err.Error()))
+		}
+		sils = append(sils, &silence)
+	}
+
+	v2.SortSilences(sils)
+
+	return sils, nil
+}
+
+// GetSilence retrieves a silence by the provided silenceID. It returns ErrSilenceNotFound if the silence is not present.
+func (am *Alertmanager) GetSilence(silenceID string) (apimodels.GettableSilence, error) {
+	sils, _, err := am.silences.Query(silence.QIDs(silenceID))
+	if err != nil {
+		return apimodels.GettableSilence{}, errors.Wrap(ErrGetSilencesInternal, err.Error())
+	}
+
+	if len(sils) == 0 {
+		am.logger.Error("failed to find silence", "err", err, "id", sils)
+		return apimodels.GettableSilence{}, ErrSilenceNotFound
+	}
+
+	sil, err := v2.GettableSilenceFromProto(sils[0])
+	if err != nil {
+		am.logger.Error("unmarshaling from protobuf failed", "err", err)
+		return apimodels.GettableSilence{}, errors.Wrap(ErrGetSilencesInternal, fmt.Sprintf("failed to convert internal silence to API silence: %v", err.Error()))
+	}
+
+	return sil, nil
+}
+
+// CreateSilence persists the provided silence and returns the silence ID if successful.
+func (am *Alertmanager) CreateSilence(ps *apimodels.PostableSilence) (string, error) {
+	sil, err := v2.PostableSilenceToProto(ps)
+	if err != nil {
+		am.logger.Error("marshaling to protobuf failed", "err", err)
+		return "", errors.Wrap(ErrCreateSilenceBadPayload, fmt.Sprintf("failed to convert API silence to internal silence: %v", err.Error()))
+	}
+
+	if sil.StartsAt.After(sil.EndsAt) || sil.StartsAt.Equal(sil.EndsAt) {
+		msg := "start time must be before end time"
+		am.logger.Error(msg, "err", "starts_at", sil.StartsAt, "ends_at", sil.EndsAt)
+		return "", errors.Wrap(ErrCreateSilenceBadPayload, msg)
+	}
+
+	if sil.EndsAt.Before(time.Now()) {
+		msg := "end time can't be in the past"
+		am.logger.Error(msg, "ends_at", sil.EndsAt)
+		return "", errors.Wrap(ErrCreateSilenceBadPayload, msg)
+	}
+
+	silenceID, err := am.silences.Set(sil)
+	if err != nil {
+		am.logger.Error("msg", "unable to save silence", "err", err)
+		if errors.Is(err, silence.ErrNotFound) {
+			return "", ErrSilenceNotFound
+		}
+		return "", errors.Wrap(ErrCreateSilenceBadPayload, fmt.Sprintf("unable to save silence: %v", err.Error()))
+	}
+
+	return silenceID, nil
+}
+
+// DeleteSilence looks for and expires the silence by the provided silenceID. It returns ErrSilenceNotFound if the silence is not present.
+func (am *Alertmanager) DeleteSilence(silenceID string) error {
+	if err := am.silences.Expire(silenceID); err != nil {
+		if errors.Is(err, silence.ErrNotFound) {
+			return ErrSilenceNotFound
+		}
+		return errors.Wrap(ErrDeleteSilenceInternal, err.Error())
+	}
+
+	return nil
+}