Contact point testing (#37308)

This commit adds contact point testing to ngalerts via a new API endpoint. This endpoint accepts JSON containing a list of receiver configurations which are validated and then tested with a notification for a test alert. The endpoint returns JSON for each receiver with a status and error message. It accepts a configurable timeout via the Request-Timeout header (in seconds) up to a maximum of 30 seconds.
2021-08-17 13:49:05 +01:00
parent afabc617ed
commit 3ca00f90b5
15 changed files with 1348 additions and 162 deletions
@@ -1,9 +1,13 @@
 package api

 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
+	"strings"
+	"time"

 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -15,12 +19,79 @@ import (
 	"github.com/grafana/grafana/pkg/util"
 )

+const (
+	defaultTestReceiversTimeout = 15 * time.Second
+	maxTestReceiversTimeout     = 30 * time.Second
+)
+
 type AlertmanagerSrv struct {
 	am    Alertmanager
 	store store.AlertingStore
 	log   log.Logger
 }

+type UnknownReceiverError struct {
+	UID string
+}
+
+func (e UnknownReceiverError) Error() string {
+	return fmt.Sprintf("unknown receiver: %s", e.UID)
+}
+
+func (srv AlertmanagerSrv) loadSecureSettings(orgId int64, receivers []*apimodels.PostableApiReceiver) error {
+	// Get the last known working configuration
+	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
+	if err := srv.store.GetLatestAlertmanagerConfiguration(&query); err != nil {
+		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
+		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
+			return fmt.Errorf("failed to get latest configuration: %w", err)
+		}
+	}
+
+	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
+	if query.Result != nil {
+		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
+		if err != nil {
+			return fmt.Errorf("failed to load latest configuration: %w", err)
+		}
+		currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
+	}
+
+	// Copy the previously known secure settings
+	for i, r := range receivers {
+		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			if gr.UID == "" { // new receiver
+				continue
+			}
+
+			cgmr, ok := currentReceiverMap[gr.UID]
+			if !ok {
+				// it tries to update a receiver that didn't previously exist
+				return UnknownReceiverError{UID: gr.UID}
+			}
+
+			// frontend sends only the secure settings that have to be updated
+			// therefore we have to copy from the last configuration only those secure settings not included in the request
+			for key := range cgmr.SecureSettings {
+				_, ok := gr.SecureSettings[key]
+				if !ok {
+					decryptedValue, err := cgmr.GetDecryptedSecret(key)
+					if err != nil {
+						return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
+					}
+
+					if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
+						receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
+					}
+
+					receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
 	return response.JSON(http.StatusOK, srv.am.GetStatus())
 }
@@ -210,46 +281,12 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 		}
 	}

-	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
-	if query.Result != nil {
-		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
-		if err != nil {
-			return ErrResp(http.StatusInternalServerError, err, "failed to load lastest configuration")
-		}
-		currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
-	}
-
-	// Copy the previously known secure settings
-	for i, r := range body.AlertmanagerConfig.Receivers {
-		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
-			if gr.UID == "" { // new receiver
-				continue
-			}
-
-			cgmr, ok := currentReceiverMap[gr.UID]
-			if !ok {
-				// it tries to update a receiver that didn't previously exist
-				return ErrResp(http.StatusBadRequest, fmt.Errorf("unknown receiver: %s", gr.UID), "")
-			}
-
-			// frontend sends only the secure settings that have to be updated
-			// therefore we have to copy from the last configuration only those secure settings not included in the request
-			for key := range cgmr.SecureSettings {
-				_, ok := body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key]
-				if !ok {
-					decryptedValue, err := cgmr.GetDecryptedSecret(key)
-					if err != nil {
-						return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", key)
-					}
-
-					if body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
-						body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
-					}
-
-					body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
-				}
-			}
+	if err := srv.loadSecureSettings(c.OrgId, body.AlertmanagerConfig.Receivers); err != nil {
+		var unknownReceiverError UnknownReceiverError
+		if errors.As(err, &unknownReceiverError) {
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}

 	if err := body.ProcessConfig(); err != nil {
@@ -265,6 +302,130 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 }

 func (srv AlertmanagerSrv) RoutePostAMAlerts(c *models.ReqContext, body apimodels.PostableAlerts) response.Response {
-	// not implemented
 	return NotImplementedResp
 }
+
+func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigParams) response.Response {
+	if !c.HasUserRole(models.ROLE_EDITOR) {
+		return accessForbiddenResp()
+	}
+
+	if err := srv.loadSecureSettings(c.OrgId, body.Receivers); err != nil {
+		var unknownReceiverError UnknownReceiverError
+		if errors.As(err, &unknownReceiverError) {
+			return ErrResp(http.StatusBadRequest, err, "")
+		}
+		return ErrResp(http.StatusInternalServerError, err, "")
+	}
+
+	if err := body.ProcessConfig(); err != nil {
+		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
+	}
+
+	ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+		c.Req.Context(),
+		c.Req.Request,
+		defaultTestReceiversTimeout,
+		maxTestReceiversTimeout)
+	if err != nil {
+		return ErrResp(http.StatusBadRequest, err, "")
+	}
+	defer cancelFunc()
+
+	result, err := srv.am.TestReceivers(ctx, body)
+	if err != nil {
+		if errors.Is(err, notifier.ErrNoReceivers) {
+			return response.Error(http.StatusBadRequest, "", err)
+		}
+		return response.Error(http.StatusInternalServerError, "", err)
+	}
+
+	return response.JSON(statusForTestReceivers(result.Receivers), newTestReceiversResult(result))
+}
+
+// contextWithTimeoutFromRequest returns a context with a deadline set from the
+// Request-Timeout header in the HTTP request. If the header is absent then the
+// context will use the default timeout. The timeout in the Request-Timeout
+// header cannot exceed the maximum timeout.
+func contextWithTimeoutFromRequest(ctx context.Context, r *http.Request, defaultTimeout, maxTimeout time.Duration) (context.Context, context.CancelFunc, error) {
+	timeout := defaultTimeout
+	if s := strings.TrimSpace(r.Header.Get("Request-Timeout")); s != "" {
+		// the timeout is measured in seconds
+		v, err := strconv.ParseInt(s, 10, 16)
+		if err != nil {
+			return nil, nil, err
+		}
+		if d := time.Duration(v) * time.Second; d < maxTimeout {
+			timeout = d
+		} else {
+			return nil, nil, fmt.Errorf("exceeded maximum timeout of %d seconds", maxTimeout)
+		}
+	}
+	ctx, cancelFunc := context.WithTimeout(ctx, timeout)
+	return ctx, cancelFunc, nil
+}
+
+func newTestReceiversResult(r *notifier.TestReceiversResult) apimodels.TestReceiversResult {
+	v := apimodels.TestReceiversResult{
+		Receivers: make([]apimodels.TestReceiverResult, len(r.Receivers)),
+		NotifedAt: r.NotifedAt,
+	}
+	for ix, next := range r.Receivers {
+		configs := make([]apimodels.TestReceiverConfigResult, len(next.Configs))
+		for jx, config := range next.Configs {
+			configs[jx].Name = config.Name
+			configs[jx].UID = config.UID
+			configs[jx].Status = config.Status
+			if config.Error != nil {
+				configs[jx].Error = config.Error.Error()
+			}
+		}
+		v.Receivers[ix].Configs = configs
+		v.Receivers[ix].Name = next.Name
+	}
+	return v
+}
+
+// statusForTestReceivers returns the appropriate status code for the response
+// for the results.
+//
+// It returns an HTTP 200 OK status code if notifications were sent to all receivers,
+// an HTTP 400 Bad Request status code if all receivers contain invalid configuration,
+// an HTTP 408 Request Timeout status code if all receivers timed out when sending
+// a test notification or an HTTP 207 Multi Status.
+func statusForTestReceivers(v []notifier.TestReceiverResult) int {
+	var (
+		numBadRequests   int
+		numTimeouts      int
+		numUnknownErrors int
+	)
+	for _, receiver := range v {
+		for _, next := range receiver.Configs {
+			if next.Error != nil {
+				var (
+					invalidReceiverErr notifier.InvalidReceiverError
+					receiverTimeoutErr notifier.ReceiverTimeoutError
+				)
+				if errors.As(next.Error, &invalidReceiverErr) {
+					numBadRequests += 1
+				} else if errors.As(next.Error, &receiverTimeoutErr) {
+					numTimeouts += 1
+				} else {
+					numUnknownErrors += 1
+				}
+			}
+		}
+	}
+	if numBadRequests == len(v) {
+		// if all receivers contain invalid configuration
+		return http.StatusBadRequest
+	} else if numTimeouts == len(v) {
+		// if all receivers contain valid configuration but timed out
+		return http.StatusRequestTimeout
+	} else if numBadRequests+numTimeouts+numUnknownErrors > 0 {
+		return http.StatusMultiStatus
+	} else {
+		// all receivers were sent a notification without error
+		return http.StatusOK
+	}
+}