Contact point testing (#37308)

This commit adds contact point testing to ngalerts via a new API
endpoint. This endpoint accepts JSON containing a list of
receiver configurations which are validated and then tested
with a notification for a test alert. The endpoint returns JSON
for each receiver with a status and error message. It accepts
a configurable timeout via the Request-Timeout header (in seconds)
up to a maximum of 30 seconds.

pkg/services/ngalert/api/api.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"net/url"
 	"time"
 
@@ -10,6 +11,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/datasources"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -43,6 +45,9 @@ type Alertmanager interface {
 	// Alerts
 	GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
 	GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error)
+
+	// Testing
+	TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigParams) (*notifier.TestReceiversResult, error)
 }
 
 // API handlers.

pkg/services/ngalert/api/api_alertmanager.go

@@ -1,9 +1,13 @@
 package api
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -15,12 +19,79 @@ import (
 	"github.com/grafana/grafana/pkg/util"
 )
 
+const (
+	defaultTestReceiversTimeout = 15 * time.Second
+	maxTestReceiversTimeout     = 30 * time.Second
+)
+
 type AlertmanagerSrv struct {
 	am    Alertmanager
 	store store.AlertingStore
 	log   log.Logger
 }
 
+type UnknownReceiverError struct {
+	UID string
+}
+
+func (e UnknownReceiverError) Error() string {
+	return fmt.Sprintf("unknown receiver: %s", e.UID)
+}
+
+func (srv AlertmanagerSrv) loadSecureSettings(orgId int64, receivers []*apimodels.PostableApiReceiver) error {
+	// Get the last known working configuration
+	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
+	if err := srv.store.GetLatestAlertmanagerConfiguration(&query); err != nil {
+		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
+		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
+			return fmt.Errorf("failed to get latest configuration: %w", err)
+		}
+	}
+
+	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
+	if query.Result != nil {
+		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
+		if err != nil {
+			return fmt.Errorf("failed to load latest configuration: %w", err)
+		}
+		currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
+	}
+
+	// Copy the previously known secure settings
+	for i, r := range receivers {
+		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			if gr.UID == "" { // new receiver
+				continue
+			}
+
+			cgmr, ok := currentReceiverMap[gr.UID]
+			if !ok {
+				// it tries to update a receiver that didn't previously exist
+				return UnknownReceiverError{UID: gr.UID}
+			}
+
+			// frontend sends only the secure settings that have to be updated
+			// therefore we have to copy from the last configuration only those secure settings not included in the request
+			for key := range cgmr.SecureSettings {
+				_, ok := gr.SecureSettings[key]
+				if !ok {
+					decryptedValue, err := cgmr.GetDecryptedSecret(key)
+					if err != nil {
+						return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
+					}
+
+					if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
+						receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
+					}
+
+					receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
 	return response.JSON(http.StatusOK, srv.am.GetStatus())
 }
@@ -210,46 +281,12 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 		}
 	}
 
-	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
-	if query.Result != nil {
-		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
-		if err != nil {
-			return ErrResp(http.StatusInternalServerError, err, "failed to load lastest configuration")
-		}
-		currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
-	}
-
-	// Copy the previously known secure settings
-	for i, r := range body.AlertmanagerConfig.Receivers {
-		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
-			if gr.UID == "" { // new receiver
-				continue
-			}
-
-			cgmr, ok := currentReceiverMap[gr.UID]
-			if !ok {
-				// it tries to update a receiver that didn't previously exist
-				return ErrResp(http.StatusBadRequest, fmt.Errorf("unknown receiver: %s", gr.UID), "")
-			}
-
-			// frontend sends only the secure settings that have to be updated
-			// therefore we have to copy from the last configuration only those secure settings not included in the request
-			for key := range cgmr.SecureSettings {
-				_, ok := body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key]
-				if !ok {
-					decryptedValue, err := cgmr.GetDecryptedSecret(key)
-					if err != nil {
-						return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", key)
-					}
-
-					if body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
-						body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
-					}
-
-					body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
-				}
-			}
+	if err := srv.loadSecureSettings(c.OrgId, body.AlertmanagerConfig.Receivers); err != nil {
+		var unknownReceiverError UnknownReceiverError
+		if errors.As(err, &unknownReceiverError) {
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	if err := body.ProcessConfig(); err != nil {
@@ -265,6 +302,130 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 }
 
 func (srv AlertmanagerSrv) RoutePostAMAlerts(c *models.ReqContext, body apimodels.PostableAlerts) response.Response {
-	// not implemented
 	return NotImplementedResp
 }
+
+func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigParams) response.Response {
+	if !c.HasUserRole(models.ROLE_EDITOR) {
+		return accessForbiddenResp()
+	}
+
+	if err := srv.loadSecureSettings(c.OrgId, body.Receivers); err != nil {
+		var unknownReceiverError UnknownReceiverError
+		if errors.As(err, &unknownReceiverError) {
+			return ErrResp(http.StatusBadRequest, err, "")
+		}
+		return ErrResp(http.StatusInternalServerError, err, "")
+	}
+
+	if err := body.ProcessConfig(); err != nil {
+		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
+	}
+
+	ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+		c.Req.Context(),
+		c.Req.Request,
+		defaultTestReceiversTimeout,
+		maxTestReceiversTimeout)
+	if err != nil {
+		return ErrResp(http.StatusBadRequest, err, "")
+	}
+	defer cancelFunc()
+
+	result, err := srv.am.TestReceivers(ctx, body)
+	if err != nil {
+		if errors.Is(err, notifier.ErrNoReceivers) {
+			return response.Error(http.StatusBadRequest, "", err)
+		}
+		return response.Error(http.StatusInternalServerError, "", err)
+	}
+
+	return response.JSON(statusForTestReceivers(result.Receivers), newTestReceiversResult(result))
+}
+
+// contextWithTimeoutFromRequest returns a context with a deadline set from the
+// Request-Timeout header in the HTTP request. If the header is absent then the
+// context will use the default timeout. The timeout in the Request-Timeout
+// header cannot exceed the maximum timeout.
+func contextWithTimeoutFromRequest(ctx context.Context, r *http.Request, defaultTimeout, maxTimeout time.Duration) (context.Context, context.CancelFunc, error) {
+	timeout := defaultTimeout
+	if s := strings.TrimSpace(r.Header.Get("Request-Timeout")); s != "" {
+		// the timeout is measured in seconds
+		v, err := strconv.ParseInt(s, 10, 16)
+		if err != nil {
+			return nil, nil, err
+		}
+		if d := time.Duration(v) * time.Second; d < maxTimeout {
+			timeout = d
+		} else {
+			return nil, nil, fmt.Errorf("exceeded maximum timeout of %d seconds", maxTimeout)
+		}
+	}
+	ctx, cancelFunc := context.WithTimeout(ctx, timeout)
+	return ctx, cancelFunc, nil
+}
+
+func newTestReceiversResult(r *notifier.TestReceiversResult) apimodels.TestReceiversResult {
+	v := apimodels.TestReceiversResult{
+		Receivers: make([]apimodels.TestReceiverResult, len(r.Receivers)),
+		NotifedAt: r.NotifedAt,
+	}
+	for ix, next := range r.Receivers {
+		configs := make([]apimodels.TestReceiverConfigResult, len(next.Configs))
+		for jx, config := range next.Configs {
+			configs[jx].Name = config.Name
+			configs[jx].UID = config.UID
+			configs[jx].Status = config.Status
+			if config.Error != nil {
+				configs[jx].Error = config.Error.Error()
+			}
+		}
+		v.Receivers[ix].Configs = configs
+		v.Receivers[ix].Name = next.Name
+	}
+	return v
+}
+
+// statusForTestReceivers returns the appropriate status code for the response
+// for the results.
+//
+// It returns an HTTP 200 OK status code if notifications were sent to all receivers,
+// an HTTP 400 Bad Request status code if all receivers contain invalid configuration,
+// an HTTP 408 Request Timeout status code if all receivers timed out when sending
+// a test notification or an HTTP 207 Multi Status.
+func statusForTestReceivers(v []notifier.TestReceiverResult) int {
+	var (
+		numBadRequests   int
+		numTimeouts      int
+		numUnknownErrors int
+	)
+	for _, receiver := range v {
+		for _, next := range receiver.Configs {
+			if next.Error != nil {
+				var (
+					invalidReceiverErr notifier.InvalidReceiverError
+					receiverTimeoutErr notifier.ReceiverTimeoutError
+				)
+				if errors.As(next.Error, &invalidReceiverErr) {
+					numBadRequests += 1
+				} else if errors.As(next.Error, &receiverTimeoutErr) {
+					numTimeouts += 1
+				} else {
+					numUnknownErrors += 1
+				}
+			}
+		}
+	}
+	if numBadRequests == len(v) {
+		// if all receivers contain invalid configuration
+		return http.StatusBadRequest
+	} else if numTimeouts == len(v) {
+		// if all receivers contain valid configuration but timed out
+		return http.StatusRequestTimeout
+	} else if numBadRequests+numTimeouts+numUnknownErrors > 0 {
+		return http.StatusMultiStatus
+	} else {
+		// all receivers were sent a notification without error
+		return http.StatusOK
+	}
+}

pkg/services/ngalert/api/api_alertmanager_test.go

@@ -0,0 +1,140 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
+	"github.com/stretchr/testify/require"
+)
+
+func TestContextWithTimeoutFromRequest(t *testing.T) {
+	t.Run("assert context has default timeout when header is absent", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://grafana.net", nil)
+		require.NoError(t, err)
+
+		now := time.Now()
+		ctx := context.Background()
+		ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+			ctx,
+			req,
+			15*time.Second,
+			30*time.Second)
+		require.NoError(t, err)
+		require.NotNil(t, cancelFunc)
+		require.NotNil(t, ctx)
+
+		deadline, ok := ctx.Deadline()
+		require.True(t, ok)
+		require.True(t, deadline.After(now))
+		require.Less(t, deadline.Sub(now).Seconds(), 30.0)
+		require.GreaterOrEqual(t, deadline.Sub(now).Seconds(), 15.0)
+	})
+
+	t.Run("assert context has timeout in request header", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://grafana.net", nil)
+		require.NoError(t, err)
+		req.Header.Set("Request-Timeout", "5")
+
+		now := time.Now()
+		ctx := context.Background()
+		ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+			ctx,
+			req,
+			15*time.Second,
+			30*time.Second)
+		require.NoError(t, err)
+		require.NotNil(t, cancelFunc)
+		require.NotNil(t, ctx)
+
+		deadline, ok := ctx.Deadline()
+		require.True(t, ok)
+		require.True(t, deadline.After(now))
+		require.Less(t, deadline.Sub(now).Seconds(), 15.0)
+		require.GreaterOrEqual(t, deadline.Sub(now).Seconds(), 5.0)
+	})
+
+	t.Run("assert timeout in request header cannot exceed max timeout", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://grafana.net", nil)
+		require.NoError(t, err)
+		req.Header.Set("Request-Timeout", "60")
+
+		ctx := context.Background()
+		ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+			ctx,
+			req,
+			15*time.Second,
+			30*time.Second)
+		require.Error(t, err, "exceeded maximum timeout")
+		require.Nil(t, cancelFunc)
+		require.Nil(t, ctx)
+	})
+}
+
+func TestStatusForTestReceivers(t *testing.T) {
+	t.Run("assert HTTP 400 Status Bad Request for no receivers", func(t *testing.T) {
+		require.Equal(t, http.StatusBadRequest, statusForTestReceivers([]notifier.TestReceiverResult{}))
+	})
+
+	t.Run("assert HTTP 400 Bad Request when all invalid receivers", func(t *testing.T) {
+		require.Equal(t, http.StatusBadRequest, statusForTestReceivers([]notifier.TestReceiverResult{{
+			Name: "test1",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test1",
+				UID:    "uid1",
+				Status: "failed",
+				Error:  notifier.InvalidReceiverError{},
+			}},
+		}, {
+			Name: "test2",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test2",
+				UID:    "uid2",
+				Status: "failed",
+				Error:  notifier.InvalidReceiverError{},
+			}},
+		}}))
+	})
+
+	t.Run("assert HTTP 408 Request Timeout when all receivers timed out", func(t *testing.T) {
+		require.Equal(t, http.StatusRequestTimeout, statusForTestReceivers([]notifier.TestReceiverResult{{
+			Name: "test1",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test1",
+				UID:    "uid1",
+				Status: "failed",
+				Error:  notifier.ReceiverTimeoutError{},
+			}},
+		}, {
+			Name: "test2",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test2",
+				UID:    "uid2",
+				Status: "failed",
+				Error:  notifier.ReceiverTimeoutError{},
+			}},
+		}}))
+	})
+
+	t.Run("assert 207 Multi Status for different errors", func(t *testing.T) {
+		require.Equal(t, http.StatusMultiStatus, statusForTestReceivers([]notifier.TestReceiverResult{{
+			Name: "test1",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test1",
+				UID:    "uid1",
+				Status: "failed",
+				Error:  notifier.InvalidReceiverError{},
+			}},
+		}, {
+			Name: "test2",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test2",
+				UID:    "uid2",
+				Status: "failed",
+				Error:  notifier.ReceiverTimeoutError{},
+			}},
+		}}))
+	})
+}

pkg/services/ngalert/api/forked_am.go

@@ -146,3 +146,12 @@ func (am *ForkedAMSvc) RoutePostAMAlerts(ctx *models.ReqContext, body apimodels.
 
 	return s.RoutePostAMAlerts(ctx, body)
 }
+
+func (am *ForkedAMSvc) RoutePostTestReceivers(ctx *models.ReqContext, body apimodels.TestReceiversConfigParams) response.Response {
+	s, err := am.getService(ctx)
+	if err != nil {
+		return ErrResp(400, err, "")
+	}
+
+	return s.RoutePostTestReceivers(ctx, body)
+}

pkg/services/ngalert/api/generated_base_api_alertmanager.go

@@ -31,6 +31,7 @@ type AlertmanagerApiService interface {
 	RouteGetSilences(*models.ReqContext) response.Response
 	RoutePostAMAlerts(*models.ReqContext, apimodels.PostableAlerts) response.Response
 	RoutePostAlertingConfig(*models.ReqContext, apimodels.PostableUserConfig) response.Response
+	RoutePostTestReceivers(*models.ReqContext, apimodels.TestReceiversConfigParams) response.Response
 }
 
 func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, m *metrics.Metrics) {
@@ -137,5 +138,15 @@ func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, m *
 				m,
 			),
 		)
+		group.Post(
+			toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/receivers/test"),
+			binding.Bind(apimodels.TestReceiversConfigParams{}),
+			metrics.Instrument(
+				http.MethodPost,
+				"/api/alertmanager/{Recipient}/config/api/v1/receivers/test",
+				srv.RoutePostTestReceivers,
+				m,
+			),
+		)
 	}, middleware.ReqSignedIn)
 }

pkg/services/ngalert/api/lotex_am.go

@@ -192,3 +192,7 @@ func (am *LotexAM) RoutePostAMAlerts(ctx *models.ReqContext, alerts apimodels.Po
 		nil,
 	)
 }
+
+func (am *LotexAM) RoutePostTestReceivers(ctx *models.ReqContext, config apimodels.TestReceiversConfigParams) response.Response {
+	return NotImplementedResp
+}

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"time"
 
 	"github.com/go-openapi/strfmt"
 	"github.com/pkg/errors"
@@ -73,6 +74,17 @@ import (
 //       200: alertGroups
 //       400: ValidationError
 
+// swagger:route POST /api/alertmanager/{Recipient}/config/api/v1/receivers/test alertmanager RoutePostTestReceivers
+//
+// Test Grafana managed receivers without saving them.
+//
+//     Responses:
+//
+//       200: Ack
+//       207: MultiStatus
+//       400: ValidationError
+//       408: Failure
+
 // swagger:route GET /api/alertmanager/{Recipient}/api/v2/silences alertmanager RouteGetSilences
 //
 // get silences
@@ -105,6 +117,40 @@ import (
 //       200: Ack
 //       400: ValidationError
 
+// swagger:model
+type TestReceiversConfig struct {
+	Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
+}
+
+// swagger:parameters RoutePostTestReceivers
+type TestReceiversConfigParams struct {
+	Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
+}
+
+func (c *TestReceiversConfigParams) ProcessConfig() error {
+	return processReceiverConfigs(c.Receivers)
+}
+
+// swagger:model
+type TestReceiversResult struct {
+	Receivers []TestReceiverResult `json:"receivers"`
+	NotifedAt time.Time            `json:"notified_at"`
+}
+
+// swagger:model
+type TestReceiverResult struct {
+	Name    string                     `json:"name"`
+	Configs []TestReceiverConfigResult `json:"grafana_managed_receiver_configs"`
+}
+
+// swagger:model
+type TestReceiverConfigResult struct {
+	Name   string `json:"name"`
+	UID    string `json:"uid"`
+	Status string `json:"status"`
+	Error  string `json:"error,omitempty"`
+}
+
 // swagger:parameters RouteCreateSilence
 type CreateSilenceParams struct {
 	// in:body
@@ -345,39 +391,7 @@ func (c *PostableUserConfig) GetGrafanaReceiverMap() map[string]*PostableGrafana
 
 // ProcessConfig parses grafana receivers, encrypts secrets and assigns UUIDs (if they are missing)
 func (c *PostableUserConfig) ProcessConfig() error {
-	seenUIDs := make(map[string]struct{})
-	// encrypt secure settings for storing them in DB
-	for _, r := range c.AlertmanagerConfig.Receivers {
-		switch r.Type() {
-		case GrafanaReceiverType:
-			for _, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
-				for k, v := range gr.SecureSettings {
-					encryptedData, err := util.Encrypt([]byte(v), setting.SecretKey)
-					if err != nil {
-						return fmt.Errorf("failed to encrypt secure settings: %w", err)
-					}
-					gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(encryptedData)
-				}
-				if gr.UID == "" {
-					retries := 5
-					for i := 0; i < retries; i++ {
-						gen := util.GenerateShortUID()
-						_, ok := seenUIDs[gen]
-						if !ok {
-							gr.UID = gen
-							break
-						}
-					}
-					if gr.UID == "" {
-						return fmt.Errorf("all %d attempts to generate UID for receiver have failed; please retry", retries)
-					}
-				}
-				seenUIDs[gr.UID] = struct{}{}
-			}
-		default:
-		}
-	}
-	return nil
+	return processReceiverConfigs(c.AlertmanagerConfig.Receivers)
 }
 
 // MarshalYAML implements yaml.Marshaller.
@@ -911,3 +925,39 @@ type GettableGrafanaReceivers struct {
 type PostableGrafanaReceivers struct {
 	GrafanaManagedReceivers []*PostableGrafanaReceiver `yaml:"grafana_managed_receiver_configs,omitempty" json:"grafana_managed_receiver_configs,omitempty"`
 }
+
+func processReceiverConfigs(c []*PostableApiReceiver) error {
+	seenUIDs := make(map[string]struct{})
+	// encrypt secure settings for storing them in DB
+	for _, r := range c {
+		switch r.Type() {
+		case GrafanaReceiverType:
+			for _, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+				for k, v := range gr.SecureSettings {
+					encryptedData, err := util.Encrypt([]byte(v), setting.SecretKey)
+					if err != nil {
+						return fmt.Errorf("failed to encrypt secure settings: %w", err)
+					}
+					gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(encryptedData)
+				}
+				if gr.UID == "" {
+					retries := 5
+					for i := 0; i < retries; i++ {
+						gen := util.GenerateShortUID()
+						_, ok := seenUIDs[gen]
+						if !ok {
+							gr.UID = gen
+							break
+						}
+					}
+					if gr.UID == "" {
+						return fmt.Errorf("all %d attempts to generate UID for receiver have failed; please retry", retries)
+					}
+				}
+				seenUIDs[gr.UID] = struct{}{}
+			}
+		default:
+		}
+	}
+	return nil
+}

pkg/services/ngalert/api/tooling/spec.json

@@ -485,6 +485,49 @@
         }
       }
     },
+    "/api/alertmanager/{Recipient}/config/api/v1/receivers/test": {
+      "post": {
+        "tags": [
+          "alertmanager"
+        ],
+        "summary": "Test Grafana managed receivers without saving them.",
+        "operationId": "RoutePostTestReceivers",
+        "parameters": [
+          {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/PostableApiReceiver"
+            },
+            "x-go-name": "Receivers",
+            "name": "receivers",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Ack",
+            "schema": {
+              "$ref": "#/definitions/Ack"
+            }
+          },
+          "207": {
+            "$ref": "#/responses/MultiStatus"
+          },
+          "400": {
+            "description": "ValidationError",
+            "schema": {
+              "$ref": "#/definitions/ValidationError"
+            }
+          },
+          "408": {
+            "description": "Failure",
+            "schema": {
+              "$ref": "#/definitions/Failure"
+            }
+          }
+        }
+      }
+    },
     "/api/prometheus/{Recipient}/api/v1/alerts": {
       "get": {
         "description": "gets the current alerts",
@@ -1707,6 +1750,7 @@
           "enum": [
             "Alerting"
           ],
+          "x-go-enum-desc": "Alerting AlertingErrState",
           "x-go-name": "ExecErrState"
         },
         "id": {
@@ -1735,6 +1779,7 @@
             "NoData",
             "OK"
           ],
+          "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
           "x-go-name": "NoDataState"
         },
         "orgId": {
@@ -2547,6 +2592,7 @@
           "enum": [
             "Alerting"
           ],
+          "x-go-enum-desc": "Alerting AlertingErrState",
           "x-go-name": "ExecErrState"
         },
         "no_data_state": {
@@ -2556,6 +2602,7 @@
             "NoData",
             "OK"
           ],
+          "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
           "x-go-name": "NoDataState"
         },
         "title": {
@@ -3229,6 +3276,76 @@
       },
       "x-go-package": "github.com/prometheus/common/config"
     },
+    "TestReceiverConfigResult": {
+      "type": "object",
+      "properties": {
+        "error": {
+          "type": "string",
+          "x-go-name": "Error"
+        },
+        "name": {
+          "type": "string",
+          "x-go-name": "Name"
+        },
+        "status": {
+          "type": "string",
+          "x-go-name": "Status"
+        },
+        "uid": {
+          "type": "string",
+          "x-go-name": "UID"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "TestReceiverResult": {
+      "type": "object",
+      "properties": {
+        "grafana_managed_receiver_configs": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TestReceiverConfigResult"
+          },
+          "x-go-name": "Configs"
+        },
+        "name": {
+          "type": "string",
+          "x-go-name": "Name"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "TestReceiversConfig": {
+      "type": "object",
+      "properties": {
+        "receivers": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/PostableApiReceiver"
+          },
+          "x-go-name": "Receivers"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "TestReceiversResult": {
+      "type": "object",
+      "properties": {
+        "notified_at": {
+          "type": "string",
+          "format": "date-time",
+          "x-go-name": "NotifedAt"
+        },
+        "receivers": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TestReceiverResult"
+          },
+          "x-go-name": "Receivers"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
     "TestRulePayload": {
       "type": "object",
       "properties": {
@@ -3483,11 +3600,12 @@
       "$ref": "#/definitions/alertGroup"
     },
     "alertGroups": {
-      "description": "AlertGroups alert groups",
       "type": "array",
       "items": {
         "$ref": "#/definitions/alertGroup"
       },
+      "x-go-name": "AlertGroups",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/alertGroups"
     },
     "alertStatus": {
@@ -3672,16 +3790,14 @@
       "$ref": "#/definitions/gettableAlert"
     },
     "gettableAlerts": {
+      "description": "GettableAlerts gettable alerts",
       "type": "array",
       "items": {
         "$ref": "#/definitions/gettableAlert"
       },
-      "x-go-name": "GettableAlerts",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/gettableAlerts"
     },
     "gettableSilence": {
-      "description": "GettableSilence gettable silence",
       "type": "object",
       "required": [
         "comment",
@@ -3734,6 +3850,8 @@
           "x-go-name": "UpdatedAt"
         }
       },
+      "x-go-name": "GettableSilence",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/gettableSilence"
     },
     "gettableSilences": {
@@ -3872,6 +3990,7 @@
       "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
     },
     "postableSilence": {
+      "description": "PostableSilence postable silence",
       "type": "object",
       "required": [
         "comment",
@@ -3912,8 +4031,6 @@
           "x-go-name": "StartsAt"
         }
       },
-      "x-go-name": "PostableSilence",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/postableSilence"
     },
     "receiver": {