Alerting: validate am config changes before persisting (#54908) (#54944)

(cherry picked from commit d88f1c8e35)

Co-authored-by: Jean-Philippe Quéméner <JohnnyQQQQ@users.noreply.github.com>

pkg/services/ngalert/provisioning/contactpoints.go

@@ -189,7 +189,7 @@ func (ecp *ContactPointService) CreateContactPoint(ctx context.Context, orgID in
 	}
 
 	err = ecp.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = ecp.amStore.UpdateAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
+		err = PersistConfig(ctx, ecp.amStore, &models.SaveAlertmanagerConfigurationCmd{
 			AlertmanagerConfiguration: string(data),
 			FetchedConfigurationHash:  revision.concurrencyToken,
 			ConfigurationVersion:      revision.version,
@@ -284,7 +284,7 @@ func (ecp *ContactPointService) UpdateContactPoint(ctx context.Context, orgID in
 		return err
 	}
 	return ecp.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = ecp.amStore.UpdateAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
+		err = PersistConfig(ctx, ecp.amStore, &models.SaveAlertmanagerConfigurationCmd{
 			AlertmanagerConfiguration: string(data),
 			FetchedConfigurationHash:  revision.concurrencyToken,
 			ConfigurationVersion:      revision.version,
@@ -344,7 +344,7 @@ func (ecp *ContactPointService) DeleteContactPoint(ctx context.Context, orgID in
 		if err != nil {
 			return err
 		}
-		return ecp.amStore.UpdateAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
+		return PersistConfig(ctx, ecp.amStore, &models.SaveAlertmanagerConfigurationCmd{
 			AlertmanagerConfiguration: string(data),
 			FetchedConfigurationHash:  revision.concurrencyToken,
 			ConfigurationVersion:      revision.version,

pkg/services/ngalert/provisioning/mute_timings.go

@@ -77,7 +77,7 @@ func (svc *MuteTimingService) CreateMuteTiming(ctx context.Context, mt definitio
 		OrgID:                     orgID,
 	}
 	err = svc.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = svc.config.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err = PersistConfig(ctx, svc.config, &cmd)
 		if err != nil {
 			return err
 		}
@@ -132,7 +132,7 @@ func (svc *MuteTimingService) UpdateMuteTiming(ctx context.Context, mt definitio
 		OrgID:                     orgID,
 	}
 	err = svc.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = svc.config.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err = PersistConfig(ctx, svc.config, &cmd)
 		if err != nil {
 			return err
 		}
@@ -181,7 +181,7 @@ func (svc *MuteTimingService) DeleteMuteTiming(ctx context.Context, name string,
 		OrgID:                     orgID,
 	}
 	return svc.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = svc.config.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err = PersistConfig(ctx, svc.config, &cmd)
 		if err != nil {
 			return err
 		}

pkg/services/ngalert/provisioning/notification_policies.go

@@ -102,7 +102,7 @@ func (nps *NotificationPolicyService) UpdatePolicyTree(ctx context.Context, orgI
 		OrgID:                     orgID,
 	}
 	err = nps.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = nps.amStore.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err = PersistConfig(ctx, nps.amStore, &cmd)
 		if err != nil {
 			return err
 		}
@@ -149,7 +149,7 @@ func (nps *NotificationPolicyService) ResetPolicyTree(ctx context.Context, orgID
 		OrgID:                     orgID,
 	}
 	err = nps.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err := nps.amStore.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err := PersistConfig(ctx, nps.amStore, &cmd)
 		if err != nil {
 			return err
 		}

pkg/services/ngalert/provisioning/persist.go

@@ -2,7 +2,10 @@ package provisioning
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
 	"github.com/grafana/grafana/pkg/services/quota"
@@ -45,3 +48,12 @@ type RuleStore interface {
 type QuotaChecker interface {
 	CheckQuotaReached(ctx context.Context, target string, scopeParams *quota.ScopeParameters) (bool, error)
 }
+
+// PersistConfig validates to config before eventually persisting it if no error occurs
+func PersistConfig(ctx context.Context, store AMConfigStore, cmd *models.SaveAlertmanagerConfigurationCmd) error {
+	cfg := &definitions.PostableUserConfig{}
+	if err := json.Unmarshal([]byte(cmd.AlertmanagerConfiguration), cfg); err != nil {
+		return fmt.Errorf("change would result in an invalid configuration state: %w", err)
+	}
+	return store.UpdateAlertmanagerConfiguration(ctx, cmd)
+}

pkg/services/ngalert/provisioning/templates.go

@@ -66,7 +66,7 @@ func (t *TemplateService) SetTemplate(ctx context.Context, orgID int64, tmpl def
 		OrgID:                     orgID,
 	}
 	err = t.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = t.config.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err = PersistConfig(ctx, t.config, &cmd)
 		if err != nil {
 			return err
 		}
@@ -104,7 +104,7 @@ func (t *TemplateService) DeleteTemplate(ctx context.Context, orgID int64, name
 		OrgID:                     orgID,
 	}
 	err = t.xact.InTransaction(ctx, func(ctx context.Context) error {
-		err = t.config.UpdateAlertmanagerConfiguration(ctx, &cmd)
+		err = PersistConfig(ctx, t.config, &cmd)
 		if err != nil {
 			return err
 		}