From c67e1d37d3ce9f1d8831086b8347f2c42de99282 Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Tue, 26 Apr 2022 11:13:18 -0400 Subject: [PATCH] [v8.5.x] Alerting: Refactor GET/POST alerting config routes to be more extensible (#48280) * Alerting: Refactor GET/POST alerting config routes to be more extensible (#47229) * Refactor GET am config to be extensible * Extract post config route * Fix tests * Remove temporary duplication * Fix broken test due to layer shift * Fix duplicated error message * Properly return 400 on config rejection * Revert weird half method extraction * Move things to notifier package and avoid redundant interface * Simplify documentation * Split encryption service and depend on minimal abstractions * Properly initialize things all the way up to the composition root * Encryption -> Crypto * Address misc feedback * Missing docstring * Few more simple polish improvements * Unify on MultiOrgAlertmanager. Discover bug in existing test * Fix rebase conflicts * Misc feedback, renames, docs * Access crypto hanging off MultiOrgAlertmanager rather than having a separate API to initialize (cherry picked from commit 758364e78bd3d5b52c8f2eeb668fa8688df2e37b) * Empty commit to kick PR build Co-authored-by: Alexander Weaver --- pkg/services/ngalert/api/api.go | 2 +- pkg/services/ngalert/api/api_alertmanager.go | 196 +++--------------- .../ngalert/api/api_alertmanager_test.go | 5 +- .../api/tooling/definitions/alertmanager.go | 3 + pkg/services/ngalert/ngalert.go | 2 +- .../ngalert/notifier/alertmanager_config.go | 119 +++++++++++ pkg/services/ngalert/notifier/crypto.go | 118 +++++++++++ .../ngalert/notifier/multiorg_alertmanager.go | 7 +- .../notifier/multiorg_alertmanager_test.go | 6 +- .../ngalert/schedule/schedule_unit_test.go | 2 +- 10 files changed, 283 insertions(+), 177 deletions(-) create mode 100644 pkg/services/ngalert/notifier/alertmanager_config.go create mode 100644 pkg/services/ngalert/notifier/crypto.go diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index be3a4623a7a..0b6123b6603 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -91,7 +91,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) { api.RegisterAlertmanagerApiEndpoints(NewForkedAM( api.DatasourceCache, NewLotexAM(proxy, logger), - &AlertmanagerSrv{store: api.AlertingStore, mam: api.MultiOrgAlertmanager, secrets: api.SecretsService, log: logger, ac: api.AccessControl}, + &AlertmanagerSrv{crypto: api.MultiOrgAlertmanager.Crypto, log: logger, ac: api.AccessControl, mam: api.MultiOrgAlertmanager}, ), m) // Register endpoints for proxying to Prometheus-compatible backends. api.RegisterPrometheusApiEndpoints(NewForkedProm( diff --git a/pkg/services/ngalert/api/api_alertmanager.go b/pkg/services/ngalert/api/api_alertmanager.go index c23cfc9985f..ee22a4145ab 100644 --- a/pkg/services/ngalert/api/api_alertmanager.go +++ b/pkg/services/ngalert/api/api_alertmanager.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/base64" "errors" "fmt" "net/http" @@ -15,10 +14,8 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/store" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) @@ -29,11 +26,10 @@ const ( ) type AlertmanagerSrv struct { - mam *notifier.MultiOrgAlertmanager - secrets secrets.Service - store AlertingStore - log log.Logger - ac accesscontrol.AccessControl + log log.Logger + ac accesscontrol.AccessControl + mam *notifier.MultiOrgAlertmanager + crypto notifier.Crypto } type UnknownReceiverError struct { @@ -44,81 +40,6 @@ func (e UnknownReceiverError) Error() string { return fmt.Sprintf("unknown receiver: %s", e.UID) } -func (srv AlertmanagerSrv) loadSecureSettings(ctx context.Context, orgId int64, receivers []*apimodels.PostableApiReceiver) error { - // Get the last known working configuration - query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId} - if err := srv.store.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil { - // If we don't have a configuration there's nothing for us to know and we should just continue saving the new one - if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) { - return fmt.Errorf("failed to get latest configuration: %w", err) - } - } - - currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver) - if query.Result != nil { - currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration)) - // If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state. - if err != nil { - srv.log.Warn("Last known alertmanager configuration was invalid. Overwriting...") - } else { - currentReceiverMap = currentConfig.GetGrafanaReceiverMap() - } - } - - // Copy the previously known secure settings - for i, r := range receivers { - for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers { - if gr.UID == "" { // new receiver - continue - } - - cgmr, ok := currentReceiverMap[gr.UID] - if !ok { - // it tries to update a receiver that didn't previously exist - return UnknownReceiverError{UID: gr.UID} - } - - // frontend sends only the secure settings that have to be updated - // therefore we have to copy from the last configuration only those secure settings not included in the request - for key := range cgmr.SecureSettings { - _, ok := gr.SecureSettings[key] - if !ok { - decryptedValue, err := srv.getDecryptedSecret(cgmr, key) - if err != nil { - return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err) - } - - if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil { - receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings)) - } - - receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue - } - } - } - } - return nil -} - -func (srv AlertmanagerSrv) getDecryptedSecret(r *apimodels.PostableGrafanaReceiver, key string) (string, error) { - storedValue, ok := r.SecureSettings[key] - if !ok { - return "", nil - } - - decodeValue, err := base64.StdEncoding.DecodeString(storedValue) - if err != nil { - return "", err - } - - decryptedValue, err := srv.secrets.Decrypt(context.Background(), decodeValue) - if err != nil { - return "", err - } - - return string(decryptedValue), nil -} - func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response { am, errResp := srv.AlertmanagerFor(c.OrgId) if errResp != nil { @@ -192,59 +113,14 @@ func (srv AlertmanagerSrv) RouteDeleteSilence(c *models.ReqContext) response.Res } func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response.Response { - query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId} - if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil { + config, err := srv.mam.GetAlertmanagerConfiguration(c.Req.Context(), c.OrgId) + if err != nil { if errors.Is(err, store.ErrNoAlertmanagerConfiguration) { return ErrResp(http.StatusNotFound, err, "") } - return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration") + return ErrResp(http.StatusInternalServerError, err, err.Error()) } - - cfg, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration)) - if err != nil { - return ErrResp(http.StatusInternalServerError, err, "failed to unmarshal alertmanager configuration") - } - - result := apimodels.GettableUserConfig{ - TemplateFiles: cfg.TemplateFiles, - AlertmanagerConfig: apimodels.GettableApiAlertingConfig{ - Config: cfg.AlertmanagerConfig.Config, - }, - } - for _, recv := range cfg.AlertmanagerConfig.Receivers { - receivers := make([]*apimodels.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers)) - for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers { - secureFields := make(map[string]bool, len(pr.SecureSettings)) - for k := range pr.SecureSettings { - decryptedValue, err := srv.getDecryptedSecret(pr, k) - if err != nil { - return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", k) - } - if decryptedValue == "" { - continue - } - secureFields[k] = true - } - gr := apimodels.GettableGrafanaReceiver{ - UID: pr.UID, - Name: pr.Name, - Type: pr.Type, - DisableResolveMessage: pr.DisableResolveMessage, - Settings: pr.Settings, - SecureFields: secureFields, - } - receivers = append(receivers, &gr) - } - gettableApiReceiver := apimodels.GettableApiReceiver{ - GettableGrafanaReceivers: apimodels.GettableGrafanaReceivers{ - GrafanaManagedReceivers: receivers, - }, - } - gettableApiReceiver.Name = recv.Name - result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver) - } - - return response.JSON(http.StatusOK, result) + return response.JSON(http.StatusOK, config) } func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.Response { @@ -334,41 +210,26 @@ func (srv AlertmanagerSrv) RouteGetSilences(c *models.ReqContext) response.Respo } func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body apimodels.PostableUserConfig) response.Response { - // Get the last known working configuration - query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId} - if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil { - // If we don't have a configuration there's nothing for us to know and we should just continue saving the new one - if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) { - return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration") - } + err := srv.mam.ApplyAlertmanagerConfiguration(c.Req.Context(), c.OrgId, body) + if err == nil { + return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"}) + } + var unknownReceiverError notifier.UnknownReceiverError + if errors.As(err, &unknownReceiverError) { + return ErrResp(http.StatusBadRequest, unknownReceiverError, "") + } + var configRejectedError notifier.AlertmanagerConfigRejectedError + if errors.As(err, &configRejectedError) { + return ErrResp(http.StatusBadRequest, configRejectedError, "") + } + if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) { + return response.Error(http.StatusNotFound, err.Error(), err) + } + if errors.Is(err, notifier.ErrAlertmanagerNotReady) { + return response.Error(http.StatusConflict, err.Error(), err) } - if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.AlertmanagerConfig.Receivers); err != nil { - var unknownReceiverError UnknownReceiverError - if errors.As(err, &unknownReceiverError) { - return ErrResp(http.StatusBadRequest, err, "") - } - return ErrResp(http.StatusInternalServerError, err, "") - } - - if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil { - return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration") - } - - am, errResp := srv.AlertmanagerFor(c.OrgId) - if errResp != nil { - // It's okay if the alertmanager isn't ready yet, we're changing its config anyway. - if !errors.Is(errResp.Err(), notifier.ErrAlertmanagerNotReady) { - return errResp - } - } - - if err := am.SaveAndApplyConfig(c.Req.Context(), &body); err != nil { - srv.log.Error("unable to save and apply alertmanager configuration", "err", err) - return ErrResp(http.StatusBadRequest, err, "failed to save and apply Alertmanager configuration") - } - - return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"}) + return ErrResp(http.StatusInternalServerError, err, "") } func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.PostableAlerts) response.Response { @@ -376,7 +237,7 @@ func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.P } func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigBodyParams) response.Response { - if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil { + if err := srv.crypto.LoadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil { var unknownReceiverError UnknownReceiverError if errors.As(err, &unknownReceiverError) { return ErrResp(http.StatusBadRequest, err, "") @@ -384,7 +245,7 @@ func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body api return ErrResp(http.StatusInternalServerError, err, "") } - if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil { + if err := body.ProcessConfig(srv.crypto.Encrypt); err != nil { return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration") } @@ -514,7 +375,6 @@ func (srv AlertmanagerSrv) AlertmanagerFor(orgID int64) (Alertmanager, *response if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) { return nil, response.Error(http.StatusNotFound, err.Error(), err) } - if errors.Is(err, notifier.ErrAlertmanagerNotReady) { return am, response.Error(http.StatusConflict, err.Error(), err) } diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index 75ee2176024..eec2b513e00 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -357,7 +357,8 @@ func createSut(t *testing.T, accessControl accesscontrol.AccessControl) Alertman accessControl = acMock.New().WithDisabled() } log := log.NewNopLogger() - return AlertmanagerSrv{mam: mam, store: &configStore, secrets: secrets, ac: accessControl, log: log} + crypto := notifier.NewCrypto(secrets, &configStore, log) + return AlertmanagerSrv{mam: mam, crypto: crypto, ac: accessControl, log: log} } func createAmConfigRequest(t *testing.T) apimodels.PostableUserConfig { @@ -395,7 +396,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager { }, // do not poll in tests. } - mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) + mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) err = mam.LoadAndSyncAlertmanagersForOrgs(context.Background()) require.NoError(t, err) diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go index 94eb137d7b7..ad75f5215b3 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go @@ -18,6 +18,7 @@ import ( "gopkg.in/yaml.v3" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/util" ) @@ -712,6 +713,8 @@ type Route struct { GroupWait *model.Duration `yaml:"group_wait,omitempty" json:"group_wait,omitempty"` GroupInterval *model.Duration `yaml:"group_interval,omitempty" json:"group_interval,omitempty"` RepeatInterval *model.Duration `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"` + + Provenance models.Provenance `yaml:"provenance,omitempty" json:"provenance,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface for Route. This is a copy of alertmanager's upstream except it removes validation on the label key. diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 9d03ecfe9cd..49158bc52b5 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -99,7 +99,7 @@ func (ng *AlertNG) init() error { decryptFn := ng.SecretsService.GetDecryptedValue multiOrgMetrics := ng.Metrics.GetMultiOrgAlertmanagerMetrics() - ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager")) + ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager"), ng.SecretsService) if err != nil { return err } diff --git a/pkg/services/ngalert/notifier/alertmanager_config.go b/pkg/services/ngalert/notifier/alertmanager_config.go new file mode 100644 index 00000000000..7efd6561589 --- /dev/null +++ b/pkg/services/ngalert/notifier/alertmanager_config.go @@ -0,0 +1,119 @@ +package notifier + +import ( + "context" + "errors" + "fmt" + + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/store" +) + +type UnknownReceiverError struct { + UID string +} + +func (e UnknownReceiverError) Error() string { + return fmt.Sprintf("unknown receiver: %s", e.UID) +} + +type AlertmanagerConfigRejectedError struct { + Inner error +} + +func (e AlertmanagerConfigRejectedError) Error() string { + return fmt.Sprintf("failed to save and apply Alertmanager configuration: %s", e.Inner.Error()) +} + +type configurationStore interface { + GetLatestAlertmanagerConfiguration(ctx context.Context, query *models.GetLatestAlertmanagerConfigurationQuery) error +} + +func (moa *MultiOrgAlertmanager) GetAlertmanagerConfiguration(ctx context.Context, org int64) (definitions.GettableUserConfig, error) { + query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org} + err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query) + if err != nil { + return definitions.GettableUserConfig{}, fmt.Errorf("failed to get latest configuration: %w", err) + } + cfg, err := Load([]byte(query.Result.AlertmanagerConfiguration)) + if err != nil { + return definitions.GettableUserConfig{}, fmt.Errorf("failed to unmarshal alertmanager configuration: %w", err) + } + + result := definitions.GettableUserConfig{ + TemplateFiles: cfg.TemplateFiles, + AlertmanagerConfig: definitions.GettableApiAlertingConfig{ + Config: cfg.AlertmanagerConfig.Config, + }, + } + + for _, recv := range cfg.AlertmanagerConfig.Receivers { + receivers := make([]*definitions.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers)) + for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers { + secureFields := make(map[string]bool, len(pr.SecureSettings)) + for k := range pr.SecureSettings { + decryptedValue, err := moa.Crypto.getDecryptedSecret(pr, k) + if err != nil { + return definitions.GettableUserConfig{}, fmt.Errorf("failed to decrypt stored secure setting: %w", err) + } + if decryptedValue == "" { + continue + } + secureFields[k] = true + } + gr := definitions.GettableGrafanaReceiver{ + UID: pr.UID, + Name: pr.Name, + Type: pr.Type, + DisableResolveMessage: pr.DisableResolveMessage, + Settings: pr.Settings, + SecureFields: secureFields, + } + receivers = append(receivers, &gr) + } + gettableApiReceiver := definitions.GettableApiReceiver{ + GettableGrafanaReceivers: definitions.GettableGrafanaReceivers{ + GrafanaManagedReceivers: receivers, + }, + } + gettableApiReceiver.Name = recv.Name + result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver) + } + + return result, nil +} + +func (moa *MultiOrgAlertmanager) ApplyAlertmanagerConfiguration(ctx context.Context, org int64, config definitions.PostableUserConfig) error { + // Get the last known working configuration + query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org} + if err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil { + // If we don't have a configuration there's nothing for us to know and we should just continue saving the new one + if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) { + return fmt.Errorf("failed to get latest configuration %w", err) + } + } + + if err := moa.Crypto.LoadSecureSettings(ctx, org, config.AlertmanagerConfig.Receivers); err != nil { + return err + } + + if err := config.ProcessConfig(moa.Crypto.Encrypt); err != nil { + return fmt.Errorf("failed to post process Alertmanager configuration: %w", err) + } + + am, err := moa.AlertmanagerFor(org) + if err != nil { + // It's okay if the alertmanager isn't ready yet, we're changing its config anyway. + if !errors.Is(err, ErrAlertmanagerNotReady) { + return err + } + } + + if err := am.SaveAndApplyConfig(ctx, &config); err != nil { + moa.logger.Error("unable to save and apply alertmanager configuration", "err", err) + return AlertmanagerConfigRejectedError{err} + } + + return nil +} diff --git a/pkg/services/ngalert/notifier/crypto.go b/pkg/services/ngalert/notifier/crypto.go new file mode 100644 index 00000000000..0d5e06a55f1 --- /dev/null +++ b/pkg/services/ngalert/notifier/crypto.go @@ -0,0 +1,118 @@ +package notifier + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/secrets" +) + +// Crypto allows decryption of Alertmanager Configuration and encryption of arbitrary payloads. +type Crypto interface { + LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error + Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) + + getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error) +} + +// alertmanagerCrypto implements decryption of Alertmanager configuration and encryption of arbitrary payloads based on Grafana's encryptions. +type alertmanagerCrypto struct { + secrets secrets.Service + configs configurationStore + log log.Logger +} + +func NewCrypto(secrets secrets.Service, configs configurationStore, log log.Logger) Crypto { + return &alertmanagerCrypto{ + secrets: secrets, + configs: configs, + log: log, + } +} + +// LoadSecureSettings adds the corresponding unencrypted secrets stored to the list of input receivers. +func (c *alertmanagerCrypto) LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error { + // Get the last known working configuration. + query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId} + if err := c.configs.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil { + // If we don't have a configuration there's nothing for us to know and we should just continue saving the new one. + if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) { + return fmt.Errorf("failed to get latest configuration: %w", err) + } + } + + currentReceiverMap := make(map[string]*definitions.PostableGrafanaReceiver) + if query.Result != nil { + currentConfig, err := Load([]byte(query.Result.AlertmanagerConfiguration)) + // If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state. + if err != nil { + c.log.Warn("Last known alertmanager configuration was invalid. Overwriting...") + } else { + currentReceiverMap = currentConfig.GetGrafanaReceiverMap() + } + } + + // Copy the previously known secure settings. + for i, r := range receivers { + for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers { + if gr.UID == "" { // new receiver + continue + } + + cgmr, ok := currentReceiverMap[gr.UID] + if !ok { + // It tries to update a receiver that didn't previously exist + return UnknownReceiverError{UID: gr.UID} + } + + // Frontend sends only the secure settings that have to be updated + // Therefore we have to copy from the last configuration only those secure settings not included in the request + for key := range cgmr.SecureSettings { + _, ok := gr.SecureSettings[key] + if !ok { + decryptedValue, err := c.getDecryptedSecret(cgmr, key) + if err != nil { + return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err) + } + + if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil { + receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings)) + } + + receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue + } + } + } + } + return nil +} + +func (c *alertmanagerCrypto) getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error) { + storedValue, ok := r.SecureSettings[key] + if !ok { + return "", nil + } + + decodeValue, err := base64.StdEncoding.DecodeString(storedValue) + if err != nil { + return "", err + } + + decryptedValue, err := c.secrets.Decrypt(context.Background(), decodeValue) + if err != nil { + return "", err + } + + return string(decryptedValue), nil +} + +// Encrypt delegates encryption to secrets.Service. +func (c *alertmanagerCrypto) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) { + return c.secrets.Encrypt(ctx, payload, opt) +} diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index 3c8a2f1d987..d34677c3e44 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels" "github.com/grafana/grafana/pkg/services/notifications" + "github.com/grafana/grafana/pkg/services/secrets" "github.com/prometheus/alertmanager/cluster" "github.com/prometheus/client_golang/prometheus" @@ -29,6 +30,8 @@ var ( ) type MultiOrgAlertmanager struct { + Crypto Crypto + alertmanagersMtx sync.RWMutex alertmanagers map[int64]*Alertmanager @@ -51,9 +54,11 @@ type MultiOrgAlertmanager struct { func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore, kvStore kvstore.KVStore, decryptFn channels.GetDecryptedValueFn, m *metrics.MultiOrgAlertmanager, - ns notifications.Service, l log.Logger, + ns notifications.Service, l log.Logger, s secrets.Service, ) (*MultiOrgAlertmanager, error) { moa := &MultiOrgAlertmanager{ + Crypto: NewCrypto(s, configStore, l), + logger: l, settings: cfg, alertmanagers: map[int64]*Alertmanager{}, diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go index 6dd2f020be0..56899182532 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go @@ -44,7 +44,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) { DisabledOrgs: map[int64]struct{}{5: {}}, }, // do not poll in tests. } - mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) + mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) ctx := context.Background() @@ -168,7 +168,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T) DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(), }, // do not poll in tests. } - mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) + mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) ctx := context.Background() @@ -218,7 +218,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) { decryptFn := secretsService.GetDecryptedValue reg := prometheus.NewPedanticRegistry() m := metrics.NewNGAlert(reg) - mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) + mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) ctx := context.Background() diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 4824f05d95f..7f5f8da4cac 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -1015,7 +1015,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac m := metrics.NewNGAlert(registry) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) + moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) schedCfg := SchedulerCfg{