public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Refactor GET/POST alerting config routes to be more extensible (#47229)

Browse Source 
* Refactor GET am config to be extensible

* Extract post config route

* Fix tests

* Remove temporary duplication

* Fix broken test due to layer shift

* Fix duplicated error message

* Properly return 400 on config rejection

* Revert weird half method extraction

* Move things to notifier package and avoid redundant interface

* Simplify documentation

* Split encryption service and depend on minimal abstractions

* Properly initialize things all the way up to the composition root

* Encryption -> Crypto

* Address misc feedback

* Missing docstring

* Few more simple polish improvements

* Unify on MultiOrgAlertmanager. Discover bug in existing test

* Fix rebase conflicts

* Misc feedback, renames, docs

* Access crypto hanging off MultiOrgAlertmanager rather than having a separate API to initialize
This commit is contained in:
Alexander Weaver
2022-04-14 13:06:21 -05:00
committed by GitHub
parent a126c07e54
commit 758364e78b
10 changed files with 283 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/notifier/alertmanager_config.go
+119
View File
@@ -0,0 +1,119 @@
package notifier
import (
"context"
"errors"
"fmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
)
type UnknownReceiverError struct {
UID string
}
func (e UnknownReceiverError) Error() string {
return fmt.Sprintf("unknown receiver: %s", e.UID)
}
type AlertmanagerConfigRejectedError struct {
Inner error
}
func (e AlertmanagerConfigRejectedError) Error() string {
return fmt.Sprintf("failed to save and apply Alertmanager configuration: %s", e.Inner.Error())
}
type configurationStore interface {
GetLatestAlertmanagerConfiguration(ctx context.Context, query *models.GetLatestAlertmanagerConfigurationQuery) error
}
func (moa *MultiOrgAlertmanager) GetAlertmanagerConfiguration(ctx context.Context, org int64) (definitions.GettableUserConfig, error) {
query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org}
err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query)
if err != nil {
return definitions.GettableUserConfig{}, fmt.Errorf("failed to get latest configuration: %w", err)
}
cfg, err := Load([]byte(query.Result.AlertmanagerConfiguration))
if err != nil {
return definitions.GettableUserConfig{}, fmt.Errorf("failed to unmarshal alertmanager configuration: %w", err)
}
result := definitions.GettableUserConfig{
TemplateFiles: cfg.TemplateFiles,
AlertmanagerConfig: definitions.GettableApiAlertingConfig{
Config: cfg.AlertmanagerConfig.Config,
},
}
for _, recv := range cfg.AlertmanagerConfig.Receivers {
receivers := make([]*definitions.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers))
for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers {
secureFields := make(map[string]bool, len(pr.SecureSettings))
for k := range pr.SecureSettings {
decryptedValue, err := moa.Crypto.getDecryptedSecret(pr, k)
if err != nil {
return definitions.GettableUserConfig{}, fmt.Errorf("failed to decrypt stored secure setting: %w", err)
}
if decryptedValue == "" {
continue
}
secureFields[k] = true
}
gr := definitions.GettableGrafanaReceiver{
UID: pr.UID,
Name: pr.Name,
Type: pr.Type,
DisableResolveMessage: pr.DisableResolveMessage,
Settings: pr.Settings,
SecureFields: secureFields,
}
receivers = append(receivers, &gr)
}
gettableApiReceiver := definitions.GettableApiReceiver{
GettableGrafanaReceivers: definitions.GettableGrafanaReceivers{
GrafanaManagedReceivers: receivers,
},
}
gettableApiReceiver.Name = recv.Name
result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver)
}
return result, nil
}
func (moa *MultiOrgAlertmanager) ApplyAlertmanagerConfiguration(ctx context.Context, org int64, config definitions.PostableUserConfig) error {
// Get the last known working configuration
query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org}
if err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
return fmt.Errorf("failed to get latest configuration %w", err)
}
}
if err := moa.Crypto.LoadSecureSettings(ctx, org, config.AlertmanagerConfig.Receivers); err != nil {
return err
}
if err := config.ProcessConfig(moa.Crypto.Encrypt); err != nil {
return fmt.Errorf("failed to post process Alertmanager configuration: %w", err)
}
am, err := moa.AlertmanagerFor(org)
if err != nil {
// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
if !errors.Is(err, ErrAlertmanagerNotReady) {
return err
}
}
if err := am.SaveAndApplyConfig(ctx, &config); err != nil {
moa.logger.Error("unable to save and apply alertmanager configuration", "err", err)
return AlertmanagerConfigRejectedError{err}
}
return nil
}
pkg/services/ngalert/notifier/crypto.go
+118
View File
@@ -0,0 +1,118 @@
package notifier
import (
"context"
"encoding/base64"
"errors"
"fmt"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/secrets"
)
// Crypto allows decryption of Alertmanager Configuration and encryption of arbitrary payloads.
type Crypto interface {
LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error
Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error)
getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error)
}
// alertmanagerCrypto implements decryption of Alertmanager configuration and encryption of arbitrary payloads based on Grafana's encryptions.
type alertmanagerCrypto struct {
secrets secrets.Service
configs configurationStore
log log.Logger
}
func NewCrypto(secrets secrets.Service, configs configurationStore, log log.Logger) Crypto {
return &alertmanagerCrypto{
secrets: secrets,
configs: configs,
log: log,
}
}
// LoadSecureSettings adds the corresponding unencrypted secrets stored to the list of input receivers.
func (c *alertmanagerCrypto) LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error {
// Get the last known working configuration.
query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
if err := c.configs.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one.
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
return fmt.Errorf("failed to get latest configuration: %w", err)
}
}
currentReceiverMap := make(map[string]*definitions.PostableGrafanaReceiver)
if query.Result != nil {
currentConfig, err := Load([]byte(query.Result.AlertmanagerConfiguration))
// If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state.
if err != nil {
c.log.Warn("Last known alertmanager configuration was invalid. Overwriting...")
} else {
currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
}
}
// Copy the previously known secure settings.
for i, r := range receivers {
for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
if gr.UID == "" { // new receiver
continue
}
cgmr, ok := currentReceiverMap[gr.UID]
if !ok {
// It tries to update a receiver that didn't previously exist
return UnknownReceiverError{UID: gr.UID}
}
// Frontend sends only the secure settings that have to be updated
// Therefore we have to copy from the last configuration only those secure settings not included in the request
for key := range cgmr.SecureSettings {
_, ok := gr.SecureSettings[key]
if !ok {
decryptedValue, err := c.getDecryptedSecret(cgmr, key)
if err != nil {
return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
}
if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
}
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
}
}
}
}
return nil
}
func (c *alertmanagerCrypto) getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error) {
storedValue, ok := r.SecureSettings[key]
if !ok {
return "", nil
}
decodeValue, err := base64.StdEncoding.DecodeString(storedValue)
if err != nil {
return "", err
}
decryptedValue, err := c.secrets.Decrypt(context.Background(), decodeValue)
if err != nil {
return "", err
}
return string(decryptedValue), nil
}
// Encrypt delegates encryption to secrets.Service.
func (c *alertmanagerCrypto) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
return c.secrets.Encrypt(ctx, payload, opt)
}
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+6 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
"github.com/grafana/grafana/pkg/services/notifications"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/prometheus/alertmanager/cluster"
"github.com/prometheus/client_golang/prometheus"
@@ -29,6 +30,8 @@ var (
)
type MultiOrgAlertmanager struct {
Crypto Crypto
alertmanagersMtx sync.RWMutex
alertmanagers map[int64]*Alertmanager
@@ -51,9 +54,11 @@ type MultiOrgAlertmanager struct {
func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore,
kvStore kvstore.KVStore, decryptFn channels.GetDecryptedValueFn, m *metrics.MultiOrgAlertmanager,
ns notifications.Service, l log.Logger,
ns notifications.Service, l log.Logger, s secrets.Service,
) (*MultiOrgAlertmanager, error) {
moa := &MultiOrgAlertmanager{
Crypto: NewCrypto(s, configStore, l),
logger: l,
settings: cfg,
alertmanagers: map[int64]*Alertmanager{},
pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
+3 -3
View File
@@ -44,7 +44,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
DisabledOrgs: map[int64]struct{}{5: {}},
}, // do not poll in tests.
}
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
ctx := context.Background()
@@ -168,7 +168,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
}, // do not poll in tests.
}
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
ctx := context.Background()
@@ -218,7 +218,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
decryptFn := secretsService.GetDecryptedValue
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
ctx := context.Background()