public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support saving extra Mimir configurations (#106721)

Browse Source
This commit is contained in:
Alexander Akhmetov
2025-06-17 23:33:16 +02:00
committed by GitHub
parent 74e800e427
commit 5a78ce8a4b
13 changed files with 583 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/tooling/api.json
+33 -7
View File
@@ -1123,6 +1123,26 @@
},
"type": "object"
},
"ExtraConfiguration": {
"properties": {
"alertmanager_config": {
"type": "string"
},
"identifier": {
"type": "string"
},
"merge_matchers": {
"$ref": "#/definitions/Matchers"
},
"template_files": {
"additionalProperties": {
"type": "string"
},
"type": "object"
}
},
"type": "object"
},
"Failure": {
"$ref": "#/definitions/ResponseDetails"
},
@@ -1854,6 +1874,12 @@
"alertmanager_config": {
"$ref": "#/definitions/GettableApiAlertingConfig"
},
"extra_config": {
"items": {
"$ref": "#/definitions/ExtraConfiguration"
},
"type": "array"
},
"template_file_provenances": {
"additionalProperties": {
"$ref": "#/definitions/Provenance"
@@ -3057,6 +3083,12 @@
"alertmanager_config": {
"$ref": "#/definitions/PostableApiAlertingConfig"
},
"extra_config": {
"items": {
"$ref": "#/definitions/ExtraConfiguration"
},
"type": "array"
},
"template_files": {
"additionalProperties": {
"type": "string"
@@ -3687,7 +3719,6 @@
"type": "object"
},
"Route": {
"description": "A Route is a node that contains definitions of how to handle alerts. This is modified\nfrom the upstream alertmanager in that it adds the ObjectMatchers property.",
"properties": {
"active_time_intervals": {
"items": {
@@ -3729,12 +3760,6 @@
},
"type": "array"
},
"object_matchers": {
"$ref": "#/definitions/ObjectMatchers"
},
"provenance": {
"$ref": "#/definitions/Provenance"
},
"receiver": {
"type": "string"
},
@@ -3748,6 +3773,7 @@
"type": "array"
}
},
"title": "A Route is a node that contains definitions of how to handle alerts.",
"type": "object"
},
"RouteExport": {
pkg/services/ngalert/api/tooling/definitions/alertmanager.go
+51 -12
View File
@@ -13,7 +13,6 @@ import (
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v3"
"github.com/grafana/alerting/definition"
@@ -649,31 +648,69 @@ type DatasourceUIDReference struct {
}
type ExtraConfiguration struct {
Identifier string `yaml:"identifier" json:"identifier"`
MergeMatchers config.Matchers `yaml:"merge_matchers" json:"merge_matchers"`
TemplateFiles map[string]string `yaml:"template_files" json:"template_files"`
AlertmanagerConfig PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"`
Identifier string `yaml:"identifier" json:"identifier"`
MergeMatchers config.Matchers `yaml:"merge_matchers" json:"merge_matchers"`
TemplateFiles map[string]string `yaml:"template_files" json:"template_files"`
AlertmanagerConfig string `yaml:"alertmanager_config" json:"alertmanager_config"`
}
func (c *ExtraConfiguration) GetAlertmanagerConfig() (PostableApiAlertingConfig, error) {
if c.AlertmanagerConfig == "" {
return PostableApiAlertingConfig{}, fmt.Errorf("no alertmanager configuration available")
}
var prometheusConfig config.Config
if err := yaml.Unmarshal([]byte(c.AlertmanagerConfig), &prometheusConfig); err != nil {
return PostableApiAlertingConfig{}, fmt.Errorf("failed to parse alertmanager config: %w", err)
}
return fromPrometheusConfig(prometheusConfig), nil
}
func (c ExtraConfiguration) Validate() error {
if c.Identifier == "" {
return errors.New("identifier is required")
}
if len(c.MergeMatchers) == 0 {
return errors.New("at least one matcher is required")
}
for _, m := range c.MergeMatchers {
if m.Type != labels.MatchEqual {
return errors.New("only matchers with type equal are supported")
}
}
err := c.AlertmanagerConfig.Validate()
// Alertmanager configuration is validated during YAML unmarshalling.
am := config.Config{}
err := yaml.Unmarshal([]byte(c.AlertmanagerConfig), &am)
if err != nil {
return fmt.Errorf("invalid alertmanager configuration: %w", err)
}
return nil
}
func fromPrometheusConfig(prometheusConfig config.Config) PostableApiAlertingConfig {
config := PostableApiAlertingConfig{
Config: Config{
Global: prometheusConfig.Global,
Route: AsGrafanaRoute(prometheusConfig.Route),
InhibitRules: prometheusConfig.InhibitRules,
Templates: prometheusConfig.Templates,
},
}
for _, receiver := range prometheusConfig.Receivers {
config.Receivers = append(config.Receivers, &PostableApiReceiver{
Receiver: receiver,
})
}
return config
}
// swagger:model
type PostableUserConfig struct {
TemplateFiles map[string]string `yaml:"template_files" json:"template_files"`
@@ -697,7 +734,13 @@ func (c *PostableUserConfig) GetMergedAlertmanagerConfig() (MergeResult, error)
if err := opts.Validate(); err != nil {
return MergeResult{}, fmt.Errorf("invalid merge options: %w", err)
}
return definition.Merge(c.AlertmanagerConfig, mimirCfg.AlertmanagerConfig, opts) // for now support only the first extra config
mcfg, err := mimirCfg.GetAlertmanagerConfig()
if err != nil {
return MergeResult{}, fmt.Errorf("failed to get mimir alertmanager config: %w", err)
}
return definition.Merge(c.AlertmanagerConfig, mcfg, opts)
}
// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of TemplateDefinitions.
@@ -738,11 +781,6 @@ func (c *PostableUserConfig) UnmarshalJSON(b []byte) error {
if len(c.ExtraConfigs) > 1 {
return errors.New("only one extra config is supported")
}
for _, extraConfig := range c.ExtraConfigs {
if err := extraConfig.Validate(); err != nil {
return fmt.Errorf("extra configuration is invalid: %w", err)
}
}
type intermediate struct {
AlertmanagerConfig map[string]interface{} `yaml:"alertmanager_config" json:"alertmanager_config"`
@@ -833,6 +871,7 @@ type GettableUserConfig struct {
TemplateFiles map[string]string `yaml:"template_files" json:"template_files"`
TemplateFileProvenances map[string]Provenance `yaml:"template_file_provenances,omitempty" json:"template_file_provenances,omitempty"`
AlertmanagerConfig GettableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"`
ExtraConfigs []ExtraConfiguration `yaml:"extra_config,omitempty" json:"extra_config,omitempty"`
// amSimple stores a map[string]interface of the decoded alertmanager config.
// This enables circumventing the underlying alertmanager secret type
pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go
+9 -21
View File
@@ -259,20 +259,10 @@ func TestPostableUserConfig_GetMergedAlertmanagerConfig(t *testing.T) {
Value: "prod",
},
},
AlertmanagerConfig: PostableApiAlertingConfig{
Config: Config{
Route: &Route{
Receiver: "mimir-receiver",
},
},
Receivers: []*PostableApiReceiver{
{
Receiver: config.Receiver{
Name: "mimir-receiver",
},
},
},
},
AlertmanagerConfig: `route:
receiver: mimir-receiver
receivers:
- name: mimir-receiver`,
},
},
},
@@ -285,13 +275,11 @@ func TestPostableUserConfig_GetMergedAlertmanagerConfig(t *testing.T) {
{
Identifier: "",
MergeMatchers: config.Matchers{},
AlertmanagerConfig: PostableApiAlertingConfig{
Config: Config{
Route: &Route{
Receiver: "test",
},
},
},
AlertmanagerConfig: `{
"route": {
"receiver": "test"
}
}`,
},
},
},
pkg/services/ngalert/api/tooling/post.json
+32
View File
@@ -1123,6 +1123,26 @@
},
"type": "object"
},
"ExtraConfiguration": {
"properties": {
"alertmanager_config": {
"type": "string"
},
"identifier": {
"type": "string"
},
"merge_matchers": {
"$ref": "#/definitions/Matchers"
},
"template_files": {
"additionalProperties": {
"type": "string"
},
"type": "object"
}
},
"type": "object"
},
"Failure": {
"$ref": "#/definitions/ResponseDetails"
},
@@ -1854,6 +1874,12 @@
"alertmanager_config": {
"$ref": "#/definitions/GettableApiAlertingConfig"
},
"extra_config": {
"items": {
"$ref": "#/definitions/ExtraConfiguration"
},
"type": "array"
},
"template_file_provenances": {
"additionalProperties": {
"$ref": "#/definitions/Provenance"
@@ -3057,6 +3083,12 @@
"alertmanager_config": {
"$ref": "#/definitions/PostableApiAlertingConfig"
},
"extra_config": {
"items": {
"$ref": "#/definitions/ExtraConfiguration"
},
"type": "array"
},
"template_files": {
"additionalProperties": {
"type": "string"
pkg/services/ngalert/api/tooling/spec.json
+32
View File
@@ -5412,6 +5412,26 @@
}
}
},
"ExtraConfiguration": {
"type": "object",
"properties": {
"alertmanager_config": {
"type": "string"
},
"identifier": {
"type": "string"
},
"merge_matchers": {
"$ref": "#/definitions/Matchers"
},
"template_files": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
},
"Failure": {
"$ref": "#/definitions/ResponseDetails"
},
@@ -6144,6 +6164,12 @@
"alertmanager_config": {
"$ref": "#/definitions/GettableApiAlertingConfig"
},
"extra_config": {
"type": "array",
"items": {
"$ref": "#/definitions/ExtraConfiguration"
}
},
"template_file_provenances": {
"type": "object",
"additionalProperties": {
@@ -7348,6 +7374,12 @@
"alertmanager_config": {
"$ref": "#/definitions/PostableApiAlertingConfig"
},
"extra_config": {
"type": "array",
"items": {
"$ref": "#/definitions/ExtraConfiguration"
}
},
"template_files": {
"type": "object",
"additionalProperties": {
pkg/services/ngalert/notifier/alertmanager.go
+16 -3
View File
@@ -55,6 +55,7 @@ type alertmanager struct {
stateStore stateStore
DefaultConfiguration string
decryptFn alertingNotify.GetDecryptedValueFn
crypto Crypto
}
// maintenanceOptions represent the options for components that need maintenance on a frequency within the Alertmanager.
@@ -86,7 +87,7 @@ func (m maintenanceOptions) MaintenanceFunc(state alertingNotify.State) (int64,
func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store AlertingStore, stateStore stateStore,
peer alertingNotify.ClusterPeer, decryptFn alertingNotify.GetDecryptedValueFn, ns notifications.Service,
m *metrics.Alertmanager, featureToggles featuremgmt.FeatureToggles,
m *metrics.Alertmanager, featureToggles featuremgmt.FeatureToggles, crypto Crypto,
) (*alertmanager, error) {
nflog, err := stateStore.GetNotificationLog(ctx)
if err != nil {
@@ -152,6 +153,7 @@ func NewAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store A
stateStore: stateStore,
logger: l.New("component", "alertmanager", opts.TenantKey, opts.TenantID), // similar to what the base does
decryptFn: decryptFn,
crypto: crypto,
}
return am, nil
@@ -206,7 +208,13 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
// Remove autogenerated config from the user config before saving it, may not be necessary as we already remove
// the autogenerated config before provenance guard. However, this is low impact and a good safety net.
RemoveAutogenConfigIfExists(cfg.AlertmanagerConfig.Route)
rawConfig, err := json.Marshal(&cfg)
err := am.crypto.EncryptExtraConfigs(ctx, cfg)
if err != nil {
return fmt.Errorf("failed to encrypt external configurations: %w", err)
}
cfgToSave, err := json.Marshal(&cfg)
if err != nil {
return fmt.Errorf("failed to serialize to the Alertmanager configuration: %w", err)
}
@@ -214,7 +222,7 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
var outerErr error
am.Base.WithLock(func() {
cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{
AlertmanagerConfiguration: string(rawConfig),
AlertmanagerConfiguration: string(cfgToSave),
ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
OrgID: am.Base.TenantID(),
LastApplied: time.Now().UTC().Unix(),
@@ -342,6 +350,11 @@ func logMergeResult(l log.Logger, m apimodels.MergeResult) {
// It returns a boolean indicating whether the user config was changed and an error.
// It is not safe to call concurrently.
func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig, skipInvalid bool) (bool, error) {
err := am.crypto.DecryptExtraConfigs(ctx, cfg)
if err != nil {
return false, fmt.Errorf("failed to decrypt external configurations: %w", err)
}
mergeResult, err := cfg.GetMergedAlertmanagerConfig()
if err != nil {
return false, fmt.Errorf("failed to get full alertmanager configuration: %w", err)
pkg/services/ngalert/notifier/alertmanager_config.go
+88
View File
@@ -30,6 +30,12 @@ var (
errutil.WithPublic(
"time interval [Name: {{ .Public.Interval }}] is used by rule",
))
msgAlertmanagerMultipleExtraConfigsUnsupported = "multiple extra configurations are not supported, found another configuration with identifier: {{ .Public.Identifier }}"
ErrAlertmanagerMultipleExtraConfigsUnsupported = errutil.Conflict("alerting.notifications.alertmanager.multipleExtraConfigsUnsupported").MustTemplate(
msgAlertmanagerMultipleExtraConfigsUnsupported,
errutil.WithPublic(msgAlertmanagerMultipleExtraConfigsUnsupported),
)
)
type UnknownReceiverError struct {
@@ -217,11 +223,17 @@ func (moa *MultiOrgAlertmanager) gettableUserConfigFromAMConfigString(ctx contex
return definitions.GettableUserConfig{}, fmt.Errorf("failed to unmarshal alertmanager configuration: %w", err)
}
err = moa.Crypto.DecryptExtraConfigs(ctx, cfg)
if err != nil {
return definitions.GettableUserConfig{}, fmt.Errorf("failed to decrypt external configurations: %w", err)
}
result := definitions.GettableUserConfig{
TemplateFiles: cfg.TemplateFiles,
AlertmanagerConfig: definitions.GettableApiAlertingConfig{
Config: cfg.AlertmanagerConfig.Config,
},
ExtraConfigs: cfg.ExtraConfigs,
}
// First we encrypt the secure settings.
@@ -339,6 +351,82 @@ func (moa *MultiOrgAlertmanager) SaveAndApplyAlertmanagerConfiguration(ctx conte
return nil
}
// modifyAndApplyExtraConfiguration is a helper function that loads the current configuration,
// applies a modification function to the ExtraConfigs, and saves the result.
func (moa *MultiOrgAlertmanager) modifyAndApplyExtraConfiguration(
ctx context.Context,
org int64,
modifyFn func([]definitions.ExtraConfiguration) ([]definitions.ExtraConfiguration, error),
) error {
currentCfg, err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, org)
if err != nil {
return fmt.Errorf("failed to get current configuration: %w", err)
}
cfg, err := Load([]byte(currentCfg.AlertmanagerConfiguration))
if err != nil {
return fmt.Errorf("failed to unmarshal current alertmanager configuration: %w", err)
}
cfg.ExtraConfigs, err = modifyFn(cfg.ExtraConfigs)
if err != nil {
return fmt.Errorf("failed to apply extra configuration: %w", err)
}
am, err := moa.AlertmanagerFor(org)
if err != nil {
// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
if !errors.Is(err, ErrAlertmanagerNotReady) {
return err
}
}
if err := am.SaveAndApplyConfig(ctx, cfg); err != nil {
moa.logger.Error("Unable to save and apply alertmanager configuration with extra config", "error", err, "org", org)
return AlertmanagerConfigRejectedError{err}
}
moa.logger.Info("Applied alertmanager configuration with extra config", "org", org)
return nil
}
// SaveAndApplyExtraConfiguration adds or replaces an ExtraConfiguration while preserving the main AlertmanagerConfig.
func (moa *MultiOrgAlertmanager) SaveAndApplyExtraConfiguration(ctx context.Context, org int64, extraConfig definitions.ExtraConfiguration) error {
modifyFunc := func(configs []definitions.ExtraConfiguration) ([]definitions.ExtraConfiguration, error) {
// for now we validate that after the update there will be just one extra config.
for _, c := range configs {
if c.Identifier != extraConfig.Identifier {
return nil, ErrAlertmanagerMultipleExtraConfigsUnsupported.Build(errutil.TemplateData{Public: map[string]interface{}{"Identifier": c.Identifier}})
}
}
return []definitions.ExtraConfiguration{extraConfig}, nil
}
err := moa.modifyAndApplyExtraConfiguration(ctx, org, modifyFunc)
if err != nil {
return err
}
moa.logger.Info("Applied alertmanager configuration with extra config", "org", org, "identifier", extraConfig.Identifier)
return nil
}
// DeleteAndApplyExtraConfiguration deletes an ExtraConfiguration by its identifier while preserving the main AlertmanagerConfig.
func (moa *MultiOrgAlertmanager) DeleteAndApplyExtraConfiguration(ctx context.Context, org int64, identifier string) error {
modifyFunc := func(configs []definitions.ExtraConfiguration) ([]definitions.ExtraConfiguration, error) {
filtered := make([]definitions.ExtraConfiguration, 0, len(configs))
for _, ec := range configs {
if ec.Identifier != identifier {
filtered = append(filtered, ec)
}
}
return filtered, nil
}
return moa.modifyAndApplyExtraConfiguration(ctx, org, modifyFunc)
}
// assignReceiverConfigsUIDs assigns missing UUIDs to receiver configs.
func assignReceiverConfigsUIDs(c []*definitions.PostableApiReceiver) error {
seenUIDs := make(map[string]struct{})
pkg/services/ngalert/notifier/alertmanager_config_test.go
+204
View File
@@ -0,0 +1,204 @@
package notifier
import (
"context"
"testing"
amconfig "github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
)
func TestMultiOrgAlertmanager_SaveAndApplyExtraConfiguration(t *testing.T) {
orgID := int64(1)
t.Run("fails when organization does not exist", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
extraConfig := definitions.ExtraConfiguration{
Identifier: "test-config",
AlertmanagerConfig: `route:
receiver: test-receiver`,
}
err := mam.SaveAndApplyExtraConfiguration(ctx, 999, extraConfig)
require.Error(t, err)
require.ErrorContains(t, err, "failed to get current configuration")
})
t.Run("save new extra configuration", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
extraConfig := definitions.ExtraConfiguration{
Identifier: "test-alertmanager-config",
MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "prod"}},
TemplateFiles: map[string]string{"test.tmpl": "{{ define \"test\" }}Test{{ end }}"},
AlertmanagerConfig: `route:
receiver: test-receiver
receivers:
- name: test-receiver`,
}
err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, extraConfig)
require.NoError(t, err)
gettableConfig, err := mam.GetAlertmanagerConfiguration(ctx, orgID, false)
require.NoError(t, err)
require.Len(t, gettableConfig.ExtraConfigs, 1)
require.Equal(t, extraConfig.Identifier, gettableConfig.ExtraConfigs[0].Identifier)
require.Equal(t, extraConfig.TemplateFiles, gettableConfig.ExtraConfigs[0].TemplateFiles)
// Test that we can get the alertmanager config from raw storage
// We need to pass a decrypt function since the config is now encrypted
amConfig, err := gettableConfig.ExtraConfigs[0].GetAlertmanagerConfig()
require.NoError(t, err)
require.Equal(t, "test-receiver", amConfig.Route.Receiver)
require.Len(t, amConfig.Receivers, 1)
require.Equal(t, "test-receiver", amConfig.Receivers[0].Name)
})
t.Run("replace existing extra configuration with same identifier", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
identifier := "test-config"
// First add a configuration
originalConfig := definitions.ExtraConfiguration{
Identifier: identifier,
MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "original"}},
AlertmanagerConfig: `route:
receiver: original-receiver
receivers:
- name: original-receiver`,
}
err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, originalConfig)
require.NoError(t, err)
// Now replace it
updatedConfig := definitions.ExtraConfiguration{
Identifier: identifier,
MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "updated"}},
TemplateFiles: map[string]string{"updated.tmpl": "{{ define \"updated\" }}Updated{{ end }}"},
AlertmanagerConfig: `route:
receiver: updated-receiver
receivers:
- name: updated-receiver`,
}
err = mam.SaveAndApplyExtraConfiguration(ctx, orgID, updatedConfig)
require.NoError(t, err)
// Verify only one config exists with updated content
gettableConfig, err := mam.GetAlertmanagerConfiguration(ctx, orgID, false)
require.NoError(t, err)
require.Len(t, gettableConfig.ExtraConfigs, 1)
require.Equal(t, identifier, gettableConfig.ExtraConfigs[0].Identifier)
require.Contains(t, gettableConfig.ExtraConfigs[0].TemplateFiles, "updated.tmpl")
})
t.Run("fail to create multiple extra configurations", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
firstConfig := definitions.ExtraConfiguration{
Identifier: "first-config",
MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "first"}},
AlertmanagerConfig: `{
"route": {
"receiver": "first-receiver"
},
"receivers": [
{
"name": "first-receiver"
}
]
}`,
}
err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, firstConfig)
require.NoError(t, err)
secondConfig := definitions.ExtraConfiguration{
Identifier: "second-config",
MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "second"}},
AlertmanagerConfig: `{
"route": {
"receiver": "second-receiver"
},
"receivers": [
{
"name": "second-receiver"
}
]
}`,
}
err = mam.SaveAndApplyExtraConfiguration(ctx, orgID, secondConfig)
require.Error(t, err)
require.ErrorContains(t, err, "multiple extra configurations are not supported")
require.ErrorContains(t, err, "first-config")
})
}
func TestMultiOrgAlertmanager_DeleteAndApplyExtraConfiguration(t *testing.T) {
orgID := int64(1)
t.Run("successfully delete existing extra configuration", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
identifier := "test-identifier"
extraConfig := definitions.ExtraConfiguration{
Identifier: identifier,
MergeMatchers: amconfig.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "env", Value: "delete"}},
AlertmanagerConfig: `route:
receiver: test-receiver
receivers:
- name: test-receiver`,
}
err := mam.SaveAndApplyExtraConfiguration(ctx, orgID, extraConfig)
require.NoError(t, err)
gettableConfig, err := mam.GetAlertmanagerConfiguration(ctx, orgID, false)
require.NoError(t, err)
require.Len(t, gettableConfig.ExtraConfigs, 1)
err = mam.DeleteAndApplyExtraConfiguration(ctx, orgID, identifier)
require.NoError(t, err)
gettableConfig, err = mam.GetAlertmanagerConfiguration(ctx, orgID, false)
require.NoError(t, err)
require.Len(t, gettableConfig.ExtraConfigs, 0)
})
t.Run("deletion of non-existent configuration", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
err := mam.DeleteAndApplyExtraConfiguration(ctx, orgID, "non-existent")
require.NoError(t, err)
})
t.Run("deletion in non-existent org fails", func(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
err := mam.DeleteAndApplyExtraConfiguration(ctx, 999, "test-config")
require.Error(t, err)
require.ErrorContains(t, err, "failed to get current configuration")
})
}
pkg/services/ngalert/notifier/alertmanager_test.go
+18 -41
View File
@@ -2,15 +2,12 @@ package notifier
import (
"context"
"net/url"
"testing"
"time"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/client_golang/prometheus"
promcfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/db"
@@ -58,8 +55,9 @@ func setupAMTest(t *testing.T) *alertmanager {
orgID := 1
stateStore := NewFileStore(int64(orgID), kvStore)
crypto := NewCrypto(secretsService, s, l)
am, err := NewAlertmanager(context.Background(), 1, cfg, s, stateStore, &NilPeer{}, decryptFn, nil, m, featuremgmt.WithFeatures())
am, err := NewAlertmanager(context.Background(), 1, cfg, s, stateStore, &NilPeer{}, decryptFn, nil, m, featuremgmt.WithFeatures(), crypto)
require.NoError(t, err)
return am
}
@@ -130,33 +128,17 @@ func TestAlertmanager_ApplyConfig(t *testing.T) {
TemplateFiles: map[string]string{
"mimir-template": "{{ define \"mimir.title\" }}Mimir Alert{{ end }}",
},
AlertmanagerConfig: definitions.PostableApiAlertingConfig{
Config: definitions.Config{
Route: &definitions.Route{
Receiver: "mimir-webhook",
GroupBy: []model.LabelName{"alertname", "cluster"},
},
},
Receivers: []*definitions.PostableApiReceiver{
{
Receiver: config.Receiver{
Name: "mimir-webhook",
WebhookConfigs: []*config.WebhookConfig{
{
URL: &config.SecretURL{
URL: &url.URL{
Scheme: "https",
Host: "webhook.example.com",
Path: "/alerts",
},
},
HTTPConfig: &promcfg.DefaultHTTPClientConfig,
},
},
},
},
},
},
AlertmanagerConfig: `route:
receiver: mimir-webhook
group_by:
- alertname
- cluster
receivers:
- name: mimir-webhook
webhook_configs:
- url: https://webhook.example.com/alerts
send_resolved: true
http_config: {}`,
},
},
},
@@ -170,13 +152,10 @@ func TestAlertmanager_ApplyConfig(t *testing.T) {
{
Identifier: "", // invalid: empty identifier
MergeMatchers: config.Matchers{},
AlertmanagerConfig: definitions.PostableApiAlertingConfig{
Config: definitions.Config{
Route: &definitions.Route{
Receiver: "test-receiver",
},
},
},
AlertmanagerConfig: `route:
receiver: test-receiver
receivers:
- name: test-receiver`,
},
},
},
@@ -190,15 +169,13 @@ func TestAlertmanager_ApplyConfig(t *testing.T) {
am := setupAMTest(t)
ctx := context.Background()
changed, err := am.applyConfig(ctx, tc.config, false)
err := am.SaveAndApplyConfig(ctx, tc.config)
if tc.expectedError != "" {
require.Error(t, err)
require.ErrorContains(t, err, tc.expectedError)
require.False(t, changed)
} else {
require.NoError(t, err)
require.True(t, changed)
templateDefs := tc.config.GetMergedTemplateDefinitions()
expectedTemplateCount := len(tc.config.TemplateFiles)
pkg/services/ngalert/notifier/crypto.go
+33
View File
@@ -18,6 +18,8 @@ import (
type Crypto interface {
LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error
Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error)
EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error
DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error
getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error)
ProcessSecureSettings(ctx context.Context, orgId int64, recvs []*definitions.PostableApiReceiver) error
@@ -233,3 +235,34 @@ func (c *alertmanagerCrypto) getDecryptedSecret(r *definitions.PostableGrafanaRe
func (c *alertmanagerCrypto) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
return c.secrets.Encrypt(ctx, payload, opt)
}
func (c *alertmanagerCrypto) EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
for i := range config.ExtraConfigs {
encryptedValue, err := c.secrets.Encrypt(ctx, []byte(config.ExtraConfigs[i].AlertmanagerConfig), secrets.WithoutScope())
if err != nil {
return fmt.Errorf("failed to encrypt extra configuration: %w", err)
}
config.ExtraConfigs[i].AlertmanagerConfig = base64.StdEncoding.EncodeToString(encryptedValue)
}
return nil
}
func (c *alertmanagerCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
for i := range config.ExtraConfigs {
encryptedValue, err := base64.StdEncoding.DecodeString(config.ExtraConfigs[i].AlertmanagerConfig)
if err != nil {
return fmt.Errorf("failed to base64 decode extra configuration: %w", err)
}
decryptedValue, err := c.secrets.Decrypt(ctx, encryptedValue)
if err != nil {
return fmt.Errorf("failed to decrypt extra configuration: %w", err)
}
config.ExtraConfigs[i].AlertmanagerConfig = string(decryptedValue)
}
return nil
}
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+1 -1
View File
@@ -160,7 +160,7 @@ func NewMultiOrgAlertmanager(
moa.factory = func(ctx context.Context, orgID int64) (Alertmanager, error) {
m := metrics.NewAlertmanagerMetrics(moa.metrics.GetOrCreateOrgRegistry(orgID), l)
stateStore := NewFileStore(orgID, kvStore)
return NewAlertmanager(ctx, orgID, moa.settings, moa.configStore, stateStore, moa.peer, moa.decryptFn, moa.ns, m, featureManager)
return NewAlertmanager(ctx, orgID, moa.settings, moa.configStore, stateStore, moa.peer, moa.decryptFn, moa.ns, m, featureManager, moa.Crypto)
}
for _, opt := range opts {
public/api-merged.json
+33 -7
View File
@@ -15355,6 +15355,26 @@
}
}
},
"ExtraConfiguration": {
"type": "object",
"properties": {
"alertmanager_config": {
"type": "string"
},
"identifier": {
"type": "string"
},
"merge_matchers": {
"$ref": "#/definitions/Matchers"
},
"template_files": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
},
"FailedUser": {
"description": "FailedUser holds the information of an user that failed",
"type": "object",
@@ -16304,6 +16324,12 @@
"alertmanager_config": {
"$ref": "#/definitions/GettableApiAlertingConfig"
},
"extra_config": {
"type": "array",
"items": {
"$ref": "#/definitions/ExtraConfiguration"
}
},
"template_file_provenances": {
"type": "object",
"additionalProperties": {
@@ -18657,6 +18683,12 @@
"alertmanager_config": {
"$ref": "#/definitions/PostableApiAlertingConfig"
},
"extra_config": {
"type": "array",
"items": {
"$ref": "#/definitions/ExtraConfiguration"
}
},
"template_files": {
"type": "object",
"additionalProperties": {
@@ -20041,8 +20073,8 @@
}
},
"Route": {
"description": "A Route is a node that contains definitions of how to handle alerts. This is modified\nfrom the upstream alertmanager in that it adds the ObjectMatchers property.",
"type": "object",
"title": "A Route is a node that contains definitions of how to handle alerts.",
"properties": {
"active_time_intervals": {
"type": "array",
@@ -20084,12 +20116,6 @@
"type": "string"
}
},
"object_matchers": {
"$ref": "#/definitions/ObjectMatchers"
},
"provenance": {
"$ref": "#/definitions/Provenance"
},
"receiver": {
"type": "string"
},
public/openapi3.json
+33 -7
View File
@@ -5406,6 +5406,26 @@
},
"type": "object"
},
"ExtraConfiguration": {
"properties": {
"alertmanager_config": {
"type": "string"
},
"identifier": {
"type": "string"
},
"merge_matchers": {
"$ref": "#/components/schemas/Matchers"
},
"template_files": {
"additionalProperties": {
"type": "string"
},
"type": "object"
}
},
"type": "object"
},
"FailedUser": {
"description": "FailedUser holds the information of an user that failed",
"properties": {
@@ -6354,6 +6374,12 @@
"alertmanager_config": {
"$ref": "#/components/schemas/GettableApiAlertingConfig"
},
"extra_config": {
"items": {
"$ref": "#/components/schemas/ExtraConfiguration"
},
"type": "array"
},
"template_file_provenances": {
"additionalProperties": {
"$ref": "#/components/schemas/Provenance"
@@ -8707,6 +8733,12 @@
"alertmanager_config": {
"$ref": "#/components/schemas/PostableApiAlertingConfig"
},
"extra_config": {
"items": {
"$ref": "#/components/schemas/ExtraConfiguration"
},
"type": "array"
},
"template_files": {
"additionalProperties": {
"type": "string"
@@ -10092,7 +10124,6 @@
"type": "object"
},
"Route": {
"description": "A Route is a node that contains definitions of how to handle alerts. This is modified\nfrom the upstream alertmanager in that it adds the ObjectMatchers property.",
"properties": {
"active_time_intervals": {
"items": {
@@ -10134,12 +10165,6 @@
},
"type": "array"
},
"object_matchers": {
"$ref": "#/components/schemas/ObjectMatchers"
},
"provenance": {
"$ref": "#/components/schemas/Provenance"
},
"receiver": {
"type": "string"
},
@@ -10153,6 +10178,7 @@
"type": "array"
}
},
"title": "A Route is a node that contains definitions of how to handle alerts.",
"type": "object"
},
"RouteExport": {