public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Send templates from extra configuration to remote Alertmanager (#107981)

Browse Source 
* extract logging of MergedResult into method

* convert GetMergedTemplateDefinitions to return PostableApiTemplate

* update mergeExtracConfigs to return GrafanaAlertmanagerConfig

* pass by value, not pointer

* add template definition to payload

* update tests

* rename to Templates

* log merge results

* fix reference in workspace
This commit is contained in:
Yuri Tseretyan
2025-07-18 15:16:17 -04:00
committed by GitHub
parent 17ba3ec321
commit 5097dd5c7d
9 changed files with 97 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.work
+1 -1
View File
@@ -24,6 +24,6 @@ use (
./pkg/semconv
)
replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250604130045-92c8f6389b36
replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6
replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56
pkg/services/ngalert/api/tooling/definitions/alertmanager.go
+36 -23
View File
@@ -5,10 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/go-openapi/strfmt"
alertingTemplates "github.com/grafana/alerting/templates"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
@@ -17,6 +17,7 @@ import (
"github.com/grafana/alerting/definition"
alertingmodels "github.com/grafana/alerting/models"
"github.com/grafana/grafana/pkg/apimachinery/errutil"
)
@@ -267,9 +268,32 @@ type (
PostableApiReceiver = definition.PostableApiReceiver
PostableGrafanaReceivers = definition.PostableGrafanaReceivers
ReceiverType = definition.ReceiverType
MergeResult = definition.MergeResult
)
type MergeResult definition.MergeResult
func (m MergeResult) LogContext() []any {
if len(m.RenamedReceivers) == 0 && len(m.RenamedTimeIntervals) == 0 {
return nil
}
logCtx := make([]any, 0, 4)
if len(m.RenamedTimeIntervals) > 0 {
rcvBuilder := strings.Builder{}
for from, to := range m.RenamedReceivers {
rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
}
logCtx = append(logCtx, "renamedReceivers", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
}
if len(m.RenamedTimeIntervals) > 0 {
rcvBuilder := strings.Builder{}
for from, to := range m.RenamedTimeIntervals {
rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
}
logCtx = append(logCtx, "renamedTimeIntervals", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
}
return logCtx
}
const (
GrafanaReceiverType = definition.GrafanaReceiverType
AlertmanagerReceiverType = definition.AlertmanagerReceiverType
@@ -779,31 +803,20 @@ func (c *PostableUserConfig) GetMergedAlertmanagerConfig() (MergeResult, error)
return MergeResult{}, fmt.Errorf("failed to get mimir alertmanager config: %w", err)
}
return definition.Merge(c.AlertmanagerConfig, mcfg, opts)
m, err := definition.Merge(c.AlertmanagerConfig, mcfg, opts)
if err != nil {
return MergeResult{}, fmt.Errorf("failed to merge alertmanager config: %w", err)
}
return MergeResult(m), nil
}
// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of TemplateDefinitions.
func (c *PostableUserConfig) GetMergedTemplateDefinitions() []alertingTemplates.TemplateDefinition {
out := make([]alertingTemplates.TemplateDefinition, 0, len(c.TemplateFiles))
for name, tmpl := range c.TemplateFiles {
out = append(out, alertingTemplates.TemplateDefinition{
Name: name,
Template: tmpl,
Kind: alertingTemplates.GrafanaKind,
})
}
if len(c.ExtraConfigs) == 0 {
// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of Templates.
func (c *PostableUserConfig) GetMergedTemplateDefinitions() []definition.PostableApiTemplate {
out := definition.TemplatesMapToPostableAPITemplates(c.TemplateFiles, definition.GrafanaTemplateKind)
if len(c.ExtraConfigs) == 0 || len(c.ExtraConfigs[0].TemplateFiles) == 0 {
return out
}
// support only one config for now
for name, tmpl := range c.ExtraConfigs[0].TemplateFiles {
out = append(out, alertingTemplates.TemplateDefinition{
Name: name,
Template: tmpl,
Kind: alertingTemplates.MimirKind,
})
}
return out
return append(out, definition.TemplatesMapToPostableAPITemplates(c.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind)...)
}
func (c *PostableUserConfig) UnmarshalJSON(b []byte) error {
pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go
+5 -5
View File
@@ -6,7 +6,7 @@ import (
"strings"
"testing"
alertingTemplates "github.com/grafana/alerting/templates"
"github.com/grafana/alerting/definition"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/common/model"
@@ -383,21 +383,21 @@ func TestPostableUserConfig_GetMergedTemplateDefinitions(t *testing.T) {
require.Len(t, result, tc.expectedTemplates)
templateMap := make(map[string]string)
kindMap := make(map[string]alertingTemplates.Kind)
kindMap := make(map[string]definition.TemplateKind)
for _, tmpl := range result {
templateMap[tmpl.Name] = tmpl.Template
templateMap[tmpl.Name] = tmpl.Content
kindMap[tmpl.Name] = tmpl.Kind
}
for name, content := range tc.config.TemplateFiles {
require.Equal(t, content, templateMap[name])
require.Equal(t, alertingTemplates.GrafanaKind, kindMap[name])
require.Equal(t, definition.GrafanaTemplateKind, kindMap[name])
}
if len(tc.config.ExtraConfigs) > 0 {
for name, content := range tc.config.ExtraConfigs[0].TemplateFiles {
require.Equal(t, content, templateMap[name])
require.Equal(t, alertingTemplates.MimirKind, kindMap[name])
require.Equal(t, definition.MimirTemplateKind, kindMap[name])
}
}
})
pkg/services/ngalert/notifier/alertmanager.go
+4 -26
View File
@@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
alertingNotify "github.com/grafana/alerting/notify"
@@ -325,29 +324,6 @@ func (am *alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu
}
}
func logMergeResult(l log.Logger, m apimodels.MergeResult) {
if len(m.RenamedReceivers) == 0 && len(m.RenamedTimeIntervals) == 0 {
return
}
logCtx := make([]any, 0, 4)
if len(m.RenamedTimeIntervals) > 0 {
rcvBuilder := strings.Builder{}
for from, to := range m.RenamedReceivers {
rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
}
logCtx = append(logCtx, "renamedReceivers", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
}
if len(m.RenamedTimeIntervals) > 0 {
rcvBuilder := strings.Builder{}
for from, to := range m.RenamedTimeIntervals {
rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
}
logCtx = append(logCtx, "renamedTimeIntervals", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
}
l.Info("Configurations merged successfully but some resources were renamed", logCtx...)
}
// applyConfig applies a new configuration by re-initializing all components using the configuration provided.
// It returns a boolean indicating whether the user config was changed and an error.
// It is not safe to call concurrently.
@@ -361,9 +337,11 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable
if err != nil {
return false, fmt.Errorf("failed to get full alertmanager configuration: %w", err)
}
logMergeResult(am.logger, mergeResult)
if logInfo := mergeResult.LogContext(); len(logInfo) > 0 {
am.logger.Info("Configurations merged successfully but some resources were renamed", logInfo...)
}
amConfig := mergeResult.Config
templates := cfg.GetMergedTemplateDefinitions()
templates := alertingNotify.PostableAPITemplatesToTemplateDefinitions(cfg.GetMergedTemplateDefinitions())
// Now add autogenerated config to the route.
err = AddAutogenConfig(ctx, am.logger, am.Store, am.Base.TenantID(), &amConfig, skipInvalid)
pkg/services/ngalert/remote/alertmanager.go
+28 -15
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/go-openapi/strfmt"
"github.com/grafana/alerting/definition"
amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
amgeneral "github.com/prometheus/alertmanager/api/v2/client/general"
@@ -290,10 +291,10 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
}
// Decrypt and merge extra configs
if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
payload, err := am.mergeExtraConfigs(ctx, decryptedCfg)
if err != nil {
return fmt.Errorf("unable to merge extra configurations: %w", err)
}
payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg)
rawPayload, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("unable to marshal decrypted configuration: %w", err)
@@ -352,27 +353,36 @@ func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn {
}
// mergeExtraConfigs decrypts and applies merged configuration if extra configs exist.
func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error {
func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) (remoteClient.GrafanaAlertmanagerConfig, error) {
if len(config.ExtraConfigs) == 0 {
return nil
return remoteClient.GrafanaAlertmanagerConfig{
TemplateFiles: config.TemplateFiles,
AlertmanagerConfig: config.AlertmanagerConfig,
Templates: nil,
}, nil
}
if err := am.crypto.DecryptExtraConfigs(ctx, config); err != nil {
return fmt.Errorf("unable to decrypt extra configs: %w", err)
return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to decrypt extra configs: %w", err)
}
mergeResult, err := config.GetMergedAlertmanagerConfig()
if err != nil {
return fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
}
config.AlertmanagerConfig = mergeResult.Config
// Clear ExtraConfigs to avoid re-processing them later
config.ExtraConfigs = nil
return nil
if logctx := mergeResult.LogContext(); len(logctx) > 0 {
am.log.Debug("Configurations merged successfully but some resources were renamed", logctx...)
}
templates := definition.TemplatesMapToPostableAPITemplates(config.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind)
return remoteClient.GrafanaAlertmanagerConfig{
// TODO keep sending Grafana templates as a map to not break old Mimir
TemplateFiles: config.TemplateFiles,
AlertmanagerConfig: mergeResult.Config,
Templates: templates,
}, nil
}
func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg *remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
am.metrics.ConfigSyncsTotal.Inc()
if err := am.mimirClient.CreateGrafanaAlertmanagerConfig(
ctx,
@@ -455,10 +465,10 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
return err
}
if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
payload, err := am.mergeExtraConfigs(ctx, decryptedCfg)
if err != nil {
return fmt.Errorf("unable to merge extra configurations: %w", err)
}
payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg)
rawCfg, err := json.Marshal(payload)
if err != nil {
return err
@@ -484,7 +494,10 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error {
return err
}
payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg)
payload := remoteClient.GrafanaAlertmanagerConfig{
TemplateFiles: c.TemplateFiles,
AlertmanagerConfig: decryptedCfg.AlertmanagerConfig,
}
rawCfg, err := json.Marshal(payload)
if err != nil {
return err
pkg/services/ngalert/remote/alertmanager_test.go
+11 -4
View File
@@ -484,12 +484,18 @@ func TestCompareAndSendConfiguration(t *testing.T) {
test, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
require.NoError(t, err)
cfgWithDecryptedSecret := PostableUserConfigToGrafanaAlertmanagerConfig(test)
cfgWithDecryptedSecret := client.GrafanaAlertmanagerConfig{
TemplateFiles: test.TemplateFiles,
AlertmanagerConfig: test.AlertmanagerConfig,
}
testAutogenRoutes, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
require.NoError(t, err)
require.NoError(t, testAutogenFn(nil, nil, 0, &testAutogenRoutes.AlertmanagerConfig, false))
cfgWithAutogenRoutes := PostableUserConfigToGrafanaAlertmanagerConfig(testAutogenRoutes)
cfgWithAutogenRoutes := client.GrafanaAlertmanagerConfig{
TemplateFiles: testAutogenRoutes.TemplateFiles,
AlertmanagerConfig: testAutogenRoutes.AlertmanagerConfig,
}
// Calculate hashes for expected configurations
cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret)
@@ -506,9 +512,10 @@ func TestCompareAndSendConfiguration(t *testing.T) {
require.NoError(t, err)
r, err := cfgWithExtraUnmerged.GetMergedAlertmanagerConfig()
require.NoError(t, err)
cfgWithExtraMerged := &client.GrafanaAlertmanagerConfig{
cfgWithExtraMerged := client.GrafanaAlertmanagerConfig{
TemplateFiles: cfgWithExtraUnmerged.TemplateFiles,
AlertmanagerConfig: r.Config,
Templates: definition.TemplatesMapToPostableAPITemplates(cfgWithExtraUnmerged.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind),
}
cfgWithExtraMergedBytes, err := json.Marshal(cfgWithExtraMerged)
require.NoError(t, err)
@@ -827,7 +834,7 @@ func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) {
// Return an empty config to ensure it gets replaced
w.Header().Add("content-type", "application/json")
require.NoError(t, json.NewEncoder(w).Encode(client.UserGrafanaConfig{
GrafanaAlertmanagerConfig: &client.GrafanaAlertmanagerConfig{},
GrafanaAlertmanagerConfig: client.GrafanaAlertmanagerConfig{},
}))
return
}
pkg/services/ngalert/remote/client/alertmanager_configuration.go
+11 -9
View File
@@ -17,24 +17,26 @@ const (
)
type GrafanaAlertmanagerConfig struct {
// TODO this needs to be deleted once Mimir is updated
TemplateFiles map[string]string `yaml:"template_files" json:"template_files"`
AlertmanagerConfig definition.PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"`
Templates []definition.PostableApiTemplate `yaml:"templates,omitempty" json:"templates,omitempty"`
}
func (u *GrafanaAlertmanagerConfig) MarshalJSON() ([]byte, error) {
// this is special marshaling that makes sure that secrets are not masked
// This is special marshaling that makes sure that secrets are not masked.
type cfg GrafanaAlertmanagerConfig
return definition.MarshalJSONWithSecrets((*cfg)(u))
}
type UserGrafanaConfig struct {
GrafanaAlertmanagerConfig *GrafanaAlertmanagerConfig `json:"configuration"`
Hash string `json:"configuration_hash"`
CreatedAt int64 `json:"created"`
Default bool `json:"default"`
Promoted bool `json:"promoted"`
ExternalURL string `json:"external_url"`
SmtpConfig SmtpConfig `json:"smtp_config"`
GrafanaAlertmanagerConfig GrafanaAlertmanagerConfig `json:"configuration"`
Hash string `json:"configuration_hash"`
CreatedAt int64 `json:"created"`
Default bool `json:"default"`
Promoted bool `json:"promoted"`
ExternalURL string `json:"external_url"`
SmtpConfig SmtpConfig `json:"smtp_config"`
// TODO: Remove once everything can be sent in the 'SmtpConfig' field.
SmtpFrom string `json:"smtp_from"`
@@ -64,7 +66,7 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana
return gc, nil
}
func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{
GrafanaAlertmanagerConfig: cfg,
Hash: hash,
pkg/services/ngalert/remote/client/mimir.go
+1 -1
View File
@@ -30,7 +30,7 @@ type MimirClient interface {
DeleteGrafanaAlertmanagerState(ctx context.Context) error
GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error)
CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error
CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error
DeleteGrafanaAlertmanagerConfig(ctx context.Context) error
TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error)
pkg/services/ngalert/remote/compat.go
-13
View File
@@ -1,13 +0,0 @@
package remote
import (
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
)
func PostableUserConfigToGrafanaAlertmanagerConfig(config *definitions.PostableUserConfig) *client.GrafanaAlertmanagerConfig {
return &client.GrafanaAlertmanagerConfig{
TemplateFiles: config.TemplateFiles,
AlertmanagerConfig: config.AlertmanagerConfig,
}
}