public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Persist silence state immediately on Create/Delete (#84705)

Browse Source 
* Alerting: Persist silence state immediately on Create/Delete

Persists the silence state to the kvstore immediately instead of waiting for the
 next maintenance run. This is used after Create/Delete to prevent silences from
 being lost when a new Alertmanager is started before the state has persisted.
 This can happen, for example, in a rolling deployment scenario.

* Fix test that requires real data

* Don't error if silence state persist fails, maintenance will correct
This commit is contained in:
Matthew Jacobson
2024-04-09 13:39:34 -04:00
committed by GitHub
parent c7e4baff75
commit f79dd7c7f9
14 changed files with 449 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/notifier/alertmanager.go
+5
View File
@@ -417,6 +417,11 @@ func (am *alertmanager) PutAlerts(_ context.Context, postableAlerts apimodels.Po
return am.Base.PutAlerts(alerts)
}
// SilenceState returns the current internal state of silences.
func (am *alertmanager) SilenceState(_ context.Context) (alertingNotify.SilenceState, error) {
return am.Base.SilenceState()
}
// AlertValidationError is the error capturing the validation errors
// faced on the alerts.
type AlertValidationError struct {
pkg/services/ngalert/notifier/alertmanager_mock/Alertmanager.go
+54
View File
@@ -711,6 +711,60 @@ func (_c *AlertmanagerMock_SaveAndApplyDefaultConfig_Call) RunAndReturn(run func
return _c
}
// SilenceState provides a mock function with given fields: _a0
func (_m *AlertmanagerMock) SilenceState(_a0 context.Context) (notify.SilenceState, error) {
ret := _m.Called(_a0)
var r0 notify.SilenceState
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (notify.SilenceState, error)); ok {
return rf(_a0)
}
if rf, ok := ret.Get(0).(func(context.Context) notify.SilenceState); ok {
r0 = rf(_a0)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(notify.SilenceState)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(_a0)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// AlertmanagerMock_SilenceState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SilenceState'
type AlertmanagerMock_SilenceState_Call struct {
*mock.Call
}
// SilenceState is a helper method to define mock.On call
// - _a0 context.Context
func (_e *AlertmanagerMock_Expecter) SilenceState(_a0 interface{}) *AlertmanagerMock_SilenceState_Call {
return &AlertmanagerMock_SilenceState_Call{Call: _e.mock.On("SilenceState", _a0)}
}
func (_c *AlertmanagerMock_SilenceState_Call) Run(run func(_a0 context.Context)) *AlertmanagerMock_SilenceState_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *AlertmanagerMock_SilenceState_Call) Return(_a0 notify.SilenceState, _a1 error) *AlertmanagerMock_SilenceState_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *AlertmanagerMock_SilenceState_Call) RunAndReturn(run func(context.Context) (notify.SilenceState, error)) *AlertmanagerMock_SilenceState_Call {
_c.Call.Return(run)
return _c
}
// StopAndWait provides a mock function with given fields:
func (_m *AlertmanagerMock) StopAndWait() {
_m.Called()
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+83 -1
View File
@@ -6,9 +6,10 @@ import (
"sync"
"time"
alertingCluster "github.com/grafana/alerting/cluster"
"github.com/prometheus/client_golang/prometheus"
alertingCluster "github.com/grafana/alerting/cluster"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/grafana/pkg/infra/kvstore"
@@ -42,6 +43,10 @@ type Alertmanager interface {
GetSilence(context.Context, string) (apimodels.GettableSilence, error)
ListSilences(context.Context, []string) (apimodels.GettableSilences, error)
// SilenceState returns the current state of silences in the Alertmanager. This is used to persist the state
// to the kvstore.
SilenceState(context.Context) (alertingNotify.SilenceState, error)
// Alerts
GetAlerts(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
GetAlertGroups(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error)
@@ -395,6 +400,83 @@ func (moa *MultiOrgAlertmanager) AlertmanagerFor(orgID int64) (Alertmanager, err
return orgAM, nil
}
// CreateSilence creates a silence in the Alertmanager for the organization provided, returning the silence ID. It will
// also persist the silence state to the kvstore immediately after creating the silence.
func (moa *MultiOrgAlertmanager) CreateSilence(ctx context.Context, orgID int64, ps *alertingNotify.PostableSilence) (string, error) {
moa.alertmanagersMtx.RLock()
defer moa.alertmanagersMtx.RUnlock()
orgAM, existing := moa.alertmanagers[orgID]
if !existing {
return "", ErrNoAlertmanagerForOrg
}
if !orgAM.Ready() {
return "", ErrAlertmanagerNotReady
}
// Need to create the silence in the AM first to get the silence ID.
silenceID, err := orgAM.CreateSilence(ctx, ps)
if err != nil {
return "", err
}
err = moa.updateSilenceState(ctx, orgAM, orgID)
if err != nil {
moa.logger.Warn("Failed to persist silence state on create, will be corrected by next maintenance run", "orgID", orgID, "silenceID", silenceID, "error", err)
}
return silenceID, nil
}
// DeleteSilence deletes a silence in the Alertmanager for the organization provided. It will also persist the silence
// state to the kvstore immediately after deleting the silence.
func (moa *MultiOrgAlertmanager) DeleteSilence(ctx context.Context, orgID int64, silenceID string) error {
moa.alertmanagersMtx.RLock()
defer moa.alertmanagersMtx.RUnlock()
orgAM, existing := moa.alertmanagers[orgID]
if !existing {
return ErrNoAlertmanagerForOrg
}
if !orgAM.Ready() {
return ErrAlertmanagerNotReady
}
err := orgAM.DeleteSilence(ctx, silenceID)
if err != nil {
return err
}
err = moa.updateSilenceState(ctx, orgAM, orgID)
if err != nil {
moa.logger.Warn("Failed to persist silence state on delete, will be corrected by next maintenance run", "orgID", orgID, "silenceID", silenceID, "error", err)
}
return nil
}
// updateSilenceState persists the silence state to the kvstore immediately instead of waiting for the next maintenance
// run. This is used after Create/Delete to prevent silences from being lost when a new Alertmanager is started before
// the state has persisted. This can happen, for example, in a rolling deployment scenario.
func (moa *MultiOrgAlertmanager) updateSilenceState(ctx context.Context, orgAM Alertmanager, orgID int64) error {
// Collect the internal silence state from the AM.
// TODO: Currently, we rely on the AM itself for the persisted silence state representation. Preferably, we would
// define the state ourselves and persist it in a format that is easy to guarantee consistency for writes to
// individual silences. In addition to the consistency benefits, this would also allow us to avoid the need for
// a network request to the AM to get the state in the case of remote alertmanagers.
silences, err := orgAM.SilenceState(ctx)
if err != nil {
return err
}
// Persist to kvstore.
fs := NewFileStore(orgID, moa.kvStore)
_, err = fs.SaveSilences(ctx, silences)
return err
}
// NilPeer and NilChannel implements the Alertmanager clustering interface.
type NilPeer struct{}
pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go
+4 -4
View File
@@ -126,8 +126,8 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
OrgID: 1,
LastApplied: time.Now().Unix(),
}))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "dGVzdAo=")) // base64-encoded string "test"
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "dGVzdAo=")) // base64-encoded string "test"
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "lwEKhgEKATISFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsDlwEKhgEKATESFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsD"))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "OgoqCgZncm91cDISEgoJcmVjZWl2ZXIyEgV0ZXN0MyoMCLSDkbAGEMvaofYCEgwIxJ+RsAYQy9qh9gI6CioKBmdyb3VwMRISCglyZWNlaXZlcjESBXRlc3QzKgwItIORsAYQy9qh9gISDAjEn5GwBhDL2qH2Ag=="))
// The sync interval (10s) has not elapsed yet, syncing should have no effect.
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
@@ -151,8 +151,8 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
OrgID: 1,
LastApplied: time.Now().Unix(),
}))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "dGVzdC0yCg==")) // base64-encoded string "test-2"
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "dGVzdC0yCg==")) // base64-encoded string "test-2"
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "lwEKhgEKAWESFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCPuEkbAGEK3AhM8CIgwIi6GRsAYQrcCEzwIqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCIuhkbAGEK3AhM8ClwEKhgEKAWISFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCPuEkbAGEK3AhM8CIgwIi6GRsAYQrcCEzwIqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCIuhkbAGEK3AhM8C"))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "OAopCgZncm91cEESEgoJcmVjZWl2ZXJBEgV0ZXN0MyoLCNmEkbAGEOzO0BUSCwjpoJGwBhDsztAVOAopCgZncm91cEISEgoJcmVjZWl2ZXJCEgV0ZXN0MyoLCNmEkbAGEOzO0BUSCwjpoJGwBhDsztAV"))
// Both state and config should be updated when shutting the Alertmanager down.
moa.StopAndWait()
pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
+156 -83
View File
@@ -6,6 +6,8 @@ import (
"testing"
"time"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
@@ -22,18 +24,7 @@ import (
)
func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
configStore := NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{})
orgStore := &FakeOrgStore{
orgs: []int64{1, 2, 3},
}
tmpDir := t.TempDir()
kvStore := ngfakes.NewFakeKVStore(t)
provStore := ngfakes.NewFakeProvisioningStore()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
cfg := &setting.Cfg{
DataPath: tmpDir,
UnifiedAlerting: setting.UnifiedAlertingSettings{
@@ -42,8 +33,9 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
DisabledOrgs: map[int64]struct{}{5: {}},
}, // do not poll in tests.
}
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService, featuremgmt.WithFeatures())
require.NoError(t, err)
mam := setupMam(t, cfg)
reg := mam.metrics.Registerer.(*prometheus.Registry)
orgStore := mam.orgStore.(*FakeOrgStore)
ctx := context.Background()
// Ensure that one Alertmanager is created per org.
@@ -61,7 +53,7 @@ grafana_alerting_discovered_configurations 3
// Configurations should be marked as successfully applied.
for _, org := range orgStore.orgs {
configs, err := configStore.GetAppliedConfigurations(ctx, org, 10)
configs, err := mam.configStore.GetAppliedConfigurations(ctx, org, 10)
require.NoError(t, err)
require.Len(t, configs, 1)
}
@@ -106,58 +98,40 @@ grafana_alerting_discovered_configurations 4
orgID := int64(6)
// Populate the kvstore with orphaned records.
err = kvStore.Set(ctx, orgID, KVNamespace, SilencesFilename, "file_1")
err := mam.kvStore.Set(ctx, orgID, KVNamespace, SilencesFilename, "file_1")
require.NoError(t, err)
err = kvStore.Set(ctx, orgID, KVNamespace, NotificationLogFilename, "file_1")
err = mam.kvStore.Set(ctx, orgID, KVNamespace, NotificationLogFilename, "file_1")
require.NoError(t, err)
// Now re run the sync job once.
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
// The organization kvstore records should be gone by now.
_, exists, _ := kvStore.Get(ctx, orgID, KVNamespace, SilencesFilename)
_, exists, _ := mam.kvStore.Get(ctx, orgID, KVNamespace, SilencesFilename)
require.False(t, exists)
_, exists, _ = kvStore.Get(ctx, orgID, KVNamespace, NotificationLogFilename)
_, exists, _ = mam.kvStore.Get(ctx, orgID, KVNamespace, NotificationLogFilename)
require.False(t, exists)
}
}
func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
// Include a broken configuration for organization 2.
var orgWithBadConfig int64 = 2
configStore := NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{
mam.configStore = NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{
2: {AlertmanagerConfiguration: brokenConfig, OrgID: orgWithBadConfig},
})
orgs := []int64{1, 2, 3}
orgStore := &FakeOrgStore{
orgs: orgs,
}
tmpDir := t.TempDir()
kvStore := ngfakes.NewFakeKVStore(t)
provStore := ngfakes.NewFakeProvisioningStore()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
cfg := &setting.Cfg{
DataPath: tmpDir,
UnifiedAlerting: setting.UnifiedAlertingSettings{
AlertmanagerConfigPollInterval: 10 * time.Minute,
DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
}, // do not poll in tests.
}
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService, featuremgmt.WithFeatures())
orgs, err := mam.orgStore.GetOrgs(ctx)
require.NoError(t, err)
ctx := context.Background()
// No successfully applied configurations should be found at first.
{
for _, org := range orgs {
configs, err := configStore.GetAppliedConfigurations(ctx, org, 10)
configs, err := mam.configStore.GetAppliedConfigurations(ctx, org, 10)
require.NoError(t, err)
require.Len(t, configs, 0)
}
@@ -173,7 +147,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
// Configurations should be marked as successfully applied for all orgs except for org 2.
for _, org := range orgs {
configs, err := configStore.GetAppliedConfigurations(ctx, org, 10)
configs, err := mam.configStore.GetAppliedConfigurations(ctx, org, 10)
require.NoError(t, err)
if org == orgWithBadConfig {
require.Len(t, configs, 0)
@@ -193,7 +167,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
// The configuration should still be marked as successfully applied for all orgs except for org 2.
for _, org := range orgs {
configs, err := configStore.GetAppliedConfigurations(ctx, org, 10)
configs, err := mam.configStore.GetAppliedConfigurations(ctx, org, 10)
require.NoError(t, err)
if org == orgWithBadConfig {
require.Len(t, configs, 0)
@@ -205,7 +179,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
// If we fix the configuration, it becomes ready.
{
configStore.configs = map[int64]*models.AlertConfiguration{} // It'll apply the default config.
mam.configStore.(*fakeConfigStore).configs = map[int64]*models.AlertConfiguration{} // It'll apply the default config.
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Len(t, mam.alertmanagers, 3)
require.True(t, mam.alertmanagers[1].Ready())
@@ -214,7 +188,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
// All configurations should be marked as successfully applied.
for _, org := range orgs {
configs, err := configStore.GetAppliedConfigurations(ctx, org, 10)
configs, err := mam.configStore.GetAppliedConfigurations(ctx, org, 10)
require.NoError(t, err)
require.NotEqual(t, 0, len(configs))
}
@@ -222,23 +196,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
}
func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
configStore := NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{})
orgStore := &FakeOrgStore{
orgs: []int64{1, 2, 3},
}
tmpDir := t.TempDir()
cfg := &setting.Cfg{
DataPath: tmpDir,
UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration()}, // do not poll in tests.
}
kvStore := ngfakes.NewFakeKVStore(t)
provStore := ngfakes.NewFakeProvisioningStore()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService, featuremgmt.WithFeatures())
require.NoError(t, err)
mam := setupMam(t, nil)
ctx := context.Background()
// Ensure that one Alertmanagers is created per org.
@@ -264,7 +222,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
}
// Let's now remove the previous queried organization.
orgStore.orgs = []int64{1, 3}
mam.orgStore.(*FakeOrgStore).orgs = []int64{1, 3}
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
{
_, err := mam.AlertmanagerFor(2)
@@ -273,24 +231,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
}
func TestMultiOrgAlertmanager_ActivateHistoricalConfiguration(t *testing.T) {
configStore := NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{})
orgStore := &FakeOrgStore{
orgs: []int64{1, 2, 3},
}
tmpDir := t.TempDir()
defaultConfig := `{"template_files":null,"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"email receiver","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}`
cfg := &setting.Cfg{
DataPath: tmpDir,
UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: defaultConfig}, // do not poll in tests.
}
kvStore := ngfakes.NewFakeKVStore(t)
provStore := ngfakes.NewFakeProvisioningStore()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService, featuremgmt.WithFeatures())
require.NoError(t, err)
mam := setupMam(t, nil)
ctx := context.Background()
// Ensure that one Alertmanager is created per org.
@@ -339,9 +280,141 @@ func TestMultiOrgAlertmanager_ActivateHistoricalConfiguration(t *testing.T) {
// Verify that the org has the old default config.
cfgs, err = mam.getLatestConfigs(ctx)
require.NoError(t, err)
require.Equal(t, defaultConfig, cfgs[2].AlertmanagerConfiguration)
require.JSONEq(t, defaultConfig, cfgs[2].AlertmanagerConfiguration)
}
func TestMultiOrgAlertmanager_Silences(t *testing.T) {
mam := setupMam(t, nil)
ctx := context.Background()
// Ensure that one Alertmanager is created per org.
{
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Len(t, mam.alertmanagers, 3)
}
am, err := mam.AlertmanagerFor(1)
require.NoError(t, err)
// Confirm no silences.
silences, err := am.ListSilences(ctx, []string{})
require.NoError(t, err)
require.Len(t, silences, 0)
// Confirm empty state.
state, err := am.SilenceState(ctx)
require.NoError(t, err)
require.Len(t, state, 0)
// Confirm empty kvstore.
v, ok, err := mam.kvStore.Get(ctx, 1, KVNamespace, SilencesFilename)
require.NoError(t, err)
require.False(t, ok)
require.Empty(t, v)
// Create 2 silences.
sid, err := mam.CreateSilence(ctx, 1, GenSilence("test"))
require.NoError(t, err)
require.NotEmpty(t, sid)
sid2, err := mam.CreateSilence(ctx, 1, GenSilence("test"))
require.NoError(t, err)
require.NotEmpty(t, sid2)
// Confirm 2 silences.
silences, err = am.ListSilences(ctx, []string{})
require.NoError(t, err)
require.Len(t, silences, 2)
// Confirm 2 states.
state, err = am.SilenceState(ctx)
require.NoError(t, err)
require.Len(t, state, 2)
// Confirm 2 silences in the kvstore.
v, ok, err = mam.kvStore.Get(ctx, 1, KVNamespace, SilencesFilename)
require.NoError(t, err)
require.True(t, ok)
decoded, err := decode(v)
require.NoError(t, err)
state, err = alertingNotify.DecodeState(bytes.NewReader(decoded))
require.NoError(t, err)
require.Len(t, state, 2)
// Delete silence.
err = mam.DeleteSilence(ctx, 1, sid)
require.NoError(t, err)
// Confirm silence is expired in memory.
silence, err := am.GetSilence(ctx, sid)
require.NoError(t, err)
require.EqualValues(t, types.SilenceStateExpired, *silence.Status.State)
// Confirm silence is expired in kvstore.
v, ok, err = mam.kvStore.Get(ctx, 1, KVNamespace, SilencesFilename)
require.NoError(t, err)
require.True(t, ok)
decoded, err = decode(v)
require.NoError(t, err)
state, err = alertingNotify.DecodeState(bytes.NewReader(decoded))
require.NoError(t, err)
require.True(t, time.Now().After(state[sid].Silence.EndsAt)) // Expired.
}
func setupMam(t *testing.T, cfg *setting.Cfg) *MultiOrgAlertmanager {
if cfg == nil {
tmpDir := t.TempDir()
cfg = &setting.Cfg{
DataPath: tmpDir,
UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: defaultConfig}, // do not poll in tests.
}
}
cs := NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{})
orgStore := &FakeOrgStore{
orgs: []int64{1, 2, 3},
}
kvStore := ngfakes.NewFakeKVStore(t)
provStore := ngfakes.NewFakeProvisioningStore()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
mam, err := NewMultiOrgAlertmanager(cfg, cs, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService, featuremgmt.WithFeatures())
require.NoError(t, err)
return mam
}
var defaultConfig = `
{
"template_files": null,
"alertmanager_config": {
"route": {
"receiver": "grafana-default-email",
"group_by": [
"grafana_folder",
"alertname"
]
},
"templates": null,
"receivers": [
{
"name": "grafana-default-email",
"grafana_managed_receiver_configs": [
{
"uid": "",
"name": "email receiver",
"type": "email",
"disableResolveMessage": false,
"settings": {
"addresses": "\u003cexample@email.com\u003e"},
"secureSettings": null
}
]
}
]
}
}`
var brokenConfig = `
"alertmanager_config": {
"route": {
pkg/services/ngalert/notifier/testing.go
+25
View File
@@ -7,14 +7,18 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"testing"
"time"
"github.com/go-openapi/strfmt"
"github.com/matttproud/golang_protobuf_extensions/pbutil"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
"github.com/prometheus/alertmanager/nflog/nflogpb"
"github.com/prometheus/alertmanager/silence/silencepb"
"github.com/prometheus/common/model"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -357,3 +361,24 @@ func createNotificationLog(groupKey string, receiverName string, sentAt, expires
ExpiresAt: expiresAt,
}
}
func GenSilence(createdBy string) *apimodels.PostableSilence {
starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
comment := "test comment"
isEqual := true
name := "test"
value := "test"
isRegex := false
matchers := amv2.Matchers{&amv2.Matcher{IsEqual: &isEqual, Name: &name, Value: &value, IsRegex: &isRegex}}
return &apimodels.PostableSilence{
Silence: amv2.Silence{
Comment: &comment,
CreatedBy: &createdBy,
Matchers: matchers,
StartsAt: &starts,
EndsAt: &ends,
},
}
}