From 36a049912872523eac161a8a8965a03640b31ae2 Mon Sep 17 00:00:00 2001 From: Santiago Date: Mon, 29 Apr 2024 18:47:25 +0200 Subject: [PATCH] Alerting: Implement CreateSilence in the forked Alertmanager (remote primary mode) (#85716) --- .../remote/forked_alertmanager_test.go | 26 ++++++++++++++----- .../remote_primary_forked_alertmanager.go | 11 +++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/pkg/services/ngalert/remote/forked_alertmanager_test.go b/pkg/services/ngalert/remote/forked_alertmanager_test.go index 2379e02719d..52a04116fbb 100644 --- a/pkg/services/ngalert/remote/forked_alertmanager_test.go +++ b/pkg/services/ngalert/remote/forked_alertmanager_test.go @@ -435,19 +435,31 @@ func TestForkedAlertmanager_ModeRemotePrimary(t *testing.T) { }) t.Run("CreateSilence", func(tt *testing.T) { - // We should create the silence in the remote Alertmanager. - _, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary) - + // We should create the silence in both Alertmanagers using the same uid. + testSilence := &apimodels.PostableSilence{} expID := "test-id" - remote.EXPECT().CreateSilence(mock.Anything, mock.Anything).Return(expID, nil).Once() - id, err := forked.CreateSilence(ctx, nil) + + internal, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary) + remote.EXPECT().CreateSilence(mock.Anything, testSilence).Return(expID, nil).Once() + internal.EXPECT().CreateSilence(mock.Anything, testSilence).Return(testSilence.ID, nil).Once() + id, err := forked.CreateSilence(ctx, testSilence) require.NoError(tt, err) + require.Equal(tt, expID, testSilence.ID) require.Equal(tt, expID, id) // If there's an error in the remote Alertmanager, the error should be returned. - remote.EXPECT().CreateSilence(mock.Anything, mock.Anything).Return("", expErr).Maybe() - _, err = forked.CreateSilence(ctx, nil) + _, remote, forked = genTestAlertmanagers(tt, modeRemotePrimary) + remote.EXPECT().CreateSilence(mock.Anything, mock.Anything).Return("", expErr).Once() + _, err = forked.CreateSilence(ctx, testSilence) require.ErrorIs(tt, expErr, err) + + // An error in the internal Alertmanager should not be returned. + internal, remote, forked = genTestAlertmanagers(tt, modeRemotePrimary) + remote.EXPECT().CreateSilence(mock.Anything, mock.Anything).Return(expID, nil).Once() + internal.EXPECT().CreateSilence(mock.Anything, mock.Anything).Return("", expErr).Once() + id, err = forked.CreateSilence(ctx, testSilence) + require.NoError(tt, err) + require.Equal(tt, expID, id) }) t.Run("DeleteSilence", func(tt *testing.T) { diff --git a/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go b/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go index 1084fff32d5..127d9d82e6f 100644 --- a/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go +++ b/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go @@ -72,7 +72,16 @@ func (fam *RemotePrimaryForkedAlertmanager) GetStatus() apimodels.GettableStatus } func (fam *RemotePrimaryForkedAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) { - return fam.remote.CreateSilence(ctx, silence) + uid, err := fam.remote.CreateSilence(ctx, silence) + if err != nil { + return "", err + } + + silence.ID = uid + if _, err := fam.internal.CreateSilence(ctx, silence); err != nil { + fam.log.Error("Error creating silence in the internal Alertmanager", "err", err, "silence", silence) + } + return uid, nil } func (fam *RemotePrimaryForkedAlertmanager) DeleteSilence(ctx context.Context, id string) error {