public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Persist silence state immediately on Create/Delete (#84705)

Browse Source 
* Alerting: Persist silence state immediately on Create/Delete

Persists the silence state to the kvstore immediately instead of waiting for the
 next maintenance run. This is used after Create/Delete to prevent silences from
 being lost when a new Alertmanager is started before the state has persisted.
 This can happen, for example, in a rolling deployment scenario.

* Fix test that requires real data

* Don't error if silence state persist fails, maintenance will correct
This commit is contained in:
Matthew Jacobson
2024-04-09 13:39:34 -04:00
committed by GitHub
parent c7e4baff75
commit f79dd7c7f9
14 changed files with 449 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/remote/alertmanager.go
+22 -5
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"net/url"
"strings"
"github.com/go-openapi/strfmt"
amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
@@ -15,6 +16,9 @@ import (
amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver"
amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/grafana/pkg/infra/log"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -22,8 +26,6 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
"github.com/grafana/grafana/pkg/services/ngalert/sender"
alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb"
)
type stateStore interface {
@@ -401,15 +403,30 @@ func (am *Alertmanager) Ready() bool {
return am.ready
}
// SilenceState returns the Alertmanager's silence state as a SilenceState. Currently, does not retrieve the state
// remotely and instead uses the value from the state store.
func (am *Alertmanager) SilenceState(ctx context.Context) (alertingNotify.SilenceState, error) {
silences, err := am.state.GetSilences(ctx)
if err != nil {
return nil, fmt.Errorf("error getting silences: %w", err)
}
return alertingNotify.DecodeState(strings.NewReader(silences))
}
// getFullState returns a base64-encoded protobuf message representing the Alertmanager's internal state.
func (am *Alertmanager) getFullState(ctx context.Context) (string, error) {
var parts []alertingClusterPB.Part
silences, err := am.state.GetSilences(ctx)
state, err := am.SilenceState(ctx)
if err != nil {
return "", fmt.Errorf("error getting silences: %w", err)
}
parts = append(parts, alertingClusterPB.Part{Key: notifier.SilencesFilename, Data: []byte(silences)})
b, err := state.MarshalBinary()
if err != nil {
return "", fmt.Errorf("error marshalling silences: %w", err)
}
parts = append(parts, alertingClusterPB.Part{Key: notifier.SilencesFilename, Data: b})
notificationLog, err := am.state.GetNotificationLog(ctx)
if err != nil {
@@ -420,7 +437,7 @@ func (am *Alertmanager) getFullState(ctx context.Context) (string, error) {
fs := alertingClusterPB.FullState{
Parts: parts,
}
b, err := fs.Marshal()
b, err = fs.Marshal()
if err != nil {
return "", fmt.Errorf("error marshaling full state: %w", err)
}
pkg/services/ngalert/remote/alertmanager_test.go
+16 -34
View File
@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"os"
@@ -40,6 +39,12 @@ import (
const testGrafanaConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}`
const testGrafanaConfigWithSecret = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"dde6ntuob69dtf","name":"WH","type":"webhook","disableResolveMessage":false,"settings":{"url":"http://localhost:8080","username":"test"},"secureSettings":{"password":"test"}}]}]}}`
// Valid Alertmanager state base64 encoded.
const testSilence1 = "lwEKhgEKATESFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsD"
const testSilence2 = "lwEKhgEKATISFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsDlwEKhgEKATESFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsD"
const testNflog1 = "OgoqCgZncm91cDESEgoJcmVjZWl2ZXIxEgV0ZXN0MyoMCIzm1bAGEPqx5uEBEgwInILWsAYQ+rHm4QE="
const testNflog2 = "OgoqCgZncm91cDISEgoJcmVjZWl2ZXIyEgV0ZXN0MyoMCLSDkbAGEMvaofYCEgwIxJ+RsAYQy9qh9gI6CioKBmdyb3VwMRISCglyZWNlaXZlcjESBXRlc3QzKgwItIORsAYQy9qh9gISDAjEn5GwBhDL2qH2Ag=="
func TestMain(m *testing.M) {
testsuite.Run(m)
}
@@ -145,8 +150,8 @@ func TestApplyConfig(t *testing.T) {
ctx := context.Background()
store := ngfakes.NewFakeKVStore(t)
fstore := notifier.NewFileStore(1, store)
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, "test"))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, "test"))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, testSilence1))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, testNflog1))
// An error response from the remote Alertmanager should result in the readiness check failing.
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
@@ -293,14 +298,12 @@ func TestIntegrationRemoteAlertmanagerApplyConfigOnlyUploadsOnce(t *testing.T) {
OrgID: 1,
}
silences := []byte("test-silences")
nflog := []byte("test-notifications")
store := ngfakes.NewFakeKVStore(t)
fstore := notifier.NewFileStore(cfg.OrgID, store)
ctx := context.Background()
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, base64.StdEncoding.EncodeToString(silences)))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, base64.StdEncoding.EncodeToString(nflog)))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, testSilence1))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, testNflog1))
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
@@ -344,8 +347,8 @@ func TestIntegrationRemoteAlertmanagerApplyConfigOnlyUploadsOnce(t *testing.T) {
// Calling `ApplyConfig` again with a changed configuration and state yields no effect.
{
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", "silences", base64.StdEncoding.EncodeToString([]byte("abc123"))))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", "notifications", base64.StdEncoding.EncodeToString([]byte("abc123"))))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", "silences", testSilence2))
require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", "notifications", testNflog2))
fakeConfig.ID = 30000000000000000
require.NoError(t, am.ApplyConfig(ctx, fakeConfig))
@@ -401,8 +404,8 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
require.Equal(t, 0, len(silences))
// Creating a silence should succeed.
testSilence := genSilence("test")
id, err := am.CreateSilence(context.Background(), &testSilence)
testSilence := notifier.GenSilence("test")
id, err := am.CreateSilence(context.Background(), testSilence)
require.NoError(t, err)
require.NotEmpty(t, id)
testSilence.ID = id
@@ -417,8 +420,8 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
require.Error(t, err)
// After creating another silence, the total amount should be 2.
testSilence2 := genSilence("test")
id, err = am.CreateSilence(context.Background(), &testSilence2)
testSilence2 := notifier.GenSilence("test")
id, err = am.CreateSilence(context.Background(), testSilence2)
require.NoError(t, err)
require.NotEmpty(t, id)
testSilence2.ID = id
@@ -552,27 +555,6 @@ func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {
require.Equal(t, "empty-receiver", *rcvs[0].Name)
}
func genSilence(createdBy string) apimodels.PostableSilence {
starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
comment := "test comment"
isEqual := true
name := "test"
value := "test"
isRegex := false
matchers := amv2.Matchers{&amv2.Matcher{IsEqual: &isEqual, Name: &name, Value: &value, IsRegex: &isRegex}}
return apimodels.PostableSilence{
Silence: amv2.Silence{
Comment: &comment,
CreatedBy: &createdBy,
Matchers: matchers,
StartsAt: &starts,
EndsAt: &ends,
},
}
}
func genAlert(active bool, labels map[string]string) amv2.PostableAlert {
endsAt := time.Now()
if active {
pkg/services/ngalert/remote/mock/remoteAlertmanager.go
+54
View File
@@ -804,6 +804,60 @@ func (_c *RemoteAlertmanagerMock_SaveAndApplyDefaultConfig_Call) RunAndReturn(ru
return _c
}
// SilenceState provides a mock function with given fields: _a0
func (_m *RemoteAlertmanagerMock) SilenceState(_a0 context.Context) (notify.SilenceState, error) {
ret := _m.Called(_a0)
var r0 notify.SilenceState
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (notify.SilenceState, error)); ok {
return rf(_a0)
}
if rf, ok := ret.Get(0).(func(context.Context) notify.SilenceState); ok {
r0 = rf(_a0)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(notify.SilenceState)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(_a0)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RemoteAlertmanagerMock_SilenceState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SilenceState'
type RemoteAlertmanagerMock_SilenceState_Call struct {
*mock.Call
}
// SilenceState is a helper method to define mock.On call
// - _a0 context.Context
func (_e *RemoteAlertmanagerMock_Expecter) SilenceState(_a0 interface{}) *RemoteAlertmanagerMock_SilenceState_Call {
return &RemoteAlertmanagerMock_SilenceState_Call{Call: _e.mock.On("SilenceState", _a0)}
}
func (_c *RemoteAlertmanagerMock_SilenceState_Call) Run(run func(_a0 context.Context)) *RemoteAlertmanagerMock_SilenceState_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *RemoteAlertmanagerMock_SilenceState_Call) Return(_a0 notify.SilenceState, _a1 error) *RemoteAlertmanagerMock_SilenceState_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *RemoteAlertmanagerMock_SilenceState_Call) RunAndReturn(run func(context.Context) (notify.SilenceState, error)) *RemoteAlertmanagerMock_SilenceState_Call {
_c.Call.Return(run)
return _c
}
// StopAndWait provides a mock function with given fields:
func (_m *RemoteAlertmanagerMock) StopAndWait() {
_m.Called()
pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go
+6
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/grafana/pkg/infra/log"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -91,6 +93,10 @@ func (fam *RemotePrimaryForkedAlertmanager) TestTemplate(ctx context.Context, c
return fam.remote.TestTemplate(ctx, c)
}
func (fam *RemotePrimaryForkedAlertmanager) SilenceState(ctx context.Context) (alertingNotify.SilenceState, error) {
return fam.remote.SilenceState(ctx)
}
func (fam *RemotePrimaryForkedAlertmanager) StopAndWait() {
fam.internal.StopAndWait()
fam.remote.StopAndWait()
pkg/services/ngalert/remote/remote_secondary_forked_alertmanager.go
+6
View File
@@ -6,6 +6,8 @@ import (
"sync"
"time"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/grafana/pkg/infra/log"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -165,6 +167,10 @@ func (fam *RemoteSecondaryForkedAlertmanager) TestTemplate(ctx context.Context,
return fam.internal.TestTemplate(ctx, c)
}
func (fam *RemoteSecondaryForkedAlertmanager) SilenceState(ctx context.Context) (alertingNotify.SilenceState, error) {
return fam.internal.SilenceState(ctx)
}
func (fam *RemoteSecondaryForkedAlertmanager) StopAndWait() {
// Stop the internal Alertmanager.
fam.internal.StopAndWait()