From 1f6575e65ee8d0eef254786c47e79bdf59a868b6 Mon Sep 17 00:00:00 2001 From: Santiago Date: Fri, 5 Jan 2024 11:05:27 +0100 Subject: [PATCH] Alerting: Test MOA in remote secondary mode (#79828) --- .../ngalert/api/api_alertmanager_test.go | 2 +- .../multiorg_alertmanager_remote_test.go | 277 ++++++++++++++++++ pkg/services/ngalert/notifier/testing.go | 4 +- pkg/services/ngalert/sender/router_test.go | 2 +- 4 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index a2445dcd2fe..2673269ca2f 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -679,7 +679,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager { }, // do not poll in tests. } - mam, err := notifier.NewMultiOrgAlertmanager(cfg, configStore, &orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) + mam, err := notifier.NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) err = mam.LoadAndSyncAlertmanagersForOrgs(context.Background()) require.NoError(t, err) diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go new file mode 100644 index 00000000000..daa3e1a5020 --- /dev/null +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go @@ -0,0 +1,277 @@ +package notifier_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/notifier" + "github.com/grafana/grafana/pkg/services/ngalert/provisioning" + "github.com/grafana/grafana/pkg/services/ngalert/remote" + remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client" + ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" + "github.com/grafana/grafana/pkg/services/secrets/fakes" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) { + ctx := context.Background() + nopLogger := log.NewNopLogger() + tenantID := "testTenantID" + password := "testPassword" + reg := prometheus.NewPedanticRegistry() + m := metrics.NewNGAlert(reg) + + // We're gonna use an test server to send configuration and state to. + fakeAM := newFakeRemoteAlertmanager(t, tenantID, password) + testsrv := httptest.NewServer(fakeAM) + + // We'll start with the default config and no values for silences and notifications. + kvStore := ngfakes.NewFakeKVStore(t) + require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "")) + require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "")) + configStore := notifier.NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{ + 1: { + OrgID: 1, + AlertmanagerConfiguration: setting.GetAlertmanagerDefaultConfiguration(), + CreatedAt: time.Now().Unix(), + Default: true, + }, + }) + + // Create the factory function for the MOA using the forked Alertmanager in remote secondary mode. + override := notifier.WithAlertmanagerOverride(func(factoryFn notifier.OrgAlertmanagerFactory) notifier.OrgAlertmanagerFactory { + return func(ctx context.Context, orgID int64) (notifier.Alertmanager, error) { + // Create internal Alertmanager. + internalAM, err := factoryFn(ctx, orgID) + require.NoError(t, err) + + // Create remote Alertmanager. + externalAMCfg := remote.AlertmanagerConfig{ + OrgID: 1, + URL: testsrv.URL, + TenantID: tenantID, + BasicAuthPassword: password, + } + // We won't be handling files on disk, we can pass an empty string as workingDirPath. + stateStore := notifier.NewFileStore(orgID, kvStore, "") + remoteAM, err := remote.NewAlertmanager(externalAMCfg, stateStore) + require.NoError(t, err) + + // Use both Alertmanager implementations in the forked Alertmanager. + cfg := remote.RemoteSecondaryConfig{ + Logger: nopLogger, + OrgID: orgID, + Store: configStore, + // Note that we're setting a sync interval of 10 seconds. + SyncInterval: 10 * time.Second, + } + return remote.NewRemoteSecondaryForkedAlertmanager(cfg, internalAM, remoteAM) + } + }) + + cfg := &setting.Cfg{ + DataPath: t.TempDir(), + UnifiedAlerting: setting.UnifiedAlertingSettings{ + AlertmanagerConfigPollInterval: 3 * time.Minute, + DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(), + }, // do not poll in tests. + } + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + moa, err := notifier.NewMultiOrgAlertmanager( + cfg, + configStore, + notifier.NewFakeOrgStore(t, []int64{1}), + kvStore, + provisioning.NewFakeProvisioningStore(), + secretsService.GetDecryptedValue, + m.GetMultiOrgAlertmanagerMetrics(), + nil, + nopLogger, + secretsService, + override, + ) + require.NoError(t, err) + + // It should send config and state on startup. + var lastConfig *remoteClient.UserGrafanaConfig + var lastState *remoteClient.UserGrafanaState + { + // We should start with no config and no state in the external Alertmanager. + require.Empty(t, fakeAM.config) + require.Empty(t, fakeAM.state) + + // On the first sync (startup), both config and state should be updated. + require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx)) + require.NotEmpty(t, fakeAM.config) + require.NotEmpty(t, fakeAM.state) + lastConfig, lastState = fakeAM.config, fakeAM.state + } + + // It should send config and state on an interval. + { + // Let's change the configuration and state. + require.NoError(t, configStore.SaveAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{ + AlertmanagerConfiguration: validConfig, + OrgID: 1, + LastApplied: time.Now().Unix(), + })) + require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "dGVzdAo=")) // base64-encoded string "test" + require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "dGVzdAo=")) // base64-encoded string "test" + + // The sync interval (10s) has not elapsed yet, syncing should have no effect. + require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx)) + require.Equal(t, fakeAM.config, lastConfig) + require.Equal(t, fakeAM.state, lastState) + + // Syncing after the sync interval elapses should update both config and state. + require.Eventually(t, func() bool { + require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx)) + return fakeAM.config != lastConfig && fakeAM.state != lastState + }, 15*time.Second, 300*time.Millisecond) + lastConfig, lastState = fakeAM.config, fakeAM.state + } + + // It should send config and state on shutdown. + { + // Let's change the configuration and state again. + require.NoError(t, configStore.SaveAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{ + AlertmanagerConfiguration: setting.GetAlertmanagerDefaultConfiguration(), + Default: true, + OrgID: 1, + LastApplied: time.Now().Unix(), + })) + require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "dGVzdC0yCg==")) // base64-encoded string "test-2" + require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "dGVzdC0yCg==")) // base64-encoded string "test-2" + + // Both state and config should be updated when shutting the Alertmanager down. + moa.StopAndWait() + require.NotEqual(t, fakeAM.config, lastConfig) + require.NotEqual(t, fakeAM.state, lastState) + } +} + +func newFakeRemoteAlertmanager(t *testing.T, user, pass string) *fakeRemoteAlertmanager { + return &fakeRemoteAlertmanager{ + t: t, + username: user, + password: pass, + } +} + +type fakeRemoteAlertmanager struct { + t *testing.T + config *remoteClient.UserGrafanaConfig + state *remoteClient.UserGrafanaState + username string + password string +} + +// ServeHTTP handles all routes we need for getting and setting state and config. +func (f *fakeRemoteAlertmanager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + + // Check that basic auth is in place. + user, pass, ok := r.BasicAuth() + require.True(f.t, ok) + require.Equal(f.t, f.username, user) + require.Equal(f.t, f.password, pass) + + switch r.Method { + // GET routes + case http.MethodGet: + switch r.RequestURI { + case "/alertmanager/-/ready": + // Make the readiness check succeed. + w.WriteHeader(http.StatusOK) + case "/api/v1/grafana/config": + f.getConfig(w) + case "/api/v1/grafana/state": + f.getState(w) + default: + w.WriteHeader(http.StatusNotFound) + } + + // POST routes + case http.MethodPost: + switch r.RequestURI { + case "/api/v1/grafana/config": + f.postConfig(w, r) + case "/api/v1/grafana/state": + f.postState(w, r) + default: + w.WriteHeader(http.StatusNotFound) + } + } +} + +type response struct { + Data any `json:"data"` + Status string `json:"status"` +} + +func (f *fakeRemoteAlertmanager) postConfig(w http.ResponseWriter, r *http.Request) { + var cfg remoteClient.UserGrafanaConfig + require.NoError(f.t, json.NewDecoder(r.Body).Decode(&cfg)) + + f.config = &cfg + w.WriteHeader(http.StatusCreated) + require.NoError(f.t, json.NewEncoder(w).Encode(response{Status: "success"})) +} + +func (f *fakeRemoteAlertmanager) getConfig(w http.ResponseWriter) { + res := response{ + Data: f.config, + Status: "success", + } + require.NoError(f.t, json.NewEncoder(w).Encode(res)) +} + +func (f *fakeRemoteAlertmanager) postState(w http.ResponseWriter, r *http.Request) { + var state remoteClient.UserGrafanaState + require.NoError(f.t, json.NewDecoder(r.Body).Decode(&state)) + + f.state = &state + w.WriteHeader(http.StatusCreated) + require.NoError(f.t, json.NewEncoder(w).Encode(response{Status: "success"})) +} + +func (f *fakeRemoteAlertmanager) getState(w http.ResponseWriter) { + res := response{ + Data: f.state, + Status: "success", + } + require.NoError(f.t, json.NewEncoder(w).Encode(res)) +} + +var validConfig = `{ + "template_files": { + "a": "template" + }, + "alertmanager_config": { + "route": { + "receiver": "grafana-default-email" + }, + "receivers": [{ + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [{ + "uid": "", + "name": "email receiver", + "type": "email", + "isDefault": true, + "settings": { + "addresses": "" + } + }] + }] + } +}` diff --git a/pkg/services/ngalert/notifier/testing.go b/pkg/services/ngalert/notifier/testing.go index 8ceb967a9ad..999c5b24c0c 100644 --- a/pkg/services/ngalert/notifier/testing.go +++ b/pkg/services/ngalert/notifier/testing.go @@ -180,10 +180,10 @@ type FakeOrgStore struct { orgs []int64 } -func NewFakeOrgStore(t *testing.T, orgs []int64) FakeOrgStore { +func NewFakeOrgStore(t *testing.T, orgs []int64) *FakeOrgStore { t.Helper() - return FakeOrgStore{ + return &FakeOrgStore{ orgs: orgs, } } diff --git a/pkg/services/ngalert/sender/router_test.go b/pkg/services/ngalert/sender/router_test.go index e9cfafeed8f..042527967df 100644 --- a/pkg/services/ngalert/sender/router_test.go +++ b/pkg/services/ngalert/sender/router_test.go @@ -411,7 +411,7 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl m := metrics.NewNGAlert(registry) secretsService := secretsManager.SetupTestService(t, fake_secrets.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - moa, err := notifier.NewMultiOrgAlertmanager(cfg, cfgStore, &orgStore, kvStore, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) + moa, err := notifier.NewMultiOrgAlertmanager(cfg, cfgStore, orgStore, kvStore, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService) require.NoError(t, err) require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(context.Background())) require.Eventually(t, func() bool {