public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Feature toggle to disallow sending alerts externally (#87982)

Browse Source 
* Define feature toggle

* Implement feature toggle
This commit is contained in:
Steve Simpson
2024-05-23 14:29:19 +02:00
committed by GitHub
parent bd2b248f0e
commit 8421919cb5
11 changed files with 2170 additions and 2036 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/sender/router.go
+13 -3
View File
@@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana/pkg/api/datasource"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
@@ -48,11 +49,12 @@ type AlertsRouter struct {
datasourceService datasources.DataSourceService
secretService secrets.Service
featureManager featuremgmt.FeatureToggles
}
func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
datasourceService datasources.DataSourceService, secretService secrets.Service, featureManager featuremgmt.FeatureToggles) *AlertsRouter {
d := &AlertsRouter{
logger: log.New("ngalert.sender.router"),
clock: clk,
@@ -71,13 +73,14 @@ func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store stor
datasourceService: datasourceService,
secretService: secretService,
featureManager: featureManager,
}
return d
}
// SyncAndApplyConfigFromDatabase looks for the admin configuration in the database
// and adjusts the sender(s) and alert handling mechanism accordingly.
func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
func (d *AlertsRouter) SyncAndApplyConfigFromDatabase(ctx context.Context) error {
cfgs, err := d.adminConfigStore.GetAdminConfigurations()
if err != nil {
return err
@@ -85,6 +88,8 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
d.logger.Debug("Attempting to sync admin configs", "count", len(cfgs))
disableExternal := d.featureManager.IsEnabled(ctx, featuremgmt.FlagAlertingDisableSendAlertsExternal)
orgsFound := make(map[int64]struct{}, len(cfgs))
d.adminConfigMtx.Lock()
for _, cfg := range cfgs {
@@ -93,6 +98,11 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
continue
}
if disableExternal && cfg.SendAlertsTo != models.InternalAlertmanager {
d.logger.Warn("Alertmanager choice in configuration will be ignored due to feature flags", "org", cfg.OrgID, "choice", cfg.SendAlertsTo)
cfg.SendAlertsTo = models.InternalAlertmanager
}
// Update the Alertmanagers choice for the organization.
d.sendAlertsTo[cfg.OrgID] = cfg.SendAlertsTo
@@ -363,7 +373,7 @@ func (d *AlertsRouter) Run(ctx context.Context) error {
for {
select {
case <-time.After(d.adminConfigPollInterval):
if err := d.SyncAndApplyConfigFromDatabase(); err != nil {
if err := d.SyncAndApplyConfigFromDatabase(ctx); err != nil {
d.logger.Error("Unable to sync admin configuration", "error", err)
}
case <-ctx.Done():
pkg/services/ngalert/sender/router_test.go
+94 -14
View File
@@ -63,14 +63,14 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
}),
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
&fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}, fake_secrets.NewFakeSecretsService())
&fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
}, nil)
// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
// when the first alert triggers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -93,7 +93,7 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
// Now, let's remove the Alertmanager from the admin configuration.
mockedGetAdminConfigurations.Return(nil, nil)
// Again, make sure we sync and verify the externalAlertmanagers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -134,7 +134,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
}
fakeDs := &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
fakeDs, fake_secrets.NewFakeSecretsService())
fakeDs, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey1.OrgID, SendAlertsTo: models.AllAlertmanagers},
@@ -142,7 +142,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
// when the first alert triggers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -167,7 +167,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
}, nil)
// If we sync again, new externalAlertmanagers must have spawned.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 2, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 2, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -216,7 +216,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
currentHash := alertsRouter.externalAlertmanagersCfgHash[ruleKey2.OrgID]
// Now, sync again.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
// The hash for org two should not be the same and we should still have two externalAlertmanagers.
require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey2.OrgID], currentHash)
@@ -236,7 +236,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
currentHash = alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID]
// Now, sync again.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
// The old configuration should not be running.
require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID], currentHash)
@@ -249,13 +249,13 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
{OrgID: ruleKey2.OrgID},
}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID], currentHash)
// Finally, remove everything.
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -293,14 +293,14 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
}),
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}}, fake_secrets.NewFakeSecretsService())
10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}}, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
}, nil)
// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
// when the first alert triggers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
require.Equal(t, models.AllAlertmanagers, alertsRouter.sendAlertsTo[ruleKey.OrgID])
@@ -325,7 +325,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
{OrgID: ruleKey.OrgID, SendAlertsTo: models.ExternalAlertmanagers},
}, nil)
// Again, make sure we sync and verify the externalAlertmanagers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -339,7 +339,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
// Again, make sure we sync and verify the externalAlertmanagers.
// externalAlertmanagers should be running even though alerts are being handled externally.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@@ -356,6 +356,86 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
require.Len(t, actualAlerts, len(expected))
}
func TestAlertmanagersChoiceWithDisableExternalFeatureToggle(t *testing.T) {
ruleKey := models.GenerateRuleKey(1)
fakeAM := NewFakeExternalAlertmanager(t)
defer fakeAM.Close()
fakeAdminConfigStore := &store.AdminConfigurationStoreMock{}
mockedGetAdminConfigurations := fakeAdminConfigStore.EXPECT().GetAdminConfigurations()
mockedClock := clock.NewMock()
mockedClock.Set(time.Now())
moa := createMultiOrgAlertmanager(t, []int64{1})
appUrl := &url.URL{
Scheme: "http",
Host: "localhost",
}
ds := datasources.DataSource{
URL: fakeAM.Server.URL,
OrgID: ruleKey.OrgID,
Type: datasources.DS_ALERTMANAGER,
JsonData: simplejson.NewFromAny(map[string]any{
"handleGrafanaManagedAlerts": true,
"implementation": "prometheus",
}),
}
var expected []*models2.PostableAlert
alerts := definitions.PostableAlerts{}
for i := 0; i < rand.Intn(5)+1; i++ {
alert := generatePostableAlert(t, mockedClock)
expected = append(expected, &alert)
alerts.PostableAlerts = append(alerts.PostableAlerts, alert)
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}},
fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal))
// Test that we only send to the internal Alertmanager even though the configuration specifies AllAlertmanagers.
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
alertsRouter.Send(context.Background(), ruleKey, alerts)
am, err := moa.AlertmanagerFor(ruleKey.OrgID)
require.NoError(t, err)
actualAlerts, err := am.GetAlerts(context.Background(), true, true, true, nil, "")
require.NoError(t, err)
require.Len(t, actualAlerts, len(expected))
// Test that we still only send to the internal alertmanager even though the configuration specifies ExternalAlertmanagers.
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.ExternalAlertmanagers},
}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
alertsRouter.Send(context.Background(), ruleKey, alerts)
am, err = moa.AlertmanagerFor(ruleKey.OrgID)
require.NoError(t, err)
actualAlerts, err = am.GetAlerts(context.Background(), true, true, true, nil, "")
require.NoError(t, err)
require.Len(t, actualAlerts, len(expected))
}
func assertAlertmanagersStatusForOrg(t *testing.T, alertsRouter *AlertsRouter, orgID int64, active, dropped int) {
t.Helper()
require.Eventuallyf(t, func() bool {