public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: create wrapper for Alertmanager to enable org level isolation (#37320)

Browse Source 
Introduces org-level isolation for the Alertmanager and its components.

Silences, Alerts and Contact points are not separated by org and are not shared between them.

Co-authored with @davidmparrott and @papagian
This commit is contained in:
David Parrott
2021-08-24 11:28:09 +01:00
committed by GitHub
parent 7ebf4027a7
commit 7fbeefc090
19 changed files with 673 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/notifier/alertmanager.go
+17 -39
View File
@@ -34,13 +34,11 @@ import (
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
)
const (
pollInterval = 1 * time.Minute
workingDir = "alerting"
workingDir = "alerting"
// How long should we keep silences and notification entries on-disk after they've served their purpose.
retentionNotificationsAndSilences = 5 * 24 * time.Hour
// maintenanceNotificationAndSilences how often should we flush and gargabe collect notifications and silences
@@ -73,18 +71,15 @@ const (
}
}
`
//TODO: temporary until fix org isolation
mainOrgID = 1
)
type Alertmanager struct {
logger log.Logger
gokitLogger gokit_log.Logger
Settings *setting.Cfg `inject:""`
SQLStore *sqlstore.SQLStore `inject:""`
Settings *setting.Cfg
Store store.AlertingStore
Metrics *metrics.Metrics `inject:""`
Metrics *metrics.Metrics
notificationLog *nflog.Log
marker types.Marker
@@ -108,18 +103,20 @@ type Alertmanager struct {
reloadConfigMtx sync.RWMutex
config *apimodels.PostableUserConfig
configHash [16]byte
orgID int64
}
func New(cfg *setting.Cfg, store store.AlertingStore, m *metrics.Metrics) (*Alertmanager, error) {
func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m *metrics.Metrics) (*Alertmanager, error) {
am := &Alertmanager{
Settings: cfg,
stopc: make(chan struct{}),
logger: log.New("alertmanager"),
logger: log.New("alertmanager", "org", orgID),
marker: types.NewMarker(m.Registerer),
stageMetrics: notify.NewMetrics(m.Registerer),
dispatcherMetrics: dispatch.NewDispatcherMetrics(m.Registerer),
Store: store,
Metrics: m,
orgID: orgID,
}
am.gokitLogger = gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger))
@@ -174,25 +171,7 @@ func (am *Alertmanager) ready() bool {
return am.config != nil
}
func (am *Alertmanager) Run(ctx context.Context) error {
// Make sure dispatcher starts. We can tolerate future reload failures.
if err := am.SyncAndApplyConfigFromDatabase(mainOrgID); err != nil {
am.logger.Error("unable to sync configuration", "err", err)
}
for {
select {
case <-ctx.Done():
return am.StopAndWait()
case <-time.After(pollInterval):
if err := am.SyncAndApplyConfigFromDatabase(mainOrgID); err != nil {
am.logger.Error("unable to sync configuration", "err", err)
}
}
}
}
func (am *Alertmanager) StopAndWait() error {
func (am *Alertmanager) StopAndWait() {
if am.dispatcher != nil {
am.dispatcher.Stop()
}
@@ -206,12 +185,11 @@ func (am *Alertmanager) StopAndWait() error {
close(am.stopc)
am.wg.Wait()
return nil
}
// SaveAndApplyDefaultConfig saves the default configuration the database and applies the configuration to the Alertmanager.
// It rollbacks the save if we fail to apply the configuration.
func (am *Alertmanager) SaveAndApplyDefaultConfig(orgID int64) error {
func (am *Alertmanager) SaveAndApplyDefaultConfig() error {
am.reloadConfigMtx.Lock()
defer am.reloadConfigMtx.Unlock()
@@ -219,7 +197,7 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(orgID int64) error {
AlertmanagerConfiguration: alertmanagerDefaultConfiguration,
Default: true,
ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
OrgID: orgID,
OrgID: am.orgID,
}
cfg, err := Load([]byte(alertmanagerDefaultConfiguration))
@@ -243,7 +221,7 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(orgID int64) error {
// SaveAndApplyConfig saves the configuration the database and applies the configuration to the Alertmanager.
// It rollbacks the save if we fail to apply the configuration.
func (am *Alertmanager) SaveAndApplyConfig(orgID int64, cfg *apimodels.PostableUserConfig) error {
func (am *Alertmanager) SaveAndApplyConfig(cfg *apimodels.PostableUserConfig) error {
rawConfig, err := json.Marshal(&cfg)
if err != nil {
return fmt.Errorf("failed to serialize to the Alertmanager configuration: %w", err)
@@ -255,7 +233,7 @@ func (am *Alertmanager) SaveAndApplyConfig(orgID int64, cfg *apimodels.PostableU
cmd := &ngmodels.SaveAlertmanagerConfigurationCmd{
AlertmanagerConfiguration: string(rawConfig),
ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
OrgID: orgID,
OrgID: am.orgID,
}
err = am.Store.SaveAlertmanagerConfigurationWithCallback(cmd, func() error {
@@ -274,12 +252,12 @@ func (am *Alertmanager) SaveAndApplyConfig(orgID int64, cfg *apimodels.PostableU
// SyncAndApplyConfigFromDatabase picks the latest config from database and restarts
// the components with the new config.
func (am *Alertmanager) SyncAndApplyConfigFromDatabase(orgID int64) error {
func (am *Alertmanager) SyncAndApplyConfigFromDatabase() error {
am.reloadConfigMtx.Lock()
defer am.reloadConfigMtx.Unlock()
// First, let's get the configuration we need from the database.
q := &ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: mainOrgID}
q := &ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: am.orgID}
if err := am.Store.GetLatestAlertmanagerConfiguration(q); err != nil {
// If there's no configuration in the database, let's use the default configuration.
if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
@@ -289,7 +267,7 @@ func (am *Alertmanager) SyncAndApplyConfigFromDatabase(orgID int64) error {
AlertmanagerConfiguration: alertmanagerDefaultConfiguration,
Default: true,
ConfigurationVersion: fmt.Sprintf("v%d", ngmodels.AlertConfigurationVersion),
OrgID: orgID,
OrgID: am.orgID,
}
if err := am.Store.SaveAlertmanagerConfiguration(savecmd); err != nil {
return err
@@ -389,7 +367,7 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
// Finally, build the integrations map using the receiver configuration and templates.
integrationsMap, err := am.buildIntegrationsMap(cfg.AlertmanagerConfig.Receivers, tmpl)
if err != nil {
return err
return fmt.Errorf("failed to build integration map: %w", err)
}
// Now, let's put together our notification pipeline
routingStage := make(notify.RoutingStage, len(integrationsMap))
@@ -433,7 +411,7 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
}
func (am *Alertmanager) WorkingDirPath() string {
return filepath.Join(am.Settings.DataPath, workingDir, strconv.Itoa(mainOrgID))
return filepath.Join(am.Settings.DataPath, workingDir, strconv.Itoa(int(am.orgID)))
}
// buildIntegrationsMap builds a map of name to the list of Grafana integration notifiers off of a list of receiver config.
pkg/services/ngalert/notifier/alertmanager_test.go
+2 -2
View File
@@ -47,14 +47,14 @@ func setupAMTest(t *testing.T) *Alertmanager {
Logger: log.New("alertmanager-test"),
}
am, err := New(cfg, store, m)
am, err := newAlertmanager(1, cfg, store, m)
require.NoError(t, err)
return am
}
func TestAlertmanager_ShouldUseDefaultConfigurationWhenNoConfiguration(t *testing.T) {
am := setupAMTest(t)
require.NoError(t, am.SyncAndApplyConfigFromDatabase(mainOrgID))
require.NoError(t, am.SyncAndApplyConfigFromDatabase())
require.NotNil(t, am.config)
}
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+147
View File
@@ -0,0 +1,147 @@
package notifier
import (
"context"
"fmt"
"sync"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/setting"
)
var (
SyncOrgsPollInterval = 1 * time.Minute
)
var (
ErrNoAlertmanagerForOrg = fmt.Errorf("Alertmanager does not exist for this organization")
ErrAlertmanagerNotReady = fmt.Errorf("Alertmanager is not ready yet")
)
type MultiOrgAlertmanager struct {
alertmanagersMtx sync.RWMutex
alertmanagers map[int64]*Alertmanager
settings *setting.Cfg
logger log.Logger
configStore store.AlertingStore
orgStore store.OrgStore
orgRegistry *metrics.OrgRegistries
}
func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore) *MultiOrgAlertmanager {
return &MultiOrgAlertmanager{
settings: cfg,
logger: log.New("multiorg.alertmanager"),
alertmanagers: map[int64]*Alertmanager{},
configStore: configStore,
orgStore: orgStore,
orgRegistry: metrics.NewOrgRegistries(),
}
}
func (moa *MultiOrgAlertmanager) Run(ctx context.Context) error {
moa.logger.Info("starting MultiOrg Alertmanager")
for {
select {
case <-ctx.Done():
moa.StopAndWait()
return nil
case <-time.After(SyncOrgsPollInterval):
if err := moa.LoadAndSyncAlertmanagersForOrgs(ctx); err != nil {
moa.logger.Error("error while synchronizing Alertmanager orgs", "err", err)
}
}
}
}
func (moa *MultiOrgAlertmanager) LoadAndSyncAlertmanagersForOrgs(ctx context.Context) error {
moa.logger.Debug("synchronizing Alertmanagers for orgs")
// First, load all the organizations from the database.
orgIDs, err := moa.orgStore.GetOrgs(ctx)
if err != nil {
return err
}
// Then, sync them by creating or deleting Alertmanagers as necessary.
moa.SyncAlertmanagersForOrgs(orgIDs)
moa.logger.Debug("done synchronizing Alertmanagers for orgs")
return nil
}
func (moa *MultiOrgAlertmanager) SyncAlertmanagersForOrgs(orgIDs []int64) {
orgsFound := make(map[int64]struct{}, len(orgIDs))
moa.alertmanagersMtx.Lock()
for _, orgID := range orgIDs {
orgsFound[orgID] = struct{}{}
existing, found := moa.alertmanagers[orgID]
if !found {
reg := moa.orgRegistry.GetOrCreateOrgRegistry(orgID)
am, err := newAlertmanager(orgID, moa.settings, moa.configStore, metrics.NewMetrics(reg))
if err != nil {
moa.logger.Error("unable to create Alertmanager for org", "org", orgID, "err", err)
}
moa.alertmanagers[orgID] = am
existing = am
}
//TODO: This will create an N+1 query
if err := existing.SyncAndApplyConfigFromDatabase(); err != nil {
moa.logger.Error("failed to apply Alertmanager config for org", "org", orgID, "err", err)
}
}
amsToStop := map[int64]*Alertmanager{}
for orgId, am := range moa.alertmanagers {
if _, exists := orgsFound[orgId]; !exists {
amsToStop[orgId] = am
delete(moa.alertmanagers, orgId)
moa.orgRegistry.RemoveOrgRegistry(orgId)
}
}
moa.alertmanagersMtx.Unlock()
// Now, we can stop the Alertmanagers without having to hold a lock.
for orgID, am := range amsToStop {
moa.logger.Info("stopping Alertmanager", "org", orgID)
am.StopAndWait()
moa.logger.Info("stopped Alertmanager", "org", orgID)
}
}
func (moa *MultiOrgAlertmanager) StopAndWait() {
moa.alertmanagersMtx.Lock()
defer moa.alertmanagersMtx.Unlock()
for _, am := range moa.alertmanagers {
am.StopAndWait()
}
}
// AlertmanagerFor returns the Alertmanager instance for the organization provided.
// When the organization does not have an active Alertmanager, it returns a ErrNoAlertmanagerForOrg.
// When the Alertmanager of the organization is not ready, it returns a ErrAlertmanagerNotReady.
func (moa *MultiOrgAlertmanager) AlertmanagerFor(orgID int64) (*Alertmanager, error) {
moa.alertmanagersMtx.RLock()
defer moa.alertmanagersMtx.RUnlock()
orgAM, existing := moa.alertmanagers[orgID]
if !existing {
return nil, ErrNoAlertmanagerForOrg
}
if !orgAM.Ready() {
return nil, ErrAlertmanagerNotReady
}
return orgAM, nil
}
pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
+92
View File
@@ -0,0 +1,92 @@
package notifier
import (
"context"
"testing"
"time"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require"
)
func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
configStore := &FakeConfigStore{
configs: map[int64]*models.AlertConfiguration{},
}
orgStore := &FakeOrgStore{
orgs: []int64{1, 2, 3},
}
SyncOrgsPollInterval = 10 * time.Minute // Don't poll in unit tests.
mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore)
ctx := context.Background()
// Ensure that one Alertmanager is created per org.
{
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Len(t, mam.alertmanagers, 3)
}
// When an org is removed, it should detect it.
{
orgStore.orgs = []int64{1, 3}
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Len(t, mam.alertmanagers, 2)
}
// if the org comes back, it should detect it.
{
orgStore.orgs = []int64{1, 2, 3, 4}
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Len(t, mam.alertmanagers, 4)
}
}
func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
configStore := &FakeConfigStore{
configs: map[int64]*models.AlertConfiguration{},
}
orgStore := &FakeOrgStore{
orgs: []int64{1, 2, 3},
}
SyncOrgsPollInterval = 10 * time.Minute // Don't poll in unit tests.
mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore)
ctx := context.Background()
// Ensure that one Alertmanagers is created per org.
{
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Len(t, mam.alertmanagers, 3)
}
// First, let's try to request an Alertmanager from an org that doesn't exist.
{
_, err := mam.AlertmanagerFor(5)
require.EqualError(t, err, ErrNoAlertmanagerForOrg.Error())
}
// Now, let's try to request an Alertmanager that is not ready.
{
// let's delete its "running config" to make it non-ready
mam.alertmanagers[1].config = nil
_, err := mam.AlertmanagerFor(1)
require.EqualError(t, err, ErrAlertmanagerNotReady.Error())
}
// With an Alertmanager that exists, it responds correctly.
{
am, err := mam.AlertmanagerFor(2)
require.NoError(t, err)
require.Equal(t, *am.GetStatus().VersionInfo.Version, "N/A")
require.Equal(t, am.orgID, int64(2))
require.NotNil(t, am.config)
}
// Let's now remove the previous queried organization.
orgStore.orgs = []int64{1, 3}
require.NoError(t, mam.LoadAndSyncAlertmanagersForOrgs(ctx))
{
_, err := mam.AlertmanagerFor(2)
require.EqualError(t, err, ErrNoAlertmanagerForOrg.Error())
}
}
pkg/services/ngalert/notifier/settings.go
-1
View File
@@ -1 +0,0 @@
package notifier
pkg/services/ngalert/notifier/testing.go
+56
View File
@@ -0,0 +1,56 @@
package notifier
import (
"context"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
)
type FakeConfigStore struct {
configs map[int64]*models.AlertConfiguration
}
func (f *FakeConfigStore) GetLatestAlertmanagerConfiguration(query *models.GetLatestAlertmanagerConfigurationQuery) error {
var ok bool
query.Result, ok = f.configs[query.OrgID]
if !ok {
return store.ErrNoAlertmanagerConfiguration
}
return nil
}
func (f *FakeConfigStore) SaveAlertmanagerConfiguration(cmd *models.SaveAlertmanagerConfigurationCmd) error {
f.configs[cmd.OrgID] = &models.AlertConfiguration{
AlertmanagerConfiguration: cmd.AlertmanagerConfiguration,
OrgID: cmd.OrgID,
ConfigurationVersion: "v1",
Default: cmd.Default,
}
return nil
}
func (f *FakeConfigStore) SaveAlertmanagerConfigurationWithCallback(cmd *models.SaveAlertmanagerConfigurationCmd, callback store.SaveCallback) error {
f.configs[cmd.OrgID] = &models.AlertConfiguration{
AlertmanagerConfiguration: cmd.AlertmanagerConfiguration,
OrgID: cmd.OrgID,
ConfigurationVersion: "v1",
Default: cmd.Default,
}
if err := callback(); err != nil {
return err
}
return nil
}
type FakeOrgStore struct {
orgs []int64
}
func (f *FakeOrgStore) GetOrgs(_ context.Context) ([]int64, error) {
return f.orgs, nil
}