Alerting: create wrapper for Alertmanager to enable org level isolation (#37320)

Introduces org-level isolation for the Alertmanager and its components. Silences, Alerts and Contact points are not separated by org and are not shared between them. Co-authored with @davidmparrott and @papagian
2021-08-24 11:28:09 +01:00
parent 7ebf4027a7
commit 7fbeefc090
19 changed files with 673 additions and 148 deletions
@@ -30,10 +30,8 @@ type Scheduler interface {

 type Alertmanager interface {
 	// Configuration
-	// temporary add orgID parameter; this will move to the Alertmanager wrapper when it will be available
-	SaveAndApplyConfig(orgID int64, config *apimodels.PostableUserConfig) error
-	// temporary add orgID parameter; this will move to the Alertmanager wrapper when it will be available
-	SaveAndApplyDefaultConfig(orgID int64) error
+	SaveAndApplyConfig(config *apimodels.PostableUserConfig) error
+	SaveAndApplyDefaultConfig() error
 	GetStatus() apimodels.GettableStatus

 	// Silences
@@ -52,19 +50,19 @@ type Alertmanager interface {

 // API handlers.
 type API struct {
-	Cfg              *setting.Cfg
-	DatasourceCache  datasources.CacheService
-	RouteRegister    routing.RouteRegister
-	DataService      *tsdb.Service
-	QuotaService     *quota.QuotaService
-	Schedule         schedule.ScheduleService
-	RuleStore        store.RuleStore
-	InstanceStore    store.InstanceStore
-	AlertingStore    store.AlertingStore
-	AdminConfigStore store.AdminConfigurationStore
-	DataProxy        *datasourceproxy.DatasourceProxyService
-	Alertmanager     Alertmanager
-	StateManager     *state.Manager
+	Cfg                  *setting.Cfg
+	DatasourceCache      datasources.CacheService
+	RouteRegister        routing.RouteRegister
+	DataService          *tsdb.Service
+	QuotaService         *quota.QuotaService
+	Schedule             schedule.ScheduleService
+	RuleStore            store.RuleStore
+	InstanceStore        store.InstanceStore
+	AlertingStore        store.AlertingStore
+	AdminConfigStore     store.AdminConfigurationStore
+	DataProxy            *datasourceproxy.DatasourceProxyService
+	MultiOrgAlertmanager *notifier.MultiOrgAlertmanager
+	StateManager         *state.Manager
 }

 // RegisterAPIEndpoints registers API handlers
@@ -78,7 +76,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.Metrics) {
 	api.RegisterAlertmanagerApiEndpoints(NewForkedAM(
 		api.DatasourceCache,
 		NewLotexAM(proxy, logger),
-		AlertmanagerSrv{store: api.AlertingStore, am: api.Alertmanager, log: logger},
+		AlertmanagerSrv{store: api.AlertingStore, mam: api.MultiOrgAlertmanager, log: logger},
 	), m)
 	// Register endpoints for proxying to Prometheus-compatible backends.
 	api.RegisterPrometheusApiEndpoints(NewForkedProm(
@@ -25,7 +25,7 @@ const (
 )

 type AlertmanagerSrv struct {
-	am    Alertmanager
+	mam   *notifier.MultiOrgAlertmanager
 	store store.AlertingStore
 	log   log.Logger
 }
@@ -93,14 +93,25 @@ func (srv AlertmanagerSrv) loadSecureSettings(orgId int64, receivers []*apimodel
 }

 func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
-	return response.JSON(http.StatusOK, srv.am.GetStatus())
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	return response.JSON(http.StatusOK, am.GetStatus())
 }

 func (srv AlertmanagerSrv) RouteCreateSilence(c *models.ReqContext, postableSilence apimodels.PostableSilence) response.Response {
 	if !c.HasUserRole(models.ROLE_EDITOR) {
 		return ErrResp(http.StatusForbidden, errors.New("permission denied"), "")
 	}
-	silenceID, err := srv.am.CreateSilence(&postableSilence)
+
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	silenceID, err := am.CreateSilence(&postableSilence)
 	if err != nil {
 		if errors.Is(err, notifier.ErrSilenceNotFound) {
 			return ErrResp(http.StatusNotFound, err, "")
@@ -119,7 +130,13 @@ func (srv AlertmanagerSrv) RouteDeleteAlertingConfig(c *models.ReqContext) respo
 	if !c.HasUserRole(models.ROLE_EDITOR) {
 		return ErrResp(http.StatusForbidden, errors.New("permission denied"), "")
 	}
-	if err := srv.am.SaveAndApplyDefaultConfig(c.OrgId); err != nil {
+
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	if err := am.SaveAndApplyDefaultConfig(); err != nil {
 		srv.log.Error("unable to save and apply default alertmanager configuration", "err", err)
 		return ErrResp(http.StatusInternalServerError, err, "failed to save and apply default Alertmanager configuration")
 	}
@@ -131,8 +148,14 @@ func (srv AlertmanagerSrv) RouteDeleteSilence(c *models.ReqContext) response.Res
 	if !c.HasUserRole(models.ROLE_EDITOR) {
 		return ErrResp(http.StatusForbidden, errors.New("permission denied"), "")
 	}
+
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
 	silenceID := c.Params(":SilenceId")
-	if err := srv.am.DeleteSilence(silenceID); err != nil {
+	if err := am.DeleteSilence(silenceID); err != nil {
 		if errors.Is(err, notifier.ErrSilenceNotFound) {
 			return ErrResp(http.StatusNotFound, err, "")
 		}
@@ -202,7 +225,12 @@ func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response
 }

 func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.Response {
-	groups, err := srv.am.GetAlertGroups(
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	groups, err := am.GetAlertGroups(
 		c.QueryBoolWithDefault("active", true),
 		c.QueryBoolWithDefault("silenced", true),
 		c.QueryBoolWithDefault("inhibited", true),
@@ -221,7 +249,12 @@ func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.
 }

 func (srv AlertmanagerSrv) RouteGetAMAlerts(c *models.ReqContext) response.Response {
-	alerts, err := srv.am.GetAlerts(
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	alerts, err := am.GetAlerts(
 		c.QueryBoolWithDefault("active", true),
 		c.QueryBoolWithDefault("silenced", true),
 		c.QueryBoolWithDefault("inhibited", true),
@@ -243,8 +276,13 @@ func (srv AlertmanagerSrv) RouteGetAMAlerts(c *models.ReqContext) response.Respo
 }

 func (srv AlertmanagerSrv) RouteGetSilence(c *models.ReqContext) response.Response {
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
 	silenceID := c.Params(":SilenceId")
-	gettableSilence, err := srv.am.GetSilence(silenceID)
+	gettableSilence, err := am.GetSilence(silenceID)
 	if err != nil {
 		if errors.Is(err, notifier.ErrSilenceNotFound) {
 			return ErrResp(http.StatusNotFound, err, "")
@@ -256,7 +294,12 @@ func (srv AlertmanagerSrv) RouteGetSilence(c *models.ReqContext) response.Respon
 }

 func (srv AlertmanagerSrv) RouteGetSilences(c *models.ReqContext) response.Response {
-	gettableSilences, err := srv.am.ListSilences(c.QueryStrings("filter"))
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	gettableSilences, err := am.ListSilences(c.QueryStrings("filter"))
 	if err != nil {
 		if errors.Is(err, notifier.ErrListSilencesBadPayload) {
 			return ErrResp(http.StatusBadRequest, err, "")
@@ -293,7 +336,12 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
 	}

-	if err := srv.am.SaveAndApplyConfig(c.OrgId, &body); err != nil {
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	if err := am.SaveAndApplyConfig(&body); err != nil {
 		srv.log.Error("unable to save and apply alertmanager configuration", "err", err)
 		return ErrResp(http.StatusBadRequest, err, "failed to save and apply Alertmanager configuration")
 	}
@@ -301,7 +349,7 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 	return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
 }

-func (srv AlertmanagerSrv) RoutePostAMAlerts(c *models.ReqContext, body apimodels.PostableAlerts) response.Response {
+func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.PostableAlerts) response.Response {
 	return NotImplementedResp
 }

@@ -332,7 +380,12 @@ func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body api
 	}
 	defer cancelFunc()

-	result, err := srv.am.TestReceivers(ctx, body)
+	am, errResp := srv.AlertmanagerFor(c.OrgId)
+	if errResp != nil {
+		return errResp
+	}
+
+	result, err := am.TestReceivers(ctx, body)
 	if err != nil {
 		if errors.Is(err, notifier.ErrNoReceivers) {
 			return response.Error(http.StatusBadRequest, "", err)
@@ -429,3 +482,21 @@ func statusForTestReceivers(v []notifier.TestReceiverResult) int {
 		return http.StatusOK
 	}
 }
+
+func (srv AlertmanagerSrv) AlertmanagerFor(orgID int64) (Alertmanager, *response.NormalResponse) {
+	am, err := srv.mam.AlertmanagerFor(orgID)
+	if err == nil {
+		return am, nil
+	}
+
+	if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
+		return nil, response.Error(http.StatusNotFound, err.Error(), nil)
+	}
+
+	if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
+		return nil, response.Error(http.StatusConflict, err.Error(), nil)
+	}
+
+	srv.log.Error("unable to obtain the org's Alertmanager", "err", err)
+	return nil, response.Error(http.StatusInternalServerError, "unable to obtain org's Alertmanager", err)
+}
@@ -2247,6 +2247,76 @@
   "type": "object",
   "x-go-package": "github.com/prometheus/common/config"
  },
+  "TestReceiverConfigResult": {
+   "properties": {
+    "error": {
+     "type": "string",
+     "x-go-name": "Error"
+    },
+    "name": {
+     "type": "string",
+     "x-go-name": "Name"
+    },
+    "status": {
+     "type": "string",
+     "x-go-name": "Status"
+    },
+    "uid": {
+     "type": "string",
+     "x-go-name": "UID"
+    }
+   },
+   "type": "object",
+   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+  },
+  "TestReceiverResult": {
+   "properties": {
+    "grafana_managed_receiver_configs": {
+     "items": {
+      "$ref": "#/definitions/TestReceiverConfigResult"
+     },
+     "type": "array",
+     "x-go-name": "Configs"
+    },
+    "name": {
+     "type": "string",
+     "x-go-name": "Name"
+    }
+   },
+   "type": "object",
+   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+  },
+  "TestReceiversConfig": {
+   "properties": {
+    "receivers": {
+     "items": {
+      "$ref": "#/definitions/PostableApiReceiver"
+     },
+     "type": "array",
+     "x-go-name": "Receivers"
+    }
+   },
+   "type": "object",
+   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+  },
+  "TestReceiversResult": {
+   "properties": {
+    "notified_at": {
+     "format": "date-time",
+     "type": "string",
+     "x-go-name": "NotifedAt"
+    },
+    "receivers": {
+     "items": {
+      "$ref": "#/definitions/TestReceiverResult"
+     },
+     "type": "array",
+     "x-go-name": "Receivers"
+    }
+   },
+   "type": "object",
+   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+  },
  "TestRulePayload": {
   "properties": {
    "expr": {
@@ -2274,6 +2344,7 @@
   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
  },
  "URL": {
+   "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
   "properties": {
    "ForceQuery": {
     "type": "boolean"
@@ -2306,9 +2377,9 @@
     "$ref": "#/definitions/Userinfo"
    }
   },
-   "title": "URL is a custom URL type that allows validation at configuration load time.",
+   "title": "A URL represents a parsed URL (technically, a URI reference).",
   "type": "object",
-   "x-go-package": "github.com/prometheus/common/config"
+   "x-go-package": "net/url"
  },
  "Userinfo": {
   "description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -2475,7 +2546,6 @@
   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
  },
  "alertGroup": {
-   "description": "AlertGroup alert group",
   "properties": {
    "alerts": {
     "description": "alerts",
@@ -2497,14 +2567,17 @@
    "labels",
    "receiver"
   ],
-   "type": "object"
+   "type": "object",
+   "x-go-name": "AlertGroup",
+   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
  },
  "alertGroups": {
-   "description": "AlertGroups alert groups",
   "items": {
    "$ref": "#/definitions/alertGroup"
   },
-   "type": "array"
+   "type": "array",
+   "x-go-name": "AlertGroups",
+   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
  },
  "alertStatus": {
   "description": "AlertStatus alert status",
@@ -2624,6 +2697,7 @@
   "$ref": "#/definitions/Duration"
  },
  "gettableAlert": {
+   "description": "GettableAlert gettable alert",
   "properties": {
    "annotations": {
     "$ref": "#/definitions/labelSet"
@@ -2682,9 +2756,7 @@
    "status",
    "updatedAt"
   ],
-   "type": "object",
-   "x-go-name": "GettableAlert",
-   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+   "type": "object"
  },
  "gettableAlerts": {
   "items": {
@@ -2928,6 +3000,7 @@
   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
  },
  "receiver": {
+   "description": "Receiver receiver",
   "properties": {
    "name": {
     "description": "name",
@@ -2938,9 +3011,7 @@
   "required": [
    "name"
   ],
-   "type": "object",
-   "x-go-name": "Receiver",
-   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+   "type": "object"
  },
  "silence": {
   "description": "Silence silence",
@@ -3523,6 +3594,49 @@
    ]
   }
  },
+  "/api/alertmanager/{Recipient}/config/api/v1/receivers/test": {
+   "post": {
+    "operationId": "RoutePostTestReceivers",
+    "parameters": [
+     {
+      "in": "query",
+      "items": {
+       "$ref": "#/definitions/PostableApiReceiver"
+      },
+      "name": "receivers",
+      "type": "array",
+      "x-go-name": "Receivers"
+     }
+    ],
+    "responses": {
+     "200": {
+      "description": "Ack",
+      "schema": {
+       "$ref": "#/definitions/Ack"
+      }
+     },
+     "207": {
+      "$ref": "#/responses/MultiStatus"
+     },
+     "400": {
+      "description": "ValidationError",
+      "schema": {
+       "$ref": "#/definitions/ValidationError"
+      }
+     },
+     "408": {
+      "description": "Failure",
+      "schema": {
+       "$ref": "#/definitions/Failure"
+      }
+     }
+    },
+    "summary": "Test Grafana managed receivers without saving them.",
+    "tags": [
+     "alertmanager"
+    ]
+   }
+  },
  "/api/prometheus/{Recipient}/api/v1/alerts": {
   "get": {
    "description": "gets the current alerts",
@@ -1750,7 +1750,6 @@
          "enum": [
            "Alerting"
          ],
-          "x-go-enum-desc": "Alerting AlertingErrState",
          "x-go-name": "ExecErrState"
        },
        "id": {
@@ -1779,7 +1778,6 @@
            "NoData",
            "OK"
          ],
-          "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
          "x-go-name": "NoDataState"
        },
        "orgId": {
@@ -2592,7 +2590,6 @@
          "enum": [
            "Alerting"
          ],
-          "x-go-enum-desc": "Alerting AlertingErrState",
          "x-go-name": "ExecErrState"
        },
        "no_data_state": {
@@ -2602,7 +2599,6 @@
            "NoData",
            "OK"
          ],
-          "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
          "x-go-name": "NoDataState"
        },
        "title": {
@@ -3373,8 +3369,9 @@
      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
    },
    "URL": {
+      "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
      "type": "object",
-      "title": "URL is a custom URL type that allows validation at configuration load time.",
+      "title": "A URL represents a parsed URL (technically, a URI reference).",
      "properties": {
        "ForceQuery": {
          "type": "boolean"
@@ -3407,7 +3404,7 @@
          "$ref": "#/definitions/Userinfo"
        }
      },
-      "x-go-package": "github.com/prometheus/common/config"
+      "x-go-package": "net/url"
    },
    "Userinfo": {
      "description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -3574,7 +3571,6 @@
      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
    },
    "alertGroup": {
-      "description": "AlertGroup alert group",
      "type": "object",
      "required": [
        "alerts",
@@ -3597,6 +3593,8 @@
          "$ref": "#/definitions/receiver"
        }
      },
+      "x-go-name": "AlertGroup",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
      "$ref": "#/definitions/alertGroup"
    },
    "alertGroups": {
@@ -3726,6 +3724,7 @@
      "$ref": "#/definitions/Duration"
    },
    "gettableAlert": {
+      "description": "GettableAlert gettable alert",
      "type": "object",
      "required": [
        "labels",
@@ -3785,19 +3784,19 @@
          "x-go-name": "UpdatedAt"
        }
      },
-      "x-go-name": "GettableAlert",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
      "$ref": "#/definitions/gettableAlert"
    },
    "gettableAlerts": {
-      "description": "GettableAlerts gettable alerts",
      "type": "array",
      "items": {
        "$ref": "#/definitions/gettableAlert"
      },
+      "x-go-name": "GettableAlerts",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
      "$ref": "#/definitions/gettableAlerts"
    },
    "gettableSilence": {
+      "description": "GettableSilence gettable silence",
      "type": "object",
      "required": [
        "comment",
@@ -3850,8 +3849,6 @@
          "x-go-name": "UpdatedAt"
        }
      },
-      "x-go-name": "GettableSilence",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
      "$ref": "#/definitions/gettableSilence"
    },
    "gettableSilences": {
@@ -3990,7 +3987,6 @@
      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
    },
    "postableSilence": {
-      "description": "PostableSilence postable silence",
      "type": "object",
      "required": [
        "comment",
@@ -4031,9 +4027,12 @@
          "x-go-name": "StartsAt"
        }
      },
+      "x-go-name": "PostableSilence",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
      "$ref": "#/definitions/postableSilence"
    },
    "receiver": {
+      "description": "Receiver receiver",
      "type": "object",
      "required": [
        "name"
@@ -4045,8 +4044,6 @@
          "x-go-name": "Name"
        }
      },
-      "x-go-name": "Receiver",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
      "$ref": "#/definitions/receiver"
    },
    "silence": {