Alerting: Provisioning API - Notification Policies (#46755)

* Base-line API for provisioning notification policies

* Wire API up, some simple tests

* Return provenance status through API

* Fix missing call

* Transactions

* Clarity in package dependencies

* Unify receivers in definitions

* Fix issue introduced by receiver change

* Drop unused internal test implementation

* FGAC hooks for provisioning routes

* Polish, swap names

* Asserting on number of exposed routes

* Don't bubble up updated object

* Integrate with new concurrency token feature in store

* Back out duplicated changes

* Remove redundant tests

* Regenerate and create unit tests for API layer

* Integration tests for auth

* Address linter errors

* Put route behind toggle

* Use alternative store API and fix feature toggle in tests

* Fixes, polish

* Fix whitespace

* Re-kick drone

* Rename services to provisioning
This commit is contained in:
Alexander Weaver
2022-04-05 16:48:51 -05:00
committed by GitHub
parent cb6124c921
commit dde0b93cf1
29 changed files with 1055 additions and 78 deletions
+11 -1
View File
@@ -11,11 +11,13 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasourceproxy"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -64,7 +66,7 @@ type API struct {
ExpressionService *expr.Service
QuotaService *quota.QuotaService
Schedule schedule.ScheduleService
TransactionManager store.TransactionManager
TransactionManager provisioning.TransactionManager
RuleStore store.RuleStore
InstanceStore store.InstanceStore
AlertingStore AlertingStore
@@ -74,6 +76,7 @@ type API struct {
StateManager *state.Manager
SecretsService secrets.Service
AccessControl accesscontrol.AccessControl
Policies *provisioning.NotificationPolicyService
}
// RegisterAPIEndpoints registers API handlers
@@ -126,4 +129,11 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
scheduler: api.Schedule,
},
), m)
if api.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagAlertProvisioning) {
api.RegisterProvisioningApiEndpoints(NewForkedProvisioningApi(&ProvisioningSrv{
log: logger,
policies: api.Policies,
}), m)
}
}
@@ -0,0 +1,51 @@
package api
import (
"context"
"errors"
"net/http"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
domain "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/util"
)
type ProvisioningSrv struct {
log log.Logger
policies NotificationPolicyService
}
type NotificationPolicyService interface {
GetPolicyTree(ctx context.Context, orgID int64) (provisioning.EmbeddedRoutingTree, error)
UpdatePolicyTree(ctx context.Context, orgID int64, tree apimodels.Route, p domain.Provenance) error
}
func (srv *ProvisioningSrv) RouteGetPolicyTree(c *models.ReqContext) response.Response {
policies, err := srv.policies.GetPolicyTree(c.Req.Context(), c.OrgId)
if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
return ErrResp(http.StatusNotFound, err, "")
}
if err != nil {
return ErrResp(http.StatusInternalServerError, err, "")
}
return response.JSON(http.StatusOK, policies)
}
func (srv *ProvisioningSrv) RoutePostPolicyTree(c *models.ReqContext, tree apimodels.Route) response.Response {
// TODO: lift validation out of definitions.Rotue.UnmarshalJSON and friends into a dedicated validator.
err := srv.policies.UpdatePolicyTree(c.Req.Context(), c.OrgId, tree, domain.ProvenanceApi)
if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
return ErrResp(http.StatusNotFound, err, "")
}
if err != nil {
return ErrResp(http.StatusInternalServerError, err, "")
}
return response.JSON(http.StatusAccepted, util.DynMap{"message": "policies updated"})
}
@@ -0,0 +1,150 @@
package api
import (
"context"
"fmt"
"net/http"
"testing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
domain "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/web"
"github.com/stretchr/testify/require"
)
func TestProvisioningApi(t *testing.T) {
t.Run("successful GET policies returns 200", func(t *testing.T) {
sut := createProvisioningSrvSut()
rc := createTestRequestCtx()
response := sut.RouteGetPolicyTree(&rc)
require.Equal(t, 200, response.Status())
})
t.Run("successful POST policies returns 202", func(t *testing.T) {
sut := createProvisioningSrvSut()
rc := createTestRequestCtx()
tree := apimodels.Route{}
response := sut.RoutePostPolicyTree(&rc, tree)
require.Equal(t, 202, response.Status())
})
// TODO: we have not lifted out validation yet. Test that we are returning errors properly once validation has been lifted.
t.Run("when org has no AM config", func(t *testing.T) {
t.Run("GET policies returns 404", func(t *testing.T) {
sut := createProvisioningSrvSut()
rc := createTestRequestCtx()
rc.SignedInUser.OrgId = 2
response := sut.RouteGetPolicyTree(&rc)
require.Equal(t, 404, response.Status())
})
t.Run("POST policies returns 404", func(t *testing.T) {
sut := createProvisioningSrvSut()
rc := createTestRequestCtx()
rc.SignedInUser.OrgId = 2
response := sut.RouteGetPolicyTree(&rc)
require.Equal(t, 404, response.Status())
})
})
t.Run("when an unspecified error occurrs", func(t *testing.T) {
t.Run("GET policies returns 500", func(t *testing.T) {
sut := createProvisioningSrvSut()
sut.policies = &fakeFailingNotificationPolicyService{}
rc := createTestRequestCtx()
response := sut.RouteGetPolicyTree(&rc)
require.Equal(t, 500, response.Status())
require.NotEmpty(t, response.Body())
require.Contains(t, string(response.Body()), "something went wrong")
})
t.Run("POST policies returns 500", func(t *testing.T) {
sut := createProvisioningSrvSut()
sut.policies = &fakeFailingNotificationPolicyService{}
rc := createTestRequestCtx()
tree := apimodels.Route{}
response := sut.RoutePostPolicyTree(&rc, tree)
require.Equal(t, 500, response.Status())
require.NotEmpty(t, response.Body())
require.Contains(t, string(response.Body()), "something went wrong")
})
})
}
func createProvisioningSrvSut() ProvisioningSrv {
return ProvisioningSrv{
log: log.NewNopLogger(),
policies: newFakeNotificationPolicyService(),
}
}
func createTestRequestCtx() models.ReqContext {
return models.ReqContext{
Context: &web.Context{
Req: &http.Request{},
},
SignedInUser: &models.SignedInUser{
OrgId: 1,
},
}
}
type fakeNotificationPolicyService struct {
tree apimodels.Route
prov domain.Provenance
}
func newFakeNotificationPolicyService() *fakeNotificationPolicyService {
return &fakeNotificationPolicyService{
tree: apimodels.Route{
Receiver: "some-receiver",
},
prov: domain.ProvenanceNone,
}
}
func (f *fakeNotificationPolicyService) GetPolicyTree(ctx context.Context, orgID int64) (provisioning.EmbeddedRoutingTree, error) {
if orgID != 1 {
return provisioning.EmbeddedRoutingTree{}, store.ErrNoAlertmanagerConfiguration
}
return provisioning.EmbeddedRoutingTree{
Route: f.tree,
Provenance: f.prov,
}, nil
}
func (f *fakeNotificationPolicyService) UpdatePolicyTree(ctx context.Context, orgID int64, tree apimodels.Route, p domain.Provenance) error {
if orgID != 1 {
return store.ErrNoAlertmanagerConfiguration
}
f.tree = tree
f.prov = p
return nil
}
type fakeFailingNotificationPolicyService struct{}
func (f *fakeFailingNotificationPolicyService) GetPolicyTree(ctx context.Context, orgID int64) (provisioning.EmbeddedRoutingTree, error) {
return provisioning.EmbeddedRoutingTree{}, fmt.Errorf("something went wrong")
}
func (f *fakeFailingNotificationPolicyService) UpdatePolicyTree(ctx context.Context, orgID int64, tree apimodels.Route, p domain.Provenance) error {
return fmt.Errorf("something went wrong")
}
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/quota"
"github.com/grafana/grafana/pkg/setting"
@@ -28,7 +29,7 @@ import (
)
type RulerSrv struct {
xactManager store.TransactionManager
xactManager provisioning.TransactionManager
store store.RuleStore
DatasourceCache datasources.CacheService
QuotaService *quota.QuotaService
@@ -179,6 +179,13 @@ func (api *API) authorize(method, path string) web.Handler {
http.MethodPost + "/api/v1/ngalert/admin_config",
http.MethodGet + "/api/v1/ngalert/alertmanagers":
return middleware.ReqOrgAdmin
// Grafana-only Provisioning Read Paths
case http.MethodGet + "/api/provisioning/policies":
return middleware.ReqSignedIn
case http.MethodPost + "/api/provisioning/policies":
return middleware.ReqEditorRole
}
if eval != nil {
@@ -47,7 +47,7 @@ func TestAuthorize(t *testing.T) {
}
paths[p] = methods
}
require.Len(t, paths, 29)
require.Len(t, paths, 30)
ac := acmock.New()
api := &API{AccessControl: ac}
@@ -0,0 +1,28 @@
package api
import (
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
)
// ForkedProvisioningApi always forwards requests to a Grafana backend.
// We do not currently support provisioning of external systems through Grafana's API.
type ForkedProvisioningApi struct {
svc *ProvisioningSrv
}
// NewForkedProvisioningApi creates a new ForkedProvisioningApi instance.
func NewForkedProvisioningApi(svc *ProvisioningSrv) *ForkedProvisioningApi {
return &ForkedProvisioningApi{
svc: svc,
}
}
func (f *ForkedProvisioningApi) forkRouteGetPolicyTree(ctx *models.ReqContext) response.Response {
return f.svc.RouteGetPolicyTree(ctx)
}
func (f *ForkedProvisioningApi) forkRoutePostPolicyTree(ctx *models.ReqContext, route apimodels.Route) response.Response {
return f.svc.RoutePostPolicyTree(ctx, route)
}
@@ -0,0 +1,62 @@
/*Package api contains base API implementation of unified alerting
*
*Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
*
*Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them.
*/
package api
import (
"net/http"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/web"
)
type ProvisioningApiForkingService interface {
RouteGetPolicyTree(*models.ReqContext) response.Response
RoutePostPolicyTree(*models.ReqContext) response.Response
}
func (f *ForkedProvisioningApi) RouteGetPolicyTree(ctx *models.ReqContext) response.Response {
return f.forkRouteGetPolicyTree(ctx)
}
func (f *ForkedProvisioningApi) RoutePostPolicyTree(ctx *models.ReqContext) response.Response {
conf := apimodels.Route{}
if err := web.Bind(ctx.Req, &conf); err != nil {
return response.Error(http.StatusBadRequest, "bad request data", err)
}
return f.forkRoutePostPolicyTree(ctx, conf)
}
func (api *API) RegisterProvisioningApiEndpoints(srv ProvisioningApiForkingService, m *metrics.API) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Get(
toMacaronPath("/api/provisioning/policies"),
api.authorize(http.MethodGet, "/api/provisioning/policies"),
metrics.Instrument(
http.MethodGet,
"/api/provisioning/policies",
srv.RouteGetPolicyTree,
m,
),
)
group.Post(
toMacaronPath("/api/provisioning/policies"),
api.authorize(http.MethodPost, "/api/provisioning/policies"),
metrics.Instrument(
http.MethodPost,
"/api/provisioning/policies",
srv.RoutePostPolicyTree,
m,
),
)
}, middleware.ReqSignedIn)
}
@@ -804,6 +804,14 @@ func AsGrafanaRoute(r *config.Route) *Route {
return gRoute
}
func (r *Route) ResourceType() string {
return "route"
}
func (r *Route) ResourceID() string {
return ""
}
// Config is the entrypoint for the embedded Alertmanager config with the exception of receivers.
// Prometheus historically uses yaml files as the method of configuration and thus some
// post-validation is included in the UnmarshalYAML method. Here we simply run this with
@@ -0,0 +1,26 @@
package definitions
// swagger:route GET /api/provisioning/policies provisioning RouteGetPolicyTree
//
// Get the notification policy tree.
//
// Responses:
// 200: Route
// 400: ValidationError
// swagger:route POST /api/provisioning/policies provisioning RoutePostPolicyTree
//
// Sets the notification policy tree.
//
// Consumes:
// - application/json
//
// Responses:
// 202: Accepted
// 400: ValidationError
// swagger:parameters RoutePostPolicyTree
type Policytree struct {
// in:body
Body Route
}
+66 -12
View File
@@ -2803,7 +2803,6 @@
"x-go-package": "github.com/prometheus/alertmanager/timeinterval"
},
"URL": {
"description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -2836,9 +2835,9 @@
"$ref": "#/definitions/Userinfo"
}
},
"title": "A URL represents a parsed URL (technically, a URI reference).",
"title": "URL is a custom URL type that allows validation at configuration load time.",
"type": "object",
"x-go-package": "net/url"
"x-go-package": "github.com/prometheus/common/config"
},
"Userinfo": {
"description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -3035,7 +3034,6 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"alertGroup": {
"description": "AlertGroup alert group",
"properties": {
"alerts": {
"description": "alerts",
@@ -3057,7 +3055,9 @@
"labels",
"receiver"
],
"type": "object"
"type": "object",
"x-go-name": "AlertGroup",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"alertGroups": {
"items": {
@@ -3185,7 +3185,6 @@
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
"description": "GettableAlert gettable alert",
"properties": {
"annotations": {
"$ref": "#/definitions/labelSet"
@@ -3244,7 +3243,9 @@
"status",
"updatedAt"
],
"type": "object"
"type": "object",
"x-go-name": "GettableAlert",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"gettableAlerts": {
"description": "GettableAlerts gettable alerts",
@@ -3254,7 +3255,6 @@
"type": "array"
},
"gettableSilence": {
"description": "GettableSilence gettable silence",
"properties": {
"comment": {
"description": "comment",
@@ -3306,7 +3306,9 @@
"status",
"updatedAt"
],
"type": "object"
"type": "object",
"x-go-name": "GettableSilence",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"gettableSilences": {
"items": {
@@ -3443,6 +3445,7 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"postableSilence": {
"description": "PostableSilence postable silence",
"properties": {
"comment": {
"description": "comment",
@@ -3482,9 +3485,7 @@
"matchers",
"startsAt"
],
"type": "object",
"x-go-name": "PostableSilence",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
"type": "object"
},
"receiver": {
"properties": {
@@ -4732,6 +4733,59 @@
]
}
},
"/api/provisioning/policies": {
"get": {
"operationId": "RouteGetPolicyTree",
"responses": {
"200": {
"description": "Route",
"schema": {
"$ref": "#/definitions/Route"
}
},
"400": {
"description": "ValidationError",
"schema": {
"$ref": "#/definitions/ValidationError"
}
}
},
"summary": "Get the notification policy tree.",
"tags": [
"provisioning"
]
},
"post": {
"consumes": [
"application/json"
],
"operationId": "RoutePostPolicyTree",
"parameters": [
{
"in": "body",
"name": "Body",
"schema": {
"$ref": "#/definitions/Route"
}
}
],
"responses": {
"202": {
"$ref": "#/responses/Accepted"
},
"400": {
"description": "ValidationError",
"schema": {
"$ref": "#/definitions/ValidationError"
}
}
},
"summary": "Sets the notification policy tree.",
"tags": [
"provisioning"
]
}
},
"/api/ruler/grafana/api/v1/rules": {
"get": {
"description": "List rule groups",
+62 -8
View File
@@ -1136,6 +1136,59 @@
}
}
},
"/api/provisioning/policies": {
"get": {
"tags": [
"provisioning"
],
"summary": "Get the notification policy tree.",
"operationId": "RouteGetPolicyTree",
"responses": {
"200": {
"description": "Route",
"schema": {
"$ref": "#/definitions/Route"
}
},
"400": {
"description": "ValidationError",
"schema": {
"$ref": "#/definitions/ValidationError"
}
}
}
},
"post": {
"consumes": [
"application/json"
],
"tags": [
"provisioning"
],
"summary": "Sets the notification policy tree.",
"operationId": "RoutePostPolicyTree",
"parameters": [
{
"name": "Body",
"in": "body",
"schema": {
"$ref": "#/definitions/Route"
}
}
],
"responses": {
"202": {
"$ref": "#/responses/Accepted"
},
"400": {
"description": "ValidationError",
"schema": {
"$ref": "#/definitions/ValidationError"
}
}
}
}
},
"/api/ruler/grafana/api/v1/rules": {
"get": {
"description": "List rule groups",
@@ -4564,9 +4617,8 @@
"x-go-package": "github.com/prometheus/alertmanager/timeinterval"
},
"URL": {
"description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"type": "object",
"title": "A URL represents a parsed URL (technically, a URI reference).",
"title": "URL is a custom URL type that allows validation at configuration load time.",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -4599,7 +4651,7 @@
"$ref": "#/definitions/Userinfo"
}
},
"x-go-package": "net/url"
"x-go-package": "github.com/prometheus/common/config"
},
"Userinfo": {
"description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -4796,7 +4848,6 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"alertGroup": {
"description": "AlertGroup alert group",
"type": "object",
"required": [
"alerts",
@@ -4819,6 +4870,8 @@
"$ref": "#/definitions/receiver"
}
},
"x-go-name": "AlertGroup",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/alertGroup"
},
"alertGroups": {
@@ -4948,7 +5001,6 @@
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
"description": "GettableAlert gettable alert",
"type": "object",
"required": [
"labels",
@@ -5008,6 +5060,8 @@
"x-go-name": "UpdatedAt"
}
},
"x-go-name": "GettableAlert",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/gettableAlert"
},
"gettableAlerts": {
@@ -5019,7 +5073,6 @@
"$ref": "#/definitions/gettableAlerts"
},
"gettableSilence": {
"description": "GettableSilence gettable silence",
"type": "object",
"required": [
"comment",
@@ -5072,6 +5125,8 @@
"x-go-name": "UpdatedAt"
}
},
"x-go-name": "GettableSilence",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/gettableSilence"
},
"gettableSilences": {
@@ -5210,6 +5265,7 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"postableSilence": {
"description": "PostableSilence postable silence",
"type": "object",
"required": [
"comment",
@@ -5250,8 +5306,6 @@
"x-go-name": "StartsAt"
}
},
"x-go-name": "PostableSilence",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/postableSilence"
},
"receiver": {
@@ -24,10 +24,10 @@ func (f *Forked{{classname}}) {{nickname}}(ctx *models.ReqContext) response.Resp
if err := web.Bind(ctx.Req, &conf); err != nil {
return response.Error(http.StatusBadRequest, "bad request data", err)
}
return f.fork{{nickname}}(ctx, conf)
return f.fork{{nickname}}(ctx, conf)
{{/bodyParams}}
{{^bodyParams}}
return f.fork{{nickname}}(ctx)
return f.fork{{nickname}}(ctx)
{{/bodyParams}}
}
{{/operation}}{{/operations}}