diff --git a/pkg/services/ngalert/api.go b/pkg/services/ngalert/api.go index 49968f1346f..28143cd3679 100644 --- a/pkg/services/ngalert/api.go +++ b/pkg/services/ngalert/api.go @@ -1,6 +1,8 @@ package ngalert import ( + "fmt" + "github.com/go-macaron/binding" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/api/response" @@ -21,6 +23,8 @@ func (ng *AlertNG) registerAPIEndpoints() { alertDefinitions.Delete("/:alertDefinitionUID", ng.validateOrgAlertDefinition, routing.Wrap(ng.deleteAlertDefinitionEndpoint)) alertDefinitions.Post("/", middleware.ReqSignedIn, binding.Bind(saveAlertDefinitionCommand{}), routing.Wrap(ng.createAlertDefinitionEndpoint)) alertDefinitions.Put("/:alertDefinitionUID", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionCommand{}), routing.Wrap(ng.updateAlertDefinitionEndpoint)) + alertDefinitions.Post("/pause", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionPausedCommand{}), routing.Wrap(ng.alertDefinitionPauseEndpoint)) + alertDefinitions.Post("/unpause", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionPausedCommand{}), routing.Wrap(ng.alertDefinitionUnpauseEndpoint)) }) ng.RouteRegister.Group("/api/ngalert/", func(schedulerRouter routing.RouteRegister) { @@ -180,3 +184,27 @@ func (ng *AlertNG) unpauseScheduler() response.Response { } return response.JSON(200, util.DynMap{"message": "alert definition scheduler unpaused"}) } + +// alertDefinitionPauseEndpoint handles POST /api/alert-definitions/pause. +func (ng *AlertNG) alertDefinitionPauseEndpoint(c *models.ReqContext, cmd updateAlertDefinitionPausedCommand) response.Response { + cmd.OrgID = c.SignedInUser.OrgId + cmd.Paused = true + + err := ng.updateAlertDefinitionPaused(&cmd) + if err != nil { + return response.Error(500, "Failed to pause alert definition", err) + } + return response.JSON(200, util.DynMap{"message": fmt.Sprintf("%d alert definitions paused", cmd.ResultCount)}) +} + +// alertDefinitionUnpauseEndpoint handles POST /api/alert-definitions/unpause. +func (ng *AlertNG) alertDefinitionUnpauseEndpoint(c *models.ReqContext, cmd updateAlertDefinitionPausedCommand) response.Response { + cmd.OrgID = c.SignedInUser.OrgId + cmd.Paused = false + + err := ng.updateAlertDefinitionPaused(&cmd) + if err != nil { + return response.Error(500, "Failed to unpause alert definition", err) + } + return response.JSON(200, util.DynMap{"message": fmt.Sprintf("%d alert definitions unpaused", cmd.ResultCount)}) +} diff --git a/pkg/services/ngalert/database.go b/pkg/services/ngalert/database.go index 4cc55c8b17b..aededd4c552 100644 --- a/pkg/services/ngalert/database.go +++ b/pkg/services/ngalert/database.go @@ -212,7 +212,7 @@ func (ng *AlertNG) getOrgAlertDefinitions(query *listAlertDefinitionsQuery) erro func (ng *AlertNG) getAlertDefinitions(query *listAlertDefinitionsQuery) error { return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { alerts := make([]*AlertDefinition, 0) - q := "SELECT uid, org_id, interval_seconds, version FROM alert_definition" + q := "SELECT uid, org_id, interval_seconds, version, paused FROM alert_definition" if err := sess.SQL(q).Find(&alerts); err != nil { return err } @@ -221,6 +221,39 @@ func (ng *AlertNG) getAlertDefinitions(query *listAlertDefinitionsQuery) error { return nil }) } + +func (ng *AlertNG) updateAlertDefinitionPaused(cmd *updateAlertDefinitionPausedCommand) error { + return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { + placeHolders := strings.Builder{} + const separator = ", " + separatorVar := separator + params := []interface{}{cmd.Paused, cmd.OrgID} + for i, UID := range cmd.UIDs { + if i == len(cmd.UIDs)-1 { + separatorVar = "" + } + placeHolders.WriteString(fmt.Sprintf("?%s", separatorVar)) + params = append(params, UID) + } + sql := fmt.Sprintf("UPDATE alert_definition SET paused = ? WHERE org_id = ? AND uid IN (%s)", placeHolders.String()) + + // prepend sql statement to params + var i interface{} + params = append(params, i) + copy(params[1:], params[0:]) + params[0] = sql + + res, err := sess.Exec(params...) + if err != nil { + return err + } + if cmd.ResultCount, err = res.RowsAffected(); err != nil { + ng.log.Debug("failed to get rows affected: %w", err) + } + return nil + }) +} + func generateNewAlertDefinitionUID(sess *sqlstore.DBSession, orgID int64) (string, error) { for i := 0; i < 3; i++ { uid := util.GenerateShortUID() diff --git a/pkg/services/ngalert/database_mig.go b/pkg/services/ngalert/database_mig.go index 38478afc3aa..ec130328244 100644 --- a/pkg/services/ngalert/database_mig.go +++ b/pkg/services/ngalert/database_mig.go @@ -46,6 +46,10 @@ func addAlertDefinitionMigrations(mg *migrator.Migrator) { } mg.AddMigration("add unique index in alert_definition on org_id and title columns", migrator.NewAddIndexMigration(alertDefinition, uniqueIndices[0])) mg.AddMigration("add unique index in alert_definition on org_id and uid columns", migrator.NewAddIndexMigration(alertDefinition, uniqueIndices[1])) + + mg.AddMigration("Add column paused in alert_definition", migrator.NewAddColumnMigration(alertDefinition, &migrator.Column{ + Name: "paused", Type: migrator.DB_Bool, Nullable: false, Default: "0", + })) } func addAlertDefinitionVersionMigrations(mg *migrator.Migrator) { diff --git a/pkg/services/ngalert/models.go b/pkg/services/ngalert/models.go index f3f5556a460..416c00b60da 100644 --- a/pkg/services/ngalert/models.go +++ b/pkg/services/ngalert/models.go @@ -21,6 +21,7 @@ type AlertDefinition struct { IntervalSeconds int64 `json:"intervalSeconds"` Version int64 `json:"version"` UID string `xorm:"uid" json:"uid"` + Paused bool `json:"paused"` } type alertDefinitionKey struct { @@ -101,3 +102,11 @@ type listAlertDefinitionsQuery struct { Result []*AlertDefinition } + +type updateAlertDefinitionPausedCommand struct { + OrgID int64 `json:"-"` + UIDs []string `json:"uids"` + Paused bool `json:"-"` + + ResultCount int64 +} diff --git a/pkg/services/ngalert/schedule.go b/pkg/services/ngalert/schedule.go index effb8e4e7d1..3a74d40e2a6 100644 --- a/pkg/services/ngalert/schedule.go +++ b/pkg/services/ngalert/schedule.go @@ -185,6 +185,10 @@ func (ng *AlertNG) alertingTicker(grafanaCtx context.Context) error { } readyToRun := make([]readyToRunItem, 0) for _, item := range alertDefinitions { + if item.Paused { + continue + } + key := item.getKey() itemVersion := item.Version newRoutine := !ng.schedule.registry.exists(key) diff --git a/pkg/services/ngalert/schedule_test.go b/pkg/services/ngalert/schedule_test.go index e86db6000fe..ca44cbe1ef0 100644 --- a/pkg/services/ngalert/schedule_test.go +++ b/pkg/services/ngalert/schedule_test.go @@ -123,6 +123,33 @@ func TestAlertingTicker(t *testing.T) { tick := advanceClock(t, mockedClock) assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...) }) + + // pause alert definition + err = ng.updateAlertDefinitionPaused(&updateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: true}) + require.NoError(t, err) + t.Logf("alert definition: %v paused", alerts[2].getKey()) + + expectedAlertDefinitionsEvaluated = []alertDefinitionKey{} + t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) { + tick := advanceClock(t, mockedClock) + assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...) + }) + + expectedAlertDefinitionsStopped = []alertDefinitionKey{alerts[2].getKey()} + t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be stopped", concatenate(expectedAlertDefinitionsStopped)), func(t *testing.T) { + assertStopRun(t, stopAppliedCh, expectedAlertDefinitionsStopped...) + }) + + // unpause alert definition + err = ng.updateAlertDefinitionPaused(&updateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: false}) + require.NoError(t, err) + t.Logf("alert definition: %v unpaused", alerts[2].getKey()) + + expectedAlertDefinitionsEvaluated = []alertDefinitionKey{alerts[0].getKey(), alerts[2].getKey()} + t.Run(fmt.Sprintf("on 9th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) { + tick := advanceClock(t, mockedClock) + assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...) + }) } func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys ...alertDefinitionKey) {