public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AlertingNG: pause/unpause definitions via the API (#30627) (#30672)

Browse Source 
* AlertingNG: pause/unpause definitions via the API

* Apply suggestions from code review

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

* Enable pausing/unpausing multiple definitions

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
(cherry picked from commit 1c158744e8)

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>
This commit is contained in:
Grot (@grafanabot)
2021-01-29 02:01:03 -06:00
committed by GitHub
parent a2e638352b
commit cd4524aba0
6 changed files with 106 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api.go
+28
View File
@@ -1,6 +1,8 @@
package ngalert
import (
"fmt"
"github.com/go-macaron/binding"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/response"
@@ -21,6 +23,8 @@ func (ng *AlertNG) registerAPIEndpoints() {
alertDefinitions.Delete("/:alertDefinitionUID", ng.validateOrgAlertDefinition, routing.Wrap(ng.deleteAlertDefinitionEndpoint))
alertDefinitions.Post("/", middleware.ReqSignedIn, binding.Bind(saveAlertDefinitionCommand{}), routing.Wrap(ng.createAlertDefinitionEndpoint))
alertDefinitions.Put("/:alertDefinitionUID", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionCommand{}), routing.Wrap(ng.updateAlertDefinitionEndpoint))
alertDefinitions.Post("/pause", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionPausedCommand{}), routing.Wrap(ng.alertDefinitionPauseEndpoint))
alertDefinitions.Post("/unpause", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionPausedCommand{}), routing.Wrap(ng.alertDefinitionUnpauseEndpoint))
})
ng.RouteRegister.Group("/api/ngalert/", func(schedulerRouter routing.RouteRegister) {
@@ -180,3 +184,27 @@ func (ng *AlertNG) unpauseScheduler() response.Response {
}
return response.JSON(200, util.DynMap{"message": "alert definition scheduler unpaused"})
}
// alertDefinitionPauseEndpoint handles POST /api/alert-definitions/pause.
func (ng *AlertNG) alertDefinitionPauseEndpoint(c *models.ReqContext, cmd updateAlertDefinitionPausedCommand) response.Response {
cmd.OrgID = c.SignedInUser.OrgId
cmd.Paused = true
err := ng.updateAlertDefinitionPaused(&cmd)
if err != nil {
return response.Error(500, "Failed to pause alert definition", err)
}
return response.JSON(200, util.DynMap{"message": fmt.Sprintf("%d alert definitions paused", cmd.ResultCount)})
}
// alertDefinitionUnpauseEndpoint handles POST /api/alert-definitions/unpause.
func (ng *AlertNG) alertDefinitionUnpauseEndpoint(c *models.ReqContext, cmd updateAlertDefinitionPausedCommand) response.Response {
cmd.OrgID = c.SignedInUser.OrgId
cmd.Paused = false
err := ng.updateAlertDefinitionPaused(&cmd)
if err != nil {
return response.Error(500, "Failed to unpause alert definition", err)
}
return response.JSON(200, util.DynMap{"message": fmt.Sprintf("%d alert definitions unpaused", cmd.ResultCount)})
}
pkg/services/ngalert/database.go
+34 -1
View File
@@ -212,7 +212,7 @@ func (ng *AlertNG) getOrgAlertDefinitions(query *listAlertDefinitionsQuery) erro
func (ng *AlertNG) getAlertDefinitions(query *listAlertDefinitionsQuery) error {
return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
alerts := make([]*AlertDefinition, 0)
q := "SELECT uid, org_id, interval_seconds, version FROM alert_definition"
q := "SELECT uid, org_id, interval_seconds, version, paused FROM alert_definition"
if err := sess.SQL(q).Find(&alerts); err != nil {
return err
}
@@ -221,6 +221,39 @@ func (ng *AlertNG) getAlertDefinitions(query *listAlertDefinitionsQuery) error {
return nil
})
}
func (ng *AlertNG) updateAlertDefinitionPaused(cmd *updateAlertDefinitionPausedCommand) error {
return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
placeHolders := strings.Builder{}
const separator = ", "
separatorVar := separator
params := []interface{}{cmd.Paused, cmd.OrgID}
for i, UID := range cmd.UIDs {
if i == len(cmd.UIDs)-1 {
separatorVar = ""
}
placeHolders.WriteString(fmt.Sprintf("?%s", separatorVar))
params = append(params, UID)
}
sql := fmt.Sprintf("UPDATE alert_definition SET paused = ? WHERE org_id = ? AND uid IN (%s)", placeHolders.String())
// prepend sql statement to params
var i interface{}
params = append(params, i)
copy(params[1:], params[0:])
params[0] = sql
res, err := sess.Exec(params...)
if err != nil {
return err
}
if cmd.ResultCount, err = res.RowsAffected(); err != nil {
ng.log.Debug("failed to get rows affected: %w", err)
}
return nil
})
}
func generateNewAlertDefinitionUID(sess *sqlstore.DBSession, orgID int64) (string, error) {
for i := 0; i < 3; i++ {
uid := util.GenerateShortUID()
pkg/services/ngalert/database_mig.go
+4
View File
@@ -46,6 +46,10 @@ func addAlertDefinitionMigrations(mg *migrator.Migrator) {
}
mg.AddMigration("add unique index in alert_definition on org_id and title columns", migrator.NewAddIndexMigration(alertDefinition, uniqueIndices[0]))
mg.AddMigration("add unique index in alert_definition on org_id and uid columns", migrator.NewAddIndexMigration(alertDefinition, uniqueIndices[1]))
mg.AddMigration("Add column paused in alert_definition", migrator.NewAddColumnMigration(alertDefinition, &migrator.Column{
Name: "paused", Type: migrator.DB_Bool, Nullable: false, Default: "0",
}))
}
func addAlertDefinitionVersionMigrations(mg *migrator.Migrator) {
pkg/services/ngalert/models.go
+9
View File
@@ -21,6 +21,7 @@ type AlertDefinition struct {
IntervalSeconds int64 `json:"intervalSeconds"`
Version int64 `json:"version"`
UID string `xorm:"uid" json:"uid"`
Paused bool `json:"paused"`
}
type alertDefinitionKey struct {
@@ -101,3 +102,11 @@ type listAlertDefinitionsQuery struct {
Result []*AlertDefinition
}
type updateAlertDefinitionPausedCommand struct {
OrgID int64 `json:"-"`
UIDs []string `json:"uids"`
Paused bool `json:"-"`
ResultCount int64
}
pkg/services/ngalert/schedule.go
+4
View File
@@ -185,6 +185,10 @@ func (ng *AlertNG) alertingTicker(grafanaCtx context.Context) error {
}
readyToRun := make([]readyToRunItem, 0)
for _, item := range alertDefinitions {
if item.Paused {
continue
}
key := item.getKey()
itemVersion := item.Version
newRoutine := !ng.schedule.registry.exists(key)
pkg/services/ngalert/schedule_test.go
+27
View File
@@ -123,6 +123,33 @@ func TestAlertingTicker(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
// pause alert definition
err = ng.updateAlertDefinitionPaused(&updateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: true})
require.NoError(t, err)
t.Logf("alert definition: %v paused", alerts[2].getKey())
expectedAlertDefinitionsEvaluated = []alertDefinitionKey{}
t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
expectedAlertDefinitionsStopped = []alertDefinitionKey{alerts[2].getKey()}
t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be stopped", concatenate(expectedAlertDefinitionsStopped)), func(t *testing.T) {
assertStopRun(t, stopAppliedCh, expectedAlertDefinitionsStopped...)
})
// unpause alert definition
err = ng.updateAlertDefinitionPaused(&updateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: false})
require.NoError(t, err)
t.Logf("alert definition: %v unpaused", alerts[2].getKey())
expectedAlertDefinitionsEvaluated = []alertDefinitionKey{alerts[0].getKey(), alerts[2].getKey()}
t.Run(fmt.Sprintf("on 9th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
tick := advanceClock(t, mockedClock)
assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
})
}
func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys ...alertDefinitionKey) {