diff --git a/devenv/dev-dashboards/testdata_alerts.json b/devenv/dev-dashboards/testdata_alerts.json index 8c2edebf155..8fd7d4d9db5 100644 --- a/devenv/dev-dashboards/testdata_alerts.json +++ b/devenv/dev-dashboards/testdata_alerts.json @@ -1,250 +1,546 @@ { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 60 + ], + "type": "gt" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "enabled": true, + "frequency": "60s", + "handler": 1, + "name": "TestData - Always OK", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 60 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Always OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": "125", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 177 + ], + "type": "gt" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "enabled": true, + "executionErrorState": "alerting", + "for": "0m", + "frequency": "60s", + "handler": 1, + "name": "TestData - Always Alerting", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "csv_metric_values", + "stringInput": "200,445,100,150,200,220,190", + "target": "" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 177 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Always Alerting", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "15m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "TestData - No data", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "no_data_points", + "stringInput": "", + "target": "" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "No data", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 177 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "15m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "1m", + "frequency": "1m", + "handler": 1, + "name": "TestData - Always Alerting with For", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 6, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "csv_metric_values", + "stringInput": "200,445,100,150,200,220,190", + "target": "" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 177 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Always Alerting with For", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], "revision": 2, - "title": "Alerting with TestData", + "schemaVersion": 16, + "style": "dark", "tags": [ "grafana-test" ], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": 255.625, - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 60 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "TestData - Always OK", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "datasource": "gdev-testdata", - "editable": true, - "error": false, - "fill": 1, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "scenario": "random_walk", - "scenarioId": "csv_metric_values", - "stringInput": "1,20,90,30,5,0", - "target": "" - } - ], - "thresholds": [ - { - "value": 60, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "critical" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Always OK", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": "125", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 177 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "TestData - Always Alerting", - "noDataState": "no_data", - "notifications": [] - }, - "aliasColors": {}, - "bars": false, - "datasource": "gdev-testdata", - "editable": true, - "error": false, - "fill": 1, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "scenario": "random_walk", - "scenarioId": "csv_metric_values", - "stringInput": "200,445,100,150,200,220,190", - "target": "" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 177 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Always Alerting", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - } - ], + "templating": { + "list": [] + }, "time": { "from": "now-6h", "to": "now" @@ -274,14 +570,8 @@ "30d" ] }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 4, - "links": [], - "gnetId": null -} + "timezone": "browser", + "title": "Alerting with TestData", + "uid": "7MeksYbmk", + "version": 1 +} \ No newline at end of file diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index ce8630d88a4..1195e2a977c 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -9,7 +9,7 @@ services: - /var/run/docker.sock:/tmp/docker.sock:ro db: - image: mysql + image: mysql:5.6 environment: MYSQL_ROOT_PASSWORD: rootpass MYSQL_DATABASE: grafana diff --git a/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet b/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet index 86ded7e79d6..e9b8abfbb9c 100644 --- a/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet +++ b/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet @@ -39,6 +39,7 @@ local alertDashboardTemplate = { "executionErrorState": "alerting", "frequency": "10s", "handler": 1, + "for": "1m", "name": "bulk alerting", "noDataState": "no_data", "notifications": [ diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md index 488619055e2..387132ecfeb 100644 --- a/docs/sources/alerting/rules.md +++ b/docs/sources/alerting/rules.md @@ -39,7 +39,7 @@ Currently alerting supports a limited form of high availability. Since v4.2.0 of ## Rule Config -{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}} + Currently only the graph panel supports alert rules but this will be added to the **Singlestat** and **Table** panels as well in a future release. @@ -48,6 +48,16 @@ panels as well in a future release. Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule. +### For + +> This setting is available in Grafana 5.4 and above. + +If an alert rule has a configured `For` and the query violates the configured threshold it will first go from `OK` to `Pending`. Going from `OK` to `Pending` Grafana will not send any notifications. Once the alert rule has been firing for more than `For` duration, it will change to `Alerting` and send alert notifications. + +Typically, it's always a good idea to use this setting since its often worse to get false positive than wait a few minutes before the alert notification triggers. + +{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}} + ### Conditions Currently the only condition type that exists is a `Query` condition that allows you to @@ -57,11 +67,11 @@ specify a query letter, time range and an aggregation function. ### Query condition example ```sql -avg() OF query(A, 5m, now) IS BELOW 14 +avg() OF query(A, 15m, now) IS BELOW 14 ``` - `avg()` Controls how the values for **each** series should be reduced to a value that can be compared against the threshold. Click on the function to change it to another aggregation function. -- `query(A, 5m, now)` The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `5m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data. +- `query(A, 15m, now)` The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `15m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data. - `IS BELOW 14` Defines the type of threshold and the threshold value. You can click on `IS BELOW` to change the type of threshold. The query used in an alert rule cannot contain any template variables. Currently we only support `AND` and `OR` operators between conditions and they are executed serially. diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index c68cee50948..66b3b504946 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -295,7 +295,7 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response { return Error(500, "", err) } - var response m.AlertStateType = m.AlertStatePending + var response m.AlertStateType = m.AlertStateUnknown pausedState := "un-paused" if cmd.Paused { response = m.AlertStatePaused diff --git a/pkg/models/alert.go b/pkg/models/alert.go index aaf9c50197a..e8171f8cf0b 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -19,6 +19,7 @@ const ( AlertStateAlerting AlertStateType = "alerting" AlertStateOK AlertStateType = "ok" AlertStatePending AlertStateType = "pending" + AlertStateUnknown AlertStateType = "unknown" ) const ( @@ -39,7 +40,12 @@ var ( ) func (s AlertStateType) IsValid() bool { - return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending + return s == AlertStateOK || + s == AlertStateNoData || + s == AlertStatePaused || + s == AlertStatePending || + s == AlertStateAlerting || + s == AlertStateUnknown } func (s NoDataOption) IsValid() bool { @@ -66,12 +72,13 @@ type Alert struct { PanelId int64 Name string Message string - Severity string + Severity string //Unused State AlertStateType - Handler int64 + Handler int64 //Unused Silenced bool ExecutionError string Frequency int64 + For time.Duration EvalData *simplejson.Json NewStateDate time.Time diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index d0441d379b7..17ed448bd2a 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -68,8 +68,13 @@ func (c *EvalContext) GetStateModel() *StateDescription { Color: "#D63232", Text: "Alerting", } + case m.AlertStateUnknown: + return &StateDescription{ + Color: "#888888", + Text: "Unknown", + } default: - panic("Unknown rule state " + c.Rule.State) + panic("Unknown rule state for alert " + c.Rule.State) } } @@ -113,7 +118,26 @@ func (c *EvalContext) GetRuleUrl() (string, error) { return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil } +// GetNewState returns the new state from the alert rule evaluation func (c *EvalContext) GetNewState() m.AlertStateType { + ns := getNewStateInternal(c) + if ns != m.AlertStateAlerting || c.Rule.For == 0 { + return ns + } + + since := time.Now().Sub(c.Rule.LastStateChange) + if c.PrevAlertState == m.AlertStatePending && since > c.Rule.For { + return m.AlertStateAlerting + } + + if c.PrevAlertState == m.AlertStateAlerting { + return m.AlertStateAlerting + } + + return m.AlertStatePending +} + +func getNewStateInternal(c *EvalContext) m.AlertStateType { if c.Error != nil { c.log.Error("Alert Rule Result Error", "ruleId", c.Rule.Id, @@ -125,11 +149,13 @@ func (c *EvalContext) GetNewState() m.AlertStateType { return c.PrevAlertState } return c.Rule.ExecutionErrorState.ToAlertState() + } - } else if c.Firing { + if c.Firing { return m.AlertStateAlerting + } - } else if c.NoDataFound { + if c.NoDataFound { c.log.Info("Alert Rule returned no data", "ruleId", c.Rule.Id, "name", c.Rule.Name, diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go index 750fa959683..4c9b88f1881 100644 --- a/pkg/services/alerting/eval_context_test.go +++ b/pkg/services/alerting/eval_context_test.go @@ -2,11 +2,11 @@ package alerting import ( "context" - "fmt" + "errors" "testing" + "time" "github.com/grafana/grafana/pkg/models" - . "github.com/smartystreets/goconvey/convey" ) func TestStateIsUpdatedWhenNeeded(t *testing.T) { @@ -31,71 +31,177 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) { }) } -func TestAlertingEvalContext(t *testing.T) { - Convey("Should compute and replace properly new rule state", t, func() { +func TestGetStateFromEvalContext(t *testing.T) { + tcs := []struct { + name string + expected models.AlertStateType + applyFn func(ec *EvalContext) + focus bool + }{ + { + name: "ok -> alerting", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.Firing = true + ec.PrevAlertState = models.AlertStateOK + }, + }, + { + name: "ok -> error(alerting)", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Error = errors.New("test error") + ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting + }, + }, + { + name: "ok -> pending. since its been firing for less than FOR", + expected: models.AlertStatePending, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Firing = true + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2) + ec.Rule.For = time.Minute * 5 + }, + }, + { + name: "ok -> pending. since it has to be pending longer than FOR and prev state is ok", + expected: models.AlertStatePending, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Firing = true + ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5)) + ec.Rule.For = time.Minute * 2 + }, + }, + { + name: "pending -> alerting. since its been firing for more than FOR and prev state is pending", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Firing = true + ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5)) + ec.Rule.For = time.Minute * 2 + }, + }, + { + name: "alerting -> alerting. should not update regardless of FOR", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateAlerting + ec.Firing = true + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5) + ec.Rule.For = time.Minute * 2 + }, + }, + { + name: "ok -> ok. should not update regardless of FOR", + expected: models.AlertStateOK, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5) + ec.Rule.For = time.Minute * 2 + }, + }, + { + name: "ok -> error(keep_last)", + expected: models.AlertStateOK, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Error = errors.New("test error") + ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState + }, + }, + { + name: "pending -> error(keep_last)", + expected: models.AlertStatePending, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Error = errors.New("test error") + ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState + }, + }, + { + name: "ok -> no_data(alerting)", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Rule.NoDataState = models.NoDataSetAlerting + ec.NoDataFound = true + }, + }, + { + name: "ok -> no_data(keep_last)", + expected: models.AlertStateOK, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStateOK + ec.Rule.NoDataState = models.NoDataKeepState + ec.NoDataFound = true + }, + }, + { + name: "pending -> no_data(keep_last)", + expected: models.AlertStatePending, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Rule.NoDataState = models.NoDataKeepState + ec.NoDataFound = true + }, + }, + { + name: "pending -> no_data(alerting) with for duration have not passed", + expected: models.AlertStatePending, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Rule.NoDataState = models.NoDataSetAlerting + ec.NoDataFound = true + ec.Rule.For = time.Minute * 5 + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2) + }, + }, + { + name: "pending -> no_data(alerting) should set alerting since time passed FOR", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Rule.NoDataState = models.NoDataSetAlerting + ec.NoDataFound = true + ec.Rule.For = time.Minute * 2 + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5) + }, + }, + { + name: "pending -> error(alerting) with for duration have not passed ", + expected: models.AlertStatePending, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting + ec.Error = errors.New("test error") + ec.Rule.For = time.Minute * 5 + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2) + }, + }, + { + name: "pending -> error(alerting) should set alerting since time passed FOR", + expected: models.AlertStateAlerting, + applyFn: func(ec *EvalContext) { + ec.PrevAlertState = models.AlertStatePending + ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting + ec.Error = errors.New("test error") + ec.Rule.For = time.Minute * 2 + ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5) + }, + }, + } + + for _, tc := range tcs { ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) - dummieError := fmt.Errorf("dummie error") - Convey("ok -> alerting", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Firing = true - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - }) - - Convey("ok -> error(alerting)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - }) - - Convey("ok -> error(keep_last)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) - }) - - Convey("pending -> error(keep_last)", func() { - ctx.PrevAlertState = models.AlertStatePending - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) - }) - - Convey("ok -> no_data(alerting)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.NoDataState = models.NoDataSetAlerting - ctx.NoDataFound = true - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - }) - - Convey("ok -> no_data(keep_last)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.NoDataState = models.NoDataKeepState - ctx.NoDataFound = true - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) - }) - - Convey("pending -> no_data(keep_last)", func() { - ctx.PrevAlertState = models.AlertStatePending - ctx.Rule.NoDataState = models.NoDataKeepState - ctx.NoDataFound = true - - ctx.Rule.State = ctx.GetNewState() - So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) - }) - }) + tc.applyFn(ctx) + have := ctx.GetNewState() + if have != tc.expected { + t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have)) + } + } } diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index 0abacc91313..e33e3dc2af3 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -2,8 +2,8 @@ package alerting import ( "errors" - "fmt" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" @@ -115,6 +115,15 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, return nil, ValidationError{Reason: "Could not parse frequency"} } + rawFor := jsonAlert.Get("for").MustString() + var forValue time.Duration + if rawFor != "" { + forValue, err = time.ParseDuration(rawFor) + if err != nil { + return nil, ValidationError{Reason: "Could not parse for"} + } + } + alert := &m.Alert{ DashboardId: e.Dash.Id, OrgId: e.OrgID, @@ -124,6 +133,7 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, Handler: jsonAlert.Get("handler").MustInt64(), Message: jsonAlert.Get("message").MustString(), Frequency: frequency, + For: forValue, } for _, condition := range jsonAlert.Get("conditions").MustArray() { diff --git a/pkg/services/alerting/extractor_test.go b/pkg/services/alerting/extractor_test.go index 0890b9e1bd1..9665a657bb7 100644 --- a/pkg/services/alerting/extractor_test.go +++ b/pkg/services/alerting/extractor_test.go @@ -3,6 +3,7 @@ package alerting import ( "io/ioutil" "testing" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" @@ -46,7 +47,7 @@ func TestAlertRuleExtraction(t *testing.T) { return nil }) - json, err := ioutil.ReadFile("./test-data/graphite-alert.json") + json, err := ioutil.ReadFile("./testdata/graphite-alert.json") So(err, ShouldBeNil) Convey("Extractor should not modify the original json", func() { @@ -118,6 +119,11 @@ func TestAlertRuleExtraction(t *testing.T) { So(alerts[1].PanelId, ShouldEqual, 4) }) + Convey("should extract for param", func() { + So(alerts[0].For, ShouldEqual, time.Minute*2) + So(alerts[1].For, ShouldEqual, time.Duration(0)) + }) + Convey("should extract name and desc", func() { So(alerts[0].Name, ShouldEqual, "name1") So(alerts[0].Message, ShouldEqual, "desc1") @@ -140,7 +146,7 @@ func TestAlertRuleExtraction(t *testing.T) { }) Convey("Panels missing id should return error", func() { - panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json") + panelWithoutId, err := ioutil.ReadFile("./testdata/panels-missing-id.json") So(err, ShouldBeNil) dashJson, err := simplejson.NewJson(panelWithoutId) @@ -156,7 +162,7 @@ func TestAlertRuleExtraction(t *testing.T) { }) Convey("Panel with id set to zero should return error", func() { - panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json") + panelWithIdZero, err := ioutil.ReadFile("./testdata/panel-with-id-0.json") So(err, ShouldBeNil) dashJson, err := simplejson.NewJson(panelWithIdZero) @@ -172,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) { }) Convey("Parse alerts from dashboard without rows", func() { - json, err := ioutil.ReadFile("./test-data/v5-dashboard.json") + json, err := ioutil.ReadFile("./testdata/v5-dashboard.json") So(err, ShouldBeNil) dashJson, err := simplejson.NewJson(json) @@ -192,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) { }) Convey("Parse and validate dashboard containing influxdb alert", func() { - json, err := ioutil.ReadFile("./test-data/influxdb-alert.json") + json, err := ioutil.ReadFile("./testdata/influxdb-alert.json") So(err, ShouldBeNil) dashJson, err := simplejson.NewJson(json) @@ -221,7 +227,7 @@ func TestAlertRuleExtraction(t *testing.T) { }) Convey("Should be able to extract collapsed panels", func() { - json, err := ioutil.ReadFile("./test-data/collapsed-panels.json") + json, err := ioutil.ReadFile("./testdata/collapsed-panels.json") So(err, ShouldBeNil) dashJson, err := simplejson.NewJson(json) @@ -242,7 +248,7 @@ func TestAlertRuleExtraction(t *testing.T) { }) Convey("Parse and validate dashboard without id and containing an alert", func() { - json, err := ioutil.ReadFile("./test-data/dash-without-id.json") + json, err := ioutil.ReadFile("./testdata/dash-without-id.json") So(err, ShouldBeNil) dashJSON, err := simplejson.NewJson(json) diff --git a/pkg/services/alerting/notifiers/alertmanager_test.go b/pkg/services/alerting/notifiers/alertmanager_test.go index 3549b536e48..7510742ed17 100644 --- a/pkg/services/alerting/notifiers/alertmanager_test.go +++ b/pkg/services/alerting/notifiers/alertmanager_test.go @@ -1,13 +1,60 @@ package notifiers import ( + "context" "testing" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" . "github.com/smartystreets/goconvey/convey" ) +func TestWhenAlertManagerShouldNotify(t *testing.T) { + tcs := []struct { + prevState m.AlertStateType + newState m.AlertStateType + + expect bool + }{ + { + prevState: m.AlertStatePending, + newState: m.AlertStateOK, + expect: false, + }, + { + prevState: m.AlertStateAlerting, + newState: m.AlertStateOK, + expect: true, + }, + { + prevState: m.AlertStateOK, + newState: m.AlertStatePending, + expect: false, + }, + { + prevState: m.AlertStateUnknown, + newState: m.AlertStatePending, + expect: false, + }, + } + + for _, tc := range tcs { + am := &AlertmanagerNotifier{log: log.New("test.logger")} + evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ + State: tc.prevState, + }) + + evalContext.Rule.State = tc.newState + + res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{}) + if res != tc.expect { + t.Errorf("got %v expected %v", res, tc.expect) + } + } +} + func TestAlertmanagerNotifier(t *testing.T) { Convey("Alertmanager notifier tests", t, func() { diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index d141d6cd257..d4a9975bcba 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -67,6 +67,16 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC } // Do not notify when we become OK for the first time. + if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK { + return false + } + + // Do not notify when we become OK for the first time. + if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending { + return false + } + + // Do not notify when we become OK from pending if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK { return false } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 5062828cb4f..3fd4447eefe 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStatePending, sendReminder: false, - state: &m.AlertNotificationState{}, expect: false, }, @@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateAlerting, prevState: m.AlertStateOK, sendReminder: false, - state: &m.AlertNotificationState{}, expect: true, }, @@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStatePending, prevState: m.AlertStateOK, sendReminder: false, - state: &m.AlertNotificationState{}, expect: false, }, @@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateOK, sendReminder: false, - state: &m.AlertNotificationState{}, expect: false, }, @@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateOK, sendReminder: true, - state: &m.AlertNotificationState{}, expect: false, }, @@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) { newState: m.AlertStateOK, prevState: m.AlertStateAlerting, sendReminder: false, - state: &m.AlertNotificationState{}, expect: true, }, @@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateAlerting, frequency: time.Minute * 10, sendReminder: true, - state: &m.AlertNotificationState{}, expect: true, }, @@ -132,6 +125,27 @@ func TestShouldSendAlertNotification(t *testing.T) { prevState: m.AlertStateOK, state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()}, + expect: true, + }, + { + name: "unknown -> ok", + prevState: m.AlertStateUnknown, + newState: m.AlertStateOK, + + expect: false, + }, + { + name: "unknown -> pending", + prevState: m.AlertStateUnknown, + newState: m.AlertStatePending, + + expect: false, + }, + { + name: "unknown -> alerting", + prevState: m.AlertStateUnknown, + newState: m.AlertStateAlerting, + expect: true, }, } @@ -141,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) { State: tc.prevState, }) + if tc.state == nil { + tc.state = &m.AlertNotificationState{} + } + evalContext.Rule.State = tc.newState nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency} diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 420ffeb9a55..ce12a8a6b96 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { // when two servers are raising. This makes sure that the server // with the last state change always sends a notification. evalContext.Rule.StateChanges = cmd.Result.StateChanges + + // Update the last state change of the alert rule in memory + evalContext.Rule.LastStateChange = time.Now() } // save annotation diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index 999611f15c4..d2a505145ac 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "strconv" + "time" "github.com/grafana/grafana/pkg/components/simplejson" @@ -18,6 +19,8 @@ type Rule struct { Frequency int64 Name string Message string + LastStateChange time.Time + For time.Duration NoDataState m.NoDataOption ExecutionErrorState m.ExecutionErrorOption State m.AlertStateType @@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.Message = ruleDef.Message model.Frequency = ruleDef.Frequency model.State = ruleDef.State + model.LastStateChange = ruleDef.NewStateDate + model.For = ruleDef.For model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) model.StateChanges = ruleDef.StateChanges diff --git a/pkg/services/alerting/test-data/collapsed-panels.json b/pkg/services/alerting/testdata/collapsed-panels.json similarity index 100% rename from pkg/services/alerting/test-data/collapsed-panels.json rename to pkg/services/alerting/testdata/collapsed-panels.json diff --git a/pkg/services/alerting/test-data/dash-without-id.json b/pkg/services/alerting/testdata/dash-without-id.json similarity index 100% rename from pkg/services/alerting/test-data/dash-without-id.json rename to pkg/services/alerting/testdata/dash-without-id.json diff --git a/pkg/services/alerting/test-data/graphite-alert.json b/pkg/services/alerting/testdata/graphite-alert.json similarity index 98% rename from pkg/services/alerting/test-data/graphite-alert.json rename to pkg/services/alerting/testdata/graphite-alert.json index 5f23e224f9a..3cb4ae1dd22 100644 --- a/pkg/services/alerting/test-data/graphite-alert.json +++ b/pkg/services/alerting/testdata/graphite-alert.json @@ -23,6 +23,7 @@ "message": "desc1", "handler": 1, "frequency": "60s", + "for": "2m", "conditions": [ { "type": "query", diff --git a/pkg/services/alerting/test-data/influxdb-alert.json b/pkg/services/alerting/testdata/influxdb-alert.json similarity index 100% rename from pkg/services/alerting/test-data/influxdb-alert.json rename to pkg/services/alerting/testdata/influxdb-alert.json diff --git a/pkg/services/alerting/test-data/panel-with-id-0.json b/pkg/services/alerting/testdata/panel-with-id-0.json similarity index 100% rename from pkg/services/alerting/test-data/panel-with-id-0.json rename to pkg/services/alerting/testdata/panel-with-id-0.json diff --git a/pkg/services/alerting/test-data/panels-missing-id.json b/pkg/services/alerting/testdata/panels-missing-id.json similarity index 100% rename from pkg/services/alerting/test-data/panels-missing-id.json rename to pkg/services/alerting/testdata/panels-missing-id.json diff --git a/pkg/services/alerting/test-data/v5-dashboard.json b/pkg/services/alerting/testdata/v5-dashboard.json similarity index 100% rename from pkg/services/alerting/test-data/v5-dashboard.json rename to pkg/services/alerting/testdata/v5-dashboard.json diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 2f17402b80c..62ab348664f 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -193,7 +193,8 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS if alertToUpdate.ContainsUpdates(alert) { alert.Updated = timeNow() alert.State = alertToUpdate.State - sess.MustCols("message") + sess.MustCols("message", "for") + _, err := sess.ID(alert.Id).Update(alert) if err != nil { return err @@ -204,7 +205,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS } else { alert.Updated = timeNow() alert.Created = timeNow() - alert.State = m.AlertStatePending + alert.State = m.AlertStateUnknown alert.NewStateDate = timeNow() _, err := sess.Insert(alert) @@ -299,7 +300,7 @@ func PauseAlert(cmd *m.PauseAlertCommand) error { params = append(params, string(m.AlertStatePaused)) params = append(params, timeNow()) } else { - params = append(params, string(m.AlertStatePending)) + params = append(params, string(m.AlertStateUnknown)) params = append(params, timeNow()) } @@ -323,7 +324,7 @@ func PauseAllAlerts(cmd *m.PauseAllAlertCommand) error { if cmd.Paused { newState = string(m.AlertStatePaused) } else { - newState = string(m.AlertStatePending) + newState = string(m.AlertStateUnknown) } res, err := sess.Exec(`UPDATE alert SET state = ?, new_state_date = ?`, newState, timeNow()) diff --git a/pkg/services/sqlstore/alert_test.go b/pkg/services/sqlstore/alert_test.go index d97deb45f0e..40867e96b4d 100644 --- a/pkg/services/sqlstore/alert_test.go +++ b/pkg/services/sqlstore/alert_test.go @@ -109,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) { So(alert.DashboardId, ShouldEqual, testDash.Id) So(alert.PanelId, ShouldEqual, 1) So(alert.Name, ShouldEqual, "Alerting title") - So(alert.State, ShouldEqual, "pending") + So(alert.State, ShouldEqual, m.AlertStateUnknown) So(alert.NewStateDate, ShouldNotBeNil) So(alert.EvalData, ShouldNotBeNil) So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test") @@ -154,7 +154,7 @@ func TestAlertingDataAccess(t *testing.T) { So(query.Result[0].Name, ShouldEqual, "Name") Convey("Alert state should not be updated", func() { - So(query.Result[0].State, ShouldEqual, "pending") + So(query.Result[0].State, ShouldEqual, m.AlertStateUnknown) }) }) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 198a47b50ff..b5aeb26483c 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -133,4 +133,8 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state)) mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0])) + + mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{ + Name: "for", Type: DB_BigInt, Nullable: true, + })) } diff --git a/public/app/core/utils/colors.ts b/public/app/core/utils/colors.ts index 16214679996..13e02b76e30 100644 --- a/public/app/core/utils/colors.ts +++ b/public/app/core/utils/colors.ts @@ -7,6 +7,7 @@ export const DEFAULT_ANNOTATION_COLOR = 'rgba(0, 211, 255, 1)'; export const OK_COLOR = 'rgba(11, 237, 50, 1)'; export const ALERTING_COLOR = 'rgba(237, 46, 24, 1)'; export const NO_DATA_COLOR = 'rgba(150, 150, 150, 1)'; +export const PENDING_COLOR = 'rgba(247, 149, 32, 1)'; export const REGION_FILL_ALPHA = 0.09; const colors = [ diff --git a/public/app/features/alerting/AlertRuleList.tsx b/public/app/features/alerting/AlertRuleList.tsx index d25fc659af5..f94134f3ee1 100644 --- a/public/app/features/alerting/AlertRuleList.tsx +++ b/public/app/features/alerting/AlertRuleList.tsx @@ -29,6 +29,7 @@ export class AlertRuleList extends PureComponent { { text: 'Alerting', value: 'alerting' }, { text: 'No Data', value: 'no_data' }, { text: 'Paused', value: 'paused' }, + { text: 'Pending', value: 'pending' }, ]; componentDidMount() { diff --git a/public/app/features/alerting/AlertTabCtrl.ts b/public/app/features/alerting/AlertTabCtrl.ts index 146b7026353..ef68ddcf4a5 100644 --- a/public/app/features/alerting/AlertTabCtrl.ts +++ b/public/app/features/alerting/AlertTabCtrl.ts @@ -169,6 +169,7 @@ export class AlertTabCtrl { alert.frequency = alert.frequency || '1m'; alert.handler = alert.handler || 1; alert.notifications = alert.notifications || []; + alert.for = alert.for || '0m'; const defaultName = this.panel.title + ' alert'; alert.name = alert.name || defaultName; @@ -217,7 +218,7 @@ export class AlertTabCtrl { buildDefaultCondition() { return { type: 'query', - query: { params: ['A', '15m', 'now'] }, + query: { params: ['A', '5m', 'now'] }, reducer: { type: 'avg', params: [] }, evaluator: { type: 'gt', params: [null] }, operator: { type: 'and' }, @@ -354,6 +355,7 @@ export class AlertTabCtrl { enable() { this.panel.alert = {}; this.initModel(); + this.panel.alert.for = '5m'; //default value for new alerts. for existing alerts we use 0m to avoid breaking changes } evaluatorParamsChanged() { diff --git a/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap b/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap index 4ae27213e1e..b753a852e92 100644 --- a/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap +++ b/public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap @@ -81,6 +81,12 @@ exports[`Render should render alert rules 1`] = ` > Paused + @@ -230,6 +236,12 @@ exports[`Render should render component 1`] = ` > Paused + diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html index cb101672aa4..2b5e9bdf0cb 100644 --- a/public/app/features/alerting/partials/alert_tab.html +++ b/public/app/features/alerting/partials/alert_tab.html @@ -1,147 +1,159 @@
- -
-
-
- {{ctrl.error}} -
+
+
+
+ {{ctrl.error}} +
-
-
Alert Config
-
- Name - - Evaluate every - -
-
+
+
Alert Config
+
+ Name + +
+
+
+ Evaluate every + +
+
+ + + + If an alert rule has a configured For and the query violates the configured threshold it will first go from OK to Pending. + Going from OK to Pending Grafana will not send any notifications. Once the alert rule has been firing for more than For duration, it will change to Alerting and send alert notifications. + +
+
+
-
-
Conditions
-
-
- - WHEN -
-
- - - OF -
-
- - -
-
- - - - -
-
- -
-
+
+
Conditions
+
+
+ + WHEN +
+
+ + + OF +
+
+ + +
+
+ + + + +
+
+ +
+
-
- -
-
+
+ +
+
-
-
- If no data or all values are null - SET STATE TO -
- -
-
+
+
+ If no data or all values are null + SET STATE TO +
+ +
+
-
- If execution error or timeout - SET STATE TO -
- -
-
+
+ If execution error or timeout + SET STATE TO +
+ +
+
-
- -
-
+
+ +
+
-
- Evaluating rule -
+
+ Evaluating rule +
-
- -
-
+
+ +
+
-
-
Notifications
-
-
- Send to - -  {{nc.name}}  - - - -
-
-
- Message - -
-
+
+
Notifications
+
+
+ Send to + +  {{nc.name}}  + + + +
+
+
+ Message + +
+
-
- -
- State history (last 50 state changes) -
+
+ +
+ State history (last 50 state changes) +
-
-
- No state changes recorded -
+
+
+ No state changes recorded +
  1. diff --git a/public/app/features/alerting/state/alertDef.ts b/public/app/features/alerting/state/alertDef.ts index 11d2aafaa7f..378be0afb91 100644 --- a/public/app/features/alerting/state/alertDef.ts +++ b/public/app/features/alerting/state/alertDef.ts @@ -99,6 +99,13 @@ function getStateDisplayModel(state) { stateClass: 'alert-state-warning', }; } + case 'unknown': { + return { + text: 'UNKNOWN', + iconClass: 'fa fa-question', + stateClass: 'alert-state-paused', + }; + } } throw { message: 'Unknown alert state' }; diff --git a/public/app/features/annotations/annotation_tooltip.ts b/public/app/features/annotations/annotation_tooltip.ts index 16c18005204..fbe85856f31 100644 --- a/public/app/features/annotations/annotation_tooltip.ts +++ b/public/app/features/annotations/annotation_tooltip.ts @@ -32,7 +32,7 @@ export function annotationTooltipDirective($sanitize, dashboardSrv, contextSrv, if (event.alertId) { const stateModel = alertDef.getStateDisplayModel(event.newState); titleStateClass = stateModel.stateClass; - title = ` ${stateModel.text}`; + title = ` ${stateModel.text}`; text = alertDef.getAlertAnnotationInfo(event); if (event.text) { text = text + '
    ' + event.text; diff --git a/public/app/features/annotations/event_manager.ts b/public/app/features/annotations/event_manager.ts index ef74ca193d4..db748e639a1 100644 --- a/public/app/features/annotations/event_manager.ts +++ b/public/app/features/annotations/event_manager.ts @@ -7,6 +7,7 @@ import { OK_COLOR, ALERTING_COLOR, NO_DATA_COLOR, + PENDING_COLOR, DEFAULT_ANNOTATION_COLOR, REGION_FILL_ALPHA, } from 'app/core/utils/colors'; @@ -71,6 +72,11 @@ export class EventManager { position: 'BOTTOM', markerSize: 5, }, + $__pending: { + color: PENDING_COLOR, + position: 'BOTTOM', + markerSize: 5, + }, $__editing: { color: DEFAULT_ANNOTATION_COLOR, position: 'BOTTOM', diff --git a/public/app/features/panel/panel_directive.ts b/public/app/features/panel/panel_directive.ts index 77ebf754b3a..aef7ca5e256 100644 --- a/public/app/features/panel/panel_directive.ts +++ b/public/app/features/panel/panel_directive.ts @@ -161,7 +161,11 @@ module.directive('grafanaPanel', ($rootScope, $document, $timeout) => { panelContainer.removeClass('panel-alert-state--' + lastAlertState); } - if (ctrl.alertState.state === 'ok' || ctrl.alertState.state === 'alerting') { + if ( + ctrl.alertState.state === 'ok' || + ctrl.alertState.state === 'alerting' || + ctrl.alertState.state === 'pending' + ) { panelContainer.addClass('panel-alert-state--' + ctrl.alertState.state); } diff --git a/public/app/plugins/panel/alertlist/editor.html b/public/app/plugins/panel/alertlist/editor.html index c48b70e02c0..a05234cea3c 100644 --- a/public/app/plugins/panel/alertlist/editor.html +++ b/public/app/plugins/panel/alertlist/editor.html @@ -50,5 +50,6 @@ +
diff --git a/public/sass/pages/_alerting.scss b/public/sass/pages/_alerting.scss index 90f2eb526f1..77752be11bc 100644 --- a/public/sass/pages/_alerting.scss +++ b/public/sass/pages/_alerting.scss @@ -66,6 +66,13 @@ content: '\e611'; } } + + &--pending { + .panel-alert-icon:before { + color: $warn; + content: '\e611'; + } + } } @keyframes alerting-panel {