From 04dd4e7f7c7877c7486e90f2a8d746aa8a825867 Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Mon, 30 Jan 2023 16:01:37 +0000 Subject: [PATCH] [v9.4.x] Alerting: Allow pausing alerts from provisioning (#62492) Alerting: Allow pausing alerts from provisioning (#62263) * Allow pausing alerts from provisioning * Update swagger * Add IsPaused to provision export endpoints * Add pause field in sample.yml * Add exception for reset state in first loop iteration of scheduler if rule is paused * Update provision definition and swagger docs * Fix provisioning export tests * Suggestion: Simplify if condition * Add more context to a comment (cherry picked from commit 7a465f42a699e9a01271dbb790e09bd697e048f8) Co-authored-by: Alex Moreno --- conf/provisioning/alerting/sample.yaml | 13 +++-- .../ngalert/api/api_provisioning_test.go | 12 ++-- pkg/services/ngalert/api/tooling/api.json | 15 +++-- .../definitions/provisioning_alert_rules.go | 4 ++ pkg/services/ngalert/api/tooling/post.json | 57 +++++++++++-------- pkg/services/ngalert/api/tooling/spec.json | 14 ++++- pkg/services/ngalert/schedule/schedule.go | 6 +- .../provisioning/alerting/file/rules_types.go | 4 ++ public/api-merged.json | 8 ++- 9 files changed, 89 insertions(+), 44 deletions(-) diff --git a/conf/provisioning/alerting/sample.yaml b/conf/provisioning/alerting/sample.yaml index 2a86bf5ea71..bce60aacd7f 100644 --- a/conf/provisioning/alerting/sample.yaml +++ b/conf/provisioning/alerting/sample.yaml @@ -11,7 +11,7 @@ apiVersion: 1 # folder: my_first_folder # # interval of the rule group evaluation # interval: 60s -# # list of rules that are part of the rule group +# # list of rules that are part of the rule group # rules: # # unique identifier for the rule # - uid: my_id_1 @@ -53,7 +53,7 @@ apiVersion: 1 # # state of the alert rule when no data is returned # # possible values: "NoData", "Alerting", "OK", default = NoData # noDataState: Alerting -# # state of the alert rule when the query execution +# # state of the alert rule when the query execution # # fails - possible values: "Error", "Alerting", "OK" # # default = Alerting # executionErrorState: Alerting @@ -62,10 +62,11 @@ apiVersion: 1 # # > map of strings to attach arbitrary custom data # annotations: # some_key: some_value -# # map of strings to filter and +# # map of strings to filter and # # route alerts # labels: # team: sre_team_1 +# isPaused: false # # List of alert rule UIDs that should be deleted # deleteRules: @@ -103,7 +104,7 @@ apiVersion: 1 # # > The labels by which incoming alerts are grouped together. For example, # # multiple alerts coming in for cluster=A and alertname=LatencyHigh would # # be batched into a single group. -# # +# # # # To aggregate by all possible labels, use the special value '...' as # # the sole label name, for example: # # group_by: ['...'] @@ -127,7 +128,7 @@ apiVersion: 1 # mute_time_intervals: # - abc # # How long to initially wait to send a notification for a group -# # of alerts. Allows to collect more initial alerts for the same group. +# # of alerts. Allows to collect more initial alerts for the same group. # # (Usually ~0s to few minutes), default = 30s # group_wait: 30s # # How long to wait before sending a notification about new alerts that @@ -138,7 +139,7 @@ apiVersion: 1 # # been sent successfully for an alert. (Usually ~3h or more), default = 4h # repeat_interval: 4h # # Zero or more child routes -# routes: +# routes: # ... # # List of orgIds that should be reset to the default policy diff --git a/pkg/services/ngalert/api/api_provisioning_test.go b/pkg/services/ngalert/api/api_provisioning_test.go index 183bf0c8d91..2d121d364be 100644 --- a/pkg/services/ngalert/api/api_provisioning_test.go +++ b/pkg/services/ngalert/api/api_provisioning_test.go @@ -480,7 +480,7 @@ func TestProvisioningApi(t *testing.T) { insertRule(t, sut, createTestAlertRule("rule1", 1)) insertRule(t, sut, createTestAlertRule("rule2", 1)) - expectedResponse := `{"apiVersion":1,"groups":[{"orgId":1,"name":"my-cool-group","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule1","title":"rule1","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s"},{"uid":"rule2","title":"rule2","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s"}]}]}` + expectedResponse := `{"apiVersion":1,"groups":[{"orgId":1,"name":"my-cool-group","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule1","title":"rule1","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s","isPaused":false},{"uid":"rule2","title":"rule2","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s","isPaused":false}]}]}` response := sut.RouteGetAlertRuleGroupExport(&rc, "folder-uid", "my-cool-group") @@ -495,7 +495,7 @@ func TestProvisioningApi(t *testing.T) { insertRule(t, sut, createTestAlertRule("rule2", 1)) rc.Context.Req.Header.Add("Accept", "application/yaml") - expectedResponse := "apiVersion: 1\ngroups:\n - orgId: 1\n name: my-cool-group\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule1\n title: rule1\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n - uid: rule2\n title: rule2\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n" + expectedResponse := "apiVersion: 1\ngroups:\n - orgId: 1\n name: my-cool-group\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule1\n title: rule1\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n isPaused: false\n - uid: rule2\n title: rule2\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n isPaused: false\n" response := sut.RouteGetAlertRuleGroupExport(&rc, "folder-uid", "my-cool-group") @@ -607,7 +607,7 @@ func TestProvisioningApi(t *testing.T) { rc := createTestRequestCtx() insertRule(t, sut, createTestAlertRule("rule1", 1)) - expectedResponse := `{"apiVersion":1,"groups":[{"orgId":1,"name":"my-cool-group","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule1","title":"rule1","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s"}]}]}` + expectedResponse := `{"apiVersion":1,"groups":[{"orgId":1,"name":"my-cool-group","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule1","title":"rule1","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s","isPaused":false}]}]}` response := sut.RouteGetAlertRuleExport(&rc, "rule1") @@ -621,7 +621,7 @@ func TestProvisioningApi(t *testing.T) { insertRule(t, sut, createTestAlertRule("rule1", 1)) rc.Context.Req.Header.Add("Accept", "application/yaml") - expectedResponse := "apiVersion: 1\ngroups:\n - orgId: 1\n name: my-cool-group\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule1\n title: rule1\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n" + expectedResponse := "apiVersion: 1\ngroups:\n - orgId: 1\n name: my-cool-group\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule1\n title: rule1\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n isPaused: false\n" response := sut.RouteGetAlertRuleExport(&rc, "rule1") @@ -725,7 +725,7 @@ func TestProvisioningApi(t *testing.T) { insertRule(t, sut, createTestAlertRuleWithFolderAndGroup("rule2", 1, "folder-uid", "groupb")) insertRule(t, sut, createTestAlertRuleWithFolderAndGroup("rule3", 1, "folder-uid2", "groupb")) - expectedResponse := `{"apiVersion":1,"groups":[{"orgId":1,"name":"groupa","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule1","title":"rule1","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s"}]},{"orgId":1,"name":"groupb","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule2","title":"rule2","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s"}]},{"orgId":1,"name":"groupb","folder":"Folder Title2","interval":"1m","rules":[{"uid":"rule3","title":"rule3","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s"}]}]}` + expectedResponse := `{"apiVersion":1,"groups":[{"orgId":1,"name":"groupa","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule1","title":"rule1","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s","isPaused":false}]},{"orgId":1,"name":"groupb","folder":"Folder Title","interval":"1m","rules":[{"uid":"rule2","title":"rule2","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s","isPaused":false}]},{"orgId":1,"name":"groupb","folder":"Folder Title2","interval":"1m","rules":[{"uid":"rule3","title":"rule3","condition":"A","data":[{"refId":"A","relativeTimeRange":{"from":0,"to":0},"datasourceUid":"","model":{"conditions":[{"evaluator":{"params":[3],"type":"gt"},"operator":{"type":"and"},"query":{"params":["A"]},"reducer":{"type":"last"},"type":"query"}],"datasource":{"type":"__expr__","uid":"-100"},"expression":"1==0","intervalMs":1000,"maxDataPoints":43200,"refId":"A","type":"math"}}],"noDataState":"OK","execErrState":"OK","for":"0s","isPaused":false}]}]}` response := sut.RouteGetAlertRulesExport(&rc) @@ -741,7 +741,7 @@ func TestProvisioningApi(t *testing.T) { insertRule(t, sut, createTestAlertRuleWithFolderAndGroup("rule3", 1, "folder-uid2", "groupb")) rc.Context.Req.Header.Add("Accept", "application/yaml") - expectedResponse := "apiVersion: 1\ngroups:\n - orgId: 1\n name: groupa\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule1\n title: rule1\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n - orgId: 1\n name: groupb\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule2\n title: rule2\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n - orgId: 1\n name: groupb\n folder: Folder Title2\n interval: 1m\n rules:\n - uid: rule3\n title: rule3\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n" + expectedResponse := "apiVersion: 1\ngroups:\n - orgId: 1\n name: groupa\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule1\n title: rule1\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n isPaused: false\n - orgId: 1\n name: groupb\n folder: Folder Title\n interval: 1m\n rules:\n - uid: rule2\n title: rule2\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n isPaused: false\n - orgId: 1\n name: groupb\n folder: Folder Title2\n interval: 1m\n rules:\n - uid: rule3\n title: rule3\n condition: A\n data:\n - refId: A\n datasourceUid: \"\"\n model:\n conditions:\n - evaluator:\n params:\n - 3\n type: gt\n operator:\n type: and\n query:\n params:\n - A\n reducer:\n type: last\n type: query\n datasource:\n type: __expr__\n uid: \"-100\"\n expression: 1==0\n intervalMs: 1000\n maxDataPoints: 43200\n refId: A\n type: math\n noDataState: OK\n execErrState: OK\n for: 0s\n isPaused: false\n" response := sut.RouteGetAlertRulesExport(&rc) diff --git a/pkg/services/ngalert/api/tooling/api.json b/pkg/services/ngalert/api/tooling/api.json index 5c39db61c9e..09c49c9cbe3 100644 --- a/pkg/services/ngalert/api/tooling/api.json +++ b/pkg/services/ngalert/api/tooling/api.json @@ -194,6 +194,9 @@ "for": { "$ref": "#/definitions/Duration" }, + "isPaused": { + "type": "boolean" + }, "labels": { "additionalProperties": { "type": "string" @@ -2276,6 +2279,10 @@ "format": "int64", "type": "integer" }, + "isPaused": { + "example": false, + "type": "boolean" + }, "labels": { "additionalProperties": { "type": "string" @@ -3277,6 +3284,7 @@ "type": "object" }, "URL": { + "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.", "properties": { "ForceQuery": { "type": "boolean" @@ -3312,7 +3320,7 @@ "$ref": "#/definitions/Userinfo" } }, - "title": "URL is a custom URL type that allows validation at configuration load time.", + "title": "A URL represents a parsed URL (technically, a URI reference).", "type": "object" }, "Userinfo": { @@ -3616,7 +3624,6 @@ "type": "object" }, "gettableAlert": { - "description": "GettableAlert gettable alert", "properties": { "annotations": { "$ref": "#/definitions/labelSet" @@ -3672,13 +3679,13 @@ "type": "object" }, "gettableAlerts": { - "description": "GettableAlerts gettable alerts", "items": { "$ref": "#/definitions/gettableAlert" }, "type": "array" }, "gettableSilence": { + "description": "GettableSilence gettable silence", "properties": { "comment": { "description": "comment", @@ -3733,6 +3740,7 @@ "type": "array" }, "integration": { + "description": "Integration integration", "properties": { "lastNotifyAttempt": { "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time", @@ -3876,7 +3884,6 @@ "type": "array" }, "postableSilence": { - "description": "PostableSilence postable silence", "properties": { "comment": { "description": "comment", diff --git a/pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go b/pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go index 74929e8b362..d273de06201 100644 --- a/pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go +++ b/pkg/services/ngalert/api/tooling/definitions/provisioning_alert_rules.go @@ -134,6 +134,8 @@ type ProvisionedAlertRule struct { Labels map[string]string `json:"labels,omitempty"` // readonly: true Provenance models.Provenance `json:"provenance,omitempty"` + // example: false + IsPaused bool `json:"isPaused"` } func (a *ProvisionedAlertRule) UpstreamModel() (models.AlertRule, error) { @@ -152,6 +154,7 @@ func (a *ProvisionedAlertRule) UpstreamModel() (models.AlertRule, error) { For: time.Duration(a.For), Annotations: a.Annotations, Labels: a.Labels, + IsPaused: a.IsPaused, }, nil } @@ -172,6 +175,7 @@ func NewAlertRule(rule models.AlertRule, provenance models.Provenance) Provision Annotations: rule.Annotations, Labels: rule.Labels, Provenance: provenance, + IsPaused: rule.IsPaused, } } diff --git a/pkg/services/ngalert/api/tooling/post.json b/pkg/services/ngalert/api/tooling/post.json index b30fb5961d6..12c7b075536 100644 --- a/pkg/services/ngalert/api/tooling/post.json +++ b/pkg/services/ngalert/api/tooling/post.json @@ -194,6 +194,9 @@ "for": { "$ref": "#/definitions/Duration" }, + "isPaused": { + "type": "boolean" + }, "labels": { "additionalProperties": { "type": "string" @@ -219,19 +222,7 @@ "type": "string" } }, - "title": "AlertRuleExport is the provisioned export of models.AlertRule.", - "type": "object" - }, - "AlertRuleFileExport": { - "properties": { - "groups": { - "items": { - "$ref": "#/definitions/AlertRuleGroupExport" - }, - "type": "array" - } - }, - "title": "AlertRuleFileExport is the provisioned export of multiple models.AlertRuleGroup.", + "title": "AlertRuleExport is the provisioned file export of models.AlertRule.", "type": "object" }, "AlertRuleGroup": { @@ -277,7 +268,7 @@ "type": "array" } }, - "title": "AlertRuleGroupExport is the provisioned export of models.AlertRuleGroup.", + "title": "AlertRuleGroupExport is the provisioned file export of AlertRuleGroupV1.", "type": "object" }, "AlertRuleGroupMetadata": { @@ -289,6 +280,22 @@ }, "type": "object" }, + "AlertingFileExport": { + "properties": { + "apiVersion": { + "format": "int64", + "type": "integer" + }, + "groups": { + "items": { + "$ref": "#/definitions/AlertRuleGroupExport" + }, + "type": "array" + } + }, + "title": "AlertingFileExport is the full provisioned file export.", + "type": "object" + }, "AlertingRule": { "description": "adapted from cortex", "properties": { @@ -2272,6 +2279,10 @@ "format": "int64", "type": "integer" }, + "isPaused": { + "example": false, + "type": "boolean" + }, "labels": { "additionalProperties": { "type": "string" @@ -3510,6 +3521,7 @@ "type": "object" }, "alertGroups": { + "description": "AlertGroups alert groups", "items": { "$ref": "#/definitions/alertGroup" }, @@ -3675,7 +3687,6 @@ "type": "array" }, "gettableSilence": { - "description": "GettableSilence gettable silence", "properties": { "comment": { "description": "comment", @@ -3724,13 +3735,13 @@ "type": "object" }, "gettableSilences": { + "description": "GettableSilences gettable silences", "items": { "$ref": "#/definitions/gettableSilence" }, "type": "array" }, "integration": { - "description": "Integration integration", "properties": { "lastNotifyAttempt": { "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time", @@ -3874,6 +3885,7 @@ "type": "array" }, "postableSilence": { + "description": "PostableSilence postable silence", "properties": { "comment": { "description": "comment", @@ -3911,7 +3923,6 @@ "type": "object" }, "receiver": { - "description": "Receiver receiver", "properties": { "active": { "description": "active", @@ -5772,9 +5783,9 @@ ], "responses": { "200": { - "description": "AlertRuleFileExport", + "description": "AlertingFileExport", "schema": { - "$ref": "#/definitions/AlertRuleFileExport" + "$ref": "#/definitions/AlertingFileExport" } }, "404": { @@ -5908,9 +5919,9 @@ ], "responses": { "200": { - "description": "AlertRuleExport", + "description": "AlertingFileExport", "schema": { - "$ref": "#/definitions/AlertRuleExport" + "$ref": "#/definitions/AlertingFileExport" } }, "404": { @@ -6157,9 +6168,9 @@ ], "responses": { "200": { - "description": "AlertRuleGroupExport", + "description": "AlertingFileExport", "schema": { - "$ref": "#/definitions/AlertRuleGroupExport" + "$ref": "#/definitions/AlertingFileExport" } }, "404": { diff --git a/pkg/services/ngalert/api/tooling/spec.json b/pkg/services/ngalert/api/tooling/spec.json index fcae8f39245..14b8d748465 100644 --- a/pkg/services/ngalert/api/tooling/spec.json +++ b/pkg/services/ngalert/api/tooling/spec.json @@ -2806,6 +2806,9 @@ "for": { "$ref": "#/definitions/Duration" }, + "isPaused": { + "type": "boolean" + }, "labels": { "type": "object", "additionalProperties": { @@ -4903,6 +4906,10 @@ "type": "integer", "format": "int64" }, + "isPaused": { + "type": "boolean", + "example": false + }, "labels": { "type": "object", "additionalProperties": { @@ -5892,8 +5899,9 @@ } }, "URL": { + "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.", "type": "object", - "title": "URL is a custom URL type that allows validation at configuration load time.", + "title": "A URL represents a parsed URL (technically, a URI reference).", "properties": { "ForceQuery": { "type": "boolean" @@ -6129,6 +6137,7 @@ "$ref": "#/definitions/alertGroup" }, "alertGroups": { + "description": "AlertGroups alert groups", "type": "array", "items": { "$ref": "#/definitions/alertGroup" @@ -6297,7 +6306,6 @@ "$ref": "#/definitions/gettableAlerts" }, "gettableSilence": { - "description": "GettableSilence gettable silence", "type": "object", "required": [ "comment", @@ -6347,6 +6355,7 @@ "$ref": "#/definitions/gettableSilence" }, "gettableSilences": { + "description": "GettableSilences gettable silences", "type": "array", "items": { "$ref": "#/definitions/gettableSilence" @@ -6498,6 +6507,7 @@ } }, "postableSilence": { + "description": "PostableSilence postable silence", "type": "object", "required": [ "comment", diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index 8cce260d003..62a6a391842 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -474,7 +474,11 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR isPaused := ctx.rule.IsPaused // fetch latest alert rule version if currentRuleVersion != newVersion { - if currentRuleVersion > 0 { // do not clean up state if the eval loop has just started. + // Do not clean up state if the eval loop has just started. + // We need to reset state if the loop has started and the alert is already paused. It can happen, + // if we have an alert with state and we do file provision with stateful Grafana, that state + // lingers in DB and won't be cleaned up until next alert rule update. + if currentRuleVersion > 0 || isPaused { logger.Debug("Got a new version of alert rule. Clear up the state and refresh extra labels", "version", currentRuleVersion, "newVersion", newVersion) resetState(grafanaCtx, isPaused) } diff --git a/pkg/services/provisioning/alerting/file/rules_types.go b/pkg/services/provisioning/alerting/file/rules_types.go index 8c31876a805..59782b08c71 100644 --- a/pkg/services/provisioning/alerting/file/rules_types.go +++ b/pkg/services/provisioning/alerting/file/rules_types.go @@ -78,6 +78,7 @@ type AlertRuleV1 struct { For values.StringValue `json:"for" yaml:"for"` Annotations values.StringMapValue `json:"annotations" yaml:"annotations"` Labels values.StringMapValue `json:"labels" yaml:"labels"` + IsPaused values.BoolValue `json:"isPaused" yaml:"isPaused"` } func (rule *AlertRuleV1) mapToModel(orgID int64) (models.AlertRule, error) { @@ -134,6 +135,7 @@ func (rule *AlertRuleV1) mapToModel(orgID int64) (models.AlertRule, error) { if len(alertRule.Data) == 0 { return models.AlertRule{}, fmt.Errorf("rule '%s' failed to parse: no data set", alertRule.Title) } + alertRule.IsPaused = rule.IsPaused.Value() return alertRule, nil } @@ -205,6 +207,7 @@ type AlertRuleExport struct { For model.Duration `json:"for" yaml:"for"` Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"` Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` + IsPaused bool `json:"isPaused" yaml:"isPaused"` } // AlertQueryExport is the provisioned export of models.AlertQuery. @@ -281,6 +284,7 @@ func newAlertRuleExport(rule models.AlertRule) (AlertRuleExport, error) { ExecErrState: rule.ExecErrState, Annotations: rule.Annotations, Labels: rule.Labels, + IsPaused: rule.IsPaused, }, nil } diff --git a/public/api-merged.json b/public/api-merged.json index 2505ee36660..b8b4d81b4d3 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -16003,6 +16003,10 @@ "type": "integer", "format": "int64" }, + "isPaused": { + "type": "boolean", + "example": true + }, "labels": { "type": "object", "additionalProperties": { @@ -18929,6 +18933,7 @@ } }, "alertGroups": { + "description": "AlertGroups alert groups", "type": "array", "items": { "$ref": "#/definitions/alertGroup" @@ -19033,7 +19038,6 @@ } }, "gettableAlert": { - "description": "GettableAlert gettable alert", "type": "object", "required": [ "labels", @@ -19144,6 +19148,7 @@ } }, "gettableSilences": { + "description": "GettableSilences gettable silences", "type": "array", "items": { "$ref": "#/definitions/gettableSilence" @@ -19293,7 +19298,6 @@ } }, "postableSilence": { - "description": "PostableSilence postable silence", "type": "object", "required": [ "comment",