public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support OK option for Error state (#47670) (#47741)

Browse Source 
* support OK state for Error
* add CHANGELOG to ngalert package
This commit is contained in:
Yuriy Tseretyan
2022-04-13 22:39:26 +02:00
committed by GitHub
parent 21404f4169
commit 0e8ba02479
3 changed files with 206 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/CHANGELOG.md
+50
View File
@@ -0,0 +1,50 @@
# Changelog
**Please note that this is not a direct replacement of Grafana's "add to changelog" label. It is _mostly_ for internal consumption of the Alerting team that owns this part of Grafana.**
## Scope Glossary
### `[ADMIN]`
The ADMIN scope denotes a change that affect the structure and layout of this repository. This includes updates to the following:
- CODEOWNERS
- README
- DotFiles (.gitignore, .git-attributes, etc)
Anything that a developer working on this repo should be aware of from a standards and practice perspective.
### `[BUGFIX]`
The BUGFIX scope denotes a change that fixes an issue with the project in question. A BUGFIX should align the behaviour of the service with the current expected behaviour of the service. If a BUGFIX introduces new unexpected behaviour to ameliorate the issue, a corresponding FEATURE or ENHANCEMENT scope should also be added to the changelog.
### `[CHANGE]`
The CHANGE scope denotes a change that changes the expected behavior of the project while not adding new functionality or fixing an underling issue. This commonly occurs when renaming things to make them more consistent or to accommodate updated versions of vendored dependencies.
### `[FEATURE]`
The FEATURE scope denotes a change that adds new functionality to the project/service.
### `[ENHANCEMENT]`
The ENHANCEMENT scope denotes a change that improves upon the current functionality of the project/service. Generally, an enhancement is something that improves upon something that is already present. Either by making it simpler, more powerful, or more performant. For Example:
An optimization on a particular process in a service that makes it more performant
Simpler syntax for setting a configuration value, like allowing 1m instead of 60 for a duration setting.
## Order
Scopes must have an order to ensure consistency and ease of search, this helps us identify which section do we need to look for what. The order must be:
1. `[CHANGE]`
2. `[FEATURE]`
3. `[BUGFIX]`
4. `[ENHANCEMENT]`
5. `[ADMIN]`
## Grafana Alerting - main / unreleased
- [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216
- [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670
- [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971
pkg/services/ngalert/state/manager_test.go
+146
View File
@@ -1397,6 +1397,152 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> normal when result is Error and ExecErrState is OK",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Data: []models.AlertQuery{{
RefID: "A",
DatasourceUID: "datasource_uid_1",
}},
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
ExecErrState: models.OkErrState,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
Error: expr.QueryError{
RefID: "A",
Err: errors.New("this is an error"),
},
State: eval.Error,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedAnnotations: 0,
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Error: nil,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Error,
Values: make(map[string]*float64),
},
},
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "alerting -> normal when result is Error and ExecErrState is OK",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Data: []models.AlertQuery{{
RefID: "A",
DatasourceUID: "datasource_uid_1",
}},
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
ExecErrState: models.OkErrState,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
Error: expr.QueryError{
RefID: "A",
Err: errors.New("this is an error"),
},
State: eval.Error,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedAnnotations: 2,
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Error: nil,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Alerting,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Error,
Values: make(map[string]*float64),
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> alerting -> error -> alerting - it should clear the error",
alertRule: &models.AlertRule{
pkg/services/ngalert/state/state.go
+10 -4
View File
@@ -2,6 +2,7 @@ package state
import (
"errors"
"fmt"
"strings"
"time"
@@ -51,8 +52,7 @@ func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]*float
}
func (a *State) resultNormal(_ *ngModels.AlertRule, result eval.Result) {
a.Error = result.Error // should be nil since state is not error
a.Error = nil // should be nil since state is not error
if a.State != eval.Normal {
a.EndsAt = result.EvaluatedAt
a.StartsAt = result.EvaluatedAt
@@ -88,9 +88,10 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
a.Error = result.Error
execErrState := eval.Error
if alertRule.ExecErrState == ngModels.AlertingErrState {
switch alertRule.ExecErrState {
case ngModels.AlertingErrState:
execErrState = eval.Alerting
} else if alertRule.ExecErrState == ngModels.ErrorErrState {
case ngModels.ErrorErrState:
// If the evaluation failed because a query returned an error then
// update the state with the Datasource UID as a label and the error
// message as an annotation so other code can use this metadata to
@@ -107,6 +108,11 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
a.Annotations["Error"] = queryError.Error()
}
execErrState = eval.Error
case ngModels.OkErrState:
a.resultNormal(alertRule, result)
return
default:
a.Error = fmt.Errorf("cannot map error to a state because option [%s] is not supported. evaluation error: %w", alertRule.ExecErrState, a.Error)
}
switch a.State {