From 9dac0c9eeb15e273bb363a0c136219033d7bf526 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Sat, 22 Feb 2025 12:36:58 +0100 Subject: [PATCH] Alerting: Add math node to the converted Prometheus rules (#101097) --- pkg/services/ngalert/prom/convert.go | 107 ++++++++++-------- pkg/services/ngalert/prom/convert_test.go | 122 +++++++++++++++++++++ pkg/services/ngalert/prom/query.go | 127 ++++++++++++++++++++++ 3 files changed, 312 insertions(+), 44 deletions(-) create mode 100644 pkg/services/ngalert/prom/query.go diff --git a/pkg/services/ngalert/prom/convert.go b/pkg/services/ngalert/prom/convert.go index 07495e0e2e1..bb9d4c3d616 100644 --- a/pkg/services/ngalert/prom/convert.go +++ b/pkg/services/ngalert/prom/convert.go @@ -1,7 +1,6 @@ package prom import ( - "encoding/json" "fmt" "time" @@ -20,6 +19,13 @@ const ( ruleUIDLabel = "__grafana_alert_rule_uid__" ) +const ( + queryRefID = "query" + prometheusMathRefID = "prometheus_math" + thresholdRefID = "threshold" +) + +// Config defines the configuration options for the Prometheus to Grafana rules converter. type Config struct { DatasourceUID string DatasourceType string @@ -31,6 +37,7 @@ type Config struct { AlertRules RulesConfig } +// RulesConfig contains configuration that applies to either recording or alerting rules. type RulesConfig struct { IsPaused bool } @@ -43,7 +50,7 @@ var ( FromTimeRange: &defaultTimeRange, EvaluationOffset: &defaultEvaluationOffset, ExecErrState: models.ErrorErrState, - NoDataState: models.NoData, + NoDataState: models.OK, } ) @@ -51,6 +58,9 @@ type Converter struct { cfg Config } +// NewConverter creates a new Converter instance with the provided configuration. +// It validates the configuration and returns an error if any required fields are missing +// or if the configuration is invalid. func NewConverter(cfg Config) (*Converter, error) { if cfg.DatasourceUID == "" { return nil, fmt.Errorf("datasource UID is required") @@ -166,15 +176,28 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr forInterval = time.Duration(*rule.For) } - queryNode, err := createAlertQueryNode(p.cfg.DatasourceUID, p.cfg.DatasourceType, rule.Expr, *p.cfg.FromTimeRange, *p.cfg.EvaluationOffset) + var query []models.AlertQuery + var title string + var isPaused bool + var record *models.Record + var err error + + isRecordingRule := rule.Record != "" + query, err = p.createQuery(rule.Expr, isRecordingRule) if err != nil { return models.AlertRule{}, err } - var title string - if rule.Record != "" { + if isRecordingRule { + record = &models.Record{ + From: queryRefID, + Metric: rule.Record, + } + + isPaused = p.cfg.RecordingRules.IsPaused title = rule.Record } else { + isPaused = p.cfg.AlertRules.IsPaused title = rule.Alert } @@ -192,14 +215,16 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr OrgID: orgID, NamespaceUID: namespaceUID, Title: title, - Data: []models.AlertQuery{queryNode}, - Condition: "A", + Data: query, + Condition: query[len(query)-1].RefID, NoDataState: p.cfg.NoDataState, ExecErrState: p.cfg.ExecErrState, Annotations: rule.Annotations, Labels: labels, For: forInterval, RuleGroup: group, + IsPaused: isPaused, + Record: record, Metadata: models.AlertRuleMetadata{ PrometheusStyleRule: &models.PrometheusStyleRule{ OriginalRuleDefinition: string(originalRuleDefinition), @@ -207,47 +232,41 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr }, } - if rule.Record != "" { - result.Record = &models.Record{ - From: "A", - Metric: rule.Record, - } - result.IsPaused = p.cfg.RecordingRules.IsPaused - } else { - result.IsPaused = p.cfg.AlertRules.IsPaused - } - return result, nil } -func createAlertQueryNode(datasourceUID, datasourceType, expr string, fromTimeRange, evaluationOffset time.Duration) (models.AlertQuery, error) { - modelData := map[string]interface{}{ - "datasource": map[string]interface{}{ - "type": datasourceType, - "uid": datasourceUID, - }, - "expr": expr, - "instant": true, - "range": false, - "refId": "A", - } - - if datasourceType == datasources.DS_LOKI { - modelData["queryType"] = "instant" - } - - modelJSON, err := json.Marshal(modelData) +// createQuery constructs the alert query nodes for a given Prometheus rule expression. +// It returns a slice of AlertQuery that represent the evaluation steps for the rule. +// +// For recording rules it generates a single query node that +// executes the PromQL query in the configured datasource. +// +// For alerting rules, it generates three query nodes: +// 1. Query Node (query): Executes the PromQL query using the configured datasource. +// 2. Math Node (prometheus_math): Applies a math expression "is_number($query) || is_nan($query) || is_inf($query)". +// 3. Threshold Node (threshold): Gets the result from the math node and checks that it's greater than 0. +// +// This is needed to ensure that we keep the Prometheus behaviour, where any returned result +// is considered alerting, and only when the query returns no data is the alert treated as normal. +func (p *Converter) createQuery(expr string, isRecordingRule bool) ([]models.AlertQuery, error) { + queryNode, err := createQueryNode(p.cfg.DatasourceUID, p.cfg.DatasourceType, expr, *p.cfg.FromTimeRange, *p.cfg.EvaluationOffset) if err != nil { - return models.AlertQuery{}, err + return nil, err } - return models.AlertQuery{ - DatasourceUID: datasourceUID, - Model: modelJSON, - RefID: "A", - RelativeTimeRange: models.RelativeTimeRange{ - From: models.Duration(fromTimeRange + evaluationOffset), - To: models.Duration(0 + evaluationOffset), - }, - }, nil + if isRecordingRule { + return []models.AlertQuery{queryNode}, nil + } + + mathNode, err := createMathNode() + if err != nil { + return nil, err + } + + thresholdNode, err := createThresholdNode() + if err != nil { + return nil, err + } + + return []models.AlertQuery{queryNode, mathNode, thresholdNode}, nil } diff --git a/pkg/services/ngalert/prom/convert_test.go b/pkg/services/ngalert/prom/convert_test.go index 1b85c0d9d92..3bad4795fcb 100644 --- a/pkg/services/ngalert/prom/convert_test.go +++ b/pkg/services/ngalert/prom/convert_test.go @@ -1,6 +1,7 @@ package prom import ( + "encoding/json" "fmt" "testing" "time" @@ -10,6 +11,8 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" + "github.com/grafana/grafana/pkg/expr" + "github.com/grafana/grafana/pkg/expr/mathexp" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/util" @@ -124,6 +127,9 @@ func TestPrometheusRulesToGrafana(t *testing.T) { if promRule.Record != "" { require.Equal(t, promRule.Record, grafanaRule.Title) + require.NotNil(t, grafanaRule.Record) + require.Equal(t, grafanaRule.Record.From, queryRefID) + require.Equal(t, promRule.Record, grafanaRule.Record.Metric) } else { require.Equal(t, promRule.Alert, grafanaRule.Title) } @@ -198,6 +204,122 @@ func TestPrometheusRulesToGrafanaWithDuplicateRuleNames(t *testing.T) { require.Equal(t, "alert (3)", group.Rules[3].Title) } +func TestCreateMathNode(t *testing.T) { + node, err := createMathNode() + require.NoError(t, err) + + require.Equal(t, expr.DatasourceUID, node.DatasourceUID) + require.Equal(t, string(expr.QueryTypeMath), node.QueryType) + require.Equal(t, "prometheus_math", node.RefID) + + var model map[string]interface{} + err = json.Unmarshal(node.Model, &model) + require.NoError(t, err) + + require.Equal(t, "prometheus_math", model["refId"]) + require.Equal(t, string(expr.QueryTypeMath), model["type"]) + require.Equal(t, "is_number($query) || is_nan($query) || is_inf($query)", model["expression"]) + + ds := model["datasource"].(map[string]interface{}) + require.Equal(t, expr.DatasourceUID, ds["name"]) + require.Equal(t, expr.DatasourceType, ds["type"]) + require.Equal(t, expr.DatasourceUID, ds["uid"]) +} + +func TestCreateThresholdNode(t *testing.T) { + node, err := createThresholdNode() + require.NoError(t, err) + + require.Equal(t, expr.DatasourceUID, node.DatasourceUID) + require.Equal(t, string(expr.QueryTypeThreshold), node.QueryType) + require.Equal(t, "threshold", node.RefID) + + var model map[string]interface{} + err = json.Unmarshal(node.Model, &model) + require.NoError(t, err) + + require.Equal(t, "threshold", model["refId"]) + require.Equal(t, string(expr.QueryTypeThreshold), model["type"]) + + ds := model["datasource"].(map[string]interface{}) + require.Equal(t, expr.DatasourceUID, ds["name"]) + require.Equal(t, expr.DatasourceType, ds["type"]) + require.Equal(t, expr.DatasourceUID, ds["uid"]) + + conditions := model["conditions"].([]interface{}) + require.Len(t, conditions, 1) + + condition := conditions[0].(map[string]interface{}) + evaluator := condition["evaluator"].(map[string]interface{}) + require.Equal(t, string(expr.ThresholdIsAbove), evaluator["type"]) + require.Equal(t, []interface{}{float64(0)}, evaluator["params"]) +} + +func TestPrometheusRulesToGrafana_NodesInRules(t *testing.T) { + cfg := Config{ + DatasourceUID: "datasource-uid", + DatasourceType: datasources.DS_PROMETHEUS, + } + converter, err := NewConverter(cfg) + require.NoError(t, err) + + t.Run("alert rule should have math and threshold nodes", func(t *testing.T) { + group := PrometheusRuleGroup{ + Name: "test", + Rules: []PrometheusRule{ + { + Alert: "alert1", + Expr: "up == 0", + }, + }, + } + + result, err := converter.PrometheusRulesToGrafana(1, "namespace", group) + require.NoError(t, err) + require.Len(t, result.Rules, 1) + require.Len(t, result.Rules[0].Data, 3) + + // First node should be query + require.Equal(t, "query", result.Rules[0].Data[0].RefID) + + // Second node should be math + require.Equal(t, "prometheus_math", result.Rules[0].Data[1].RefID) + require.Equal(t, string(expr.QueryTypeMath), result.Rules[0].Data[1].QueryType) + // Check that the math expression is valid + var model map[string]interface{} + err = json.Unmarshal(result.Rules[0].Data[1].Model, &model) + require.NoError(t, err) + require.Equal(t, "is_number($query) || is_nan($query) || is_inf($query)", model["expression"]) + // The math expression should be parsed successfully + _, err = mathexp.New(model["expression"].(string)) + require.NoError(t, err) + + // Third node should be threshold + require.Equal(t, "threshold", result.Rules[0].Data[2].RefID) + require.Equal(t, string(expr.QueryTypeThreshold), result.Rules[0].Data[2].QueryType) + }) + + t.Run("recording rule should only have query node", func(t *testing.T) { + group := PrometheusRuleGroup{ + Name: "test", + Rules: []PrometheusRule{ + { + Record: "metric", + Expr: "sum(rate(http_requests_total[5m]))", + }, + }, + } + + result, err := converter.PrometheusRulesToGrafana(1, "namespace", group) + require.NoError(t, err) + require.Len(t, result.Rules, 1) + require.Len(t, result.Rules[0].Data, 1) + + // Should only have query node + require.Equal(t, "query", result.Rules[0].Data[0].RefID) + }) +} + func TestPrometheusRulesToGrafana_UID(t *testing.T) { orgID := int64(1) namespace := "some-namespace" diff --git a/pkg/services/ngalert/prom/query.go b/pkg/services/ngalert/prom/query.go new file mode 100644 index 00000000000..74aed34f83f --- /dev/null +++ b/pkg/services/ngalert/prom/query.go @@ -0,0 +1,127 @@ +package prom + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/expr" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/ngalert/models" +) + +type CommonQueryModel struct { + Datasource datasources.DataSource `json:"datasource"` + RefID string `json:"refId"` + Type expr.QueryType `json:"type"` +} + +func createQueryNode(datasourceUID, datasourceType, expr string, fromTimeRange, evaluationOffset time.Duration) (models.AlertQuery, error) { + modelData := map[string]interface{}{ + "datasource": map[string]interface{}{ + "type": datasourceType, + "uid": datasourceUID, + }, + "expr": expr, + "instant": true, + "range": false, + "refId": queryRefID, + } + + if datasourceType == datasources.DS_LOKI { + modelData["queryType"] = "instant" + } + + modelJSON, err := json.Marshal(modelData) + if err != nil { + return models.AlertQuery{}, err + } + + return models.AlertQuery{ + DatasourceUID: datasourceUID, + Model: modelJSON, + RefID: queryRefID, + RelativeTimeRange: models.RelativeTimeRange{ + From: models.Duration(fromTimeRange + evaluationOffset), + To: models.Duration(0 + evaluationOffset), + }, + }, nil +} + +type MathQueryModel struct { + expr.MathQuery + CommonQueryModel +} + +func createMathNode() (models.AlertQuery, error) { + ds, err := expr.DataSourceModelFromNodeType(expr.TypeCMDNode) + if err != nil { + return models.AlertQuery{}, err + } + + model := MathQueryModel{ + CommonQueryModel: CommonQueryModel{ + Datasource: *ds, + RefID: prometheusMathRefID, + Type: expr.QueryTypeMath, + }, + MathQuery: expr.MathQuery{ + Expression: fmt.Sprintf("is_number($%[1]s) || is_nan($%[1]s) || is_inf($%[1]s)", queryRefID), + }, + } + + modelJSON, err := json.Marshal(model) + if err != nil { + return models.AlertQuery{}, err + } + + return models.AlertQuery{ + DatasourceUID: expr.DatasourceUID, + Model: modelJSON, + RefID: prometheusMathRefID, + QueryType: string(model.Type), + }, nil +} + +type ThresholdQueryModel struct { + expr.ThresholdQuery + CommonQueryModel +} + +func createThresholdNode() (models.AlertQuery, error) { + ds, err := expr.DataSourceModelFromNodeType(expr.TypeCMDNode) + if err != nil { + return models.AlertQuery{}, err + } + + model := ThresholdQueryModel{ + CommonQueryModel: CommonQueryModel{ + Datasource: *ds, + RefID: thresholdRefID, + Type: expr.QueryTypeThreshold, + }, + ThresholdQuery: expr.ThresholdQuery{ + Expression: prometheusMathRefID, + Conditions: []expr.ThresholdConditionJSON{ + { + Evaluator: expr.ConditionEvalJSON{ + Type: expr.ThresholdIsAbove, + Params: []float64{0}, + }, + }, + }, + }, + } + + modelJSON, err := json.Marshal(model) + if err != nil { + return models.AlertQuery{}, err + } + + return models.AlertQuery{ + DatasourceUID: expr.DatasourceUID, + Model: modelJSON, + RefID: thresholdRefID, + QueryType: string(model.Type), + }, nil +}