Alerting: Add math node to the converted Prometheus rules (#101097)

2025-02-22 12:36:58 +01:00
parent 5a6d9a99f3
commit 9dac0c9eeb
3 changed files with 312 additions and 44 deletions
@@ -1,7 +1,6 @@
 package prom

 import (
-	"encoding/json"
 	"fmt"
 	"time"

@@ -20,6 +19,13 @@ const (
 	ruleUIDLabel = "__grafana_alert_rule_uid__"
 )

+const (
+	queryRefID          = "query"
+	prometheusMathRefID = "prometheus_math"
+	thresholdRefID      = "threshold"
+)
+
+// Config defines the configuration options for the Prometheus to Grafana rules converter.
 type Config struct {
 	DatasourceUID    string
 	DatasourceType   string
@@ -31,6 +37,7 @@ type Config struct {
 	AlertRules       RulesConfig
 }

+// RulesConfig contains configuration that applies to either recording or alerting rules.
 type RulesConfig struct {
 	IsPaused bool
 }
@@ -43,7 +50,7 @@ var (
 		FromTimeRange:    &defaultTimeRange,
 		EvaluationOffset: &defaultEvaluationOffset,
 		ExecErrState:     models.ErrorErrState,
-		NoDataState:      models.NoData,
+		NoDataState:      models.OK,
 	}
 )

@@ -51,6 +58,9 @@ type Converter struct {
 	cfg Config
 }

+// NewConverter creates a new Converter instance with the provided configuration.
+// It validates the configuration and returns an error if any required fields are missing
+// or if the configuration is invalid.
 func NewConverter(cfg Config) (*Converter, error) {
 	if cfg.DatasourceUID == "" {
 		return nil, fmt.Errorf("datasource UID is required")
@@ -166,15 +176,28 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr
 		forInterval = time.Duration(*rule.For)
 	}

-	queryNode, err := createAlertQueryNode(p.cfg.DatasourceUID, p.cfg.DatasourceType, rule.Expr, *p.cfg.FromTimeRange, *p.cfg.EvaluationOffset)
+	var query []models.AlertQuery
+	var title string
+	var isPaused bool
+	var record *models.Record
+	var err error
+
+	isRecordingRule := rule.Record != ""
+	query, err = p.createQuery(rule.Expr, isRecordingRule)
 	if err != nil {
 		return models.AlertRule{}, err
 	}

-	var title string
-	if rule.Record != "" {
+	if isRecordingRule {
+		record = &models.Record{
+			From:   queryRefID,
+			Metric: rule.Record,
+		}
+
+		isPaused = p.cfg.RecordingRules.IsPaused
 		title = rule.Record
 	} else {
+		isPaused = p.cfg.AlertRules.IsPaused
 		title = rule.Alert
 	}

@@ -192,14 +215,16 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr
 		OrgID:        orgID,
 		NamespaceUID: namespaceUID,
 		Title:        title,
-		Data:         []models.AlertQuery{queryNode},
-		Condition:    "A",
+		Data:         query,
+		Condition:    query[len(query)-1].RefID,
 		NoDataState:  p.cfg.NoDataState,
 		ExecErrState: p.cfg.ExecErrState,
 		Annotations:  rule.Annotations,
 		Labels:       labels,
 		For:          forInterval,
 		RuleGroup:    group,
+		IsPaused:     isPaused,
+		Record:       record,
 		Metadata: models.AlertRuleMetadata{
 			PrometheusStyleRule: &models.PrometheusStyleRule{
 				OriginalRuleDefinition: string(originalRuleDefinition),
@@ -207,47 +232,41 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr
 		},
 	}

-	if rule.Record != "" {
-		result.Record = &models.Record{
-			From:   "A",
-			Metric: rule.Record,
-		}
-		result.IsPaused = p.cfg.RecordingRules.IsPaused
-	} else {
-		result.IsPaused = p.cfg.AlertRules.IsPaused
-	}
-
 	return result, nil
 }

-func createAlertQueryNode(datasourceUID, datasourceType, expr string, fromTimeRange, evaluationOffset time.Duration) (models.AlertQuery, error) {
-	modelData := map[string]interface{}{
-		"datasource": map[string]interface{}{
-			"type": datasourceType,
-			"uid":  datasourceUID,
-		},
-		"expr":    expr,
-		"instant": true,
-		"range":   false,
-		"refId":   "A",
-	}
-
-	if datasourceType == datasources.DS_LOKI {
-		modelData["queryType"] = "instant"
-	}
-
-	modelJSON, err := json.Marshal(modelData)
+// createQuery constructs the alert query nodes for a given Prometheus rule expression.
+// It returns a slice of AlertQuery that represent the evaluation steps for the rule.
+//
+// For recording rules it generates a single query node that
+// executes the PromQL query in the configured datasource.
+//
+// For alerting rules, it generates three query nodes:
+//  1. Query Node (query): Executes the PromQL query using the configured datasource.
+//  2. Math Node (prometheus_math): Applies a math expression "is_number($query) || is_nan($query) || is_inf($query)".
+//  3. Threshold Node (threshold): Gets the result from the math node and checks that it's greater than 0.
+//
+// This is needed to ensure that we keep the Prometheus behaviour, where any returned result
+// is considered alerting, and only when the query returns no data is the alert treated as normal.
+func (p *Converter) createQuery(expr string, isRecordingRule bool) ([]models.AlertQuery, error) {
+	queryNode, err := createQueryNode(p.cfg.DatasourceUID, p.cfg.DatasourceType, expr, *p.cfg.FromTimeRange, *p.cfg.EvaluationOffset)
 	if err != nil {
-		return models.AlertQuery{}, err
+		return nil, err
 	}

-	return models.AlertQuery{
-		DatasourceUID: datasourceUID,
-		Model:         modelJSON,
-		RefID:         "A",
-		RelativeTimeRange: models.RelativeTimeRange{
-			From: models.Duration(fromTimeRange + evaluationOffset),
-			To:   models.Duration(0 + evaluationOffset),
-		},
-	}, nil
+	if isRecordingRule {
+		return []models.AlertQuery{queryNode}, nil
+	}
+
+	mathNode, err := createMathNode()
+	if err != nil {
+		return nil, err
+	}
+
+	thresholdNode, err := createThresholdNode()
+	if err != nil {
+		return nil, err
+	}
+
+	return []models.AlertQuery{queryNode, mathNode, thresholdNode}, nil
 }
@@ -1,6 +1,7 @@
 package prom

 import (
+	"encoding/json"
 	"fmt"
 	"testing"
 	"time"
@@ -10,6 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"

+	"github.com/grafana/grafana/pkg/expr"
+	"github.com/grafana/grafana/pkg/expr/mathexp"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/util"
@@ -124,6 +127,9 @@ func TestPrometheusRulesToGrafana(t *testing.T) {

 				if promRule.Record != "" {
 					require.Equal(t, promRule.Record, grafanaRule.Title)
+					require.NotNil(t, grafanaRule.Record)
+					require.Equal(t, grafanaRule.Record.From, queryRefID)
+					require.Equal(t, promRule.Record, grafanaRule.Record.Metric)
 				} else {
 					require.Equal(t, promRule.Alert, grafanaRule.Title)
 				}
@@ -198,6 +204,122 @@ func TestPrometheusRulesToGrafanaWithDuplicateRuleNames(t *testing.T) {
 	require.Equal(t, "alert (3)", group.Rules[3].Title)
 }

+func TestCreateMathNode(t *testing.T) {
+	node, err := createMathNode()
+	require.NoError(t, err)
+
+	require.Equal(t, expr.DatasourceUID, node.DatasourceUID)
+	require.Equal(t, string(expr.QueryTypeMath), node.QueryType)
+	require.Equal(t, "prometheus_math", node.RefID)
+
+	var model map[string]interface{}
+	err = json.Unmarshal(node.Model, &model)
+	require.NoError(t, err)
+
+	require.Equal(t, "prometheus_math", model["refId"])
+	require.Equal(t, string(expr.QueryTypeMath), model["type"])
+	require.Equal(t, "is_number($query) || is_nan($query) || is_inf($query)", model["expression"])
+
+	ds := model["datasource"].(map[string]interface{})
+	require.Equal(t, expr.DatasourceUID, ds["name"])
+	require.Equal(t, expr.DatasourceType, ds["type"])
+	require.Equal(t, expr.DatasourceUID, ds["uid"])
+}
+
+func TestCreateThresholdNode(t *testing.T) {
+	node, err := createThresholdNode()
+	require.NoError(t, err)
+
+	require.Equal(t, expr.DatasourceUID, node.DatasourceUID)
+	require.Equal(t, string(expr.QueryTypeThreshold), node.QueryType)
+	require.Equal(t, "threshold", node.RefID)
+
+	var model map[string]interface{}
+	err = json.Unmarshal(node.Model, &model)
+	require.NoError(t, err)
+
+	require.Equal(t, "threshold", model["refId"])
+	require.Equal(t, string(expr.QueryTypeThreshold), model["type"])
+
+	ds := model["datasource"].(map[string]interface{})
+	require.Equal(t, expr.DatasourceUID, ds["name"])
+	require.Equal(t, expr.DatasourceType, ds["type"])
+	require.Equal(t, expr.DatasourceUID, ds["uid"])
+
+	conditions := model["conditions"].([]interface{})
+	require.Len(t, conditions, 1)
+
+	condition := conditions[0].(map[string]interface{})
+	evaluator := condition["evaluator"].(map[string]interface{})
+	require.Equal(t, string(expr.ThresholdIsAbove), evaluator["type"])
+	require.Equal(t, []interface{}{float64(0)}, evaluator["params"])
+}
+
+func TestPrometheusRulesToGrafana_NodesInRules(t *testing.T) {
+	cfg := Config{
+		DatasourceUID:  "datasource-uid",
+		DatasourceType: datasources.DS_PROMETHEUS,
+	}
+	converter, err := NewConverter(cfg)
+	require.NoError(t, err)
+
+	t.Run("alert rule should have math and threshold nodes", func(t *testing.T) {
+		group := PrometheusRuleGroup{
+			Name: "test",
+			Rules: []PrometheusRule{
+				{
+					Alert: "alert1",
+					Expr:  "up == 0",
+				},
+			},
+		}
+
+		result, err := converter.PrometheusRulesToGrafana(1, "namespace", group)
+		require.NoError(t, err)
+		require.Len(t, result.Rules, 1)
+		require.Len(t, result.Rules[0].Data, 3)
+
+		// First node should be query
+		require.Equal(t, "query", result.Rules[0].Data[0].RefID)
+
+		// Second node should be math
+		require.Equal(t, "prometheus_math", result.Rules[0].Data[1].RefID)
+		require.Equal(t, string(expr.QueryTypeMath), result.Rules[0].Data[1].QueryType)
+		// Check that the math expression is valid
+		var model map[string]interface{}
+		err = json.Unmarshal(result.Rules[0].Data[1].Model, &model)
+		require.NoError(t, err)
+		require.Equal(t, "is_number($query) || is_nan($query) || is_inf($query)", model["expression"])
+		// The math expression should be parsed successfully
+		_, err = mathexp.New(model["expression"].(string))
+		require.NoError(t, err)
+
+		// Third node should be threshold
+		require.Equal(t, "threshold", result.Rules[0].Data[2].RefID)
+		require.Equal(t, string(expr.QueryTypeThreshold), result.Rules[0].Data[2].QueryType)
+	})
+
+	t.Run("recording rule should only have query node", func(t *testing.T) {
+		group := PrometheusRuleGroup{
+			Name: "test",
+			Rules: []PrometheusRule{
+				{
+					Record: "metric",
+					Expr:   "sum(rate(http_requests_total[5m]))",
+				},
+			},
+		}
+
+		result, err := converter.PrometheusRulesToGrafana(1, "namespace", group)
+		require.NoError(t, err)
+		require.Len(t, result.Rules, 1)
+		require.Len(t, result.Rules[0].Data, 1)
+
+		// Should only have query node
+		require.Equal(t, "query", result.Rules[0].Data[0].RefID)
+	})
+}
+
 func TestPrometheusRulesToGrafana_UID(t *testing.T) {
 	orgID := int64(1)
 	namespace := "some-namespace"
@@ -0,0 +1,127 @@
+package prom
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/expr"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
+)
+
+type CommonQueryModel struct {
+	Datasource datasources.DataSource `json:"datasource"`
+	RefID      string                 `json:"refId"`
+	Type       expr.QueryType         `json:"type"`
+}
+
+func createQueryNode(datasourceUID, datasourceType, expr string, fromTimeRange, evaluationOffset time.Duration) (models.AlertQuery, error) {
+	modelData := map[string]interface{}{
+		"datasource": map[string]interface{}{
+			"type": datasourceType,
+			"uid":  datasourceUID,
+		},
+		"expr":    expr,
+		"instant": true,
+		"range":   false,
+		"refId":   queryRefID,
+	}
+
+	if datasourceType == datasources.DS_LOKI {
+		modelData["queryType"] = "instant"
+	}
+
+	modelJSON, err := json.Marshal(modelData)
+	if err != nil {
+		return models.AlertQuery{}, err
+	}
+
+	return models.AlertQuery{
+		DatasourceUID: datasourceUID,
+		Model:         modelJSON,
+		RefID:         queryRefID,
+		RelativeTimeRange: models.RelativeTimeRange{
+			From: models.Duration(fromTimeRange + evaluationOffset),
+			To:   models.Duration(0 + evaluationOffset),
+		},
+	}, nil
+}
+
+type MathQueryModel struct {
+	expr.MathQuery
+	CommonQueryModel
+}
+
+func createMathNode() (models.AlertQuery, error) {
+	ds, err := expr.DataSourceModelFromNodeType(expr.TypeCMDNode)
+	if err != nil {
+		return models.AlertQuery{}, err
+	}
+
+	model := MathQueryModel{
+		CommonQueryModel: CommonQueryModel{
+			Datasource: *ds,
+			RefID:      prometheusMathRefID,
+			Type:       expr.QueryTypeMath,
+		},
+		MathQuery: expr.MathQuery{
+			Expression: fmt.Sprintf("is_number($%[1]s) || is_nan($%[1]s) || is_inf($%[1]s)", queryRefID),
+		},
+	}
+
+	modelJSON, err := json.Marshal(model)
+	if err != nil {
+		return models.AlertQuery{}, err
+	}
+
+	return models.AlertQuery{
+		DatasourceUID: expr.DatasourceUID,
+		Model:         modelJSON,
+		RefID:         prometheusMathRefID,
+		QueryType:     string(model.Type),
+	}, nil
+}
+
+type ThresholdQueryModel struct {
+	expr.ThresholdQuery
+	CommonQueryModel
+}
+
+func createThresholdNode() (models.AlertQuery, error) {
+	ds, err := expr.DataSourceModelFromNodeType(expr.TypeCMDNode)
+	if err != nil {
+		return models.AlertQuery{}, err
+	}
+
+	model := ThresholdQueryModel{
+		CommonQueryModel: CommonQueryModel{
+			Datasource: *ds,
+			RefID:      thresholdRefID,
+			Type:       expr.QueryTypeThreshold,
+		},
+		ThresholdQuery: expr.ThresholdQuery{
+			Expression: prometheusMathRefID,
+			Conditions: []expr.ThresholdConditionJSON{
+				{
+					Evaluator: expr.ConditionEvalJSON{
+						Type:   expr.ThresholdIsAbove,
+						Params: []float64{0},
+					},
+				},
+			},
+		},
+	}
+
+	modelJSON, err := json.Marshal(model)
+	if err != nil {
+		return models.AlertQuery{}, err
+	}
+
+	return models.AlertQuery{
+		DatasourceUID: expr.DatasourceUID,
+		Model:         modelJSON,
+		RefID:         thresholdRefID,
+		QueryType:     string(model.Type),
+	}, nil
+}