AlertingNG: manage and evaluate alert definitions via the API (#28377)

* Alerting NG: prototype v2 (WIP) * Separate eval package * Modify eval alert definition endpoint * Disable migration if ngalert is not enabled * Remove premature test * Fix lint issues * Delete obsolete struct * Apply suggestions from code review * Update pkg/services/ngalert/ngalert.go Co-authored-by: Kyle Brandt <kyle@grafana.com> * Add API endpoint for listing alert definitions * Introduce index for alert_definition table * make ds object for expression to avoid panic * wrap error * Update pkg/services/ngalert/eval/eval.go * Swith to backend.DataQuery * Export TransformWrapper callback * Fix lint issues * Update pkg/services/ngalert/ngalert.go Co-authored-by: Kyle Brandt <kyle@grafana.com> * Validate alert definitions before storing them * Introduce AlertQuery * Add test * Add QueryType in AlertQuery * Accept only float64 (seconds) durations * Apply suggestions from code review * Get rid of bus * Do not export symbols * Fix failing test * Fix failure due to service initialization order Introduce MediumHigh service priority and assign it to backendplugin service * Fix test * Apply suggestions from code review * Fix renamed reference Co-authored-by: Kyle Brandt <kyle@grafana.com>
2020-11-12 15:11:30 +02:00
parent a1e80af800
commit 43f580c299
21 changed files with 1558 additions and 411 deletions
@@ -0,0 +1,215 @@
+// Package eval executes the condition for an alert definition, evaluates the condition results, and
+// returns the alert instance states.
+package eval
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/tsdb"
+)
+
+// invalidEvalResultFormatError is an error for invalid format of the alert definition evaluation results.
+type invalidEvalResultFormatError struct {
+	refID  string
+	reason string
+	err    error
+}
+
+func (e *invalidEvalResultFormatError) Error() string {
+	s := fmt.Sprintf("invalid format of evaluation results for the alert definition %s: %s", e.refID, e.reason)
+	if e.err != nil {
+		s = fmt.Sprintf("%s: %s", s, e.err.Error())
+	}
+	return s
+}
+
+func (e *invalidEvalResultFormatError) Unwrap() error {
+	return e.err
+}
+
+// Condition contains backend expressions and queries and the RefID
+// of the query or expression that will be evaluated.
+type Condition struct {
+	RefID string `json:"refId"`
+
+	QueriesAndExpressions []AlertQuery `json:"queriesAndExpressions"`
+}
+
+// ExecutionResults contains the unevaluated results from executing
+// a condition.
+type ExecutionResults struct {
+	AlertDefinitionID int64
+
+	Error error
+
+	Results data.Frames
+}
+
+// Results is a slice of evaluated alert instances states.
+type Results []result
+
+// result contains the evaluated state of an alert instance
+// identified by its labels.
+type result struct {
+	Instance data.Labels
+	State    state // Enum
+}
+
+// state is an enum of the evaluation state for an alert instance.
+type state int
+
+const (
+	// Normal is the eval state for an alert instance condition
+	// that evaluated to false.
+	Normal state = iota
+
+	// Alerting is the eval state for an alert instance condition
+	// that evaluated to false.
+	Alerting
+)
+
+func (s state) String() string {
+	return [...]string{"Normal", "Alerting"}[s]
+}
+
+// IsValid checks the condition's validity.
+func (c Condition) IsValid() bool {
+	// TODO search for refIDs in QueriesAndExpressions
+	return len(c.QueriesAndExpressions) != 0
+}
+
+// AlertExecCtx is the context provided for executing an alert condition.
+type AlertExecCtx struct {
+	AlertDefitionID int64
+	SignedInUser    *models.SignedInUser
+
+	Ctx context.Context
+}
+
+// Execute runs the Condition's expressions or queries.
+func (c *Condition) Execute(ctx AlertExecCtx, fromStr, toStr string) (*ExecutionResults, error) {
+	result := ExecutionResults{}
+	if !c.IsValid() {
+		return nil, fmt.Errorf("invalid conditions")
+	}
+
+	pbQuery := &pluginv2.QueryDataRequest{
+		PluginContext: &pluginv2.PluginContext{
+			// TODO: Things probably
+		},
+		Queries: []*pluginv2.DataQuery{},
+	}
+
+	for i := range c.QueriesAndExpressions {
+		q := c.QueriesAndExpressions[i]
+		model, err := q.getModel()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get query model: %w", err)
+		}
+		intervalMS, err := q.getIntervalMS()
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve intervalMs from the model: %w", err)
+		}
+
+		maxDatapoints, err := q.getMaxDatapoints()
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve maxDatapoints from the model: %w", err)
+		}
+
+		pbQuery.Queries = append(pbQuery.Queries, &pluginv2.DataQuery{
+			Json:          model,
+			IntervalMS:    intervalMS,
+			RefId:         q.RefID,
+			MaxDataPoints: maxDatapoints,
+			QueryType:     q.QueryType,
+			TimeRange:     q.RelativeTimeRange.toTimeRange(time.Now()),
+		})
+	}
+
+	tw := plugins.Transform
+	pbRes, err := tw.TransformClient.TransformData(ctx.Ctx, pbQuery, tw.Callback)
+	if err != nil {
+		return &result, err
+	}
+
+	for refID, res := range pbRes.Responses {
+		if refID != c.RefID {
+			continue
+		}
+		df := tsdb.NewEncodedDataFrames(res.Frames)
+		result.Results, err = df.Decoded()
+		if err != nil {
+			result.Error = err
+			return &result, err
+		}
+	}
+
+	if len(result.Results) == 0 {
+		err = fmt.Errorf("no GEL results")
+		result.Error = err
+		return &result, err
+	}
+
+	return &result, nil
+}
+
+// EvaluateExecutionResult takes the ExecutionResult, and returns a frame where
+// each column is a string type that holds a string representing its state.
+func EvaluateExecutionResult(results *ExecutionResults) (Results, error) {
+	evalResults := make([]result, 0)
+	labels := make(map[string]bool)
+	for _, f := range results.Results {
+		rowLen, err := f.RowLen()
+		if err != nil {
+			return nil, &invalidEvalResultFormatError{refID: f.RefID, reason: "unable to get frame row length", err: err}
+		}
+		if rowLen > 1 {
+			return nil, &invalidEvalResultFormatError{refID: f.RefID, reason: fmt.Sprintf("unexpected row length: %d instead of 1", rowLen)}
+		}
+
+		if len(f.Fields) > 1 {
+			return nil, &invalidEvalResultFormatError{refID: f.RefID, reason: fmt.Sprintf("unexpected field length: %d instead of 1", len(f.Fields))}
+		}
+
+		if f.Fields[0].Type() != data.FieldTypeNullableFloat64 {
+			return nil, &invalidEvalResultFormatError{refID: f.RefID, reason: fmt.Sprintf("invalid field type: %d", f.Fields[0].Type())}
+		}
+
+		labelsStr := f.Fields[0].Labels.String()
+		_, ok := labels[labelsStr]
+		if ok {
+			return nil, &invalidEvalResultFormatError{refID: f.RefID, reason: fmt.Sprintf("frame cannot uniquely be identified by its labels: %s", labelsStr)}
+		}
+		labels[labelsStr] = true
+
+		state := Normal
+		val, err := f.Fields[0].FloatAt(0)
+		if err != nil || val != 0 {
+			state = Alerting
+		}
+
+		evalResults = append(evalResults, result{
+			Instance: f.Fields[0].Labels,
+			State:    state,
+		})
+	}
+	return evalResults, nil
+}
+
+// AsDataFrame forms the EvalResults in Frame suitable for displaying in the table panel of the front end.
+// This may be temporary, as there might be a fair amount we want to display in the frontend, and it might not make sense to store that in data.Frame.
+// For the first pass, I would expect a Frame with a single row, and a column for each instance with a boolean value.
+func (evalResults Results) AsDataFrame() data.Frame {
+	fields := make([]*data.Field, 0)
+	for _, evalResult := range evalResults {
+		fields = append(fields, data.NewField("", evalResult.Instance, []bool{evalResult.State != Normal}))
+	}
+	f := data.NewFrame("", fields...)
+	return *f
+}