public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add traceability headers for alert queries (#57127)

Browse Source 
* Define EvaluationContext

* Refactor ConditionEval to use new context struct

* Refactor QueriesAndExpressionsEval to use EvaluationContext

* Remove dead field from AlertExecCtx

* Refactor Validate to use EvaluationContext

* Get rid of privately used AlertExecCtx

* Move EvaluationContext to new file and add helper

* Add builder pattern and bind rule info to context

* Extract header logic and add rule UID header

* Fix missing call
This commit is contained in:
Alexander Weaver
2022-10-19 14:19:43 -05:00
committed by GitHub
parent 85cda0db69
commit 4eb8e4ff66
8 changed files with 153 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/eval/eval.go
+38 -40
View File
@@ -3,7 +3,6 @@
package eval
import (
"context"
"errors"
"fmt"
"runtime/debug"
@@ -17,7 +16,6 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -27,11 +25,11 @@ import (
//go:generate mockery --name Evaluator --structname FakeEvaluator --inpackage --filename evaluator_mock.go --with-expecter
type Evaluator interface {
// ConditionEval executes conditions and evaluates the result.
ConditionEval(ctx context.Context, user *user.SignedInUser, condition models.Condition, now time.Time) Results
ConditionEval(ctx EvaluationContext, condition models.Condition) Results
// QueriesAndExpressionsEval executes queries and expressions and returns the result.
QueriesAndExpressionsEval(ctx context.Context, user *user.SignedInUser, data []models.AlertQuery, now time.Time) (*backend.QueryDataResponse, error)
QueriesAndExpressionsEval(ctx EvaluationContext, data []models.AlertQuery) (*backend.QueryDataResponse, error)
// Validate validates that the condition is correct. Returns nil if the condition is correct. Otherwise, error that describes the failure
Validate(ctx context.Context, user *user.SignedInUser, condition models.Condition) error
Validate(ctx EvaluationContext, condition models.Condition) error
}
type evaluatorImpl struct {
@@ -159,24 +157,31 @@ func (s State) String() string {
return [...]string{"Normal", "Alerting", "Pending", "NoData", "Error"}[s]
}
// AlertExecCtx is the context provided for executing an alert condition.
type AlertExecCtx struct {
User *user.SignedInUser
ExpressionsEnabled bool
Log log.Logger
func buildDatasourceHeaders(ctx EvaluationContext) map[string]string {
headers := map[string]string{
// Many data sources check this in query method as sometimes alerting needs special considerations.
// Several existing systems also compare against the value of this header. Altering this constitutes a breaking change.
//
// Note: The spelling of this headers is intentionally degenerate from the others for compatibility reasons.
// When sent over a network, the key of this header is canonicalized to "Fromalert".
// However, some datasources still compare against the string "FromAlert".
"FromAlert": "true",
Ctx context.Context
"X-Cache-Skip": "true",
}
if ctx.RuleUID != "" {
headers["X-Rule-Uid"] = ctx.RuleUID
}
return headers
}
// getExprRequest validates the condition, gets the datasource information and creates an expr.Request from it.
func getExprRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time, dsCacheService datasources.CacheService) (*expr.Request, error) {
func getExprRequest(ctx EvaluationContext, data []models.AlertQuery, dsCacheService datasources.CacheService) (*expr.Request, error) {
req := &expr.Request{
OrgId: ctx.User.OrgID,
Headers: map[string]string{
// Some data sources check this in query method as sometimes alerting needs special considerations.
"FromAlert": "true",
"X-Cache-Skip": "true",
},
OrgId: ctx.User.OrgID,
Headers: buildDatasourceHeaders(ctx),
}
datasources := make(map[string]*datasources.DataSource, len(data))
@@ -211,8 +216,8 @@ func getExprRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time, d
req.Queries = append(req.Queries, expr.Query{
TimeRange: expr.TimeRange{
From: q.RelativeTimeRange.ToTimeRange(now).From,
To: q.RelativeTimeRange.ToTimeRange(now).To,
From: q.RelativeTimeRange.ToTimeRange(ctx.At).From,
To: q.RelativeTimeRange.ToTimeRange(ctx.At).To,
},
DataSource: ds,
JSON: model,
@@ -311,10 +316,10 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
return result
}
func executeQueriesAndExpressions(ctx AlertExecCtx, data []models.AlertQuery, now time.Time, exprService *expr.Service, dsCacheService datasources.CacheService) (resp *backend.QueryDataResponse, err error) {
func executeQueriesAndExpressions(ctx EvaluationContext, data []models.AlertQuery, exprService *expr.Service, dsCacheService datasources.CacheService, log log.Logger) (resp *backend.QueryDataResponse, err error) {
defer func() {
if e := recover(); e != nil {
ctx.Log.Error("alert rule panic", "error", e, "stack", string(debug.Stack()))
log.Error("alert rule panic", "error", e, "stack", string(debug.Stack()))
panicErr := fmt.Errorf("alert rule panic; please check the logs for the full stack")
if err != nil {
err = fmt.Errorf("queries and expressions execution failed: %w; %v", err, panicErr.Error())
@@ -324,7 +329,7 @@ func executeQueriesAndExpressions(ctx AlertExecCtx, data []models.AlertQuery, no
}
}()
queryDataReq, err := getExprRequest(ctx, data, now, dsCacheService)
queryDataReq, err := getExprRequest(ctx, data, dsCacheService)
if err != nil {
return nil, err
}
@@ -560,25 +565,23 @@ func (evalResults Results) AsDataFrame() data.Frame {
}
// ConditionEval executes conditions and evaluates the result.
func (e *evaluatorImpl) ConditionEval(ctx context.Context, user *user.SignedInUser, condition models.Condition, now time.Time) Results {
execResp, err := e.QueriesAndExpressionsEval(ctx, user, condition.Data, now)
func (e *evaluatorImpl) ConditionEval(ctx EvaluationContext, condition models.Condition) Results {
execResp, err := e.QueriesAndExpressionsEval(ctx, condition.Data)
var execResults ExecutionResults
if err != nil {
execResults = ExecutionResults{Error: err}
} else {
execResults = queryDataResponseToExecutionResults(condition, execResp)
}
return evaluateExecutionResult(execResults, now)
return evaluateExecutionResult(execResults, ctx.At)
}
// QueriesAndExpressionsEval executes queries and expressions and returns the result.
func (e *evaluatorImpl) QueriesAndExpressionsEval(ctx context.Context, user *user.SignedInUser, data []models.AlertQuery, now time.Time) (*backend.QueryDataResponse, error) {
alertCtx, cancelFn := context.WithTimeout(ctx, e.cfg.UnifiedAlerting.EvaluationTimeout)
func (e *evaluatorImpl) QueriesAndExpressionsEval(ctx EvaluationContext, data []models.AlertQuery) (*backend.QueryDataResponse, error) {
timeoutCtx, cancelFn := ctx.WithTimeout(e.cfg.UnifiedAlerting.EvaluationTimeout)
defer cancelFn()
alertExecCtx := AlertExecCtx{User: user, Ctx: alertCtx, ExpressionsEnabled: e.cfg.ExpressionsEnabled, Log: e.log}
execResult, err := executeQueriesAndExpressions(alertExecCtx, data, now, e.expressionService, e.dataSourceCache)
execResult, err := executeQueriesAndExpressions(timeoutCtx, data, e.expressionService, e.dataSourceCache, e.log)
if err != nil {
return nil, fmt.Errorf("failed to execute conditions: %w", err)
}
@@ -586,14 +589,7 @@ func (e *evaluatorImpl) QueriesAndExpressionsEval(ctx context.Context, user *use
return execResult, nil
}
func (e *evaluatorImpl) Validate(ctx context.Context, user *user.SignedInUser, condition models.Condition) error {
evalctx := AlertExecCtx{
User: user,
ExpressionsEnabled: e.cfg.ExpressionsEnabled,
Log: e.log,
Ctx: ctx,
}
func (e *evaluatorImpl) Validate(ctx EvaluationContext, condition models.Condition) error {
if len(condition.Data) == 0 {
return errors.New("expression list is empty. must be at least 1 expression")
}
@@ -601,7 +597,9 @@ func (e *evaluatorImpl) Validate(ctx context.Context, user *user.SignedInUser, c
return errors.New("condition must not be empty")
}
req, err := getExprRequest(evalctx, condition.Data, time.Now(), e.dataSourceCache)
ctx.At = time.Now()
req, err := getExprRequest(ctx, condition.Data, e.dataSourceCache)
if err != nil {
return err
}