SSE: Use errutil to show better error messages in prod (#71658)

- include public message
- propagate data source query errors so they are shown as well to which fixes #70026

pkg/services/ngalert/state/manager_test.go

@@ -1722,11 +1722,8 @@ func TestProcessEvalResults(t *testing.T) {
 				},
 				{
 					eval.Result{
-						Instance: data.Labels{"instance_label": "test"},
-						Error: expr.QueryError{
-							RefID: "A",
-							Err:   errors.New("this is an error"),
-						},
+						Instance:           data.Labels{"instance_label": "test"},
+						Error:              expr.MakeQueryError("A", "", errors.New("this is an error")),
 						State:              eval.Error,
 						EvaluatedAt:        evaluationTime.Add(10 * time.Second),
 						EvaluationDuration: evaluationDuration,
@@ -1750,10 +1747,7 @@ func TestProcessEvalResults(t *testing.T) {
 					},
 					Values: make(map[string]float64),
 					State:  eval.Error,
-					Error: expr.QueryError{
-						RefID: "A",
-						Err:   errors.New("this is an error"),
-					},
+					Error:  expr.MakeQueryError("A", "", errors.New("this is an error")),
 					Results: []state.Evaluation{
 						{
 							EvaluationTime:  evaluationTime,
@@ -1770,7 +1764,7 @@ func TestProcessEvalResults(t *testing.T) {
 					EndsAt:             evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3),
 					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
 					EvaluationDuration: evaluationDuration,
-					Annotations:        map[string]string{"annotation": "test", "Error": "failed to execute query A: this is an error"},
+					Annotations:        map[string]string{"annotation": "test", "Error": "[sse.dataQueryError] failed to execute query [A]: this is an error"},
 				},
 			},
 		},
@@ -1802,11 +1796,8 @@ func TestProcessEvalResults(t *testing.T) {
 				},
 				{
 					eval.Result{
-						Instance: data.Labels{"instance_label": "test"},
-						Error: expr.QueryError{
-							RefID: "A",
-							Err:   errors.New("this is an error"),
-						},
+						Instance:           data.Labels{"instance_label": "test"},
+						Error:              expr.MakeQueryError("A", "", errors.New("this is an error")),
 						State:              eval.Error,
 						EvaluatedAt:        evaluationTime.Add(10 * time.Second),
 						EvaluationDuration: evaluationDuration,
@@ -1878,11 +1869,8 @@ func TestProcessEvalResults(t *testing.T) {
 				},
 				{
 					eval.Result{
-						Instance: data.Labels{"instance_label": "test"},
-						Error: expr.QueryError{
-							RefID: "A",
-							Err:   errors.New("this is an error"),
-						},
+						Instance:           data.Labels{"instance_label": "test"},
+						Error:              expr.MakeQueryError("A", "", errors.New("this is an error")),
 						State:              eval.Error,
 						EvaluatedAt:        evaluationTime.Add(10 * time.Second),
 						EvaluationDuration: evaluationDuration,

pkg/services/ngalert/state/state.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/screenshot"
+	"github.com/grafana/grafana/pkg/util/errutil"
 )
 
 type State struct {
@@ -297,10 +298,11 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge
 				state.Annotations["Error"] = result.Error.Error()
 				// If the evaluation failed because a query returned an error then add the Ref ID and
 				// Datasource UID as labels
-				var queryError expr.QueryError
-				if errors.As(state.Error, &queryError) {
+				var utilError errutil.Error
+				if errors.As(state.Error, &utilError) &&
+					(errors.Is(state.Error, expr.QueryError) || errors.Is(state.Error, expr.ConversionError)) {
 					for _, next := range rule.Data {
-						if next.RefID == queryError.RefID {
+						if next.RefID == utilError.PublicPayload["refId"].(string) {
 							state.Labels["ref_id"] = next.RefID
 							state.Labels["datasource_uid"] = next.DatasourceUID
 							break