Alerting: Create DatasourceError alert if evaluation returns error (#41869) (#42281)

* Alerting: Create DatasourceError alert if evaluation returns error

* Alerting: Add docs for DatasourceError alert

* Alerting: Fix DatasourceError alert does not have dashboard_uid label

* Alerting: Add break when datasource_uid found

* Alerting: Update TestProcessEvalResults

(cherry picked from commit 1b26d4d88e)

Co-authored-by: George Robinson <george.robinson@grafana.com>

pkg/services/sqlstore/migrations/ualert/alert_rule.go

@@ -131,6 +131,10 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string
 		m.mg.Logger.Error("alert migration error: failed to create silence", "rule_name", ar.Title, "err", err)
 	}
 
+	if err := m.addErrorSilence(da, ar); err != nil {
+		m.mg.Logger.Error("alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err)
+	}
+
 	if err := m.addNoDataSilence(da, ar); err != nil {
 		m.mg.Logger.Error("alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err)
 	}
@@ -215,7 +219,9 @@ func transExecErr(s string) (string, error) {
 	case "", "alerting":
 		return "Alerting", nil
 	case "keep_state":
-		return "Alerting", nil
+		// Keep last state is translated to error as we now emit a
+		// DatasourceError alert when the state is error
+		return "Error", nil
 	}
 	return "", fmt.Errorf("unrecognized Execution Error setting %v", s)
 }

pkg/services/sqlstore/migrations/ualert/silences.go

@@ -22,6 +22,8 @@ import (
 const (
 	// Should be the same as 'NoDataAlertName' in pkg/services/schedule/compat.go.
 	NoDataAlertName = "DatasourceNoData"
+
+	ErrorAlertName = "DatasourceError"
 )
 
 func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
@@ -61,6 +63,45 @@ func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
 	return nil
 }
 
+func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error {
+	if da.ParsedSettings.ExecutionErrorState != "keep_state" {
+		return nil
+	}
+
+	uid, err := uuid.NewV4()
+	if err != nil {
+		return errors.New("failed to create uuid for silence")
+	}
+
+	s := &pb.MeshSilence{
+		Silence: &pb.Silence{
+			Id: uid.String(),
+			Matchers: []*pb.Matcher{
+				{
+					Type:    pb.Matcher_EQUAL,
+					Name:    model.AlertNameLabel,
+					Pattern: ErrorAlertName,
+				},
+				{
+					Type:    pb.Matcher_EQUAL,
+					Name:    "rule_uid",
+					Pattern: rule.UID,
+				},
+			},
+			StartsAt:  time.Now(),
+			EndsAt:    time.Now().AddDate(1, 0, 0), // 1 year
+			CreatedBy: "Grafana Migration",
+			Comment:   fmt.Sprintf("Created during migration to unified alerting to silence Error state for alert rule ID '%s' and Title '%s' because the option 'Keep Last State' was selected for Error state", rule.UID, rule.Title),
+		},
+		ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year
+	}
+	if _, ok := m.silences[da.OrgId]; !ok {
+		m.silences[da.OrgId] = make([]*pb.MeshSilence, 0)
+	}
+	m.silences[da.OrgId] = append(m.silences[da.OrgId], s)
+	return nil
+}
+
 func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error {
 	if da.ParsedSettings.NoDataState != "keep_state" {
 		return nil