Alerting: Special alert instance if rule is in state NoData (#40540) (#41525)

* do not suppress NoData state
* extract conversion of state to postable alert + tests
* create a special alert instance if nodata
* use NoData when converting from Keep Last State instead of Alerting
* add silence during migration if NoData is mapped to KeepLastState.

(cherry picked from commit 610643a668)

Co-authored-by: Yuriy Tseretyan <yuriy.tseretyan@grafana.com>

pkg/services/sqlstore/migrations/ualert/alert_rule.go

@@ -130,6 +130,10 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string
 		m.mg.Logger.Error("alert migration error: failed to create silence", "rule_name", ar.Title, "err", err)
 	}
 
+	if err := m.addNoDataSilence(da, ar); err != nil {
+		m.mg.Logger.Error("alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err)
+	}
+
 	return ar, nil
 }
 
@@ -200,7 +204,7 @@ func transNoData(s string) (string, error) {
 	case "alerting":
 		return "Alerting", nil
 	case "keep_state":
-		return "Alerting", nil
+		return "NoData", nil // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert.
 	}
 	return "", fmt.Errorf("unrecognized No Data setting %v", s)
 }

pkg/services/sqlstore/migrations/ualert/silences.go

@@ -14,10 +14,16 @@ import (
 	"github.com/gofrs/uuid"
 	"github.com/matttproud/golang_protobuf_extensions/pbutil"
 	pb "github.com/prometheus/alertmanager/silence/silencepb"
+	"github.com/prometheus/common/model"
 
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
 )
 
+const (
+	// Should be the same as 'NoDataAlertName' in pkg/services/schedule/compat.go.
+	NoDataAlertName = "DatasourceNoData"
+)
+
 func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
 	if da.State != "paused" {
 		return nil
@@ -55,6 +61,46 @@ func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
 	return nil
 }
 
+func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error {
+	if da.ParsedSettings.NoDataState != "keep_state" {
+		return nil
+	}
+
+	uid, err := uuid.NewV4()
+	if err != nil {
+		return errors.New("failed to create uuid for silence")
+	}
+
+	s := &pb.MeshSilence{
+		Silence: &pb.Silence{
+			Id: uid.String(),
+			Matchers: []*pb.Matcher{
+				{
+					Type:    pb.Matcher_EQUAL,
+					Name:    model.AlertNameLabel,
+					Pattern: NoDataAlertName,
+				},
+				{
+					Type:    pb.Matcher_EQUAL,
+					Name:    "rule_uid",
+					Pattern: rule.UID,
+				},
+			},
+			StartsAt:  time.Now(),
+			EndsAt:    time.Now().AddDate(1, 0, 0), // 1 year.
+			CreatedBy: "Grafana Migration",
+			Comment:   fmt.Sprintf("Created during migration to unified alerting to silence NoData state for alert rule ID '%s' and Title '%s' because the option 'Keep Last State' was selected for NoData state", rule.UID, rule.Title),
+		},
+		ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year.
+	}
+	_, ok := m.silences[da.OrgId]
+	if !ok {
+		m.silences[da.OrgId] = make([]*pb.MeshSilence, 0)
+	}
+	m.silences[da.OrgId] = append(m.silences[da.OrgId], s)
+	return nil
+}
+
 func (m *migration) writeSilencesFile(orgID int64) error {
 	var buf bytes.Buffer
 	orgSilences, ok := m.silences[orgID]