Alerting: Fix database unavailable removes rules from scheduler (#49874)

pkg/services/ngalert/schedule/fetcher.go

@@ -2,24 +2,55 @@ package schedule
 
 import (
 	"context"
+	"fmt"
+	"hash/fnv"
+	"sort"
 	"time"
 
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 )
 
-func (sch *schedule) getAlertRules(ctx context.Context, disabledOrgs []int64) []*models.SchedulableAlertRule {
+// hashUIDs returns a fnv64 hash of the UIDs for all alert rules.
+// The order of the alert rules does not matter as hashUIDs sorts
+// the UIDs in increasing order.
+func hashUIDs(alertRules []*models.SchedulableAlertRule) uint64 {
+	h := fnv.New64()
+	for _, uid := range sortedUIDs(alertRules) {
+		// We can ignore err as fnv64 does not return an error
+		// nolint:errcheck,gosec
+		h.Write([]byte(uid))
+	}
+	return h.Sum64()
+}
+
+// sortedUIDs returns a slice of sorted UIDs.
+func sortedUIDs(alertRules []*models.SchedulableAlertRule) []string {
+	uids := make([]string, 0, len(alertRules))
+	for _, alertRule := range alertRules {
+		uids = append(uids, alertRule.UID)
+	}
+	sort.Strings(uids)
+	return uids
+}
+
+// updateSchedulableAlertRules updates the alert rules for the scheduler.
+// It returns an error if the database is unavailable or the query returned
+// an error.
+func (sch *schedule) updateSchedulableAlertRules(ctx context.Context, disabledOrgs []int64) error {
 	start := time.Now()
 	defer func() {
-		sch.metrics.GetAlertRulesDuration.Observe(time.Since(start).Seconds())
+		sch.metrics.UpdateSchedulableAlertRulesDuration.Observe(
+			time.Since(start).Seconds())
 	}()
 
 	q := models.GetAlertRulesForSchedulingQuery{
 		ExcludeOrgIDs: disabledOrgs,
 	}
-	err := sch.ruleStore.GetAlertRulesForScheduling(ctx, &q)
-	if err != nil {
-		sch.log.Error("failed to fetch alert definitions", "err", err)
-		return nil
+	if err := sch.ruleStore.GetAlertRulesForScheduling(ctx, &q); err != nil {
+		return fmt.Errorf("failed to get alert rules: %w", err)
 	}
-	return q.Result
+	sch.schedulableAlertRules.set(q.Result)
+	sch.metrics.SchedulableAlertRules.Set(float64(len(q.Result)))
+	sch.metrics.SchedulableAlertRulesHash.Set(float64(hashUIDs(q.Result)))
+	return nil
 }