Alerting: Update scheduler to get updates only from database (#64635)

* stop using the scheduler's Update and Delete methods all communication must be via the database
* update scheduler's registry to calculate diff before re-setting the cache
* update fetcher to return the diff generated by registry
* update processTick to update rule eval routine if the rule was updated and it is not going to be evaluated at this tick.
* remove references to the scheduler from api package
* remove unused methods in the scheduler

pkg/services/ngalert/schedule/fetcher.go

@@ -34,9 +34,9 @@ func sortedUIDs(alertRules []*models.AlertRule) []string {
 }
 
 // updateSchedulableAlertRules updates the alert rules for the scheduler.
-// It returns an error if the database is unavailable or the query returned
-// an error.
-func (sch *schedule) updateSchedulableAlertRules(ctx context.Context) error {
+// It returns diff that contains rule keys that were updated since the last poll,
+// and an error if the database query encountered problems.
+func (sch *schedule) updateSchedulableAlertRules(ctx context.Context) (diff, error) {
 	start := time.Now()
 	defer func() {
 		sch.metrics.UpdateSchedulableAlertRulesDuration.Observe(
@@ -46,21 +46,21 @@ func (sch *schedule) updateSchedulableAlertRules(ctx context.Context) error {
 	if !sch.schedulableAlertRules.isEmpty() {
 		keys, err := sch.ruleStore.GetAlertRulesKeysForScheduling(ctx)
 		if err != nil {
-			return err
+			return diff{}, err
 		}
 		if !sch.schedulableAlertRules.needsUpdate(keys) {
 			sch.log.Debug("No changes detected. Skip updating")
-			return nil
+			return diff{}, nil
 		}
 	}
-
+	// At this point, we know we need to re-fetch rules as there are changes.
 	q := models.GetAlertRulesForSchedulingQuery{
 		PopulateFolders: !sch.disableGrafanaFolder,
 	}
 	if err := sch.ruleStore.GetAlertRulesForScheduling(ctx, &q); err != nil {
-		return fmt.Errorf("failed to get alert rules: %w", err)
+		return diff{}, fmt.Errorf("failed to get alert rules: %w", err)
 	}
-	sch.log.Debug("Alert rules fetched", "rulesCount", len(q.ResultRules), "foldersCount", len(q.ResultFoldersTitles))
-	sch.schedulableAlertRules.set(q.ResultRules, q.ResultFoldersTitles)
-	return nil
+	d := sch.schedulableAlertRules.set(q.ResultRules, q.ResultFoldersTitles)
+	sch.log.Debug("Alert rules fetched", "rulesCount", len(q.ResultRules), "foldersCount", len(q.ResultFoldersTitles), "updatedRules", len(d.updated))
+	return d, nil
 }