Alerting: Persist alerts on evaluation and shutdown. Warm cache from DB on startup (#32576)

* Initial commit for state tracking

* basic state transition logic and tests

* constructor. test and interface fixup

* use new sig for sch.definitionRoutine()

* test fixup

* make the linter happy

* more minor linting cleanup

* Alerting: Send alerts from state tracker to notifier

* Add evaluation time and test

Add evaluation time and test

* Add cleanup routine and logging

* Pull in compact.go and reconcile differences

* Save alert transitions and save all state on shutdown

* pr feedback

* WIP

* WIP

* Persist alerts on evaluation and shutdown. Warm cache on startup

* Filter non-firing alerts before sending to notifier

Co-authored-by: Josue Abreu <josue@grafana.com>

pkg/services/ngalert/store/instance_database.go

@@ -91,7 +91,8 @@ func (st DBstore) SaveAlertInstance(cmd *models.SaveAlertInstanceCommand) error
 			Labels:            cmd.Labels,
 			LabelsHash:        labelsHash,
 			CurrentState:      cmd.State,
-			CurrentStateSince: TimeNow(),
+			CurrentStateSince: cmd.CurrentStateSince,
+			CurrentStateEnd:   cmd.CurrentStateEnd,
 			LastEvalTime:      cmd.LastEvalTime,
 		}
 
@@ -99,12 +100,12 @@ func (st DBstore) SaveAlertInstance(cmd *models.SaveAlertInstanceCommand) error
 			return err
 		}
 
-		params := append(make([]interface{}, 0), alertInstance.DefinitionOrgID, alertInstance.DefinitionUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentStateSince.Unix(), alertInstance.LastEvalTime.Unix())
+		params := append(make([]interface{}, 0), alertInstance.DefinitionOrgID, alertInstance.DefinitionUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix())
 
 		upsertSQL := st.SQLStore.Dialect.UpsertSQL(
 			"alert_instance",
 			[]string{"def_org_id", "def_uid", "labels_hash"},
-			[]string{"def_org_id", "def_uid", "labels", "labels_hash", "current_state", "current_state_since", "last_eval_time"})
+			[]string{"def_org_id", "def_uid", "labels", "labels_hash", "current_state", "current_state_since", "current_state_end", "last_eval_time"})
 		_, err = sess.SQL(upsertSQL, params...).Query()
 		if err != nil {
 			return err
@@ -113,3 +114,26 @@ func (st DBstore) SaveAlertInstance(cmd *models.SaveAlertInstanceCommand) error
 		return nil
 	})
 }
+
+func (st DBstore) FetchOrgIds(cmd *models.FetchUniqueOrgIdsQuery) error {
+	return st.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
+		orgIds := make([]*models.FetchUniqueOrgIdsQueryResult, 0)
+
+		s := strings.Builder{}
+		params := make([]interface{}, 0)
+
+		addToQuery := func(stmt string, p ...interface{}) {
+			s.WriteString(stmt)
+			params = append(params, p...)
+		}
+
+		addToQuery("SELECT DISTINCT def_org_id FROM alert_instance")
+
+		if err := sess.SQL(s.String(), params...).Find(&orgIds); err != nil {
+			return err
+		}
+
+		cmd.Result = orgIds
+		return nil
+	})
+}