Alerting: Persist alerts on evaluation and shutdown. Warm cache from DB on startup (#32576)

* Initial commit for state tracking

* basic state transition logic and tests

* constructor. test and interface fixup

* use new sig for sch.definitionRoutine()

* test fixup

* make the linter happy

* more minor linting cleanup

* Alerting: Send alerts from state tracker to notifier

* Add evaluation time and test

Add evaluation time and test

* Add cleanup routine and logging

* Pull in compact.go and reconcile differences

* Save alert transitions and save all state on shutdown

* pr feedback

* WIP

* WIP

* Persist alerts on evaluation and shutdown. Warm cache on startup

* Filter non-firing alerts before sending to notifier

Co-authored-by: Josue Abreu <josue@grafana.com>

pkg/services/ngalert/models/instance.go

@@ -13,6 +13,7 @@ type AlertInstance struct {
 	LabelsHash        string
 	CurrentState      InstanceStateType
 	CurrentStateSince time.Time
+	CurrentStateEnd   time.Time
 	LastEvalTime      time.Time
 }
 
@@ -35,11 +36,13 @@ func (i InstanceStateType) IsValid() bool {
 
 // SaveAlertInstanceCommand is the query for saving a new alert instance.
 type SaveAlertInstanceCommand struct {
-	DefinitionOrgID int64
-	DefinitionUID   string
-	Labels          InstanceLabels
-	State           InstanceStateType
-	LastEvalTime    time.Time
+	DefinitionOrgID   int64
+	DefinitionUID     string
+	Labels            InstanceLabels
+	State             InstanceStateType
+	LastEvalTime      time.Time
+	CurrentStateSince time.Time
+	CurrentStateEnd   time.Time
 }
 
 // GetAlertInstanceQuery is the query for retrieving/deleting an alert definition by ID.
@@ -61,6 +64,10 @@ type ListAlertInstancesQuery struct {
 	Result []*ListAlertInstancesQueryResult
 }
 
+type FetchUniqueOrgIdsQuery struct {
+	Result []*FetchUniqueOrgIdsQueryResult
+}
+
 // ListAlertInstancesQueryResult represents the result of listAlertInstancesQuery.
 type ListAlertInstancesQueryResult struct {
 	DefinitionOrgID   int64             `xorm:"def_org_id" json:"definitionOrgId"`
@@ -70,9 +77,14 @@ type ListAlertInstancesQueryResult struct {
 	LabelsHash        string            `json:"labeHash"`
 	CurrentState      InstanceStateType `json:"currentState"`
 	CurrentStateSince time.Time         `json:"currentStateSince"`
+	CurrentStateEnd   time.Time         `json:"currentStateEnd"`
 	LastEvalTime      time.Time         `json:"lastEvalTime"`
 }
 
+type FetchUniqueOrgIdsQueryResult struct {
+	DefinitionOrgID int64 `xorm:"def_org_id" json:"definitionOrgId"`
+}
+
 // ValidateAlertInstance validates that the alert instance contains an alert definition id,
 // and state.
 func ValidateAlertInstance(alertInstance *AlertInstance) error {