Alerting: Support hysteresis command expression (#75189)

Backend: 

* Update the Grafana Alerting engine to provide feedback to HysteresisCommand. The feedback information is stored in state.Manager as a fingerprint of each state. The fingerprint is persisted to the database. Only fingerprints that belong to Pending and Alerting states are considered as "loaded" and provided back to the command.
   - add ResultFingerprint to state.State. It's different from other fingerprints we store in the state because it is calculated from the result labels.
  -  add rule_fingerprint column to alert_instance
   - update alerting evaluator to accept AlertingResultsReader via context, and update scheduler to provide it.
   - add AlertingResultsFromRuleState that implements the new interface in eval package
   - update getExprRequest to patch the hysteresis command.

* Only one "Recovery Threshold" query is allowed to be used in the alert rule and it must be the Condition.


Frontend:

* Add hysteresis option to Threshold in UI. It's called "Recovery Threshold"
* Add test for getUnloadEvaluatorTypeFromCondition
* Hide hysteresis in panel expressions

* Refactor isInvalid and add test for it
* Remove unnecesary React.memo
* Add tests for updateEvaluatorConditions

---------

Co-authored-by: Sonia Aguilar <soniaaguilarpeiron@gmail.com>

pkg/services/ngalert/backtesting/engine.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/auth/identity"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 )
 
@@ -35,6 +36,7 @@ type backtestingEvaluator interface {
 
 type stateManager interface {
 	ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *models.AlertRule, results eval.Results, extraLabels data.Labels) []state.StateTransition
+	schedule.RuleStateProvider
 }
 
 type Engine struct {
@@ -74,13 +76,16 @@ func (e *Engine) Test(ctx context.Context, user identity.Requester, rule *models
 	}
 	length := int(to.Sub(from).Seconds()) / int(rule.IntervalSeconds)
 
-	evaluator, err := backtestingEvaluatorFactory(ruleCtx, e.evalFactory, user, rule.GetEvalCondition())
+	stateManager := e.createStateManager()
+
+	evaluator, err := backtestingEvaluatorFactory(ruleCtx, e.evalFactory, user, rule.GetEvalCondition(), &schedule.AlertingResultsFromRuleState{
+		Manager: stateManager,
+		Rule:    rule,
+	})
 	if err != nil {
 		return nil, errors.Join(ErrInvalidInputData, err)
 	}
 
-	stateManager := e.createStateManager()
-
 	logger.Info("Start testing alert rule", "from", from, "to", to, "interval", rule.IntervalSeconds, "evaluations", length)
 
 	start := time.Now()
@@ -126,7 +131,7 @@ func (e *Engine) Test(ctx context.Context, user identity.Requester, rule *models
 	return result, nil
 }
 
-func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition) (backtestingEvaluator, error) {
+func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition, reader eval.AlertingResultsReader) (backtestingEvaluator, error) {
 	for _, q := range condition.Data {
 		if q.DatasourceUID == "__data__" || q.QueryType == "__data__" {
 			if len(condition.Data) != 1 {
@@ -152,9 +157,7 @@ func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFact
 		}
 	}
 
-	evaluator, err := evalFactory.Create(eval.EvaluationContext{Ctx: ctx,
-		User: user,
-	}, condition)
+	evaluator, err := evalFactory.Create(eval.NewContextWithPreviousResults(ctx, user, reader), condition)
 
 	if err != nil {
 		return nil, err

pkg/services/ngalert/backtesting/engine_test.go

@@ -145,7 +145,7 @@ func TestNewBacktestingEvaluator(t *testing.T) {
 
 		for _, testCase := range testCases {
 			t.Run(testCase.name, func(t *testing.T) {
-				e, err := newBacktestingEvaluator(context.Background(), evalFactory, nil, testCase.condition)
+				e, err := newBacktestingEvaluator(context.Background(), evalFactory, nil, testCase.condition, nil)
 				if testCase.error {
 					require.Error(t, err)
 					return
@@ -175,7 +175,7 @@ func TestEvaluatorTest(t *testing.T) {
 	}
 	manager := &fakeStateManager{}
 
-	backtestingEvaluatorFactory = func(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition) (backtestingEvaluator, error) {
+	backtestingEvaluatorFactory = func(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition, r eval.AlertingResultsReader) (backtestingEvaluator, error) {
 		return evaluator, nil
 	}
 
@@ -386,6 +386,10 @@ func (f *fakeStateManager) ProcessEvalResults(_ context.Context, evaluatedAt tim
 	return f.stateCallback(evaluatedAt)
 }
 
+func (f *fakeStateManager) GetStatesForRuleUID(orgID int64, alertRuleUID string) []*state.State {
+	return nil
+}
+
 type fakeBacktestingEvaluator struct {
 	evalCallback func(now time.Time) (eval.Results, error)
 }

pkg/services/ngalert/eval/context.go

@@ -3,13 +3,22 @@ package eval
 import (
 	"context"
 
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+
 	"github.com/grafana/grafana/pkg/services/auth/identity"
 )
 
+// AlertingResultsReader provides fingerprints of results that are in alerting state.
+// It is used during the evaluation of queries.
+type AlertingResultsReader interface {
+	Read() map[data.Fingerprint]struct{}
+}
+
 // EvaluationContext represents the context in which a condition is evaluated.
 type EvaluationContext struct {
-	Ctx  context.Context
-	User identity.Requester
+	Ctx                   context.Context
+	User                  identity.Requester
+	AlertingResultsReader AlertingResultsReader
 }
 
 func NewContext(ctx context.Context, user identity.Requester) EvaluationContext {
@@ -18,3 +27,11 @@ func NewContext(ctx context.Context, user identity.Requester) EvaluationContext
 		User: user,
 	}
 }
+
+func NewContextWithPreviousResults(ctx context.Context, user identity.Requester, reader AlertingResultsReader) EvaluationContext {
+	return EvaluationContext{
+		Ctx:                   ctx,
+		User:                  user,
+		AlertingResultsReader: reader,
+	}
+}

pkg/services/ngalert/eval/eval.go

@@ -298,30 +298,16 @@ func buildDatasourceHeaders(ctx context.Context) map[string]string {
 }
 
 // getExprRequest validates the condition, gets the datasource information and creates an expr.Request from it.
-func getExprRequest(ctx EvaluationContext, data []models.AlertQuery, dsCacheService datasources.CacheService) (*expr.Request, error) {
+func getExprRequest(ctx EvaluationContext, condition models.Condition, dsCacheService datasources.CacheService, reader AlertingResultsReader) (*expr.Request, error) {
 	req := &expr.Request{
 		OrgId:   ctx.User.GetOrgID(),
 		Headers: buildDatasourceHeaders(ctx.Ctx),
 		User:    ctx.User,
 	}
+	datasources := make(map[string]*datasources.DataSource, len(condition.Data))
 
-	datasources := make(map[string]*datasources.DataSource, len(data))
-
-	for _, q := range data {
-		model, err := q.GetModel()
-		if err != nil {
-			return nil, fmt.Errorf("failed to get query model from '%s': %w", q.RefID, err)
-		}
-		interval, err := q.GetIntervalDuration()
-		if err != nil {
-			return nil, fmt.Errorf("failed to retrieve intervalMs from '%s': %w", q.RefID, err)
-		}
-
-		maxDatapoints, err := q.GetMaxDatapoints()
-		if err != nil {
-			return nil, fmt.Errorf("failed to retrieve maxDatapoints from '%s': %w", q.RefID, err)
-		}
-
+	for _, q := range condition.Data {
+		var err error
 		ds, ok := datasources[q.DatasourceUID]
 		if !ok {
 			switch nodeType := expr.NodeTypeFromDatasourceUID(q.DatasourceUID); nodeType {
@@ -336,6 +322,45 @@ func getExprRequest(ctx EvaluationContext, data []models.AlertQuery, dsCacheServ
 			datasources[q.DatasourceUID] = ds
 		}
 
+		// TODO rewrite the code below and remove the mutable component from AlertQuery
+
+		// if the query is command expression and it's a hysteresis, patch it with the current state
+		// it's important to do this before GetModel
+		if ds.Type == expr.DatasourceType {
+			isHysteresis, err := q.IsHysteresisExpression()
+			if err != nil {
+				return nil, fmt.Errorf("failed to build query '%s': %w", q.RefID, err)
+			}
+			if isHysteresis {
+				// make sure we allow hysteresis expressions to be specified only as the alert condition.
+				// This guarantees us that the AlertResultsReader can be correctly applied to the expression tree.
+				if q.RefID != condition.Condition {
+					return nil, fmt.Errorf("recovery threshold '%s' is only allowed to be the alert condition", q.RefID)
+				}
+				if reader != nil {
+					logger.FromContext(ctx.Ctx).Debug("Detected hysteresis threshold command. Populating with the results")
+					err = q.PatchHysteresisExpression(reader.Read())
+					if err != nil {
+						return nil, fmt.Errorf("failed to amend hysteresis command '%s': %w", q.RefID, err)
+					}
+				}
+			}
+		}
+
+		model, err := q.GetModel()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get query model from '%s': %w", q.RefID, err)
+		}
+		interval, err := q.GetIntervalDuration()
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve intervalMs from '%s': %w", q.RefID, err)
+		}
+
+		maxDatapoints, err := q.GetMaxDatapoints()
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve maxDatapoints from '%s': %w", q.RefID, err)
+		}
+
 		req.Queries = append(req.Queries, expr.Query{
 			TimeRange:     q.RelativeTimeRange.ToTimeRange(),
 			DataSource:    ds,
@@ -724,7 +749,7 @@ func (evalResults Results) AsDataFrame() data.Frame {
 }
 
 func (e *evaluatorImpl) Validate(ctx EvaluationContext, condition models.Condition) error {
-	req, err := getExprRequest(ctx, condition.Data, e.dataSourceCache)
+	req, err := getExprRequest(ctx, condition, e.dataSourceCache, ctx.AlertingResultsReader)
 	if err != nil {
 		return err
 	}
@@ -760,7 +785,7 @@ func (e *evaluatorImpl) Create(ctx EvaluationContext, condition models.Condition
 	if len(condition.Condition) == 0 {
 		return nil, errors.New("condition must not be empty")
 	}
-	req, err := getExprRequest(ctx, condition.Data, e.dataSourceCache)
+	req, err := getExprRequest(ctx, condition, e.dataSourceCache, ctx.AlertingResultsReader)
 	if err != nil {
 		return nil, err
 	}

pkg/services/ngalert/eval/eval_test.go

@@ -524,6 +524,59 @@ func TestValidate(t *testing.T) {
 				}
 			},
 		},
+		{
+			name:  "fail if hysteresis command is not the condition",
+			error: true,
+			condition: func(services services) models.Condition {
+				dsQuery := models.GenerateAlertQuery()
+				ds := &datasources.DataSource{
+					UID:  dsQuery.DatasourceUID,
+					Type: util.GenerateShortUID(),
+				}
+				services.cache.DataSources = append(services.cache.DataSources, ds)
+				services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
+					JSONData: plugins.JSONData{
+						ID:      ds.Type,
+						Backend: true,
+					},
+				})
+
+				return models.Condition{
+					Condition: "C",
+					Data: []models.AlertQuery{
+						dsQuery,
+						models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
+						models.CreateClassicConditionExpression("C", "B", "last", "gt", rand.Int()),
+					},
+				}
+			},
+		},
+		{
+			name:  "pass if hysteresis command and it is the condition",
+			error: false,
+			condition: func(services services) models.Condition {
+				dsQuery := models.GenerateAlertQuery()
+				ds := &datasources.DataSource{
+					UID:  dsQuery.DatasourceUID,
+					Type: util.GenerateShortUID(),
+				}
+				services.cache.DataSources = append(services.cache.DataSources, ds)
+				services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
+					JSONData: plugins.JSONData{
+						ID:      ds.Type,
+						Backend: true,
+					},
+				})
+
+				return models.Condition{
+					Condition: "B",
+					Data: []models.AlertQuery{
+						dsQuery,
+						models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
+					},
+				}
+			},
+		},
 	}
 
 	for _, testCase := range testCases {
@@ -550,6 +603,133 @@ func TestValidate(t *testing.T) {
 	}
 }
 
+func TestCreate_HysteresisCommand(t *testing.T) {
+	type services struct {
+		cache        *fakes.FakeCacheService
+		pluginsStore *pluginstore.FakePluginStore
+	}
+
+	testCases := []struct {
+		name      string
+		reader    AlertingResultsReader
+		condition func(services services) models.Condition
+		error     bool
+	}{
+		{
+			name:  "fail if hysteresis command is not the condition",
+			error: true,
+			condition: func(services services) models.Condition {
+				dsQuery := models.GenerateAlertQuery()
+				ds := &datasources.DataSource{
+					UID:  dsQuery.DatasourceUID,
+					Type: util.GenerateShortUID(),
+				}
+				services.cache.DataSources = append(services.cache.DataSources, ds)
+				services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
+					JSONData: plugins.JSONData{
+						ID:      ds.Type,
+						Backend: true,
+					},
+				})
+
+				return models.Condition{
+					Condition: "C",
+					Data: []models.AlertQuery{
+						dsQuery,
+						models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
+						models.CreateClassicConditionExpression("C", "B", "last", "gt", rand.Int()),
+					},
+				}
+			},
+		},
+		{
+			name:   "populate with loaded metrics",
+			error:  false,
+			reader: FakeLoadedMetricsReader{fingerprints: map[data.Fingerprint]struct{}{1: {}, 2: {}, 3: {}}},
+			condition: func(services services) models.Condition {
+				dsQuery := models.GenerateAlertQuery()
+				ds := &datasources.DataSource{
+					UID:  dsQuery.DatasourceUID,
+					Type: util.GenerateShortUID(),
+				}
+				services.cache.DataSources = append(services.cache.DataSources, ds)
+				services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
+					JSONData: plugins.JSONData{
+						ID:      ds.Type,
+						Backend: true,
+					},
+				})
+
+				return models.Condition{
+					Condition: "B",
+					Data: []models.AlertQuery{
+						dsQuery,
+						models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
+					},
+				}
+			},
+		},
+		{
+			name:   "do nothing if reader is not specified",
+			error:  false,
+			reader: nil,
+			condition: func(services services) models.Condition {
+				dsQuery := models.GenerateAlertQuery()
+				ds := &datasources.DataSource{
+					UID:  dsQuery.DatasourceUID,
+					Type: util.GenerateShortUID(),
+				}
+				services.cache.DataSources = append(services.cache.DataSources, ds)
+				services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
+					JSONData: plugins.JSONData{
+						ID:      ds.Type,
+						Backend: true,
+					},
+				})
+
+				return models.Condition{
+					Condition: "B",
+					Data: []models.AlertQuery{
+						dsQuery,
+						models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
+					},
+				}
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		u := &user.SignedInUser{}
+
+		t.Run(testCase.name, func(t *testing.T) {
+			cacheService := &fakes.FakeCacheService{}
+			store := &pluginstore.FakePluginStore{}
+			condition := testCase.condition(services{
+				cache:        cacheService,
+				pluginsStore: store,
+			})
+			evaluator := NewEvaluatorFactory(setting.UnifiedAlertingSettings{}, cacheService, expr.ProvideService(&setting.Cfg{ExpressionsEnabled: true}, nil, nil, featuremgmt.WithFeatures(featuremgmt.FlagRecoveryThreshold), nil, tracing.InitializeTracerForTest()), store)
+			evalCtx := NewContextWithPreviousResults(context.Background(), u, testCase.reader)
+
+			eval, err := evaluator.Create(evalCtx, condition)
+			if testCase.error {
+				require.Error(t, err)
+				return
+			}
+			require.IsType(t, &conditionEvaluator{}, eval)
+			ce := eval.(*conditionEvaluator)
+
+			cmds := expr.GetCommandsFromPipeline[*expr.HysteresisCommand](ce.pipeline)
+			require.Len(t, cmds, 1)
+			if testCase.reader == nil {
+				require.Empty(t, cmds[0].LoadedDimensions)
+			} else {
+				require.EqualValues(t, testCase.reader.Read(), cmds[0].LoadedDimensions)
+			}
+		})
+	}
+}
+
 func TestEvaluate(t *testing.T) {
 	cases := []struct {
 		name     string

pkg/services/ngalert/eval/testing.go

@@ -79,3 +79,11 @@ func WithLabels(labels data.Labels) ResultMutator {
 		r.Instance = labels
 	}
 }
+
+type FakeLoadedMetricsReader struct {
+	fingerprints map[data.Fingerprint]struct{}
+}
+
+func (f FakeLoadedMetricsReader) Read() map[data.Fingerprint]struct{} {
+	return f.fingerprints
+}

pkg/services/ngalert/models/alert_query.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+
 	"github.com/grafana/grafana/pkg/expr"
 )
 
@@ -116,6 +118,28 @@ func (aq *AlertQuery) IsExpression() (bool, error) {
 	return expr.NodeTypeFromDatasourceUID(aq.DatasourceUID) == expr.TypeCMDNode, nil
 }
 
+// IsHysteresisExpression returns true if the model describes a hysteresis command expression. Returns error if the Model is not a valid JSON
+func (aq *AlertQuery) IsHysteresisExpression() (bool, error) {
+	if aq.modelProps == nil {
+		err := aq.setModelProps()
+		if err != nil {
+			return false, err
+		}
+	}
+	return expr.IsHysteresisExpression(aq.modelProps), nil
+}
+
+// PatchHysteresisExpression updates the AlertQuery to include loaded metrics into hysteresis
+func (aq *AlertQuery) PatchHysteresisExpression(loadedMetrics map[data.Fingerprint]struct{}) error {
+	if aq.modelProps == nil {
+		err := aq.setModelProps()
+		if err != nil {
+			return err
+		}
+	}
+	return expr.SetLoadedDimensionsToHysteresisCommand(aq.modelProps, loadedMetrics)
+}
+
 // setMaxDatapoints sets the model maxDataPoints if it's missing or invalid
 func (aq *AlertQuery) setMaxDatapoints() error {
 	if aq.modelProps == nil {

pkg/services/ngalert/models/alert_query_test.go

@@ -7,9 +7,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/grafana/grafana/pkg/expr"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/expr"
 )
 
 func TestAlertQuery(t *testing.T) {
@@ -17,6 +19,7 @@ func TestAlertQuery(t *testing.T) {
 		desc                 string
 		alertQuery           AlertQuery
 		expectedIsExpression bool
+		expectedIsHysteresis bool
 		expectedDatasource   string
 		expectedMaxPoints    int64
 		expectedIntervalMS   int64
@@ -133,6 +136,64 @@ func TestAlertQuery(t *testing.T) {
 			expectedMaxPoints:    int64(defaultMaxDataPoints),
 			expectedIntervalMS:   int64(defaultIntervalMS),
 		},
+		{
+			desc: "given a query with threshold expression",
+			alertQuery: AlertQuery{
+				RefID:         "A",
+				DatasourceUID: expr.DatasourceType,
+				Model: json.RawMessage(`{
+	                "type": "threshold",
+					"queryType": "metricQuery",
+					"extraParam": "some text",
+	                "conditions": [
+		                {
+		                  "evaluator": {
+		                    "params": [
+		                      4
+		                    ],
+		                    "type": "gt"
+		                  }
+		                }
+		              ]
+				}`),
+			},
+			expectedIsExpression: true,
+			expectedIsHysteresis: false,
+			expectedMaxPoints:    int64(defaultMaxDataPoints),
+			expectedIntervalMS:   int64(defaultIntervalMS),
+		},
+		{
+			desc: "given a query with hysteresis expression",
+			alertQuery: AlertQuery{
+				RefID:         "A",
+				DatasourceUID: expr.DatasourceType,
+				Model: json.RawMessage(`{
+					"type": "threshold",
+					"queryType": "metricQuery",
+					"extraParam": "some text",
+					"conditions": [
+		                {
+		                  "evaluator": {
+		                    "params": [
+		                      4
+		                    ],
+		                    "type": "gt"
+		                  },
+		                  "unloadEvaluator": {
+		                    "params": [
+		                      2
+		                    ],
+		                    "type": "lt"
+		                  }
+		                }
+		              ]
+				}`),
+			},
+			expectedMaxPoints:    int64(defaultMaxDataPoints),
+			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression: true,
+			expectedIsHysteresis: true,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -143,6 +204,12 @@ func TestAlertQuery(t *testing.T) {
 				assert.Equal(t, tc.expectedIsExpression, isExpression)
 			})
 
+			t.Run("can recognize if it's a hysteresis expression", func(t *testing.T) {
+				isExpression, err := tc.alertQuery.IsHysteresisExpression()
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectedIsHysteresis, isExpression)
+			})
+
 			t.Run("can set queryType for expression", func(t *testing.T) {
 				err := tc.alertQuery.setQueryType()
 				require.NoError(t, err)
@@ -186,6 +253,17 @@ func TestAlertQuery(t *testing.T) {
 				require.True(t, ok)
 				require.Equal(t, "some text", extraParam)
 			})
+
+			if tc.expectedIsHysteresis {
+				t.Run("can patch the command with loaded metrics", func(t *testing.T) {
+					require.NoError(t, tc.alertQuery.PatchHysteresisExpression(map[data.Fingerprint]struct{}{1: {}, 2: {}, 3: {}}))
+					data, ok := tc.alertQuery.modelProps["conditions"].([]any)[0].(map[string]any)["loadedDimensions"]
+					require.True(t, ok)
+					require.NotNil(t, data)
+					_, err := tc.alertQuery.GetModel()
+					require.NoError(t, err)
+				})
+			}
 		})
 	}
 }

pkg/services/ngalert/models/instance.go

@@ -14,6 +14,7 @@ type AlertInstance struct {
 	CurrentStateSince time.Time
 	CurrentStateEnd   time.Time
 	LastEvalTime      time.Time
+	ResultFingerprint string
 }
 
 type AlertInstanceKey struct {

pkg/services/ngalert/models/testing.go

@@ -6,10 +6,12 @@ import (
 	"math/rand"
 	"slices"
 	"sync"
+	"testing"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/expr"
 	"github.com/grafana/grafana/pkg/services/datasources"
@@ -511,6 +513,46 @@ func CreateLokiQuery(refID string, expr string, intervalMs int64, maxDataPoints
 	}
 }
 
+func CreateHysteresisExpression(t *testing.T, refID string, inputRefID string, threshold int, recoveryThreshold int) AlertQuery {
+	t.Helper()
+	q := AlertQuery{
+		RefID:         refID,
+		QueryType:     expr.DatasourceType,
+		DatasourceUID: expr.DatasourceUID,
+		Model: json.RawMessage(fmt.Sprintf(`
+		{
+			"refId": "%[1]s",
+            "type": "threshold",
+            "datasource": {
+                "uid": "%[5]s",
+                "type": "%[6]s"
+            },
+			"expression": "%[2]s",
+            "conditions": [
+                {
+                    "type": "query",
+                    "evaluator": {
+                        "params": [
+                            %[3]d
+                        ],
+                        "type": "gt"
+                    },
+					"unloadEvaluator": {
+                        "params": [
+                            %[4]d
+                        ],
+                        "type": "lt"
+					}
+                }
+            ]
+		}`, refID, inputRefID, threshold, recoveryThreshold, expr.DatasourceUID, expr.DatasourceType)),
+	}
+	h, err := q.IsHysteresisExpression()
+	require.NoError(t, err)
+	require.Truef(t, h, "test model is expected to be a hysteresis expression")
+	return q
+}
+
 type AlertInstanceMutator func(*AlertInstance)
 
 // AlertInstanceGen provides a factory function that generates a random AlertInstance.

pkg/services/ngalert/schedule/loaded_metrics_reader.go

@@ -0,0 +1,44 @@
+package schedule
+
+import (
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/state"
+)
+
+var _ eval.AlertingResultsReader = AlertingResultsFromRuleState{}
+
+func (sch *schedule) newLoadedMetricsReader(rule *ngmodels.AlertRule) eval.AlertingResultsReader {
+	return &AlertingResultsFromRuleState{
+		Manager: sch.stateManager,
+		Rule:    rule,
+	}
+}
+
+type RuleStateProvider interface {
+	GetStatesForRuleUID(orgID int64, alertRuleUID string) []*state.State
+}
+
+// AlertingResultsFromRuleState implements eval.AlertingResultsReader that gets the data from state manager.
+// It returns results fingerprints only for Alerting and Pending states that have empty StateReason.
+type AlertingResultsFromRuleState struct {
+	Manager RuleStateProvider
+	Rule    *ngmodels.AlertRule
+}
+
+func (n AlertingResultsFromRuleState) Read() map[data.Fingerprint]struct{} {
+	states := n.Manager.GetStatesForRuleUID(n.Rule.OrgID, n.Rule.UID)
+
+	active := map[data.Fingerprint]struct{}{}
+	for _, st := range states {
+		if st.StateReason != "" {
+			continue
+		}
+		if st.State == eval.Alerting || st.State == eval.Pending {
+			active[st.ResultFingerprint] = struct{}{}
+		}
+	}
+	return active
+}

pkg/services/ngalert/schedule/loaded_metrics_reader_test.go

@@ -0,0 +1,65 @@
+package schedule
+
+import (
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/state"
+)
+
+func TestLoadedResultsFromRuleState(t *testing.T) {
+	rule := ngmodels.AlertRuleGen()()
+	p := &FakeRuleStateProvider{
+		map[ngmodels.AlertRuleKey][]*state.State{
+			rule.GetKey(): {
+				{State: eval.Alerting, ResultFingerprint: data.Fingerprint(1)},
+				{State: eval.Pending, ResultFingerprint: data.Fingerprint(2)},
+				{State: eval.Normal, ResultFingerprint: data.Fingerprint(3)},
+				{State: eval.NoData, ResultFingerprint: data.Fingerprint(4)},
+				{State: eval.Error, ResultFingerprint: data.Fingerprint(5)},
+			},
+		},
+	}
+
+	reader := AlertingResultsFromRuleState{
+		Manager: p,
+		Rule:    rule,
+	}
+
+	t.Run("should return pending and alerting states", func(t *testing.T) {
+		loaded := reader.Read()
+		require.Len(t, loaded, 2)
+		require.Contains(t, loaded, data.Fingerprint(1))
+		require.Contains(t, loaded, data.Fingerprint(2))
+	})
+
+	t.Run("should not return any states with reason", func(t *testing.T) {
+		for _, s := range p.states[rule.GetKey()] {
+			s.StateReason = uuid.NewString()
+		}
+		loaded := reader.Read()
+		require.Empty(t, loaded)
+	})
+
+	t.Run("empty if no states", func(t *testing.T) {
+		p.states[rule.GetKey()] = nil
+		loaded := reader.Read()
+		require.Empty(t, loaded)
+	})
+}
+
+type FakeRuleStateProvider struct {
+	states map[ngmodels.AlertRuleKey][]*state.State
+}
+
+func (f FakeRuleStateProvider) GetStatesForRuleUID(orgID int64, UID string) []*state.State {
+	return f.states[ngmodels.AlertRuleKey{
+		OrgID: orgID,
+		UID:   UID,
+	}]
+}

pkg/services/ngalert/schedule/schedule.go

@@ -382,7 +382,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 		logger := logger.New("version", e.rule.Version, "fingerprint", f, "attempt", attempt, "now", e.scheduledAt).FromContext(ctx)
 		start := sch.clock.Now()
 
-		evalCtx := eval.NewContext(ctx, SchedulerUserFor(e.rule.OrgID))
+		evalCtx := eval.NewContextWithPreviousResults(ctx, SchedulerUserFor(e.rule.OrgID), sch.newLoadedMetricsReader(e.rule))
 		ruleEval, err := sch.evaluatorFactory.Create(evalCtx, e.rule.GetEvalCondition())
 		var results eval.Results
 		var dur time.Duration

pkg/services/ngalert/state/cache.go

@@ -190,6 +190,7 @@ func calculateState(ctx context.Context, log log.Logger, alertRule *ngModels.Ale
 		Values:             values,
 		StartsAt:           result.EvaluatedAt,
 		EndsAt:             result.EvaluatedAt,
+		ResultFingerprint:  result.Instance.Fingerprint(),
 	}
 	return newState
 }

pkg/services/ngalert/state/manager.go

@@ -3,6 +3,7 @@ package state
 import (
 	"context"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/benbjohnson/clock"
@@ -158,6 +159,14 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
 			if err != nil {
 				st.log.Error("Error getting cacheId for entry", "error", err)
 			}
+			var resultFp data.Fingerprint
+			if entry.ResultFingerprint != "" {
+				fp, err := strconv.ParseUint(entry.ResultFingerprint, 16, 64)
+				if err != nil {
+					st.log.Error("Failed to parse result fingerprint of alert instance", "error", err, "ruleUID", entry.RuleUID)
+				}
+				resultFp = data.Fingerprint(fp)
+			}
 			rulesStates.states[cacheID] = &State{
 				AlertRuleUID:         entry.RuleUID,
 				OrgID:                entry.RuleOrgID,
@@ -170,6 +179,7 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
 				EndsAt:               entry.CurrentStateEnd,
 				LastEvaluationTime:   entry.LastEvalTime,
 				Annotations:          ruleForEntry.Annotations,
+				ResultFingerprint:    resultFp,
 			}
 			statesCount++
 		}
@@ -458,6 +468,7 @@ func (st *Manager) saveAlertStates(ctx context.Context, logger log.Logger, state
 			LastEvalTime:      s.LastEvaluationTime,
 			CurrentStateSince: s.StartsAt,
 			CurrentStateEnd:   s.EndsAt,
+			ResultFingerprint: s.ResultFingerprint.String(),
 		}
 
 		err = st.instanceStore.SaveAlertInstance(ctx, instance)

pkg/services/ngalert/state/manager_private_test.go

@@ -286,6 +286,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 		"system + rule + datasource-error": mergeLabels(mergeLabels(expectedDatasourceErrorLabels, baseRule.Labels), systemLabels),
 	}
 
+	resultFingerprints := map[string]data.Fingerprint{
+		"system + rule":                    data.Labels{}.Fingerprint(),
+		"system + rule + labels1":          labels1.Fingerprint(),
+		"system + rule + labels2":          labels2.Fingerprint(),
+		"system + rule + labels3":          labels3.Fingerprint(),
+		"system + rule + no-data":          noDataLabels.Fingerprint(),
+		"system + rule + datasource-error": data.Labels{}.Fingerprint(),
+	}
+
 	patchState := func(r *ngmodels.AlertRule, s *State) {
 		// patch all optional fields of the expected state
 		setCacheID(s)
@@ -304,6 +313,14 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 		if s.Values == nil {
 			s.Values = make(map[string]float64)
 		}
+		if s.ResultFingerprint == data.Fingerprint(0) {
+			for key, set := range labels {
+				if set.Fingerprint() == s.Labels.Fingerprint() {
+					s.ResultFingerprint = resultFingerprints[key]
+					break
+				}
+			}
+		}
 	}
 
 	executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition, applyNoDataErrorToAllStates bool) {

pkg/services/ngalert/state/manager_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"math/rand"
 	"sort"
 	"strings"
@@ -58,6 +59,7 @@ func TestWarmStateCache(t *testing.T) {
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
+			ResultFingerprint:  data.Fingerprint(math.MaxUint64),
 		}, {
 			AlertRuleUID: rule.UID,
 			OrgID:        rule.OrgID,
@@ -70,6 +72,7 @@ func TestWarmStateCache(t *testing.T) {
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
+			ResultFingerprint:  data.Fingerprint(math.MaxUint64 - 1),
 		},
 		{
 			AlertRuleUID: rule.UID,
@@ -83,6 +86,7 @@ func TestWarmStateCache(t *testing.T) {
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
+			ResultFingerprint:  data.Fingerprint(0),
 		},
 		{
 			AlertRuleUID: rule.UID,
@@ -96,6 +100,7 @@ func TestWarmStateCache(t *testing.T) {
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
+			ResultFingerprint:  data.Fingerprint(1),
 		},
 		{
 			AlertRuleUID: rule.UID,
@@ -109,6 +114,7 @@ func TestWarmStateCache(t *testing.T) {
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
+			ResultFingerprint:  data.Fingerprint(2),
 		},
 	}
 
@@ -127,6 +133,7 @@ func TestWarmStateCache(t *testing.T) {
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
 		Labels:            labels,
+		ResultFingerprint: data.Fingerprint(math.MaxUint64).String(),
 	})
 
 	labels = models.InstanceLabels{"test2": "testValue2"}
@@ -142,6 +149,7 @@ func TestWarmStateCache(t *testing.T) {
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
 		Labels:            labels,
+		ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1).String(),
 	})
 
 	labels = models.InstanceLabels{"test3": "testValue3"}
@@ -157,6 +165,7 @@ func TestWarmStateCache(t *testing.T) {
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
 		Labels:            labels,
+		ResultFingerprint: data.Fingerprint(0).String(),
 	})
 
 	labels = models.InstanceLabels{"test4": "testValue4"}
@@ -172,6 +181,7 @@ func TestWarmStateCache(t *testing.T) {
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
 		Labels:            labels,
+		ResultFingerprint: data.Fingerprint(1).String(),
 	})
 
 	labels = models.InstanceLabels{"test5": "testValue5"}
@@ -187,6 +197,7 @@ func TestWarmStateCache(t *testing.T) {
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
 		Labels:            labels,
+		ResultFingerprint: data.Fingerprint(2).String(),
 	})
 	for _, instance := range instances {
 		_ = dbstore.SaveAlertInstance(ctx, instance)
@@ -384,8 +395,9 @@ func TestProcessEvalResults(t *testing.T) {
 			},
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 					},
@@ -407,8 +419,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 					},
@@ -417,8 +430,9 @@ func TestProcessEvalResults(t *testing.T) {
 					LastEvaluationTime: t1,
 				},
 				{
-					Labels: labels["system + rule + labels2"],
-					State:  eval.Alerting,
+					Labels:            labels["system + rule + labels2"],
+					ResultFingerprint: labels2.Fingerprint(),
+					State:             eval.Alerting,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Alerting),
 					},
@@ -441,8 +455,9 @@ func TestProcessEvalResults(t *testing.T) {
 			},
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(tn(6), eval.Normal),
@@ -467,8 +482,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Alerting,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Alerting,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.Alerting),
@@ -499,8 +515,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 2,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Alerting,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Alerting,
 					Results: []state.Evaluation{
 						newEvaluation(t3, eval.Alerting),
 						newEvaluation(tn(4), eval.Alerting),
@@ -534,8 +551,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 3, // Normal -> Pending, Pending -> NoData, NoData -> Pending
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Pending,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Pending,
 					Results: []state.Evaluation{
 						newEvaluation(tn(4), eval.Alerting),
 						newEvaluation(tn(5), eval.Alerting),
@@ -566,8 +584,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 3,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.NoData,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.NoData,
 					Results: []state.Evaluation{
 						newEvaluation(t3, eval.Alerting),
 						newEvaluation(tn(4), eval.NoData),
@@ -592,7 +611,8 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
 
 					State: eval.Pending,
 					Results: []state.Evaluation{
@@ -619,8 +639,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Pending,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Pending,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Alerting),
 						newEvaluation(t2, eval.Alerting),
@@ -645,9 +666,10 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Pending,
-					StateReason: eval.NoData.String(),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Pending,
+					StateReason:       eval.NoData.String(),
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.NoData),
@@ -681,9 +703,10 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 2,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Alerting,
-					StateReason: eval.NoData.String(),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Alerting,
+					StateReason:       eval.NoData.String(),
 					Results: []state.Evaluation{
 						newEvaluation(t3, eval.NoData),
 						newEvaluation(tn(4), eval.NoData),
@@ -709,8 +732,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.NoData,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.NoData,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.NoData),
@@ -735,8 +759,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 					},
@@ -745,8 +770,9 @@ func TestProcessEvalResults(t *testing.T) {
 					LastEvaluationTime: t1,
 				},
 				{
-					Labels: labels["system + rule"],
-					State:  eval.NoData,
+					Labels:            labels["system + rule"],
+					ResultFingerprint: data.Labels{}.Fingerprint(),
+					State:             eval.NoData,
 					Results: []state.Evaluation{
 						newEvaluation(t2, eval.NoData),
 					},
@@ -771,8 +797,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 					},
@@ -781,8 +808,9 @@ func TestProcessEvalResults(t *testing.T) {
 					LastEvaluationTime: t1,
 				},
 				{
-					Labels: labels["system + rule + labels2"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels2"],
+					ResultFingerprint: labels2.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 					},
@@ -791,8 +819,9 @@ func TestProcessEvalResults(t *testing.T) {
 					LastEvaluationTime: t1,
 				},
 				{
-					Labels: labels["system + rule"],
-					State:  eval.NoData,
+					Labels:            labels["system + rule"],
+					ResultFingerprint: data.Labels{}.Fingerprint(),
+					State:             eval.NoData,
 					Results: []state.Evaluation{
 						newEvaluation(t2, eval.NoData),
 					},
@@ -819,8 +848,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Normal,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t3, eval.Normal),
@@ -830,8 +860,9 @@ func TestProcessEvalResults(t *testing.T) {
 					LastEvaluationTime: t3,
 				},
 				{
-					Labels: labels["system + rule + no-data"],
-					State:  eval.NoData,
+					Labels:            labels["system + rule + no-data"],
+					ResultFingerprint: noDataLabels.Fingerprint(),
+					State:             eval.NoData,
 					Results: []state.Evaluation{
 						newEvaluation(t2, eval.NoData),
 					},
@@ -855,9 +886,10 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 0,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Normal,
-					StateReason: eval.NoData.String(),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
+					StateReason:       eval.NoData.String(),
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.NoData),
@@ -882,10 +914,11 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Pending,
-					StateReason: eval.Error.String(),
-					Error:       errors.New("with_state_error"),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Pending,
+					StateReason:       eval.Error.String(),
+					Error:             errors.New("with_state_error"),
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.Error),
@@ -919,10 +952,11 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 2,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Alerting,
-					StateReason: eval.Error.String(),
-					Error:       errors.New("with_state_error"),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Alerting,
+					StateReason:       eval.Error.String(),
+					Error:             errors.New("with_state_error"),
 					Results: []state.Evaluation{
 						newEvaluation(t3, eval.Error),
 						newEvaluation(tn(4), eval.Error),
@@ -960,8 +994,9 @@ func TestProcessEvalResults(t *testing.T) {
 						"datasource_uid": "datasource_uid_1",
 						"ref_id":         "A",
 					}),
-					State: eval.Error,
-					Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Error,
+					Error:             expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.Error),
@@ -988,9 +1023,10 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 1,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Normal,
-					StateReason: eval.Error.String(),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
+					StateReason:       eval.Error.String(),
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Normal),
 						newEvaluation(t2, eval.Error),
@@ -1015,9 +1051,10 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 2,
 			expectedStates: []*state.State{
 				{
-					Labels:      labels["system + rule + labels1"],
-					State:       eval.Normal,
-					StateReason: eval.Error.String(),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Normal,
+					StateReason:       eval.Error.String(),
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Alerting),
 						newEvaluation(t2, eval.Error),
@@ -1054,9 +1091,10 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 3,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Error,
-					Error:  fmt.Errorf("with_state_error"),
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Error,
+					Error:             fmt.Errorf("with_state_error"),
 					Results: []state.Evaluation{
 						newEvaluation(tn(5), eval.Error),
 						newEvaluation(tn(6), eval.Error),
@@ -1087,8 +1125,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 3,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.Pending,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.Pending,
 					Results: []state.Evaluation{
 						newEvaluation(tn(4), eval.Alerting),
 						newEvaluation(tn(5), eval.Error),
@@ -1120,8 +1159,9 @@ func TestProcessEvalResults(t *testing.T) {
 			expectedAnnotations: 3,
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule + labels1"],
-					State:  eval.NoData,
+					Labels:            labels["system + rule + labels1"],
+					ResultFingerprint: labels1.Fingerprint(),
+					State:             eval.NoData,
 					Results: []state.Evaluation{
 						newEvaluation(tn(4), eval.Alerting),
 						newEvaluation(tn(5), eval.Error),
@@ -1166,6 +1206,11 @@ func TestProcessEvalResults(t *testing.T) {
 					LastEvaluationTime: t1,
 					EvaluationDuration: evaluationDuration,
 					Annotations:        map[string]string{"summary": "grafana is down in us-central-1 cluster -> prod namespace"},
+					ResultFingerprint: data.Labels{
+						"cluster":   "us-central-1",
+						"namespace": "prod",
+						"pod":       "grafana",
+					}.Fingerprint(),
 				},
 			},
 		},
@@ -1186,8 +1231,9 @@ func TestProcessEvalResults(t *testing.T) {
 			},
 			expectedStates: []*state.State{
 				{
-					Labels: labels["system + rule"],
-					State:  eval.Alerting,
+					Labels:            labels["system + rule"],
+					ResultFingerprint: data.Labels{}.Fingerprint(),
+					State:             eval.Alerting,
 					Results: []state.Evaluation{
 						newEvaluation(t1, eval.Alerting),
 						newEvaluation(t2, eval.Error),
@@ -1454,6 +1500,7 @@ func TestStaleResultsHandler(t *testing.T) {
 					LastEvaluationTime: evaluationTime,
 					EvaluationDuration: 0,
 					Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
+					ResultFingerprint:  data.Labels{"test1": "testValue1"}.Fingerprint(),
 				},
 			},
 			startingStateCount: 2,

pkg/services/ngalert/state/state.go

@@ -34,6 +34,9 @@ type State struct {
 	// StateReason is a textual description to explain why the state has its current state.
 	StateReason string
 
+	// ResultFingerprint is a hash of labels of the result before it is processed by
+	ResultFingerprint data.Fingerprint
+
 	// Results contains the result of the current and previous evaluations.
 	Results []Evaluation
 

pkg/services/ngalert/store/instance_database.go

@@ -54,12 +54,12 @@ func (st DBstore) SaveAlertInstance(ctx context.Context, alertInstance models.Al
 		if err != nil {
 			return err
 		}
-		params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix())
+		params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix(), alertInstance.ResultFingerprint)
 
 		upsertSQL := st.SQLStore.GetDialect().UpsertSQL(
 			"alert_instance",
 			[]string{"rule_org_id", "rule_uid", "labels_hash"},
-			[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time"})
+			[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "result_fingerprint"})
 		_, err = sess.SQL(upsertSQL, params...).Query()
 		if err != nil {
 			return err

pkg/services/sqlstore/migrations/ualert/tables.go

@@ -189,6 +189,10 @@ func alertInstanceMigration(mg *migrator.Migrator) {
 		migrator.NewAddColumnMigration(alertInstance, &migrator.Column{
 			Name: "current_reason", Type: migrator.DB_NVarchar, Length: DefaultFieldMaxLength, Nullable: true,
 		}))
+
+	mg.AddMigration("add result_fingerprint column to alert_instance", migrator.NewAddColumnMigration(alertInstance, &migrator.Column{
+		Name: "result_fingerprint", Type: migrator.DB_NVarchar, Length: 16, Nullable: true,
+	}))
 }
 
 func addAlertRuleMigrations(mg *migrator.Migrator, defaultIntervalSeconds int64) {

pkg/tests/api/alerting/api_ruler_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/uuid"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1315,3 +1316,77 @@ func TestIntegrationRulePause(t *testing.T) {
 		})
 	}
 }
+
+func TestIntegrationHysteresisRule(t *testing.T) {
+	testinfra.SQLiteIntegrationTest(t)
+
+	// Setup Grafana and its Database. Scheduler is set to evaluate every 1 second
+	dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		DisableLegacyAlerting:        true,
+		EnableUnifiedAlerting:        true,
+		DisableAnonymous:             true,
+		AppModeProduction:            true,
+		NGAlertSchedulerBaseInterval: 1 * time.Second,
+		EnableFeatureToggles:         []string{featuremgmt.FlagConfigurableSchedulerTick, featuremgmt.FlagRecoveryThreshold},
+	})
+
+	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, p)
+
+	// Create a user to make authenticated requests
+	createUser(t, store, user.CreateUserCommand{
+		DefaultOrgRole: string(org.RoleAdmin),
+		Password:       "password",
+		Login:          "grafana",
+	})
+
+	apiClient := newAlertingApiClient(grafanaListedAddr, "grafana", "password")
+
+	folder := "hysteresis"
+	testDs := apiClient.CreateTestDatasource(t)
+	apiClient.CreateFolder(t, folder, folder)
+
+	bodyRaw, err := testData.ReadFile("test-data/hysteresis_rule.json")
+	require.NoError(t, err)
+
+	var postData apimodels.PostableRuleGroupConfig
+	require.NoError(t, json.Unmarshal(bodyRaw, &postData))
+	for _, rule := range postData.Rules {
+		for i := range rule.GrafanaManagedAlert.Data {
+			rule.GrafanaManagedAlert.Data[i].DatasourceUID = strings.ReplaceAll(rule.GrafanaManagedAlert.Data[i].DatasourceUID, "REPLACE_ME", testDs.Body.Datasource.UID)
+		}
+	}
+	changes, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &postData)
+	require.Equalf(t, http.StatusAccepted, status, body)
+	require.Len(t, changes.Created, 1)
+	ruleUid := changes.Created[0]
+
+	var frame data.Frame
+	require.Eventuallyf(t, func() bool {
+		frame, status, body = apiClient.GetRuleHistoryWithStatus(t, ruleUid)
+		require.Equalf(t, http.StatusOK, status, body)
+		return frame.Rows() > 1
+	}, 15*time.Second, 1*time.Second, "Alert state history expected to have more than one record but got %d. Body: %s", frame.Rows(), body)
+
+	f, _ := frame.FieldByName("next")
+
+	alertingIdx := 0
+	normalIdx := 1
+	if f.At(alertingIdx).(string) != "Alerting" {
+		alertingIdx = 1
+		normalIdx = 0
+	}
+
+	assert.Equalf(t, "Alerting", f.At(alertingIdx).(string), body)
+	assert.Equalf(t, "Normal", f.At(normalIdx).(string), body)
+
+	type HistoryData struct {
+		Values map[string]int64
+	}
+
+	f, _ = frame.FieldByName("data")
+	var d HistoryData
+	require.NoErrorf(t, json.Unmarshal([]byte(f.At(alertingIdx).(string)), &d), body)
+	assert.EqualValuesf(t, 5, d.Values["B"], body)
+	require.NoErrorf(t, json.Unmarshal([]byte(f.At(normalIdx).(string)), &d), body)
+	assert.EqualValuesf(t, 1, d.Values["B"], body)
+}

pkg/tests/api/alerting/test-data/hysteresis_rule.json

@@ -0,0 +1,71 @@
+{
+  "name": "Default",
+  "interval": "1s",
+  "rules": [
+    {
+      "grafana_alert": {
+        "title": "Hysteresis Test",
+        "condition": "C",
+        "no_data_state": "NoData",
+        "exec_err_state": "Error",
+        "data": [
+          {
+            "refId": "A",
+            "datasourceUid": "REPLACE_ME",
+            "queryType": "",
+            "relativeTimeRange": {
+              "from": 600,
+              "to": 0
+            },
+            "model": {
+              "refId": "A",
+              "scenarioId": "predictable_csv_wave",
+              "csvWave": [
+                {
+                  "timeStep": 1,
+                  "valuesCSV": "5,3,2,1"
+                }
+              ],
+              "seriesCount": 1
+            }
+          },
+          {
+            "refId": "B",
+            "datasourceUid": "__expr__",
+            "model": {
+              "refId": "B",
+              "type": "reduce",
+              "reducer": "last",
+              "expression": "A"
+            }
+          },
+          {
+            "refId": "C",
+            "datasourceUid": "__expr__",
+            "model": {
+              "refId": "C",
+              "type": "threshold",
+              "conditions": [
+                {
+                  "evaluator": {
+                    "params": [
+                      4
+                    ],
+                    "type": "gt"
+                  },
+                  "unloadEvaluator": {
+                    "params": [
+                      2
+                    ],
+                    "type": "lt"
+                  }
+                }
+              ],
+              "expression": "B"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

pkg/tests/api/alerting/testing.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -695,6 +696,20 @@ func (a apiClient) UpdateRouteWithStatus(t *testing.T, route apimodels.Route) (i
 	return resp.StatusCode, string(body)
 }
 
+func (a apiClient) GetRuleHistoryWithStatus(t *testing.T, ruleUID string) (data.Frame, int, string) {
+	t.Helper()
+	u, err := url.Parse(fmt.Sprintf("%s/api/v1/rules/history", a.url))
+	require.NoError(t, err)
+	q := url.Values{}
+	q.Set("ruleUID", ruleUID)
+	u.RawQuery = q.Encode()
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	require.NoError(t, err)
+
+	return sendRequest[data.Frame](t, req, http.StatusOK)
+}
+
 func sendRequest[T any](t *testing.T, req *http.Request, successStatusCode int) (T, int, string) {
 	client := &http.Client{}
 	resp, err := client.Do(req)

pkg/tests/testinfra/testinfra.go

@@ -394,6 +394,15 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) {
 		require.NoError(t, err)
 		_, err = logSection.NewKey("query_retries", fmt.Sprintf("%d", queryRetries))
 		require.NoError(t, err)
+
+		if o.NGAlertSchedulerBaseInterval > 0 {
+			unifiedAlertingSection, err := getOrCreateSection("unified_alerting")
+			require.NoError(t, err)
+			_, err = unifiedAlertingSection.NewKey("scheduler_tick_interval", o.NGAlertSchedulerBaseInterval.String())
+			require.NoError(t, err)
+			_, err = unifiedAlertingSection.NewKey("min_interval", o.NGAlertSchedulerBaseInterval.String())
+			require.NoError(t, err)
+		}
 	}
 
 	cfgPath := filepath.Join(cfgDir, "test.ini")
@@ -419,6 +428,7 @@ type GrafanaOpts struct {
 	EnableFeatureToggles                  []string
 	NGAlertAdminConfigPollInterval        time.Duration
 	NGAlertAlertmanagerConfigPollInterval time.Duration
+	NGAlertSchedulerBaseInterval          time.Duration
 	AnonymousUserRole                     org.RoleType
 	EnableQuota                           bool
 	DashboardOrgQuota                     *int64