From 69c8200fc95935949af4fdb3cbeb009fe37a978f Mon Sep 17 00:00:00 2001
From: Yuri Tseretyan <yuriy.tseretyan@grafana.com>
Date: Wed, 9 Aug 2023 12:21:12 -0400
Subject: [PATCH] Alerting: Add more tests for state manager ProcessEvalResults
 (#73019)

Co-authored-by: Matthew Jacobson <matthew.jacobson@grafana.com>
---
 .../ngalert/state/manager_private_test.go     | 2494 +++++++++++++++++
 1 file changed, 2494 insertions(+)

diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go
index e4ddb5ff6b4..ff03833545c 100644
--- a/pkg/services/ngalert/state/manager_private_test.go
+++ b/pkg/services/ngalert/state/manager_private_test.go
@@ -2,20 +2,32 @@ package state
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/rand"
+	"sort"
 	"testing"
 	"time"
 
+	"github.com/benbjohnson/clock"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/grafana/grafana/pkg/expr"
 	"github.com/grafana/grafana/pkg/infra/log/logtest"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+
 	"github.com/grafana/grafana/pkg/util"
 )
 
+var testMetrics = metrics.NewNGAlert(prometheus.NewPedanticRegistry()).GetStateMetrics()
+
 // Not for parallel tests.
 type CountingImageService struct {
 	Called int
@@ -147,3 +159,2485 @@ func TestManager_saveAlertStates(t *testing.T) {
 		}
 	})
 }
+
+// TestProcessEvalResults_StateTransitions tests how state.Manager's ProcessEvalResults processes results and creates or changes states.
+// In other words, it tests the state transition.
+//
+// The tests use a micro-framework that has the following features:
+//  1. It uses a base rule definition and allows each test case mutate its copy.
+//  2. Expected State definition omits several fields which are patched before assertion
+//     if they are not specified explicitly (see function "patchState" for patched fields).
+//     This allows specifications to be more condense and mention only important fields.
+//  3. Expected State definition uses some shortcut functions to make the specification more clear.
+//     Expected labels are populated from a labels map where keys = description of what labels included in its values.
+//     This allows us to specify the list of labels expected to be in the state in one line, e.g. "system + rule + labels1"
+//     Evaluations are populated using function `newEvaluation` that pre-set all important fields.
+//  4. Each test case can contain multiple consecutive evaluations at different times with assertions at every interval.
+//     The framework offers variables t1, t2, t3 and function tN(n) that provide timestamps of different evaluations.
+//  5. NoData and Error tests require assertions for all possible execution options for the same input.
+//
+// # Naming convention for tests cases.
+//
+// The tests are formatted to the input characteristics, such as rule definition,
+// result format (multi- or single- dimensional) and at which times the assertions are defined.
+//
+// <time>[(<labelSet>:)<eval.State>] (and <rule_modifications>) at <asserted_time>
+//
+//	Where:
+//	  - <time> can be t1, t2 or t3, i.e. timestamp of evaluation.
+//	  - <labelSet> indicates the label set of the normal result. It can be 1,2,3 which corresponds to labels1, labels2 and labels3, or {} for results without labels.
+//	    In the case of NoData or Error labelSet is omitted
+//	  - <rule_modifications> rule modifications.
+//	  - <asserted_time> at which time intervals the test executes assertions. Can be t1,t2 or t3.
+//
+// For example:
+//
+//	t1[1:normal] t2[1:alerting] and 'for'=2 at t2
+//	t1[{}:alerting] t2[{}:normal] t3[NoData] at t2,t3
+func TestProcessEvalResults_StateTransitions(t *testing.T) {
+	evaluationDuration := 10 * time.Millisecond
+	evaluationInterval := 10 * time.Second
+
+	tN := func(n int) time.Time {
+		return time.Time{}.Add(time.Duration(n) * evaluationInterval)
+	}
+	t1 := tN(1)
+	t2 := tN(2)
+	t3 := tN(3)
+
+	baseRule := &ngmodels.AlertRule{
+		OrgID: 1,
+		Title: "test_title",
+		UID:   "test_alert_rule_uid",
+		Data: []ngmodels.AlertQuery{{
+			RefID:         "A",
+			DatasourceUID: "datasource_uid_1",
+		}, {
+			RefID:         "B",
+			DatasourceUID: expr.DatasourceType,
+		}},
+		NamespaceUID:    "test_namespace_uid",
+		Annotations:     map[string]string{"annotation": "test"},
+		Labels:          map[string]string{"label": "test"},
+		IntervalSeconds: int64(evaluationInterval.Seconds()),
+		NoDataState:     ngmodels.NoData,
+		ExecErrState:    ngmodels.ErrorErrState,
+	}
+
+	baseRuleWith := func(mutators ...ngmodels.AlertRuleMutator) *ngmodels.AlertRule {
+		r := ngmodels.CopyRule(baseRule)
+		for _, mutator := range mutators {
+			mutator(r)
+		}
+		return r
+	}
+
+	newEvaluation := func(evalTime time.Time, evalState eval.State) Evaluation {
+		return Evaluation{
+			EvaluationTime:  evalTime,
+			EvaluationState: evalState,
+			Values:          make(map[string]*float64),
+		}
+	}
+
+	newResult := func(mutators ...eval.ResultMutator) eval.Result {
+		r := eval.Result{
+			State:              eval.Normal,
+			EvaluationDuration: evaluationDuration,
+		}
+		for _, mutator := range mutators {
+			mutator(&r)
+		}
+		return r
+	}
+
+	genericError := errors.New("test-error")
+	datasourceError := expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))
+	expectedDatasourceErrorLabels := data.Labels{
+		"datasource_uid": "datasource_uid_1",
+		"ref_id":         "A",
+	}
+
+	labels1 := data.Labels{
+		"instance_label": "test-1",
+	}
+	labels2 := data.Labels{
+		"instance_label": "test-2",
+	}
+	labels3 := data.Labels{
+		"instance_label": "test-3",
+	}
+	systemLabels := data.Labels{
+		"system": "owned",
+	}
+	noDataLabels := data.Labels{
+		"datasource_uid": "1",
+		"ref_id":         "A",
+	}
+
+	labels := map[string]data.Labels{
+		"system + rule":                    mergeLabels(baseRule.Labels, systemLabels),
+		"system + rule + labels1":          mergeLabels(mergeLabels(labels1, baseRule.Labels), systemLabels),
+		"system + rule + labels2":          mergeLabels(mergeLabels(labels2, baseRule.Labels), systemLabels),
+		"system + rule + labels3":          mergeLabels(mergeLabels(labels3, baseRule.Labels), systemLabels),
+		"system + rule + no-data":          mergeLabels(mergeLabels(noDataLabels, baseRule.Labels), systemLabels),
+		"system + rule + datasource-error": mergeLabels(mergeLabels(expectedDatasourceErrorLabels, baseRule.Labels), systemLabels),
+	}
+
+	patchState := func(r *ngmodels.AlertRule, s *State) {
+		// patch all optional fields of the expected state
+		setCacheID(s)
+		if s.AlertRuleUID == "" {
+			s.AlertRuleUID = r.UID
+		}
+		if s.OrgID == 0 {
+			s.OrgID = r.OrgID
+		}
+		if s.Annotations == nil {
+			s.Annotations = r.Annotations
+		}
+		if s.EvaluationDuration == 0 {
+			s.EvaluationDuration = evaluationDuration
+		}
+		if s.Values == nil {
+			s.Values = make(map[string]float64)
+		}
+	}
+
+	executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition) {
+		clk := clock.NewMock()
+
+		cfg := ManagerCfg{
+			Metrics:                 testMetrics,
+			ExternalURL:             nil,
+			InstanceStore:           &FakeInstanceStore{},
+			Images:                  &NotAvailableImageService{},
+			Clock:                   clk,
+			Historian:               &FakeHistorian{},
+			MaxStateSaveConcurrency: 1,
+		}
+		st := NewManager(cfg)
+
+		tss := make([]time.Time, 0, len(resultsAtTime))
+		for ts, results := range resultsAtTime {
+			for i := range results {
+				results[i].EvaluatedAt = ts
+			}
+			tss = append(tss, ts)
+		}
+		sort.Slice(tss, func(i, j int) bool {
+			return tss[i].Before(tss[j])
+		})
+
+		for _, ts := range tss {
+			results := resultsAtTime[ts]
+			clk.Set(ts)
+			actual := st.ProcessEvalResults(context.Background(), ts, alertRule, results, systemLabels)
+
+			expectedTransitions, ok := expectedTransitionsAtTime[ts]
+			if !ok { // skip if nothing to assert
+				continue
+			}
+			expectedTransitionsMap := make(map[string]StateTransition, len(expectedTransitions))
+			for i := range expectedTransitions {
+				patchState(alertRule, expectedTransitions[i].State)
+				expectedTransitionsMap[expectedTransitions[i].CacheID] = expectedTransitions[i]
+			}
+
+			tn := ts.Sub(t1)/evaluationInterval + 1
+			for _, transition := range actual {
+				expected, ok := expectedTransitionsMap[transition.CacheID]
+				if !ok {
+					assert.Failf(t, fmt.Sprintf("transition is not expected at time [t%d]", tn), "CacheID: %s.\nTransition: %s->%s", transition.CacheID, transition.PreviousFormatted(), transition.Formatted())
+				}
+				delete(expectedTransitionsMap, transition.CacheID)
+				if !assert.ObjectsAreEqual(expected, transition) {
+					assert.Failf(t, fmt.Sprintf("expected and actual transitions at time [t%d] are not equal", tn), "CacheID: %s\nDiff: %s", transition.CacheID, cmp.Diff(expected, transition, cmpopts.EquateErrors()))
+				}
+			}
+			if len(expectedTransitionsMap) > 0 {
+				vals := make([]string, 0, len(expectedTransitionsMap))
+				for _, s := range expectedTransitionsMap {
+					vals = append(vals, s.CacheID)
+				}
+				assert.Failf(t, fmt.Sprintf("some expected states do not exist at time [t%d]", tn), "States: %v", vals)
+			}
+		}
+	}
+
+	type testCase struct {
+		desc                string
+		alertRule           *ngmodels.AlertRule
+		results             map[time.Time]eval.Results
+		expectedTransitions map[time.Time][]StateTransition
+	}
+
+	testCases := []testCase{
+		{
+			desc:      "t1[1:normal,2:normal] at t1",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels2)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t1,
+						},
+					},
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels2"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[1:alerting,2:normal] at t1",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels2)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Alerting,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1.Add(ResendDelay * 3),
+							LastEvaluationTime: t1,
+						},
+					},
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels2"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[1:alerting,2:normal] and 'for'>0 at t1",
+			alertRule: baseRuleWith(ngmodels.WithForNTimes(3)),
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels2)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Pending,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1.Add(ResendDelay * 3),
+							LastEvaluationTime: t1,
+						},
+					},
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels2"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[1:normal,2:normal] t2[1:alerting,2:normal] at t2",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels2)),
+				},
+				t2: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels2)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t2: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Alerting,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+								newEvaluation(t2, eval.Alerting),
+							},
+							StartsAt:           t2,
+							EndsAt:             t2.Add(ResendDelay * 3),
+							LastEvaluationTime: t2,
+						},
+					},
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels2"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+								newEvaluation(t2, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t2,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[1:alerting] t2[1:alerting] t3[1:alerting] and 'for'=2 at t1,t2,t3",
+			alertRule: baseRuleWith(ngmodels.WithForNTimes(2)),
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+				},
+				t2: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+				},
+				t3: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Pending,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1.Add(ResendDelay * 3),
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+				t2: {
+					{
+						PreviousState: eval.Pending,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Pending,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+								newEvaluation(t2, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1.Add(ResendDelay * 3), // TODO probably it should be t1 (semantic of Normal)?
+							LastEvaluationTime: t2,
+						},
+					},
+				},
+				t3: {
+					{
+						PreviousState: eval.Pending,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Alerting,
+							Results: []Evaluation{
+								newEvaluation(t2, eval.Alerting),
+								newEvaluation(t3, eval.Alerting),
+							},
+							StartsAt:           t3,
+							EndsAt:             t3.Add(ResendDelay * 3),
+							LastEvaluationTime: t3,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[1:alerting], t2[1:normal] and 'for'=2 at t2",
+			alertRule: baseRuleWith(ngmodels.WithForNTimes(2)),
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+				},
+				t2: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t2: {
+					{
+						PreviousState: eval.Pending,
+						State: &State{
+							Labels: labels["system + rule + labels1"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+								newEvaluation(t2, eval.Normal),
+							},
+							StartsAt:           t2,
+							EndsAt:             t2,
+							LastEvaluationTime: t2,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1:[1:alerting] t2[1:normal] at t2",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+				},
+				t2: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t2: {{
+					PreviousState: eval.Alerting,
+					State: &State{
+						Labels: labels["system + rule + labels1"],
+						State:  eval.Normal,
+						Results: []Evaluation{
+							newEvaluation(t1, eval.Alerting),
+							newEvaluation(t2, eval.Normal),
+						},
+						StartsAt:           t2,
+						EndsAt:             t2,
+						LastEvaluationTime: t2,
+						Resolved:           true,
+					},
+				},
+				},
+			},
+		},
+		{
+			desc:      "t1[1:normal,2:alerting,3:normal] t2[3:normal] t3[3:normal] at t2,t3",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels2)),
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels3)),
+				},
+				t2: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels3)),
+				},
+				t3: {
+					newResult(eval.WithState(eval.Normal), eval.WithLabels(labels3)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t2: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels3"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+								newEvaluation(t2, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t2,
+						},
+					},
+				},
+				t3: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels:      labels["system + rule + labels1"],
+							State:       eval.Normal,
+							StateReason: ngmodels.StateReasonMissingSeries,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t3,
+							LastEvaluationTime: t3,
+						},
+					},
+					{
+						PreviousState: eval.Alerting,
+						State: &State{
+							Labels:      labels["system + rule + labels2"],
+							State:       eval.Normal,
+							StateReason: ngmodels.StateReasonMissingSeries,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t3,
+							LastEvaluationTime: t3,
+							Resolved:           true,
+						},
+					},
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule + labels3"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+								newEvaluation(t2, eval.Normal),
+								newEvaluation(t3, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t3,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[{}:normal] at t1",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Normal)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule"],
+							State:  eval.Normal,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1,
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[{}:alerting] at t1",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule"],
+							State:  eval.Alerting,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1.Add(ResendDelay * 3),
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[{}:alerting] and 'for'>0  at t1",
+			alertRule: baseRuleWith(ngmodels.WithForNTimes(3)),
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Alerting)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t1: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule"],
+							State:  eval.Pending,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Alerting),
+							},
+							StartsAt:           t1,
+							EndsAt:             t1.Add(ResendDelay * 3),
+							LastEvaluationTime: t1,
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:      "t1[{}:normal] t2[{}:alerting] at t2",
+			alertRule: baseRule,
+			results: map[time.Time]eval.Results{
+				t1: {
+					newResult(eval.WithState(eval.Normal)),
+				},
+				t2: {
+					newResult(eval.WithState(eval.Alerting)),
+				},
+			},
+			expectedTransitions: map[time.Time][]StateTransition{
+				t2: {
+					{
+						PreviousState: eval.Normal,
+						State: &State{
+							Labels: labels["system + rule"],
+							State:  eval.Alerting,
+							Results: []Evaluation{
+								newEvaluation(t1, eval.Normal),
+								newEvaluation(t2, eval.Alerting),
+							},
+							StartsAt:           t2,
+							EndsAt:             t2.Add(ResendDelay * 3),
+							LastEvaluationTime: t2,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions)
+		})
+	}
+
+	t.Run("no-data", func(t *testing.T) {
+		rules := map[ngmodels.NoDataState]*ngmodels.AlertRule{
+			ngmodels.NoData:   baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.NoData)),
+			ngmodels.Alerting: baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.Alerting)),
+			ngmodels.OK:       baseRuleWith(ngmodels.WithNoDataExecAs(ngmodels.OK)),
+		}
+
+		type noDataTestCase struct {
+			desc                string
+			ruleMutators        []ngmodels.AlertRuleMutator
+			results             map[time.Time]eval.Results
+			expectedTransitions map[ngmodels.NoDataState]map[time.Time][]StateTransition
+		}
+
+		executeForEachRule := func(t *testing.T, tc noDataTestCase) {
+			t.Helper()
+			for stateExec, rule := range rules {
+				r := rule
+				if len(tc.ruleMutators) > 0 {
+					r = ngmodels.CopyRule(r)
+					for _, mutateRule := range tc.ruleMutators {
+						mutateRule(r)
+					}
+				}
+				t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) {
+					expectedTransitions, ok := tc.expectedTransitions[stateExec]
+					if !ok {
+						require.Fail(t, "no expected state transitions")
+					}
+					executeTest(t, r, tc.results, expectedTransitions)
+				})
+			}
+		}
+
+		testCases := []noDataTestCase{
+			{
+				desc: "t1[NoData] at t1",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[1:normal] t2[NoData] at t2",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[1:normal,2:alerting] t2[NoData] t3[NoData] at t3",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+						newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels2)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+							{
+								PreviousState: eval.NoData,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc:         "t1[1:normal,2:alerting] t2[NoData] t3[NoData] and 'for'=1 at t2*,t3",
+				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(1)},
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+						newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels2)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.NoData,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Pending,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Pending,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc:         "t1[1:alerting] t2[NoData] t3[1:alerting] and 'for'=2 at t3",
+				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(2)},
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t3: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[NoData] t2[1:normal] t3[1:normal] at t3",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Normal),
+										newEvaluation(t3, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.NoData,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Normal),
+										newEvaluation(t3, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Normal),
+										newEvaluation(t3, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[{}:normal] t2[NoData] at t2",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Normal)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t3",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Alerting)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t3: {
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+							{
+								PreviousState: eval.NoData,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + no-data"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc:         "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t3",
+				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(2)},
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Alerting)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.NoData), eval.WithLabels(noDataLabels)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.Alerting)),
+					},
+				},
+				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.NoData: {
+						t3: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.desc, func(t *testing.T) {
+				executeForEachRule(t, tc)
+			})
+		}
+	})
+
+	t.Run("error", func(t *testing.T) {
+		rules := map[ngmodels.ExecutionErrorState]*ngmodels.AlertRule{
+			ngmodels.ErrorErrState:    baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.ErrorErrState)),
+			ngmodels.AlertingErrState: baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.AlertingErrState)),
+			ngmodels.OkErrState:       baseRuleWith(ngmodels.WithErrorExecAs(ngmodels.OkErrState)),
+		}
+
+		cacheID := func(lbls data.Labels) string {
+			l := ngmodels.InstanceLabels(lbls)
+			r, err := l.StringKey()
+			if err != nil {
+				panic(err)
+			}
+			return r
+		}
+
+		type errorTestCase struct {
+			desc                string
+			ruleMutators        []ngmodels.AlertRuleMutator
+			results             map[time.Time]eval.Results
+			expectedTransitions map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition
+		}
+
+		executeForEachRule := func(t *testing.T, tc errorTestCase) {
+			t.Helper()
+			for stateExec, rule := range rules {
+				r := rule
+				if len(tc.ruleMutators) > 0 {
+					r = ngmodels.CopyRule(r)
+					for _, mutateRule := range tc.ruleMutators {
+						mutateRule(r)
+					}
+				}
+				t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) {
+					expectedTransitions, ok := tc.expectedTransitions[stateExec]
+					if !ok {
+						require.Fail(t, "no expected state transitions")
+					}
+					executeTest(t, r, tc.results, expectedTransitions)
+				})
+			}
+		}
+
+		testCases := []errorTestCase{
+			{
+				desc: "t1[QueryError] at t1",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithError(datasourceError)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									CacheID: cacheID(labels["system + rule"]),
+									Labels:  labels["system + rule + datasource-error"],
+									State:   eval.Error,
+									Error:   datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[GenericError] at t1",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithError(genericError)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Error,
+									Error:  genericError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": genericError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       genericError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc:         "t1[1:alerting] t2[QueryError] and 'for'=1 at t2",
+				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(1)},
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
+					},
+					t2: {
+						newResult(eval.WithError(datasourceError)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									CacheID: cacheID(labels["system + rule"]),
+									Labels:  labels["system + rule + datasource-error"],
+									State:   eval.Error,
+									Error:   datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Pending,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[1:normal] t2[QueryError] at t2",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					},
+					t2: {
+						newResult(eval.WithError(datasourceError)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									CacheID: cacheID(labels["system + rule"]),
+									Labels:  labels["system + rule + datasource-error"],
+									State:   eval.Error,
+									Error:   datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[QueryError] t2[1:normal] t3[1:normal] at t3",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithError(datasourceError)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Normal),
+										newEvaluation(t3, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState: eval.Error,
+								State: &State{
+									CacheID:     cacheID(labels["system + rule"]),
+									Labels:      labels["system + rule + datasource-error"],
+									Error:       datasourceError,
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Normal),
+										newEvaluation(t3, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.Error.String(),
+								State: &State{
+									Labels:      labels["system + rule"],
+									Error:       datasourceError,
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+									Resolved:           true,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t3: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Normal),
+										newEvaluation(t3, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.Error.String(),
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: ngmodels.StateReasonMissingSeries,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[{}:normal] t2[QueryError] at t2",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Normal)),
+					},
+					t2: {
+						newResult(eval.WithError(datasourceError)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									CacheID: cacheID(labels["system + rule"]),
+									Labels:  labels["system + rule + datasource-error"],
+									State:   eval.Error,
+									Error:   datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc:         "t1[{}:alerting] t2[QueryError] and 'for'=1 at t1*,t2",
+				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(1)},
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Alerting)),
+					},
+					t2: {
+						newResult(eval.WithError(datasourceError)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t1: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Pending,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3),
+									LastEvaluationTime: t1,
+								},
+							},
+						},
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									CacheID: cacheID(labels["system + rule"]),
+									Labels:  labels["system + rule + datasource-error"],
+									State:   eval.Error,
+									Error:   datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc:         "t1[{}:alerting] t2[QueryError] t3[{}:alerting] and 'for'=2 at t2,t3",
+				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(2)},
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithState(eval.Alerting)),
+					},
+					t2: {
+						newResult(eval.WithError(datasourceError)),
+					},
+					t3: {
+						newResult(eval.WithState(eval.Alerting)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									CacheID: cacheID(labels["system + rule"]),
+									Labels:  labels["system + rule + datasource-error"],
+									State:   eval.Error,
+									Error:   datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+									Annotations: mergeLabels(baseRule.Annotations, data.Labels{
+										"Error": datasourceError.Error(),
+									}),
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState: eval.Error,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Pending,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Pending,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1.Add(ResendDelay * 3), // TODO probably it should be t1 (semantic of Normal)?
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Pending,
+								PreviousStateReason: eval.Error.String(),
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.Error.String(),
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Pending,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				desc: "t1[QueryError] t2[{}:normal] at t2",
+				results: map[time.Time]eval.Results{
+					t1: {
+						newResult(eval.WithError(datasourceError)),
+					},
+					t2: {
+						newResult(eval.WithState(eval.Normal)),
+					},
+				},
+				expectedTransitions: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.ErrorErrState: {
+						t2: {
+							{
+								PreviousState: eval.Error,
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+										newEvaluation(t2, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.Error.String(),
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+										newEvaluation(t2, eval.Normal),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+									Resolved:           true,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.Error.String(),
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Normal,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Error),
+										newEvaluation(t2, eval.Normal),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.desc, func(t *testing.T) {
+				executeForEachRule(t, tc)
+			})
+		}
+	})
+}
+
+func setCacheID(s *State) *State {
+	if s.CacheID != "" {
+		return s
+	}
+	il := ngmodels.InstanceLabels(s.Labels)
+	id, err := il.StringKey()
+	if err != nil {
+		panic(err)
+	}
+	s.CacheID = id
+	return s
+}