public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Include a list of ref_Id and aggregated datasource UIDs to alerts when state reason is NoData (#88819)

Browse Source 
* include a list of ref_Id and datasource UID to alerts when state reason is NoData. 

---------

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>
This commit is contained in:
Nihal
2024-07-30 12:55:59 -04:00
committed by GitHub
parent ac0b4bb34d
commit 9ad9b4989b
2 changed files with 114 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/state/manager.go
+27 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"net/url"
"strconv"
"strings"
"time"
"github.com/benbjohnson/clock"
@@ -369,9 +370,34 @@ func (st *Manager) updateLastSentAt(states StateTransitions, evaluatedAt time.Ti
func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger) []StateTransition {
if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK || alertRule.NoDataState == ngModels.KeepLast) { // If it is no data, check the mapping and switch all results to the new state
// TODO aggregate UID of datasources that returned NoData into one and provide as auxiliary info, probably annotation
// aggregate UID of datasources that returned NoData into one and provide as auxiliary info via annotationa. See: https://github.com/grafana/grafana/issues/88184
var refIds strings.Builder
var datasourceUIDs strings.Builder
// for deduplication of datasourceUIDs
dsUIDSet := make(map[string]bool)
for i, result := range results {
if refid, ok := result.Instance["ref_id"]; ok {
if i > 0 {
refIds.WriteString(",")
}
refIds.WriteString(refid)
}
if dsUID, ok := result.Instance["datasource_uid"]; ok {
if !dsUIDSet[dsUID] {
if i > 0 {
refIds.WriteString(",")
}
datasourceUIDs.WriteString(dsUID)
dsUIDSet[dsUID] = true
}
}
}
transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger)
if len(transitions) > 0 {
for _, t := range transitions {
t.State.Annotations["datasource_uid"] = datasourceUIDs.String()
t.State.Annotations["ref_id"] = refIds.String()
}
return transitions // if there are no current states for the rule. Create ones for each result
}
}
pkg/services/ngalert/state/manager_private_test.go
+87 -1
View File
@@ -197,6 +197,11 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
"ref_id": "A",
}
noDataAnnotations := data.Labels{
"datasource_uid": "1",
"ref_id": "A",
}
labels := map[string]data.Labels{
"system + rule": mergeLabels(baseRule.Labels, systemLabels),
"system + rule + labels1": mergeLabels(mergeLabels(labels1, baseRule.Labels), systemLabels),
@@ -986,6 +991,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}),
@@ -1004,6 +1010,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}),
@@ -1021,6 +1028,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluationWithValues(t2, eval.NoData, map[string]float64{"A": float64(-1)}),
@@ -1243,6 +1251,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1256,6 +1265,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1273,6 +1283,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
State: &State{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t2,
@@ -1287,6 +1298,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
State: &State{
Labels: labels["system + rule + labels2"],
State: eval.Alerting,
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
StartsAt: t1,
@@ -1303,6 +1315,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1315,6 +1328,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1332,6 +1346,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1345,6 +1360,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1363,6 +1379,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1375,6 +1392,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1391,6 +1409,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1404,6 +1423,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1633,6 +1653,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Pending,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1645,6 +1666,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1661,6 +1683,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1675,6 +1698,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1692,6 +1716,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1704,6 +1729,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1719,6 +1745,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1732,6 +1759,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1748,6 +1776,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1760,6 +1789,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -1776,6 +1806,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels1"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -1789,6 +1820,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule + labels2"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -2149,6 +2181,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -2166,6 +2199,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -2182,6 +2216,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Normal,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -2354,6 +2389,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -2370,6 +2406,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -2387,6 +2424,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -2404,6 +2442,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: eval.NoData.String(),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -2422,6 +2461,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousState: eval.Alerting,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
@@ -2438,6 +2478,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
PreviousStateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Alerting,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t3, eval.NoData),
@@ -2452,7 +2493,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
{
desc: "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t2*,t3",
desc: "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t2,t3",
ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.RuleMuts.WithForNTimes(2)},
results: map[time.Time]eval.Results{
t1: {
@@ -2547,6 +2588,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
ngmodels.Alerting: {
t2: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Pending,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Pending,
@@ -2564,6 +2620,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
ngmodels.OK: {
t2: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Normal,
@@ -2580,6 +2651,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
},
},
ngmodels.KeepLast: {
t2: {
{
PreviousState: eval.Pending,
State: &State{
Labels: labels["system + rule"],
Annotations: mergeLabels(baseRule.Annotations, noDataAnnotations),
State: eval.Pending,
StateReason: ngmodels.ConcatReasons(eval.NoData.String(), ngmodels.StateReasonKeepLast),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
},
t3: {
{
PreviousState: eval.Pending,