diff --git a/pkg/services/ngalert/models/alert_rule.go b/pkg/services/ngalert/models/alert_rule.go index 803bc65ba4f..b497b101e4d 100644 --- a/pkg/services/ngalert/models/alert_rule.go +++ b/pkg/services/ngalert/models/alert_rule.go @@ -124,6 +124,9 @@ func (r RuleType) String() string { } const ( + // When adding new constants with __*__ pattern that are labels, + // you need to add them to PrivateLabelsToFilter below if they should be filtered out from recording rules. + // Annotations are actually a set of labels, so technically this is the label name of an annotation. DashboardUIDAnnotation = "__dashboardUid__" PanelIDAnnotation = "__panelId__" @@ -199,6 +202,15 @@ var ( AutogeneratedRouteReceiverNameLabel: {}, AutogeneratedRouteSettingsHashLabel: {}, } + + // PrivateLabelsToFilter are internal labels that should be filtered out from recording rules. + // These labels are used internally by Grafana and should not appear in the final metrics. + PrivateLabelsToFilter = map[string]struct{}{ + ConvertedPrometheusRuleLabel: {}, + AutogeneratedRouteLabel: {}, + AutogeneratedRouteReceiverNameLabel: {}, + AutogeneratedRouteSettingsHashLabel: {}, + } ) // AlertRuleGroup is the base model for a rule group in unified alerting. @@ -407,6 +419,25 @@ func WithoutInternalLabels() LabelOption { } } +// WithoutPrivateLabels returns a new map without private labels. +// It filters out labels that are in PrivateLabelsToFilter set. +func WithoutPrivateLabels(labels map[string]string) map[string]string { + if labels == nil { + return nil + } + + result := make(map[string]string, len(labels)) + for k, v := range labels { + // Check if it's in the explicit filter list + if _, shouldFilter := PrivateLabelsToFilter[k]; shouldFilter { + continue + } + + result[k] = v + } + return result +} + func (alertRule *AlertRule) ImportedPrometheusRule() bool { _, hasConvertedPrometheusRuleLabel := alertRule.GetLabels()[ConvertedPrometheusRuleLabel] return hasConvertedPrometheusRuleLabel || alertRule.HasPrometheusRuleDefinition() diff --git a/pkg/services/ngalert/models/alert_rule_test.go b/pkg/services/ngalert/models/alert_rule_test.go index 326bd40cb4a..f6935ececbb 100644 --- a/pkg/services/ngalert/models/alert_rule_test.go +++ b/pkg/services/ngalert/models/alert_rule_test.go @@ -1348,3 +1348,54 @@ func TestAlertRule_ImportedPrometheusRule(t *testing.T) { }) } } + +func TestWithoutPrivateLabels(t *testing.T) { + tests := []struct { + name string + input map[string]string + expected map[string]string + }{ + { + name: "nil map", + input: nil, + expected: nil, + }, + { + name: "empty map", + input: map[string]string{}, + expected: map[string]string{}, + }, + { + name: "removes only specific private labels", + input: map[string]string{ + ConvertedPrometheusRuleLabel: "removed", + AutogeneratedRouteLabel: "removed", + AutogeneratedRouteReceiverNameLabel: "removed", + AutogeneratedRouteSettingsHashLabel: "removed", + DashboardUIDAnnotation: "kept", + PanelIDAnnotation: "kept", + "__custom_label__": "kept", // User-defined labels with __ are kept + "normal_label": "kept", + "another_label": "kept", + }, + expected: map[string]string{ + "__custom_label__": "kept", + "normal_label": "kept", + "another_label": "kept", + DashboardUIDAnnotation: "kept", + PanelIDAnnotation: "kept", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + inputCopy := maps.Clone(tt.input) + + result := WithoutPrivateLabels(tt.input) + + require.Equal(t, tt.expected, result) + require.Equal(t, inputCopy, tt.input, "input map should not be modified") + }) + } +} diff --git a/pkg/services/ngalert/schedule/recording_rule.go b/pkg/services/ngalert/schedule/recording_rule.go index f0992873981..d931c502327 100644 --- a/pkg/services/ngalert/schedule/recording_rule.go +++ b/pkg/services/ngalert/schedule/recording_rule.go @@ -272,8 +272,9 @@ func (r *recordingRule) tryEvaluation(ctx context.Context, ev *Evaluation, logge return nil } + filteredLabels := ngmodels.WithoutPrivateLabels(ev.rule.Labels) writeStart := r.clock.Now() - err = r.writer.WriteDatasource(ctx, ev.rule.Record.TargetDatasourceUID, ev.rule.Record.Metric, ev.scheduledAt, frames, ev.rule.OrgID, ev.rule.Labels) + err = r.writer.WriteDatasource(ctx, ev.rule.Record.TargetDatasourceUID, ev.rule.Record.Metric, ev.scheduledAt, frames, ev.rule.OrgID, filteredLabels) writeDur := r.clock.Now().Sub(writeStart) if err != nil { diff --git a/pkg/services/ngalert/schedule/recording_rule_test.go b/pkg/services/ngalert/schedule/recording_rule_test.go index 56cdf939dca..115da2e73d6 100644 --- a/pkg/services/ngalert/schedule/recording_rule_test.go +++ b/pkg/services/ngalert/schedule/recording_rule_test.go @@ -750,6 +750,54 @@ func testRecordingRule_Integration(t *testing.T, writeTarget *writer.TestRemoteW require.Equal(t, "error", status.Health) }) }) + + t.Run("rule with private labels filtered", func(t *testing.T) { + writeTarget.Reset() + rule := gen.With(withQueryForHealth("ok")).GenerateRef() + rule.Record.TargetDatasourceUID = dsUID + rule.Labels = map[string]string{ + "normal_label": "value1", + "another_label": "value2", + models.AutogeneratedRouteLabel: "filtered", + models.AutogeneratedRouteReceiverNameLabel: "filtered", + "__user_custom__": "not_filtered", + "only_end__": "not_filtered", + } + + ruleStore.PutRule(context.Background(), rule) + folderTitle := ruleStore.getNamespaceTitle(rule.NamespaceUID) + ruleFactory := ruleFactoryFromScheduler(sch) + + process := ruleFactory.new(context.Background(), rule) + evalDoneChan := make(chan time.Time) + process.(*recordingRule).evalAppliedHook = func(_ models.AlertRuleKey, t time.Time) { + evalDoneChan <- t + } + now := time.Now() + + go func() { + _ = process.Run() + }() + + process.Eval(&Evaluation{ + scheduledAt: now, + rule: rule, + folderTitle: folderTitle, + }) + _ = waitForTimeChannel(t, evalDoneChan) + + t.Run("write was performed with filtered labels", func(t *testing.T) { + require.Equal(t, 1, writeTarget.RequestsCount) + require.NotEmpty(t, writeTarget.LastRequestBody) + + // Check that the body doesn't contain the private labels + require.NotContains(t, writeTarget.LastRequestBody, models.AutogeneratedRouteLabel) + require.NotContains(t, writeTarget.LastRequestBody, models.AutogeneratedRouteReceiverNameLabel) + + require.Contains(t, writeTarget.LastRequestBody, "__user_custom__") + require.Contains(t, writeTarget.LastRequestBody, rule.Record.Metric) + }) + }) } func withQueryForHealth(health string) models.AlertRuleMutator {