Alerting: Filter out private labels before writing recording rules (#109295)

pkg/services/ngalert/models/alert_rule.go

@@ -124,6 +124,9 @@ func (r RuleType) String() string {
 }
 
 const (
+	// When adding new constants with __*__ pattern that are labels,
+	// you need to add them to PrivateLabelsToFilter below if they should be filtered out from recording rules.
+
 	// Annotations are actually a set of labels, so technically this is the label name of an annotation.
 	DashboardUIDAnnotation = "__dashboardUid__"
 	PanelIDAnnotation      = "__panelId__"
@@ -199,6 +202,15 @@ var (
 		AutogeneratedRouteReceiverNameLabel: {},
 		AutogeneratedRouteSettingsHashLabel: {},
 	}
+
+	// PrivateLabelsToFilter are internal labels that should be filtered out from recording rules.
+	// These labels are used internally by Grafana and should not appear in the final metrics.
+	PrivateLabelsToFilter = map[string]struct{}{
+		ConvertedPrometheusRuleLabel:        {},
+		AutogeneratedRouteLabel:             {},
+		AutogeneratedRouteReceiverNameLabel: {},
+		AutogeneratedRouteSettingsHashLabel: {},
+	}
 )
 
 // AlertRuleGroup is the base model for a rule group in unified alerting.
@@ -407,6 +419,25 @@ func WithoutInternalLabels() LabelOption {
 	}
 }
 
+// WithoutPrivateLabels returns a new map without private labels.
+// It filters out labels that are in PrivateLabelsToFilter set.
+func WithoutPrivateLabels(labels map[string]string) map[string]string {
+	if labels == nil {
+		return nil
+	}
+
+	result := make(map[string]string, len(labels))
+	for k, v := range labels {
+		// Check if it's in the explicit filter list
+		if _, shouldFilter := PrivateLabelsToFilter[k]; shouldFilter {
+			continue
+		}
+
+		result[k] = v
+	}
+	return result
+}
+
 func (alertRule *AlertRule) ImportedPrometheusRule() bool {
 	_, hasConvertedPrometheusRuleLabel := alertRule.GetLabels()[ConvertedPrometheusRuleLabel]
 	return hasConvertedPrometheusRuleLabel || alertRule.HasPrometheusRuleDefinition()

pkg/services/ngalert/models/alert_rule_test.go

@@ -1348,3 +1348,54 @@ func TestAlertRule_ImportedPrometheusRule(t *testing.T) {
 		})
 	}
 }
+
+func TestWithoutPrivateLabels(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    map[string]string
+		expected map[string]string
+	}{
+		{
+			name:     "nil map",
+			input:    nil,
+			expected: nil,
+		},
+		{
+			name:     "empty map",
+			input:    map[string]string{},
+			expected: map[string]string{},
+		},
+		{
+			name: "removes only specific private labels",
+			input: map[string]string{
+				ConvertedPrometheusRuleLabel:        "removed",
+				AutogeneratedRouteLabel:             "removed",
+				AutogeneratedRouteReceiverNameLabel: "removed",
+				AutogeneratedRouteSettingsHashLabel: "removed",
+				DashboardUIDAnnotation:              "kept",
+				PanelIDAnnotation:                   "kept",
+				"__custom_label__":                  "kept", // User-defined labels with __ are kept
+				"normal_label":                      "kept",
+				"another_label":                     "kept",
+			},
+			expected: map[string]string{
+				"__custom_label__":     "kept",
+				"normal_label":         "kept",
+				"another_label":        "kept",
+				DashboardUIDAnnotation: "kept",
+				PanelIDAnnotation:      "kept",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			inputCopy := maps.Clone(tt.input)
+
+			result := WithoutPrivateLabels(tt.input)
+
+			require.Equal(t, tt.expected, result)
+			require.Equal(t, inputCopy, tt.input, "input map should not be modified")
+		})
+	}
+}

pkg/services/ngalert/schedule/recording_rule.go

@@ -272,8 +272,9 @@ func (r *recordingRule) tryEvaluation(ctx context.Context, ev *Evaluation, logge
 		return nil
 	}
 
+	filteredLabels := ngmodels.WithoutPrivateLabels(ev.rule.Labels)
 	writeStart := r.clock.Now()
-	err = r.writer.WriteDatasource(ctx, ev.rule.Record.TargetDatasourceUID, ev.rule.Record.Metric, ev.scheduledAt, frames, ev.rule.OrgID, ev.rule.Labels)
+	err = r.writer.WriteDatasource(ctx, ev.rule.Record.TargetDatasourceUID, ev.rule.Record.Metric, ev.scheduledAt, frames, ev.rule.OrgID, filteredLabels)
 	writeDur := r.clock.Now().Sub(writeStart)
 
 	if err != nil {

pkg/services/ngalert/schedule/recording_rule_test.go

@@ -750,6 +750,54 @@ func testRecordingRule_Integration(t *testing.T, writeTarget *writer.TestRemoteW
 			require.Equal(t, "error", status.Health)
 		})
 	})
+
+	t.Run("rule with private labels filtered", func(t *testing.T) {
+		writeTarget.Reset()
+		rule := gen.With(withQueryForHealth("ok")).GenerateRef()
+		rule.Record.TargetDatasourceUID = dsUID
+		rule.Labels = map[string]string{
+			"normal_label":                             "value1",
+			"another_label":                            "value2",
+			models.AutogeneratedRouteLabel:             "filtered",
+			models.AutogeneratedRouteReceiverNameLabel: "filtered",
+			"__user_custom__":                          "not_filtered",
+			"only_end__":                               "not_filtered",
+		}
+
+		ruleStore.PutRule(context.Background(), rule)
+		folderTitle := ruleStore.getNamespaceTitle(rule.NamespaceUID)
+		ruleFactory := ruleFactoryFromScheduler(sch)
+
+		process := ruleFactory.new(context.Background(), rule)
+		evalDoneChan := make(chan time.Time)
+		process.(*recordingRule).evalAppliedHook = func(_ models.AlertRuleKey, t time.Time) {
+			evalDoneChan <- t
+		}
+		now := time.Now()
+
+		go func() {
+			_ = process.Run()
+		}()
+
+		process.Eval(&Evaluation{
+			scheduledAt: now,
+			rule:        rule,
+			folderTitle: folderTitle,
+		})
+		_ = waitForTimeChannel(t, evalDoneChan)
+
+		t.Run("write was performed with filtered labels", func(t *testing.T) {
+			require.Equal(t, 1, writeTarget.RequestsCount)
+			require.NotEmpty(t, writeTarget.LastRequestBody)
+
+			// Check that the body doesn't contain the private labels
+			require.NotContains(t, writeTarget.LastRequestBody, models.AutogeneratedRouteLabel)
+			require.NotContains(t, writeTarget.LastRequestBody, models.AutogeneratedRouteReceiverNameLabel)
+
+			require.Contains(t, writeTarget.LastRequestBody, "__user_custom__")
+			require.Contains(t, writeTarget.LastRequestBody, rule.Record.Metric)
+		})
+	})
 }
 
 func withQueryForHealth(health string) models.AlertRuleMutator {