Alerting: Sanitize Prometheus state history labels before writing (#107181)

pkg/services/ngalert/state/historian/prometheus.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"maps"
 	"math"
 	"strings"
 	"time"
@@ -12,6 +11,7 @@ import (
 	"github.com/grafana/dataplane/sdata/numeric"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	promValue "github.com/prometheus/prometheus/model/value"
+	"github.com/prometheus/prometheus/util/strutil"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
@@ -190,7 +190,10 @@ func (b *RemotePrometheusBackend) framesFor(ctx context.Context, rule history_mo
 
 	for i, sample := range samples {
 		labels := make(data.Labels, len(baseLabels)+2)
-		maps.Copy(labels, baseLabels)
+		for k, v := range baseLabels {
+			sanitizedKey := strutil.SanitizeFullLabelName(k)
+			labels[sanitizedKey] = v
+		}
 		labels[alertStateLabel] = sample.promState
 		labels[grafanaAlertStateLabel] = sample.grafanaState
 

pkg/services/ngalert/state/historian/prometheus_test.go

@@ -263,6 +263,24 @@ func TestPrometheusBackend_Record(t *testing.T) {
 			ruleMeta: ruleMeta,
 			states:   []state.StateTransition{createTransition(eval.Normal, eval.Normal, orgID, now)},
 		},
+		{
+			name:     "labels with invalid characters are sanitized",
+			ruleMeta: ruleMeta,
+			states: []state.StateTransition{
+				{
+					State: &state.State{
+						AlertRuleUID:       "rule-uid-sanitize",
+						OrgID:              orgID,
+						Labels:             data.Labels{"valid-label": "value1", "invalid.label": "value2", "label-with-dash": "value3", "123starts-with-number": "value4", "has spaces": "value5"},
+						State:              eval.Alerting,
+						LastEvaluationTime: now,
+					},
+				},
+			},
+			expectedFrames: data.Frames{
+				createExpectedFrame(t, "rule-uid-sanitize", "test rule", "firing", "alerting", data.Labels{"valid_label": "value1", "invalid_label": "value2", "label_with_dash": "value3", "_23starts_with_number": "value4", "has_spaces": "value5"}, 1.0),
+			},
+		},
 	}
 
 	for _, tc := range testCases {