Alerting: Handle err-mimir-max-label-names-per-series as a user error in the prom writer (#101214)

pkg/services/ngalert/writer/prom.go

@@ -25,10 +25,11 @@ const backendType = "prometheus"
 
 const (
 	// Fixed error messages
-	MimirDuplicateTimestampError = "err-mimir-sample-duplicate-timestamp"
-	MimirInvalidLabelError       = "err-mimir-label-invalid"
-	MimirMaxSeriesPerUserError   = "err-mimir-max-series-per-user"
-	MimirLabelValueTooLongError  = "err-mimir-label-value-too-long"
+	MimirDuplicateTimestampError     = "err-mimir-sample-duplicate-timestamp"
+	MimirInvalidLabelError           = "err-mimir-label-invalid"
+	MimirLabelValueTooLongError      = "err-mimir-label-value-too-long"
+	MimirMaxLabelNamesPerSeriesError = "err-mimir-max-label-names-per-series"
+	MimirMaxSeriesPerUserError       = "err-mimir-max-series-per-user"
 
 	// Best effort error messages
 	PrometheusDuplicateTimestampError = "duplicate sample for timestamp"
@@ -267,16 +268,12 @@ func checkWriteError(writeErr promremote.WriteError) (err error, ignored bool) {
 			}
 		}
 
-		if strings.Contains(msg, MimirInvalidLabelError) {
-			return errors.Join(ErrRejectedWrite, writeErr), false
-		}
-
-		// this can happen when user exceeded defined maximum of
-		if strings.Contains(msg, MimirMaxSeriesPerUserError) {
-			return errors.Join(ErrRejectedWrite, writeErr), false
-		}
-
-		if strings.Contains(msg, MimirLabelValueTooLongError) {
+		// Check for expected user errors.
+		switch {
+		case strings.Contains(msg, MimirInvalidLabelError),
+			strings.Contains(msg, MimirMaxSeriesPerUserError),
+			strings.Contains(msg, MimirMaxLabelNamesPerSeriesError),
+			strings.Contains(msg, MimirLabelValueTooLongError):
 			return errors.Join(ErrRejectedWrite, writeErr), false
 		}
 

pkg/services/ngalert/writer/prom_test.go

@@ -240,7 +240,23 @@ func TestPrometheusWriter_Write(t *testing.T) {
 	})
 
 	t.Run("too long labels fit under the client error category", func(t *testing.T) {
-		msg := "received a series whose label value length exceeds the limit, label: 'label-1', value: 'value-1' (truncated) series: 'some_series (err-mimir-label-value-too-long). To adjust the related per-tenant limit, configure -validation.max-length-label-value, or contact your service administrator."
+		msg := "received a series whose label value length exceeds the limit, label: 'label-1', value: 'value-1' (truncated) series: 'some_series' (err-mimir-label-value-too-long). To adjust the related per-tenant limit, configure -validation.max-length-label-value, or contact your service administrator."
+		clientErr := testClientWriteError{
+			statusCode: http.StatusBadRequest,
+			msg:        &msg,
+		}
+		client.writeSeriesFunc = func(ctx context.Context, ts promremote.TSList, opts promremote.WriteOptions) (promremote.WriteResult, promremote.WriteError) {
+			return promremote.WriteResult{}, clientErr
+		}
+
+		err := writer.Write(ctx, "test", now, frames, 1, map[string]string{"extra": "label"})
+
+		require.Error(t, err)
+		require.ErrorIs(t, err, ErrRejectedWrite)
+	})
+
+	t.Run("too many labels fit under the client error category", func(t *testing.T) {
+		msg := "received a series whose number of labels exceeds the limit (actual: 50, limit: 40) series: 'some_series' (err-mimir-max-label-names-per-series). To adjust the related per-tenant limit, configure -validation.max-label-names-per-series, or contact your service administrator."
 		clientErr := testClientWriteError{
 			statusCode: http.StatusBadRequest,
 			msg:        &msg,