Alerting: Ensure state history client has external labels set (#113101)

* Ensure state history client has external labels set

* Run `make update-workspace`

* Add dep owner

go.mod

@@ -104,6 +104,7 @@ require (
 	github.com/grafana/grafana-google-sdk-go v0.4.2 // @grafana/partner-datasources
 	github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79 // @grafana/grafana-backend-group
 	github.com/grafana/grafana-plugin-sdk-go v0.281.0 // @grafana/plugins-platform-backend
+	github.com/grafana/loki/pkg/push v0.0.0-20250823105456-332df2b20000 // @grafana/alerting-backend
 	github.com/grafana/loki/v3 v3.2.1 // @grafana/observability-logs
 	github.com/grafana/nanogit v0.0.0-20250723104447-68f58f5ecec0 // indirect; @grafana/grafana-git-ui-sync-team
 	github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group
@@ -446,7 +447,6 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/grafana/jsonparser v0.0.0-20240425183733-ea80629e1a32 // indirect
-	github.com/grafana/loki/pkg/push v0.0.0-20250823105456-332df2b20000 // indirect
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/grafana/sqlds/v4 v4.2.7 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect

pkg/services/ngalert/ngalert.go

@@ -668,6 +668,8 @@ func configureHistorianBackend(
 		if err != nil {
 			return nil, fmt.Errorf("invalid remote loki configuration: %w", err)
 		}
+		// Use external labels from state history config
+		lcfg.ExternalLabels = cfg.ExternalLabels
 		req := lokiclient.NewRequester()
 		logCtx := log.WithContextualAttributes(ctx, []any{"backend", "loki"})
 		lokiBackendLogger := log.New("ngalert.state.historian").FromContext(logCtx)

pkg/services/ngalert/ngalert_test.go

@@ -3,10 +3,17 @@ package ngalert
 import (
 	"bytes"
 	"context"
+	"io"
 	"math/rand"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/loki/pkg/push"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
@@ -20,9 +27,11 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
 	acfakes "github.com/grafana/grafana/pkg/services/ngalert/accesscontrol/fakes"
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
+	history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
 	"github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
 	"github.com/grafana/grafana/pkg/setting"
@@ -151,6 +160,80 @@ func TestConfigureHistorianBackend(t *testing.T) {
 		require.NoError(t, err)
 	})
 
+	t.Run("Loki backend sends external labels in Record calls", func(t *testing.T) {
+		var receivedRequest *http.Request
+		var receivedBody []byte
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			receivedRequest = r
+			body, _ := io.ReadAll(r.Body)
+			receivedBody = body
+			w.WriteHeader(http.StatusNoContent)
+		}))
+		defer server.Close()
+
+		met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
+		logger := log.NewNopLogger()
+		tracer := tracing.InitializeTracerForTest()
+		cfg := setting.UnifiedAlertingStateHistorySettings{
+			Enabled: true,
+			Backend: "loki",
+			LokiSettings: setting.UnifiedAlertingLokiSettings{
+				LokiReadURL:  server.URL,
+				LokiWriteURL: server.URL,
+			},
+			ExternalLabels: map[string]string{
+				"test_label": "test_value",
+				"cluster":    "prod",
+			},
+		}
+		ac := &acfakes.FakeRuleService{}
+
+		h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer, ac, nil, nil, nil, nil, nil)
+		require.NoError(t, err)
+		require.NotNil(t, h)
+
+		rule := history_model.RuleMeta{
+			OrgID:        1,
+			UID:          "test-rule-uid",
+			Group:        "test-group",
+			NamespaceUID: "test-namespace",
+			Title:        "Test Rule",
+		}
+		states := []state.StateTransition{
+			{
+				PreviousState: eval.Normal,
+				State: &state.State{
+					State:              eval.Alerting,
+					Labels:             data.Labels{"instance": "test-instance"},
+					LastEvaluationTime: time.Now(),
+				},
+			},
+		}
+
+		errCh := h.Record(context.Background(), rule, states)
+		err = <-errCh
+		require.NoError(t, err)
+
+		require.NotNil(t, receivedRequest, "Expected HTTP request to be sent to Loki")
+		require.Contains(t, receivedRequest.URL.Path, "/loki/api/v1/push")
+
+		// Loki uses snappy-compressed protobuf encoding
+		decompressed, err := snappy.Decode(nil, receivedBody)
+		require.NoError(t, err)
+
+		var req push.PushRequest
+		err = proto.Unmarshal(decompressed, &req)
+		require.NoError(t, err)
+
+		require.Len(t, req.Streams, 1, "Expected exactly one stream")
+		stream := req.Streams[0]
+
+		require.Contains(t, stream.Labels, `test_label="test_value"`)
+		require.Contains(t, stream.Labels, `cluster="prod"`)
+		require.Contains(t, stream.Labels, `from="state-history"`)
+		require.Contains(t, stream.Labels, `orgID="1"`)
+	})
+
 	t.Run("fail initialization if prometheus backend missing datasource UID", func(t *testing.T) {
 		met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
 		logger := log.NewNopLogger()