From 75fb83282659bd727de20386df895e5b9ec1ab0d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 28 Oct 2025 11:35:54 -0400 Subject: [PATCH] Alerting: Ensure state history client has external labels set (#113101) * Ensure state history client has external labels set * Run `make update-workspace` * Add dep owner --- go.mod | 2 +- pkg/services/ngalert/ngalert.go | 2 + pkg/services/ngalert/ngalert_test.go | 83 ++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 08ca8ca8112..6c3cf08f214 100644 --- a/go.mod +++ b/go.mod @@ -104,6 +104,7 @@ require ( github.com/grafana/grafana-google-sdk-go v0.4.2 // @grafana/partner-datasources github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79 // @grafana/grafana-backend-group github.com/grafana/grafana-plugin-sdk-go v0.281.0 // @grafana/plugins-platform-backend + github.com/grafana/loki/pkg/push v0.0.0-20250823105456-332df2b20000 // @grafana/alerting-backend github.com/grafana/loki/v3 v3.2.1 // @grafana/observability-logs github.com/grafana/nanogit v0.0.0-20250723104447-68f58f5ecec0 // indirect; @grafana/grafana-git-ui-sync-team github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group @@ -446,7 +447,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/grafana/jsonparser v0.0.0-20240425183733-ea80629e1a32 // indirect - github.com/grafana/loki/pkg/push v0.0.0-20250823105456-332df2b20000 // indirect github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect github.com/grafana/sqlds/v4 v4.2.7 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 77484ba43c4..ebb7e1061c2 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -668,6 +668,8 @@ func configureHistorianBackend( if err != nil { return nil, fmt.Errorf("invalid remote loki configuration: %w", err) } + // Use external labels from state history config + lcfg.ExternalLabels = cfg.ExternalLabels req := lokiclient.NewRequester() logCtx := log.WithContextualAttributes(ctx, []any{"backend", "loki"}) lokiBackendLogger := log.New("ngalert.state.historian").FromContext(logCtx) diff --git a/pkg/services/ngalert/ngalert_test.go b/pkg/services/ngalert/ngalert_test.go index 640a81821c9..c9adb2603cc 100644 --- a/pkg/services/ngalert/ngalert_test.go +++ b/pkg/services/ngalert/ngalert_test.go @@ -3,10 +3,17 @@ package ngalert import ( "bytes" "context" + "io" "math/rand" + "net/http" + "net/http/httptest" "testing" "time" + "github.com/gogo/protobuf/proto" + "github.com/golang/snappy" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/loki/pkg/push" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" @@ -20,9 +27,11 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" acfakes "github.com/grafana/grafana/pkg/services/ngalert/accesscontrol/fakes" + "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" + history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model" "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/setting" @@ -151,6 +160,80 @@ func TestConfigureHistorianBackend(t *testing.T) { require.NoError(t, err) }) + t.Run("Loki backend sends external labels in Record calls", func(t *testing.T) { + var receivedRequest *http.Request + var receivedBody []byte + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedRequest = r + body, _ := io.ReadAll(r.Body) + receivedBody = body + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem) + logger := log.NewNopLogger() + tracer := tracing.InitializeTracerForTest() + cfg := setting.UnifiedAlertingStateHistorySettings{ + Enabled: true, + Backend: "loki", + LokiSettings: setting.UnifiedAlertingLokiSettings{ + LokiReadURL: server.URL, + LokiWriteURL: server.URL, + }, + ExternalLabels: map[string]string{ + "test_label": "test_value", + "cluster": "prod", + }, + } + ac := &acfakes.FakeRuleService{} + + h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer, ac, nil, nil, nil, nil, nil) + require.NoError(t, err) + require.NotNil(t, h) + + rule := history_model.RuleMeta{ + OrgID: 1, + UID: "test-rule-uid", + Group: "test-group", + NamespaceUID: "test-namespace", + Title: "Test Rule", + } + states := []state.StateTransition{ + { + PreviousState: eval.Normal, + State: &state.State{ + State: eval.Alerting, + Labels: data.Labels{"instance": "test-instance"}, + LastEvaluationTime: time.Now(), + }, + }, + } + + errCh := h.Record(context.Background(), rule, states) + err = <-errCh + require.NoError(t, err) + + require.NotNil(t, receivedRequest, "Expected HTTP request to be sent to Loki") + require.Contains(t, receivedRequest.URL.Path, "/loki/api/v1/push") + + // Loki uses snappy-compressed protobuf encoding + decompressed, err := snappy.Decode(nil, receivedBody) + require.NoError(t, err) + + var req push.PushRequest + err = proto.Unmarshal(decompressed, &req) + require.NoError(t, err) + + require.Len(t, req.Streams, 1, "Expected exactly one stream") + stream := req.Streams[0] + + require.Contains(t, stream.Labels, `test_label="test_value"`) + require.Contains(t, stream.Labels, `cluster="prod"`) + require.Contains(t, stream.Labels, `from="state-history"`) + require.Contains(t, stream.Labels, `orgID="1"`) + }) + t.Run("fail initialization if prometheus backend missing datasource UID", func(t *testing.T) { met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem) logger := log.NewNopLogger()