diff --git a/pkg/services/ngalert/writer/datasourcewriter.go b/pkg/services/ngalert/writer/datasourcewriter.go index fc9a3087c7b..19af0db0a5b 100644 --- a/pkg/services/ngalert/writer/datasourcewriter.go +++ b/pkg/services/ngalert/writer/datasourcewriter.go @@ -32,6 +32,13 @@ const ( cacheCleanupInterval = 10 * time.Minute ) +type backendType string + +const ( + grafanaCloudPromType backendType = "grafanacloud-prom" + prometheusType backendType = "prometheus" +) + type DatasourceWriterConfig struct { // Timeout is the maximum time to wait for a remote write to succeed. Timeout time.Duration @@ -203,6 +210,13 @@ func (w *DatasourceWriter) makeWriter(ctx context.Context, orgID int64, dsUID st headers.Add(k, v) } + var backend backendType + if dsUID == string(grafanaCloudPromType) { + backend = grafanaCloudPromType + } else { + backend = prometheusType + } + cfg := PrometheusWriterConfig{ URL: u.String(), HTTPOptions: httpclient.Options{ @@ -212,7 +226,8 @@ func (w *DatasourceWriter) makeWriter(ctx context.Context, orgID int64, dsUID st Header: headers, ProxyOptions: ho.ProxyOptions, }, - Timeout: w.cfg.Timeout, + Timeout: w.cfg.Timeout, + BackendType: backend, } if err != nil { return nil, err diff --git a/pkg/services/ngalert/writer/datasourcewriter_test.go b/pkg/services/ngalert/writer/datasourcewriter_test.go index 09fc1ac6455..f06d2d95de4 100644 --- a/pkg/services/ngalert/writer/datasourcewriter_test.go +++ b/pkg/services/ngalert/writer/datasourcewriter_test.go @@ -2,7 +2,9 @@ package writer import ( "context" + "fmt" "net/http" + "strings" "testing" "time" @@ -10,6 +12,7 @@ import ( sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -127,7 +130,7 @@ func TestDatasourceWriter(t *testing.T) { series := []map[string]string{{"foo": "1"}, {"foo": "2"}, {"foo": "3"}, {"foo": "4"}} frames := frameGenFromLabels(t, data.FrameTypeNumericWide, series) - datasources := setupDataSources(t) + testDS := setupDataSources(t) cfg := DatasourceWriterConfig{ Timeout: time.Second * 5, @@ -136,26 +139,26 @@ func TestDatasourceWriter(t *testing.T) { met := metrics.NewRemoteWriterMetrics(prometheus.NewRegistry()) pluginContextProvider := &mockPluginContextProvider{} - writer := NewDatasourceWriter(cfg, datasources, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met) + writer := NewDatasourceWriter(cfg, testDS, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met) t.Run("when writing a prometheus datasource then the request is made to the expected endpoint", func(t *testing.T) { - datasources.Reset() + testDS.Reset() err := writer.WriteDatasource(context.Background(), "prom-1", "metric", time.Now(), frames, 1, map[string]string{}) require.NoError(t, err) - assert.Equal(t, 1, datasources.prom1.RequestsCount) - assert.Equal(t, 0, datasources.prom2.RequestsCount) + assert.Equal(t, 1, testDS.prom1.RequestsCount) + assert.Equal(t, 0, testDS.prom2.RequestsCount) err = writer.WriteDatasource(context.Background(), "prom-2", "metric", time.Now(), frames, 1, map[string]string{}) require.NoError(t, err) - assert.Equal(t, 1, datasources.prom1.RequestsCount) - assert.Equal(t, 1, datasources.prom2.RequestsCount) + assert.Equal(t, 1, testDS.prom1.RequestsCount) + assert.Equal(t, 1, testDS.prom2.RequestsCount) }) t.Run("when writing an unknown datasource then an error is returned", func(t *testing.T) { - datasources.Reset() + testDS.Reset() err := writer.WriteDatasource(context.Background(), "prom-unknown", "metric", time.Now(), frames, 1, map[string]string{}) require.Error(t, err) @@ -163,7 +166,7 @@ func TestDatasourceWriter(t *testing.T) { }) t.Run("when writing a non-prometheus datasource then an error is returned", func(t *testing.T) { - datasources.Reset() + testDS.Reset() err := writer.WriteDatasource(context.Background(), "loki-1", "metric", time.Now(), frames, 1, map[string]string{}) require.Error(t, err) @@ -171,14 +174,14 @@ func TestDatasourceWriter(t *testing.T) { }) t.Run("when writing with an empty datasource uid then the default is written", func(t *testing.T) { - datasources.Reset() + testDS.Reset() err := writer.WriteDatasource(context.Background(), "", "metric", time.Now(), frames, 1, map[string]string{}) require.NoError(t, err) }) t.Run("when custom headers are configured, they are passed to the request", func(t *testing.T) { - datasources.Reset() + testDS.Reset() header1 := "X-Custom-Header" header2 := "X-Another-Header" @@ -192,17 +195,17 @@ func TestDatasourceWriter(t *testing.T) { DefaultDatasourceUID: "prom-2", CustomHeaders: headers, } - writer = NewDatasourceWriter(cfg, datasources, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met) + writer = NewDatasourceWriter(cfg, testDS, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met) err := writer.WriteDatasource(context.Background(), "prom-1", "metric", time.Now(), frames, 1, map[string]string{}) require.NoError(t, err) - assert.Equal(t, headers[header1], datasources.prom1.LastHeaders.Get(header1)) - assert.Equal(t, headers[header2], datasources.prom1.LastHeaders.Get(header2)) + assert.Equal(t, headers[header1], testDS.prom1.LastHeaders.Get(header1)) + assert.Equal(t, headers[header2], testDS.prom1.LastHeaders.Get(header2)) }) t.Run("when PDC is enabled proxy options are passed to HTTP client provider", func(t *testing.T) { - datasources.Reset() + testDS.Reset() mockProvider := newMockHTTPClientProvider() @@ -212,7 +215,7 @@ func TestDatasourceWriter(t *testing.T) { } met := metrics.NewRemoteWriterMetrics(prometheus.NewRegistry()) - writer := NewDatasourceWriter(cfg, datasources, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met) + writer := NewDatasourceWriter(cfg, testDS, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met) err := writer.WriteDatasource(context.Background(), "prom-3", "metric", time.Now(), frames, 1, map[string]string{}) require.NoError(t, err) @@ -228,7 +231,7 @@ func TestDatasourceWriter(t *testing.T) { }) t.Run("when PDC is disabled proxy options are not set", func(t *testing.T) { - datasources.Reset() + testDS.Reset() mockProvider := newMockHTTPClientProvider() @@ -238,7 +241,7 @@ func TestDatasourceWriter(t *testing.T) { } met := metrics.NewRemoteWriterMetrics(prometheus.NewRegistry()) - writer := NewDatasourceWriter(cfg, datasources, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met) + writer := NewDatasourceWriter(cfg, testDS, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met) err := writer.WriteDatasource(context.Background(), "prom-1", "metric", time.Now(), frames, 1, map[string]string{}) require.NoError(t, err) @@ -247,6 +250,62 @@ func TestDatasourceWriter(t *testing.T) { require.NotNil(t, mockProvider.lastOptions) require.Nil(t, mockProvider.lastOptions.ProxyOptions) }) + + t.Run("datasource uses correct backend type in metrics", func(t *testing.T) { + testCases := []struct { + name string + datasourceUID string + expectedBackendType string + }{ + { + name: "grafanacloud-prom uses special backend type", + datasourceUID: string(grafanaCloudPromType), + expectedBackendType: string(grafanaCloudPromType), + }, + { + name: "prometheus uses default backend type", + datasourceUID: "prom-1", + expectedBackendType: "prometheus", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + testDS.Reset() + + if tc.datasourceUID == string(grafanaCloudPromType) { + gcProm, _ := testDS.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ + Name: string(grafanaCloudPromType), + UID: string(grafanaCloudPromType), + Type: datasources.DS_PROMETHEUS, + JsonData: simplejson.MustJson([]byte(`{"prometheusType":"Prometheus"}`)), + }) + gcProm.URL = testDS.prom1.srv.URL + } + + cfg := DatasourceWriterConfig{ + Timeout: time.Second * 5, + DefaultDatasourceUID: "prom-2", + } + + reg := prometheus.NewRegistry() + met := metrics.NewRemoteWriterMetrics(reg) + writer := NewDatasourceWriter(cfg, testDS, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met) + + err := writer.WriteDatasource(context.Background(), tc.datasourceUID, "metric", time.Now(), frames, 1, map[string]string{}) + require.NoError(t, err) + + expectedMetric := fmt.Sprintf(` + # HELP grafana_alerting_remote_writer_writes_total The total number of remote writes attempted. + # TYPE grafana_alerting_remote_writer_writes_total counter + grafana_alerting_remote_writer_writes_total{backend="%s",org="1",status_code="200"} 1 + `, tc.expectedBackendType) + require.NoError(t, testutil.CollectAndCompare(met.WritesTotal, + strings.NewReader(expectedMetric), + "grafana_alerting_remote_writer_writes_total")) + }) + } + }) } func TestDatasourceWriterGetRemoteWriteURL(t *testing.T) { diff --git a/pkg/services/ngalert/writer/prom.go b/pkg/services/ngalert/writer/prom.go index 64fb525ff44..7ade2fc1e09 100644 --- a/pkg/services/ngalert/writer/prom.go +++ b/pkg/services/ngalert/writer/prom.go @@ -20,8 +20,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" ) -const backendType = "prometheus" - const ( // Network error strings networkErrDialTCP = "dial tcp" @@ -236,16 +234,18 @@ type HttpClientProvider interface { } type PrometheusWriter struct { - client promremote.Client - clock clock.Clock - logger log.Logger - metrics *metrics.RemoteWriter + client promremote.Client + clock clock.Clock + logger log.Logger + metrics *metrics.RemoteWriter + backendType backendType } type PrometheusWriterConfig struct { URL string HTTPOptions httpclient.Options Timeout time.Duration + BackendType backendType } func NewPrometheusWriter( @@ -272,11 +272,19 @@ func NewPrometheusWriter( return nil, err } + var backend backendType + if cfg.BackendType != "" { + backend = cfg.BackendType + } else { + backend = prometheusType + } + return &PrometheusWriter{ - client: client, - clock: clock, - logger: l, - metrics: metrics, + client: client, + clock: clock, + logger: l, + metrics: metrics, + backendType: backend, }, nil } @@ -295,7 +303,7 @@ func (w PrometheusWriter) WriteDatasource(ctx context.Context, dsUID string, nam // Write writes the given frames to the Prometheus remote write endpoint. func (w PrometheusWriter) Write(ctx context.Context, name string, t time.Time, frames data.Frames, orgID int64, extraLabels map[string]string) error { l := w.logger.FromContext(ctx) - lvs := []string{fmt.Sprint(orgID), backendType} + lvs := []string{fmt.Sprint(orgID), string(w.backendType)} points, err := PointsFromFrames(name, t, frames, extraLabels) if err != nil {