public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Set specific backend type in remote_writer_writes_total metric for grafanacloud-prom datasource (#109516)

Browse Source
This commit is contained in:
Alexander Akhmetov
2025-08-20 13:04:00 +02:00
committed by GitHub
parent 91c23988ef
commit 38672c7936
3 changed files with 112 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/writer/datasourcewriter.go
+16 -1
View File
@@ -32,6 +32,13 @@ const (
cacheCleanupInterval = 10 * time.Minute
)
type backendType string
const (
grafanaCloudPromType backendType = "grafanacloud-prom"
prometheusType backendType = "prometheus"
)
type DatasourceWriterConfig struct {
// Timeout is the maximum time to wait for a remote write to succeed.
Timeout time.Duration
@@ -203,6 +210,13 @@ func (w *DatasourceWriter) makeWriter(ctx context.Context, orgID int64, dsUID st
headers.Add(k, v)
}
var backend backendType
if dsUID == string(grafanaCloudPromType) {
backend = grafanaCloudPromType
} else {
backend = prometheusType
}
cfg := PrometheusWriterConfig{
URL: u.String(),
HTTPOptions: httpclient.Options{
@@ -212,7 +226,8 @@ func (w *DatasourceWriter) makeWriter(ctx context.Context, orgID int64, dsUID st
Header: headers,
ProxyOptions: ho.ProxyOptions,
},
Timeout: w.cfg.Timeout,
Timeout: w.cfg.Timeout,
BackendType: backend,
}
if err != nil {
return nil, err
pkg/services/ngalert/writer/datasourcewriter_test.go
+77 -18
View File
@@ -2,7 +2,9 @@ package writer
import (
"context"
"fmt"
"net/http"
"strings"
"testing"
"time"
@@ -10,6 +12,7 @@ import (
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -127,7 +130,7 @@ func TestDatasourceWriter(t *testing.T) {
series := []map[string]string{{"foo": "1"}, {"foo": "2"}, {"foo": "3"}, {"foo": "4"}}
frames := frameGenFromLabels(t, data.FrameTypeNumericWide, series)
datasources := setupDataSources(t)
testDS := setupDataSources(t)
cfg := DatasourceWriterConfig{
Timeout: time.Second * 5,
@@ -136,26 +139,26 @@ func TestDatasourceWriter(t *testing.T) {
met := metrics.NewRemoteWriterMetrics(prometheus.NewRegistry())
pluginContextProvider := &mockPluginContextProvider{}
writer := NewDatasourceWriter(cfg, datasources, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met)
writer := NewDatasourceWriter(cfg, testDS, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met)
t.Run("when writing a prometheus datasource then the request is made to the expected endpoint", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
err := writer.WriteDatasource(context.Background(), "prom-1", "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
assert.Equal(t, 1, datasources.prom1.RequestsCount)
assert.Equal(t, 0, datasources.prom2.RequestsCount)
assert.Equal(t, 1, testDS.prom1.RequestsCount)
assert.Equal(t, 0, testDS.prom2.RequestsCount)
err = writer.WriteDatasource(context.Background(), "prom-2", "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
assert.Equal(t, 1, datasources.prom1.RequestsCount)
assert.Equal(t, 1, datasources.prom2.RequestsCount)
assert.Equal(t, 1, testDS.prom1.RequestsCount)
assert.Equal(t, 1, testDS.prom2.RequestsCount)
})
t.Run("when writing an unknown datasource then an error is returned", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
err := writer.WriteDatasource(context.Background(), "prom-unknown", "metric", time.Now(), frames, 1, map[string]string{})
require.Error(t, err)
@@ -163,7 +166,7 @@ func TestDatasourceWriter(t *testing.T) {
})
t.Run("when writing a non-prometheus datasource then an error is returned", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
err := writer.WriteDatasource(context.Background(), "loki-1", "metric", time.Now(), frames, 1, map[string]string{})
require.Error(t, err)
@@ -171,14 +174,14 @@ func TestDatasourceWriter(t *testing.T) {
})
t.Run("when writing with an empty datasource uid then the default is written", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
err := writer.WriteDatasource(context.Background(), "", "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
})
t.Run("when custom headers are configured, they are passed to the request", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
header1 := "X-Custom-Header"
header2 := "X-Another-Header"
@@ -192,17 +195,17 @@ func TestDatasourceWriter(t *testing.T) {
DefaultDatasourceUID: "prom-2",
CustomHeaders: headers,
}
writer = NewDatasourceWriter(cfg, datasources, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met)
writer = NewDatasourceWriter(cfg, testDS, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met)
err := writer.WriteDatasource(context.Background(), "prom-1", "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
assert.Equal(t, headers[header1], datasources.prom1.LastHeaders.Get(header1))
assert.Equal(t, headers[header2], datasources.prom1.LastHeaders.Get(header2))
assert.Equal(t, headers[header1], testDS.prom1.LastHeaders.Get(header1))
assert.Equal(t, headers[header2], testDS.prom1.LastHeaders.Get(header2))
})
t.Run("when PDC is enabled proxy options are passed to HTTP client provider", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
mockProvider := newMockHTTPClientProvider()
@@ -212,7 +215,7 @@ func TestDatasourceWriter(t *testing.T) {
}
met := metrics.NewRemoteWriterMetrics(prometheus.NewRegistry())
writer := NewDatasourceWriter(cfg, datasources, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met)
writer := NewDatasourceWriter(cfg, testDS, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met)
err := writer.WriteDatasource(context.Background(), "prom-3", "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
@@ -228,7 +231,7 @@ func TestDatasourceWriter(t *testing.T) {
})
t.Run("when PDC is disabled proxy options are not set", func(t *testing.T) {
datasources.Reset()
testDS.Reset()
mockProvider := newMockHTTPClientProvider()
@@ -238,7 +241,7 @@ func TestDatasourceWriter(t *testing.T) {
}
met := metrics.NewRemoteWriterMetrics(prometheus.NewRegistry())
writer := NewDatasourceWriter(cfg, datasources, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met)
writer := NewDatasourceWriter(cfg, testDS, mockProvider, &mockPluginContextProvider{}, clock.New(), log.New("test"), met)
err := writer.WriteDatasource(context.Background(), "prom-1", "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
@@ -247,6 +250,62 @@ func TestDatasourceWriter(t *testing.T) {
require.NotNil(t, mockProvider.lastOptions)
require.Nil(t, mockProvider.lastOptions.ProxyOptions)
})
t.Run("datasource uses correct backend type in metrics", func(t *testing.T) {
testCases := []struct {
name string
datasourceUID string
expectedBackendType string
}{
{
name: "grafanacloud-prom uses special backend type",
datasourceUID: string(grafanaCloudPromType),
expectedBackendType: string(grafanaCloudPromType),
},
{
name: "prometheus uses default backend type",
datasourceUID: "prom-1",
expectedBackendType: "prometheus",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
testDS.Reset()
if tc.datasourceUID == string(grafanaCloudPromType) {
gcProm, _ := testDS.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
Name: string(grafanaCloudPromType),
UID: string(grafanaCloudPromType),
Type: datasources.DS_PROMETHEUS,
JsonData: simplejson.MustJson([]byte(`{"prometheusType":"Prometheus"}`)),
})
gcProm.URL = testDS.prom1.srv.URL
}
cfg := DatasourceWriterConfig{
Timeout: time.Second * 5,
DefaultDatasourceUID: "prom-2",
}
reg := prometheus.NewRegistry()
met := metrics.NewRemoteWriterMetrics(reg)
writer := NewDatasourceWriter(cfg, testDS, httpclient.NewProvider(), pluginContextProvider, clock.New(), log.New("test"), met)
err := writer.WriteDatasource(context.Background(), tc.datasourceUID, "metric", time.Now(), frames, 1, map[string]string{})
require.NoError(t, err)
expectedMetric := fmt.Sprintf(`
# HELP grafana_alerting_remote_writer_writes_total The total number of remote writes attempted.
# TYPE grafana_alerting_remote_writer_writes_total counter
grafana_alerting_remote_writer_writes_total{backend="%s",org="1",status_code="200"} 1
`, tc.expectedBackendType)
require.NoError(t, testutil.CollectAndCompare(met.WritesTotal,
strings.NewReader(expectedMetric),
"grafana_alerting_remote_writer_writes_total"))
})
}
})
}
func TestDatasourceWriterGetRemoteWriteURL(t *testing.T) {
pkg/services/ngalert/writer/prom.go
+19 -11
View File
@@ -20,8 +20,6 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
)
const backendType = "prometheus"
const (
// Network error strings
networkErrDialTCP = "dial tcp"
@@ -236,16 +234,18 @@ type HttpClientProvider interface {
}
type PrometheusWriter struct {
client promremote.Client
clock clock.Clock
logger log.Logger
metrics *metrics.RemoteWriter
client promremote.Client
clock clock.Clock
logger log.Logger
metrics *metrics.RemoteWriter
backendType backendType
}
type PrometheusWriterConfig struct {
URL string
HTTPOptions httpclient.Options
Timeout time.Duration
BackendType backendType
}
func NewPrometheusWriter(
@@ -272,11 +272,19 @@ func NewPrometheusWriter(
return nil, err
}
var backend backendType
if cfg.BackendType != "" {
backend = cfg.BackendType
} else {
backend = prometheusType
}
return &PrometheusWriter{
client: client,
clock: clock,
logger: l,
metrics: metrics,
client: client,
clock: clock,
logger: l,
metrics: metrics,
backendType: backend,
}, nil
}
@@ -295,7 +303,7 @@ func (w PrometheusWriter) WriteDatasource(ctx context.Context, dsUID string, nam
// Write writes the given frames to the Prometheus remote write endpoint.
func (w PrometheusWriter) Write(ctx context.Context, name string, t time.Time, frames data.Frames, orgID int64, extraLabels map[string]string) error {
l := w.logger.FromContext(ctx)
lvs := []string{fmt.Sprint(orgID), backendType}
lvs := []string{fmt.Sprint(orgID), string(w.backendType)}
points, err := PointsFromFrames(name, t, frames, extraLabels)
if err != nil {