public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Instrument outbound requests for Loki Historian and Remote Alertmanager with tracing (#89185)

Browse Source 
* Add TracedClient

* Handle errors and status codes

* Wire up tracing to normal ASH and loki annotation mapping

* Add tracing to remote alertmanager

* one more spot

* and not or

* More consistency with other grafana traces, lower cardinality name
This commit is contained in:
Alexander Weaver
2024-06-14 13:24:12 -05:00
committed by GitHub
parent 6262c56132
commit 8491e02caf
16 changed files with 124 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/remote/client/alertmanager.go
+5 -3
View File
@@ -9,6 +9,7 @@ import (
httptransport "github.com/go-openapi/runtime/client"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
amclient "github.com/prometheus/alertmanager/api/v2/client"
@@ -31,7 +32,7 @@ type Alertmanager struct {
logger log.Logger
}
func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
// First, add the authentication middleware.
c := &http.Client{Transport: &MimirAuthRoundTripper{
TenantID: cfg.TenantID,
@@ -40,14 +41,15 @@ func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanage
}}
tc := client.NewTimedClient(c, metrics.RequestLatency)
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
apiEndpoint := *cfg.URL
// Next, make sure you set the right path.
u := apiEndpoint.JoinPath(alertmanagerAPIMountPath, amclient.DefaultBasePath)
// Create an Alertmanager client using the timed client as the transport.
// Create an Alertmanager client using the instrumented client as the transport.
r := httptransport.New(u.Host, u.Path, []string{u.Scheme})
r.Transport = tc
r.Transport = trc
return &Alertmanager{
logger: cfg.Logger,
pkg/services/ngalert/remote/client/mimir.go
+5 -2
View File
@@ -12,6 +12,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -68,7 +69,7 @@ func (e *errorResponse) Error() string {
return e.Error2
}
func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Mimir, error) {
rt := &MimirAuthRoundTripper{
TenantID: cfg.TenantID,
Password: cfg.Password,
@@ -78,10 +79,12 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
c := &http.Client{
Transport: rt,
}
tc := client.NewTimedClient(c, metrics.RequestLatency)
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
return &Mimir{
endpoint: cfg.URL,
client: client.NewTimedClient(c, metrics.RequestLatency),
client: trc,
logger: cfg.Logger,
metrics: metrics,
promoteConfig: cfg.PromoteConfig,