Alerting: Instrument outgoing state history requests using weaveworks/common (#63689)

* Alerting: Instrument outgoing state history requests using weaveworks/common (#63600)

* Loki backend and client depend on a requester

* Instrument all requests to loki using weaveworks TimedClient

* Construct collector in metrics package

(cherry picked from commit e77621649d)

* Revert all changes to gomod and gosum

pkg/services/ngalert/metrics/historian.go

@@ -0,0 +1,23 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/weaveworks/common/instrument"
+)
+
+type Historian struct {
+	WriteDuration *instrument.HistogramCollector
+}
+
+func NewHistorianMetrics(r prometheus.Registerer) *Historian {
+	return &Historian{
+		WriteDuration: instrument.NewHistogramCollector(promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "state_history_request_duration_seconds",
+			Help:      "Histogram of request durations to the state history store.",
+			Buckets:   instrument.DefBuckets,
+		}, instrument.HistogramCollectorBuckets)),
+	}
+}

pkg/services/ngalert/metrics/ngalert.go

@@ -29,6 +29,7 @@ type NGAlert struct {
 	stateMetrics                *State
 	multiOrgAlertmanagerMetrics *MultiOrgAlertmanager
 	apiMetrics                  *API
+	historianMetrics            *Historian
 }
 
 // NewNGAlert manages the metrics of all the alerting components.
@@ -39,6 +40,7 @@ func NewNGAlert(r prometheus.Registerer) *NGAlert {
 		stateMetrics:                NewStateMetrics(r),
 		multiOrgAlertmanagerMetrics: NewMultiOrgAlertmanagerMetrics(r),
 		apiMetrics:                  NewAPIMetrics(r),
+		historianMetrics:            NewHistorianMetrics(r),
 	}
 }
 
@@ -67,3 +69,7 @@ func (moa *MultiOrgAlertmanager) RemoveOrgRegistry(id int64) {
 func (moa *MultiOrgAlertmanager) GetOrCreateOrgRegistry(id int64) prometheus.Registerer {
 	return moa.registries.GetOrCreateOrgRegistry(id)
 }
+
+func (ng *NGAlert) GetHistorianMetrics() *Historian {
+	return ng.historianMetrics
+}

pkg/services/ngalert/ngalert.go

@@ -210,7 +210,7 @@ func (ng *AlertNG) init() error {
 		Tracer:               ng.tracer,
 	}
 
-	history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store)
+	history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics())
 	if err != nil {
 		return err
 	}
@@ -389,7 +389,7 @@ type Historian interface {
 	state.Historian
 }
 
-func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore) (Historian, error) {
+func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian) (Historian, error) {
 	if !cfg.Enabled {
 		return historian.NewNopHistorian(), nil
 	}
@@ -402,7 +402,8 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
 		if err != nil {
 			return nil, fmt.Errorf("invalid remote loki configuration: %w", err)
 		}
-		backend := historian.NewRemoteLokiBackend(lcfg)
+		req := historian.NewRequester()
+		backend := historian.NewRemoteLokiBackend(lcfg, req, met)
 
 		testConnCtx, cancelFunc := context.WithTimeout(ctx, 10*time.Second)
 		defer cancelFunc()

pkg/services/ngalert/state/historian/loki.go

@@ -9,10 +9,12 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/weaveworks/common/http/client"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
@@ -43,14 +45,16 @@ type remoteLokiClient interface {
 type RemoteLokiBackend struct {
 	client         remoteLokiClient
 	externalLabels map[string]string
+	metrics        *metrics.Historian
 	log            log.Logger
 }
 
-func NewRemoteLokiBackend(cfg LokiConfig) *RemoteLokiBackend {
+func NewRemoteLokiBackend(cfg LokiConfig, req client.Requester, metrics *metrics.Historian) *RemoteLokiBackend {
 	logger := log.New("ngalert.state.historian", "backend", "loki")
 	return &RemoteLokiBackend{
-		client:         newLokiClient(cfg, logger),
+		client:         newLokiClient(cfg, req, metrics, logger),
 		externalLabels: cfg.ExternalLabels,
+		metrics:        metrics,
 		log:            logger,
 	}
 }

pkg/services/ngalert/state/historian/loki_http.go

@@ -11,11 +11,19 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/weaveworks/common/http/client"
 )
 
 const defaultClientTimeout = 30 * time.Second
 
+func NewRequester() client.Requester {
+	return &http.Client{
+		Timeout: defaultClientTimeout,
+	}
+}
+
 type LokiConfig struct {
 	ReadPathURL       *url.URL
 	WritePathURL      *url.URL
@@ -53,7 +61,7 @@ func NewLokiConfig(cfg setting.UnifiedAlertingStateHistorySettings) (LokiConfig,
 }
 
 type httpLokiClient struct {
-	client http.Client
+	client client.Requester
 	cfg    LokiConfig
 	log    log.Logger
 }
@@ -80,13 +88,12 @@ type Selector struct {
 	Value string
 }
 
-func newLokiClient(cfg LokiConfig, logger log.Logger) *httpLokiClient {
+func newLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger) *httpLokiClient {
+	tc := client.NewTimedClient(req, metrics.WriteDuration)
 	return &httpLokiClient{
-		client: http.Client{
-			Timeout: defaultClientTimeout,
-		},
-		cfg: cfg,
-		log: logger.New("protocol", "http"),
+		client: tc,
+		cfg:    cfg,
+		log:    logger.New("protocol", "http"),
 	}
 }
 

pkg/services/ngalert/state/historian/loki_http_test.go

@@ -6,7 +6,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -83,7 +85,7 @@ func TestLokiHTTPClient(t *testing.T) {
 		client := newLokiClient(LokiConfig{
 			ReadPathURL:  url,
 			WritePathURL: url,
-		}, log.NewNopLogger())
+		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry()), log.NewNopLogger())
 
 		// Unauthorized request should fail against Grafana Cloud.
 		err = client.ping(context.Background())
@@ -110,7 +112,7 @@ func TestLokiHTTPClient(t *testing.T) {
 			WritePathURL:      url,
 			BasicAuthUser:     "<your_username>",
 			BasicAuthPassword: "<your_password>",
-		}, log.NewNopLogger())
+		}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry()), log.NewNopLogger())
 
 		// When running on prem, you might need to set the tenant id,
 		// so the x-scope-orgid header is set.