From 1ac89ea04026473505810a3a7637b1a2f60e74b6 Mon Sep 17 00:00:00 2001
From: Alexander Weaver <weaver.alex.d@gmail.com>
Date: Tue, 17 Jan 2023 13:58:52 -0600
Subject: [PATCH] Alerting: Add client configuration for remote Loki historian
 backend and test connection (#61114)

* Create loki client type and ping method

* Expose TestConnection on client

* Configure and ping Loki URL

* Close response body reader if present

* Add 30 second timeout

* Remove duplicate close
---
 pkg/services/ngalert/ngalert.go               | 10 +++-
 pkg/services/ngalert/state/historian/loki.go  | 22 ++++++--
 .../ngalert/state/historian/loki_http.go      | 54 +++++++++++++++++++
 pkg/setting/setting_unified_alerting.go       | 10 ++--
 4 files changed, 87 insertions(+), 9 deletions(-)
 create mode 100644 pkg/services/ngalert/state/historian/loki_http.go

diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go
index 993b35bd2da..fa9622b919a 100644
--- a/pkg/services/ngalert/ngalert.go
+++ b/pkg/services/ngalert/ngalert.go
@@ -387,7 +387,15 @@ func configureHistorianBackend(cfg setting.UnifiedAlertingStateHistorySettings,
 		return historian.NewAnnotationBackend(ar, ds), nil
 	}
 	if cfg.Backend == "loki" {
-		return historian.NewRemoteLokiBackend(), nil
+		baseURL, err := url.Parse(cfg.LokiRemoteURL)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse remote loki URL: %w", err)
+		}
+		backend := historian.NewRemoteLokiBackend(baseURL)
+		if err := backend.TestConnection(); err != nil {
+			return nil, fmt.Errorf("failed to ping the remote loki historian: %w", err)
+		}
+		return backend, nil
 	}
 	if cfg.Backend == "sql" {
 		return historian.NewSqlBackend(), nil
diff --git a/pkg/services/ngalert/state/historian/loki.go b/pkg/services/ngalert/state/historian/loki.go
index a1ebacddc06..038f4e98065 100644
--- a/pkg/services/ngalert/state/historian/loki.go
+++ b/pkg/services/ngalert/state/historian/loki.go
@@ -2,21 +2,35 @@ package historian
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 )
 
-type RemoteLokiBackend struct {
-	log log.Logger
+type remoteLokiClient interface {
+	ping() error
 }
 
-func NewRemoteLokiBackend() *RemoteLokiBackend {
+type RemoteLokiBackend struct {
+	client remoteLokiClient
+	log    log.Logger
+}
+
+func NewRemoteLokiBackend(url *url.URL) *RemoteLokiBackend {
+	logger := log.New("ngalert.state.historian", "backend", "loki")
 	return &RemoteLokiBackend{
-		log: log.New("ngalert.state.historian"),
+		client: newLokiClient(url, logger),
+		log:    logger,
 	}
 }
 
+func (h *RemoteLokiBackend) TestConnection() error {
+	return h.client.ping()
+}
+
 func (h *RemoteLokiBackend) RecordStatesAsync(ctx context.Context, _ *models.AlertRule, _ []state.StateTransition) {
+	logger := h.log.FromContext(ctx)
+	logger.Debug("Remote Loki state history backend was called with states")
 }
diff --git a/pkg/services/ngalert/state/historian/loki_http.go b/pkg/services/ngalert/state/historian/loki_http.go
new file mode 100644
index 00000000000..78232dde025
--- /dev/null
+++ b/pkg/services/ngalert/state/historian/loki_http.go
@@ -0,0 +1,54 @@
+package historian
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+const defaultClientTimeout = 30 * time.Second
+
+type httpLokiClient struct {
+	client http.Client
+	url    *url.URL
+	log    log.Logger
+}
+
+func newLokiClient(u *url.URL, logger log.Logger) *httpLokiClient {
+	return &httpLokiClient{
+		client: http.Client{
+			Timeout: defaultClientTimeout,
+		},
+		url: u,
+		log: logger.New("protocol", "http"),
+	}
+}
+
+func (c *httpLokiClient) ping() error {
+	uri := c.url.JoinPath("/loki/api/v1/status/buildinfo")
+	req, err := http.NewRequest(http.MethodGet, uri.String(), nil)
+	if err != nil {
+		return fmt.Errorf("error creating request: %w", err)
+	}
+
+	res, err := c.client.Do(req)
+	if res != nil {
+		defer func() {
+			if err := res.Body.Close(); err != nil {
+				c.log.Warn("Failed to close response body", "err", err)
+			}
+		}()
+	}
+	if err != nil {
+		return fmt.Errorf("error sending request: %w", err)
+	}
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return fmt.Errorf("request to the loki buildinfo endpoint returned a non-200 status code: %d", res.StatusCode)
+	}
+	c.log.Debug("Request to Loki buildinfo endpoint succeeded", "status", res.StatusCode)
+	return nil
+}
diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go
index b9441ca5d5a..9663b44426d 100644
--- a/pkg/setting/setting_unified_alerting.go
+++ b/pkg/setting/setting_unified_alerting.go
@@ -101,8 +101,9 @@ type UnifiedAlertingReservedLabelSettings struct {
 }
 
 type UnifiedAlertingStateHistorySettings struct {
-	Enabled bool
-	Backend string
+	Enabled       bool
+	Backend       string
+	LokiRemoteURL string
 }
 
 // IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
@@ -313,8 +314,9 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 
 	stateHistory := iniFile.Section("unified_alerting.state_history")
 	uaCfgStateHistory := UnifiedAlertingStateHistorySettings{
-		Enabled: stateHistory.Key("enabled").MustBool(stateHistoryDefaultEnabled),
-		Backend: stateHistory.Key("backend").MustString("annotations"),
+		Enabled:       stateHistory.Key("enabled").MustBool(stateHistoryDefaultEnabled),
+		Backend:       stateHistory.Key("backend").MustString("annotations"),
+		LokiRemoteURL: stateHistory.Key("loki_remote_url").MustString(""),
 	}
 	uaCfg.StateHistory = uaCfgStateHistory