Alerting: Introduce a Mimir client as part of the Remote Alertmanager (#78357)

* Alerting: Introduce a Mimir client as part of the Remote Alertmanager This is our first attempt at making Grafana communicate use Mimir as a backend - it uses a new set of APIs that we've developed on the Mimir side to upload the grafana configuration and alertmanager state so that it can then be ported over. Codewise, we've introduced a couple of things: A client to isolate in its own package all the communication that happens with Mimir A few changes to the remote/alertmanager to include uploading the configuration and state when it starts A few refactors that align a bit better with the design approach that we're thinking An integration tests again these newly developed APIs using a custom image --------- Signed-off-by: gotjosh <josue.abreu@gmail.com> Co-authored-by: Santiago <santiagohernandez.1997@gmail.com>
2023-11-23 16:59:36 +00:00
parent eedc19f9f0
commit 23fe8f4e9c
10 changed files with 547 additions and 92 deletions
@@ -0,0 +1,59 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+const (
+	grafanaAlertmanagerConfigPath = "/api/v1/grafana/config"
+)
+
+type UserGrafanaConfig struct {
+	ID                        int64  `json:"id"`
+	GrafanaAlertmanagerConfig string `json:"configuration"`
+	Hash                      string `json:"configuration_hash"`
+	CreatedAt                 int64  `json:"created"`
+	Default                   bool   `json:"default"`
+}
+
+func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error) {
+	gc := &UserGrafanaConfig{}
+	response := successResponse{
+		Data: gc,
+	}
+	// nolint:bodyclose
+	// closed within `do`
+	_, err := mc.do(ctx, grafanaAlertmanagerConfigPath, http.MethodGet, nil, &response)
+	if err != nil {
+		return nil, err
+	}
+
+	if response.Status != "success" {
+		return nil, fmt.Errorf("returned non-success `status` from the MimirAPI: %s", response.Status)
+	}
+
+	return gc, nil
+}
+
+func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, c, hash string, id, created int64, d bool) error {
+	payload, err := json.Marshal(&UserGrafanaConfig{
+		ID:                        id,
+		GrafanaAlertmanagerConfig: c,
+		Hash:                      hash,
+		CreatedAt:                 created,
+		Default:                   d,
+	})
+	if err != nil {
+		return err
+	}
+
+	return mc.doOK(ctx, grafanaAlertmanagerConfigPath, http.MethodPost, bytes.NewBuffer(payload))
+}
+
+func (mc *Mimir) DeleteGrafanaAlertmanagerConfig(ctx context.Context) error {
+	return mc.doOK(ctx, grafanaAlertmanagerConfigPath, http.MethodDelete, nil)
+}
@@ -0,0 +1,51 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+const (
+	grafanaAlertmanagerStatePath = "/grafana/state"
+)
+
+type UserGrafanaState struct {
+	State string `json:"state"`
+}
+
+func (mc *Mimir) GetGrafanaAlertmanagerState(ctx context.Context) (*UserGrafanaState, error) {
+	gs := &UserGrafanaState{}
+	response := successResponse{
+		Data: gs,
+	}
+	// nolint:bodyclose
+	// closed within `do`
+	_, err := mc.do(ctx, grafanaAlertmanagerStatePath, http.MethodGet, nil, &response)
+	if err != nil {
+		return nil, err
+	}
+
+	if response.Status != "success" {
+		return nil, fmt.Errorf("returned non-success `status` from the MimirAPI: %s", response.Status)
+	}
+
+	return gs, nil
+}
+
+func (mc *Mimir) CreateGrafanaAlertmanagerState(ctx context.Context, state string) error {
+	payload, err := json.Marshal(&UserGrafanaState{
+		State: state,
+	})
+	if err != nil {
+		return err
+	}
+
+	return mc.doOK(ctx, grafanaAlertmanagerStatePath, http.MethodPost, bytes.NewBuffer(payload))
+}
+
+func (mc *Mimir) DeleteGrafanaAlertmanagerState(ctx context.Context) error {
+	return mc.doOK(ctx, grafanaAlertmanagerStatePath, http.MethodDelete, nil)
+}
@@ -0,0 +1,179 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+// MimirClient contains all the methods to query the migration critical endpoints of Mimir instance, it's an interface to allow multiple implementations.
+type MimirClient interface {
+	GetGrafanaAlertmanagerState(ctx context.Context) (*UserGrafanaState, error)
+	CreateGrafanaAlertmanagerState(ctx context.Context, s string) error
+	DeleteGrafanaAlertmanagerState(ctx context.Context) error
+
+	GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error)
+	CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration string, hash string, id int64, at int64, d bool) error
+	DeleteGrafanaAlertmanagerConfig(ctx context.Context) error
+}
+
+type Mimir struct {
+	endpoint *url.URL
+	client   http.Client
+	logger   log.Logger
+}
+
+type Config struct {
+	Address  string
+	TenantID string
+	Password string
+
+	Logger log.Logger
+}
+
+// successResponse represents a successful response from the Mimir API.
+type successResponse struct {
+	Status string `json:"status"`
+	Data   any    `json:"data"`
+}
+
+// errorResponse represents an error from the Mimir API.
+type errorResponse struct {
+	Status string `json:"status"`
+	Error1 string `json:"error"`
+	Error2 string `json:"Error"`
+}
+
+func (e *errorResponse) Error() string {
+	if e.Error1 != "" {
+		return e.Error1
+	}
+
+	return e.Error2
+}
+
+func New(cfg *Config) (*Mimir, error) {
+	endpoint, err := url.Parse(cfg.Address)
+	if err != nil {
+		return nil, err
+	}
+
+	rt := &MimirAuthRoundTripper{
+		TenantID: cfg.TenantID,
+		Password: cfg.Password,
+		Next:     http.DefaultTransport,
+	}
+
+	c := http.Client{
+		Transport: rt,
+	}
+
+	return &Mimir{
+		endpoint: endpoint,
+		client:   c,
+		logger:   cfg.Logger,
+	}, nil
+}
+
+// do execute an HTTP requests against the specified path and method using the specified payload.
+// It returns the HTTP response.
+func (mc *Mimir) do(ctx context.Context, p, method string, payload io.Reader, out any) (*http.Response, error) {
+	pathURL, err := url.Parse(p)
+	if err != nil {
+		return nil, err
+	}
+
+	endpoint := *mc.endpoint
+	endpoint.Path = path.Join(endpoint.Path, pathURL.Path)
+
+	r, err := http.NewRequestWithContext(ctx, method, endpoint.String(), payload)
+	if err != nil {
+		return nil, err
+	}
+
+	r.Header.Set("Accept", "application/json")
+	r.Header.Set("Content-Type", "application/json")
+
+	resp, err := mc.client.Do(r)
+	if err != nil {
+		msg := "Unable to fulfill request to the Mimir API"
+		mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method)
+		return nil, fmt.Errorf("%s: %w", msg, err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			mc.logger.Error("Error closing HTTP body", "err", err, "url", r.URL.String(), "method", r.Method)
+		}
+	}()
+
+	ct := resp.Header.Get("Content-Type")
+	if !strings.HasPrefix(ct, "application/json") {
+		msg := "Response content-type is not application/json"
+		mc.logger.Error(msg, "content-type", "url", r.URL.String(), "method", r.Method, ct, "status", resp.StatusCode)
+		return nil, fmt.Errorf("%s: %s", msg, ct)
+	}
+
+	if out == nil {
+		return resp, nil
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		msg := "Failed to read the request body"
+		mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode)
+		return nil, fmt.Errorf("%s: %w", msg, err)
+	}
+
+	if resp.StatusCode/100 != 2 {
+		errResponse := &errorResponse{}
+		err = json.Unmarshal(body, errResponse)
+
+		if err == nil && errResponse.Error() != "" {
+			msg := "Error response from the Mimir API"
+			mc.logger.Error(msg, "err", errResponse, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode)
+			return nil, fmt.Errorf("%s: %w", msg, errResponse)
+		}
+
+		msg := "Failed to decode non-2xx JSON response"
+		mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode)
+		return nil, fmt.Errorf("%s: %w", msg, err)
+	}
+
+	if err = json.Unmarshal(body, out); err != nil {
+		msg := "Failed to decode 2xx JSON response"
+		mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode)
+		return nil, fmt.Errorf("%s: %w", msg, err)
+	}
+
+	return resp, nil
+}
+
+func (mc *Mimir) doOK(ctx context.Context, p, method string, payload io.Reader) error {
+	var sr successResponse
+	resp, err := mc.do(ctx, p, method, payload, &sr)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			mc.logger.Error("Error closing HTTP body", "err", err)
+		}
+	}()
+
+	switch sr.Status {
+	case "success":
+		return nil
+	case "error":
+		return errors.New("received an 2xx status code but the request body reflected an error")
+	default:
+		return fmt.Errorf("received an unknown status from the request body: %s", sr.Status)
+	}
+}
@@ -0,0 +1,28 @@
+package client
+
+import (
+	"net/http"
+)
+
+const mimirTenantHeader = "X-Scope-OrgID"
+
+type MimirAuthRoundTripper struct {
+	TenantID string
+	Password string
+	Next     http.RoundTripper
+}
+
+// RoundTrip implements the http.RoundTripper interface
+// It adds an `X-Scope-OrgID` header with the TenantID if only provided with a tenantID or sets HTTP Basic Authentication if both
+// a tenantID and a password are provided.
+func (r *MimirAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if r.TenantID != "" && r.Password == "" {
+		req.Header.Set(mimirTenantHeader, r.TenantID)
+	}
+
+	if r.TenantID != "" && r.Password != "" {
+		req.SetBasicAuth(r.TenantID, r.Password)
+	}
+
+	return r.Next.RoundTrip(req)
+}