Merge remote-tracking branch 'origin/main' into ds-apiserver-with-configs

2025-07-02 13:37:12 -07:00
parent fb0aaa321e b6eacc929a
commit 775ed81b58
22 changed files with 2036 additions and 49 deletions
@@ -11,6 +11,7 @@ import (

 	"github.com/grafana/e2e"
 	gapi "github.com/grafana/grafana-api-golang-client"
+	"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	"github.com/stretchr/testify/require"
 )

@@ -43,6 +44,7 @@ type AlertmanagerScenario struct {
 	Webhook  *WebhookService
 	Postgres *PostgresService
 	Loki     *LokiService
+	Mimir    *MimirService
 }

 func NewAlertmanagerScenario() (*AlertmanagerScenario, error) {
@@ -381,3 +383,10 @@ func mapInstancePeers(is []string) map[string][]string {

 	return mIs
 }
+
+func (s *AlertmanagerScenario) NewMimirClient(tenantID string) (client.MimirClient, error) {
+	if s.Mimir == nil {
+		return nil, fmt.Errorf("mimir service not started")
+	}
+	return NewMimirClient("http://"+s.Mimir.HTTPEndpoint(), tenantID)
+}
@@ -0,0 +1,67 @@
+package alertmanager
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/grafana/e2e"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
+	"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	mimirImage = "grafana/mimir:r348-017076d8"
+
+	mimirBinary   = "/bin/mimir"
+	mimirHTTPPort = 33667
+	mimirGRPCPort = 33668
+)
+
+type MimirService struct {
+	*e2e.HTTPService
+}
+
+func NewMimirService(name string) *MimirService {
+	flags := map[string]string{
+		"-target":                                                  "alertmanager",
+		"-server.http-listen-port":                                 fmt.Sprintf("%d", mimirHTTPPort),
+		"-server.grpc-listen-port":                                 fmt.Sprintf("%d", mimirGRPCPort),
+		"-alertmanager.web.external-url":                           "http://localhost:8080/alertmanager",
+		"-alertmanager-storage.backend":                            "filesystem",
+		"-alertmanager-storage.filesystem.dir":                     "/tmp/mimir/alertmanager",
+		"-alertmanager.grafana-alertmanager-compatibility-enabled": "true",
+	}
+
+	return &MimirService{
+		HTTPService: e2e.NewHTTPService(
+			name,
+			mimirImage,
+			e2e.NewCommandWithoutEntrypoint(mimirBinary, e2e.BuildArgs(flags)...),
+			e2e.NewHTTPReadinessProbe(mimirHTTPPort, "/ready", 200, 299),
+			mimirHTTPPort,
+		),
+	}
+}
+
+func NewMimirClient(mimirURL, tenantID string) (client.MimirClient, error) {
+	u, err := url.Parse(mimirURL)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &client.Config{
+		URL:      u,
+		TenantID: tenantID,
+		Password: "", // No password needed for test
+		Logger:   log.NewNopLogger(),
+	}
+
+	registry := prometheus.NewRegistry()
+	metrics := metrics.NewRemoteAlertmanagerMetrics(registry)
+	tracer := tracing.InitializeTracerForTest()
+
+	return client.New(cfg, metrics, tracer)
+}
@@ -0,0 +1,262 @@
+package alerting
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/tests/alertmanager"
+	"github.com/grafana/grafana/pkg/tests/testinfra"
+)
+
+// TestIntegrationRemoteAlertmanagerConfigUpload tests that when we post an alertmanager
+// configuration to Grafana with remote alertmanager enabled, it gets uploaded to the remote Mimir.
+func TestIntegrationRemoteAlertmanagerConfigUpload(t *testing.T) {
+	testinfra.SQLiteIntegrationTest(t)
+
+	s, err := alertmanager.NewAlertmanagerScenario()
+	require.NoError(t, err)
+	defer s.Close()
+
+	s.Mimir = alertmanager.NewMimirService("mimir")
+	require.NoError(t, s.StartAndWaitReady(s.Mimir))
+
+	mimirEndpoint := "http://" + s.Mimir.HTTPEndpoint()
+
+	dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
+		AppModeProduction:     true,
+		EnableFeatureToggles: []string{
+			"alertmanagerRemotePrimary",
+			"alertingImportAlertmanagerAPI",
+		},
+		RemoteAlertmanagerURL: mimirEndpoint,
+	})
+
+	grafanaListedAddr, _ := testinfra.StartGrafanaEnv(t, dir, gpath)
+
+	apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
+	mimirClient, err := alertmanager.NewMimirClient(mimirEndpoint, "1")
+	require.NoError(t, err)
+
+	// Wait for Grafana to be ready
+	require.Eventually(t, func() bool {
+		_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
+		return status == http.StatusOK
+	}, 30*time.Second, time.Second, "Grafana failed to start")
+
+	// Check that the initial Mimir config contains the default Grafana configuration
+	initialMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, initialMimirConfig) // Grafana automatically syncs default config to remote alertmanager
+	require.NotNil(t, initialMimirConfig.GrafanaAlertmanagerConfig)
+
+	// Initially there is just the default grafana-default-email receiver
+	receivers := initialMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
+	require.Len(t, receivers, 1)
+	require.Equal(t, "grafana-default-email", receivers[0].Name)
+
+	// Now upload a new extra config and check that it gets uploaded to Mimir
+	testAlertmanagerConfigYAML := `
+route:
+  group_by: ['alertname'] 
+  group_wait: 10s
+  group_interval: 10s
+  repeat_interval: 1h
+  receiver: extra-slack
+
+receivers:
+- name: extra-slack
+  slack_configs:
+  - api_url: 'http://localhost/slack'
+    channel: '#alerts'
+    title: 'Alerts'
+`
+
+	headers := map[string]string{
+		"Content-Type":                         "application/yaml",
+		"X-Grafana-Alerting-Config-Identifier": "external-system",
+		"X-Grafana-Alerting-Merge-Matchers":    "environment=production,team=backend",
+	}
+
+	amConfig := apimodels.AlertmanagerUserConfig{
+		AlertmanagerConfig: testAlertmanagerConfigYAML,
+		TemplateFiles: map[string]string{
+			"test.tmpl": `{{ define "test.template" }}Test template for remote sync{{ end }}`,
+		},
+	}
+
+	// Post the configuration to Grafana
+	response := apiClient.ConvertPrometheusPostAlertmanagerConfig(t, amConfig, headers)
+	require.Equal(t, "success", response.Status)
+
+	_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
+	require.Equal(t, http.StatusOK, status)
+
+	// Check that the configuration was successfully sent to Mimir and contains the new receiver
+	finalMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, finalMimirConfig)
+	require.NotNil(t, finalMimirConfig.GrafanaAlertmanagerConfig)
+
+	receivers = finalMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
+	require.Len(t, receivers, 2)
+
+	var foundDefault, foundExtraSlack bool
+	for _, receiver := range receivers {
+		switch receiver.Name {
+		case "grafana-default-email":
+			foundDefault = true
+			require.Len(t, receiver.GrafanaManagedReceivers, 1)
+			require.Equal(t, "email receiver", receiver.GrafanaManagedReceivers[0].Name)
+			require.Equal(t, "email", receiver.GrafanaManagedReceivers[0].Type)
+		case "extra-slack":
+			foundExtraSlack = true
+			require.Len(t, receiver.SlackConfigs, 1)
+			require.NotNil(t, receiver.SlackConfigs[0].APIURL)
+			require.Equal(t, "#alerts", receiver.SlackConfigs[0].Channel)
+		}
+	}
+	require.True(t, foundDefault, "Default receiver not found")
+	require.True(t, foundExtraSlack, "Extra slack receiver not found")
+}
+
+// TestIntegrationRemoteAlertmanagerHistoricalConfigActivation tests that when we activate
+// a historical alertmanager configuration with extra configs, it gets properly decrypted
+// and uploaded to the remote Mimir.
+func TestIntegrationRemoteAlertmanagerHistoricalConfigActivation(t *testing.T) {
+	testinfra.SQLiteIntegrationTest(t)
+
+	s, err := alertmanager.NewAlertmanagerScenario()
+	require.NoError(t, err)
+	defer s.Close()
+
+	s.Mimir = alertmanager.NewMimirService("mimir")
+	require.NoError(t, s.StartAndWaitReady(s.Mimir))
+
+	mimirEndpoint := "http://" + s.Mimir.HTTPEndpoint()
+
+	dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
+		AppModeProduction:     true,
+		EnableFeatureToggles: []string{
+			"alertmanagerRemotePrimary",
+			"alertingImportAlertmanagerAPI",
+		},
+		RemoteAlertmanagerURL: mimirEndpoint,
+	})
+
+	grafanaListedAddr, _ := testinfra.StartGrafanaEnv(t, dir, gpath)
+
+	apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
+	mimirClient, err := alertmanager.NewMimirClient(mimirEndpoint, "1")
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
+		return status == http.StatusOK
+	}, 30*time.Second, time.Second, "Grafana failed to start")
+
+	// Upload configuration with extra configs
+	testAlertmanagerConfigYAML := `
+route:
+  group_by: ['alertname'] 
+  group_wait: 10s
+  group_interval: 10s
+  repeat_interval: 1h
+  receiver: old-slack
+
+receivers:
+- name: old-slack
+  slack_configs:
+  - api_url: 'http://localhost/slack'
+    channel: '#alerts'
+`
+
+	headers := map[string]string{
+		"Content-Type":                         "application/yaml",
+		"X-Grafana-Alerting-Config-Identifier": "historical-system",
+		"X-Grafana-Alerting-Merge-Matchers":    "environment=test,team=platform",
+	}
+
+	amConfig := apimodels.AlertmanagerUserConfig{
+		AlertmanagerConfig: testAlertmanagerConfigYAML,
+		TemplateFiles: map[string]string{
+			"historical.tmpl": `{{ define "historical.template" }}Historical template{{ end }}`,
+		},
+	}
+
+	response := apiClient.ConvertPrometheusPostAlertmanagerConfig(t, amConfig, headers)
+	require.Equal(t, "success", response.Status)
+
+	// Get the configuration history to find the most recent config
+	historyResponse := getAlertmanagerConfigHistory(t, apiClient)
+	require.NotEmpty(t, historyResponse)
+
+	var mostRecentID int64
+	for _, entry := range historyResponse {
+		if entry.ID > mostRecentID {
+			mostRecentID = entry.ID
+		}
+	}
+	require.Greater(t, mostRecentID, int64(0), "Should have found a historical configuration")
+
+	// Activate the historical configuration
+	activateHistoricalConfiguration(t, apiClient, mostRecentID)
+
+	// Verify the configuration
+	finalMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, finalMimirConfig)
+	require.NotNil(t, finalMimirConfig.GrafanaAlertmanagerConfig)
+
+	receivers := finalMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
+	require.Len(t, receivers, 2)
+
+	found := false
+	for _, receiver := range receivers {
+		if receiver.Name == "old-slack" {
+			found = true
+			require.Len(t, receiver.SlackConfigs, 1)
+			break
+		}
+	}
+	require.True(t, found)
+}
+
+func getAlertmanagerConfigHistory(t *testing.T, client apiClient) []apimodels.GettableHistoricUserConfig {
+	t.Helper()
+	u, err := url.Parse(fmt.Sprintf("%s/api/alertmanager/grafana/config/history", client.url))
+	require.NoError(t, err)
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	require.NoError(t, err)
+
+	history, _, _ := sendRequestJSON[[]apimodels.GettableHistoricUserConfig](t, req, http.StatusOK)
+	return history
+}
+
+func activateHistoricalConfiguration(t *testing.T, client apiClient, configID int64) {
+	t.Helper()
+	u, err := url.Parse(fmt.Sprintf("%s/api/alertmanager/grafana/config/history/%d/_activate", client.url, configID))
+	require.NoError(t, err)
+
+	req, err := http.NewRequest(http.MethodPost, u.String(), nil)
+	require.NoError(t, err)
+
+	response, statusCode, body := sendRequestJSON[map[string]string](t, req, http.StatusAccepted)
+	if statusCode != http.StatusAccepted {
+		t.Fatalf("Expected status code %d but got %d. Response body: %s", http.StatusAccepted, statusCode, body)
+	}
+	require.Equal(t, "configuration activated", response["message"])
+}
@@ -477,6 +477,17 @@ func CreateGrafDir(t *testing.T, opts GrafanaOpts) (string, string) {
 		_, err = grafanaComSection.NewKey("api_url", opts.GrafanaComAPIURL)
 		require.NoError(t, err)
 	}
+
+	if opts.RemoteAlertmanagerURL != "" {
+		remoteAlertmanagerSection, err := getOrCreateSection("remote.alertmanager")
+		require.NoError(t, err)
+		_, err = remoteAlertmanagerSection.NewKey("enabled", "true")
+		require.NoError(t, err)
+		_, err = remoteAlertmanagerSection.NewKey("url", opts.RemoteAlertmanagerURL)
+		require.NoError(t, err)
+		_, err = remoteAlertmanagerSection.NewKey("tenant", "1")
+		require.NoError(t, err)
+	}
 	if opts.GrafanaComSSOAPIToken != "" {
 		grafanaComSection, err := getOrCreateSection("grafana_com")
 		require.NoError(t, err)
@@ -571,6 +582,9 @@ type GrafanaOpts struct {

 	// When "unified-grpc" is selected it will also start the grpc server
 	APIServerStorageType options.StorageType
+
+	// Remote alertmanager configuration
+	RemoteAlertmanagerURL string
 }

 func CreateUser(t *testing.T, store db.DB, cfg *setting.Cfg, cmd user.CreateUserCommand) *user.User {