Alerting: Send merged configuration to the remote alertmanager (#107004)
This commit is contained in:
Alexander Akhmetov
committed by
GitHub
GitHub
parent
d57155a19b
commit
b483a04aec
@@ -0,0 +1,262 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/tests/alertmanager"
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra"
|
||||
)
|
||||
|
||||
// TestIntegrationRemoteAlertmanagerConfigUpload tests that when we post an alertmanager
|
||||
// configuration to Grafana with remote alertmanager enabled, it gets uploaded to the remote Mimir.
|
||||
func TestIntegrationRemoteAlertmanagerConfigUpload(t *testing.T) {
|
||||
testinfra.SQLiteIntegrationTest(t)
|
||||
|
||||
s, err := alertmanager.NewAlertmanagerScenario()
|
||||
require.NoError(t, err)
|
||||
defer s.Close()
|
||||
|
||||
s.Mimir = alertmanager.NewMimirService("mimir")
|
||||
require.NoError(t, s.StartAndWaitReady(s.Mimir))
|
||||
|
||||
mimirEndpoint := "http://" + s.Mimir.HTTPEndpoint()
|
||||
|
||||
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
||||
DisableLegacyAlerting: true,
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{
|
||||
"alertmanagerRemotePrimary",
|
||||
"alertingImportAlertmanagerAPI",
|
||||
},
|
||||
RemoteAlertmanagerURL: mimirEndpoint,
|
||||
})
|
||||
|
||||
grafanaListedAddr, _ := testinfra.StartGrafanaEnv(t, dir, gpath)
|
||||
|
||||
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
|
||||
mimirClient, err := alertmanager.NewMimirClient(mimirEndpoint, "1")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for Grafana to be ready
|
||||
require.Eventually(t, func() bool {
|
||||
_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
|
||||
return status == http.StatusOK
|
||||
}, 30*time.Second, time.Second, "Grafana failed to start")
|
||||
|
||||
// Check that the initial Mimir config contains the default Grafana configuration
|
||||
initialMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, initialMimirConfig) // Grafana automatically syncs default config to remote alertmanager
|
||||
require.NotNil(t, initialMimirConfig.GrafanaAlertmanagerConfig)
|
||||
|
||||
// Initially there is just the default grafana-default-email receiver
|
||||
receivers := initialMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
|
||||
require.Len(t, receivers, 1)
|
||||
require.Equal(t, "grafana-default-email", receivers[0].Name)
|
||||
|
||||
// Now upload a new extra config and check that it gets uploaded to Mimir
|
||||
testAlertmanagerConfigYAML := `
|
||||
route:
|
||||
group_by: ['alertname']
|
||||
group_wait: 10s
|
||||
group_interval: 10s
|
||||
repeat_interval: 1h
|
||||
receiver: extra-slack
|
||||
|
||||
receivers:
|
||||
- name: extra-slack
|
||||
slack_configs:
|
||||
- api_url: 'http://localhost/slack'
|
||||
channel: '#alerts'
|
||||
title: 'Alerts'
|
||||
`
|
||||
|
||||
headers := map[string]string{
|
||||
"Content-Type": "application/yaml",
|
||||
"X-Grafana-Alerting-Config-Identifier": "external-system",
|
||||
"X-Grafana-Alerting-Merge-Matchers": "environment=production,team=backend",
|
||||
}
|
||||
|
||||
amConfig := apimodels.AlertmanagerUserConfig{
|
||||
AlertmanagerConfig: testAlertmanagerConfigYAML,
|
||||
TemplateFiles: map[string]string{
|
||||
"test.tmpl": `{{ define "test.template" }}Test template for remote sync{{ end }}`,
|
||||
},
|
||||
}
|
||||
|
||||
// Post the configuration to Grafana
|
||||
response := apiClient.ConvertPrometheusPostAlertmanagerConfig(t, amConfig, headers)
|
||||
require.Equal(t, "success", response.Status)
|
||||
|
||||
_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
|
||||
// Check that the configuration was successfully sent to Mimir and contains the new receiver
|
||||
finalMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, finalMimirConfig)
|
||||
require.NotNil(t, finalMimirConfig.GrafanaAlertmanagerConfig)
|
||||
|
||||
receivers = finalMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
|
||||
require.Len(t, receivers, 2)
|
||||
|
||||
var foundDefault, foundExtraSlack bool
|
||||
for _, receiver := range receivers {
|
||||
switch receiver.Name {
|
||||
case "grafana-default-email":
|
||||
foundDefault = true
|
||||
require.Len(t, receiver.GrafanaManagedReceivers, 1)
|
||||
require.Equal(t, "email receiver", receiver.GrafanaManagedReceivers[0].Name)
|
||||
require.Equal(t, "email", receiver.GrafanaManagedReceivers[0].Type)
|
||||
case "extra-slack":
|
||||
foundExtraSlack = true
|
||||
require.Len(t, receiver.SlackConfigs, 1)
|
||||
require.NotNil(t, receiver.SlackConfigs[0].APIURL)
|
||||
require.Equal(t, "#alerts", receiver.SlackConfigs[0].Channel)
|
||||
}
|
||||
}
|
||||
require.True(t, foundDefault, "Default receiver not found")
|
||||
require.True(t, foundExtraSlack, "Extra slack receiver not found")
|
||||
}
|
||||
|
||||
// TestIntegrationRemoteAlertmanagerHistoricalConfigActivation tests that when we activate
|
||||
// a historical alertmanager configuration with extra configs, it gets properly decrypted
|
||||
// and uploaded to the remote Mimir.
|
||||
func TestIntegrationRemoteAlertmanagerHistoricalConfigActivation(t *testing.T) {
|
||||
testinfra.SQLiteIntegrationTest(t)
|
||||
|
||||
s, err := alertmanager.NewAlertmanagerScenario()
|
||||
require.NoError(t, err)
|
||||
defer s.Close()
|
||||
|
||||
s.Mimir = alertmanager.NewMimirService("mimir")
|
||||
require.NoError(t, s.StartAndWaitReady(s.Mimir))
|
||||
|
||||
mimirEndpoint := "http://" + s.Mimir.HTTPEndpoint()
|
||||
|
||||
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
||||
DisableLegacyAlerting: true,
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{
|
||||
"alertmanagerRemotePrimary",
|
||||
"alertingImportAlertmanagerAPI",
|
||||
},
|
||||
RemoteAlertmanagerURL: mimirEndpoint,
|
||||
})
|
||||
|
||||
grafanaListedAddr, _ := testinfra.StartGrafanaEnv(t, dir, gpath)
|
||||
|
||||
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
|
||||
mimirClient, err := alertmanager.NewMimirClient(mimirEndpoint, "1")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
_, status, _ := apiClient.GetAlertmanagerConfigWithStatus(t)
|
||||
return status == http.StatusOK
|
||||
}, 30*time.Second, time.Second, "Grafana failed to start")
|
||||
|
||||
// Upload configuration with extra configs
|
||||
testAlertmanagerConfigYAML := `
|
||||
route:
|
||||
group_by: ['alertname']
|
||||
group_wait: 10s
|
||||
group_interval: 10s
|
||||
repeat_interval: 1h
|
||||
receiver: old-slack
|
||||
|
||||
receivers:
|
||||
- name: old-slack
|
||||
slack_configs:
|
||||
- api_url: 'http://localhost/slack'
|
||||
channel: '#alerts'
|
||||
`
|
||||
|
||||
headers := map[string]string{
|
||||
"Content-Type": "application/yaml",
|
||||
"X-Grafana-Alerting-Config-Identifier": "historical-system",
|
||||
"X-Grafana-Alerting-Merge-Matchers": "environment=test,team=platform",
|
||||
}
|
||||
|
||||
amConfig := apimodels.AlertmanagerUserConfig{
|
||||
AlertmanagerConfig: testAlertmanagerConfigYAML,
|
||||
TemplateFiles: map[string]string{
|
||||
"historical.tmpl": `{{ define "historical.template" }}Historical template{{ end }}`,
|
||||
},
|
||||
}
|
||||
|
||||
response := apiClient.ConvertPrometheusPostAlertmanagerConfig(t, amConfig, headers)
|
||||
require.Equal(t, "success", response.Status)
|
||||
|
||||
// Get the configuration history to find the most recent config
|
||||
historyResponse := getAlertmanagerConfigHistory(t, apiClient)
|
||||
require.NotEmpty(t, historyResponse)
|
||||
|
||||
var mostRecentID int64
|
||||
for _, entry := range historyResponse {
|
||||
if entry.ID > mostRecentID {
|
||||
mostRecentID = entry.ID
|
||||
}
|
||||
}
|
||||
require.Greater(t, mostRecentID, int64(0), "Should have found a historical configuration")
|
||||
|
||||
// Activate the historical configuration
|
||||
activateHistoricalConfiguration(t, apiClient, mostRecentID)
|
||||
|
||||
// Verify the configuration
|
||||
finalMimirConfig, err := mimirClient.GetGrafanaAlertmanagerConfig(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, finalMimirConfig)
|
||||
require.NotNil(t, finalMimirConfig.GrafanaAlertmanagerConfig)
|
||||
|
||||
receivers := finalMimirConfig.GrafanaAlertmanagerConfig.AlertmanagerConfig.Receivers
|
||||
require.Len(t, receivers, 2)
|
||||
|
||||
found := false
|
||||
for _, receiver := range receivers {
|
||||
if receiver.Name == "old-slack" {
|
||||
found = true
|
||||
require.Len(t, receiver.SlackConfigs, 1)
|
||||
break
|
||||
}
|
||||
}
|
||||
require.True(t, found)
|
||||
}
|
||||
|
||||
func getAlertmanagerConfigHistory(t *testing.T, client apiClient) []apimodels.GettableHistoricUserConfig {
|
||||
t.Helper()
|
||||
u, err := url.Parse(fmt.Sprintf("%s/api/alertmanager/grafana/config/history", client.url))
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
history, _, _ := sendRequestJSON[[]apimodels.GettableHistoricUserConfig](t, req, http.StatusOK)
|
||||
return history
|
||||
}
|
||||
|
||||
func activateHistoricalConfiguration(t *testing.T, client apiClient, configID int64) {
|
||||
t.Helper()
|
||||
u, err := url.Parse(fmt.Sprintf("%s/api/alertmanager/grafana/config/history/%d/_activate", client.url, configID))
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, u.String(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
response, statusCode, body := sendRequestJSON[map[string]string](t, req, http.StatusAccepted)
|
||||
if statusCode != http.StatusAccepted {
|
||||
t.Fatalf("Expected status code %d but got %d. Response body: %s", http.StatusAccepted, statusCode, body)
|
||||
}
|
||||
require.Equal(t, "configuration activated", response["message"])
|
||||
}
|
||||
Reference in New Issue
Block a user