Alerting: API paths for cortextool to import Loki rules (#101409)

Alerting: Legacy rules paths for cortextool
This commit is contained in:
Alexander Akhmetov
2025-02-27 17:20:49 +01:00
committed by GitHub
parent 843d876f16
commit ef86582dfc
12 changed files with 1049 additions and 255 deletions
@@ -99,283 +99,315 @@ var (
)
func TestIntegrationConvertPrometheusEndpoints(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
runTest := func(t *testing.T, enableLokiPaths bool) {
testinfra.SQLiteIntegrationTest(t)
// Setup Grafana and its Database
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"alertingConversionAPI"},
// Setup Grafana and its Database
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"alertingConversionAPI"},
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, gpath)
// Create users to make authenticated requests
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "admin",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
apiClient.prometheusConversionUseLokiPaths = enableLokiPaths
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleViewer),
Password: "password",
Login: "viewer",
})
viewerClient := newAlertingApiClient(grafanaListedAddr, "viewer", "password")
namespace1 := "test-namespace-1"
namespace2 := "test-namespace-2"
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
t.Run("create rule groups and get them back", func(t *testing.T) {
_, status, body := apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup2, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
// create a third group in a different namespace
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace2, ds.Body.Datasource.UID, promGroup3, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
// And a non-provisioned rule in another namespace
namespace3UID := util.GenerateShortUID()
apiClient.CreateFolder(t, namespace3UID, "folder")
createRule(t, apiClient, namespace3UID)
// Now get the first group
group1 := apiClient.ConvertPrometheusGetRuleGroupRules(t, namespace1, promGroup1.Name)
require.Equal(t, promGroup1, group1)
// Get namespace1
ns1 := apiClient.ConvertPrometheusGetNamespaceRules(t, namespace1)
expectedNs1 := map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup1, promGroup2},
}
require.Equal(t, expectedNs1, ns1)
// Get all namespaces
namespaces := apiClient.ConvertPrometheusGetAllRules(t)
expectedNamespaces := map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup1, promGroup2},
namespace2: {promGroup3},
}
require.Equal(t, expectedNamespaces, namespaces)
})
t.Run("without permissions to create folders cannot create rule groups either", func(t *testing.T) {
_, status, raw := viewerClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusForbidden, status, raw)
})
t.Run("delete one rule group", func(t *testing.T) {
_, status, body := apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup2, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace2, ds.Body.Datasource.UID, promGroup3, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
apiClient.ConvertPrometheusDeleteRuleGroup(t, namespace1, promGroup1.Name)
// Check that the promGroup2 and promGroup3 are still there
namespaces := apiClient.ConvertPrometheusGetAllRules(t)
expectedNamespaces := map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup2},
namespace2: {promGroup3},
}
require.Equal(t, expectedNamespaces, namespaces)
// Delete the second namespace
apiClient.ConvertPrometheusDeleteNamespace(t, namespace2)
// Check that only the first namespace is left
namespaces = apiClient.ConvertPrometheusGetAllRules(t)
expectedNamespaces = map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup2},
}
require.Equal(t, expectedNamespaces, namespaces)
})
}
t.Run("with the mimirtool paths", func(t *testing.T) {
runTest(t, false)
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, gpath)
// Create users to make authenticated requests
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "admin",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleViewer),
Password: "password",
Login: "viewer",
})
viewerClient := newAlertingApiClient(grafanaListedAddr, "viewer", "password")
namespace1 := "test-namespace-1"
namespace2 := "test-namespace-2"
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
t.Run("create rule groups and get them back", func(t *testing.T) {
_, status, body := apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup2, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
// create a third group in a different namespace
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace2, ds.Body.Datasource.UID, promGroup3, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
// And a non-provisioned rule in another namespace
namespace3UID := util.GenerateShortUID()
apiClient.CreateFolder(t, namespace3UID, "folder")
createRule(t, apiClient, namespace3UID)
// Now get the first group
group1 := apiClient.ConvertPrometheusGetRuleGroupRules(t, namespace1, promGroup1.Name)
require.Equal(t, promGroup1, group1)
// Get namespace1
ns1 := apiClient.ConvertPrometheusGetNamespaceRules(t, namespace1)
expectedNs1 := map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup1, promGroup2},
}
require.Equal(t, expectedNs1, ns1)
// Get all namespaces
namespaces := apiClient.ConvertPrometheusGetAllRules(t)
expectedNamespaces := map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup1, promGroup2},
namespace2: {promGroup3},
}
require.Equal(t, expectedNamespaces, namespaces)
})
t.Run("without permissions to create folders cannot create rule groups either", func(t *testing.T) {
_, status, raw := viewerClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusForbidden, status, raw)
})
t.Run("delete one rule group", func(t *testing.T) {
_, status, body := apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup2, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
_, status, body = apiClient.ConvertPrometheusPostRuleGroup(t, namespace2, ds.Body.Datasource.UID, promGroup3, nil)
requireStatusCode(t, http.StatusAccepted, status, body)
apiClient.ConvertPrometheusDeleteRuleGroup(t, namespace1, promGroup1.Name)
// Check that the promGroup2 and promGroup3 are still there
namespaces := apiClient.ConvertPrometheusGetAllRules(t)
expectedNamespaces := map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup2},
namespace2: {promGroup3},
}
require.Equal(t, expectedNamespaces, namespaces)
// Delete the second namespace
apiClient.ConvertPrometheusDeleteNamespace(t, namespace2)
// Check that only the first namespace is left
namespaces = apiClient.ConvertPrometheusGetAllRules(t)
expectedNamespaces = map[string][]apimodels.PrometheusRuleGroup{
namespace1: {promGroup2},
}
require.Equal(t, expectedNamespaces, namespaces)
t.Run("with the cortextool Loki paths", func(t *testing.T) {
runTest(t, true)
})
}
func TestIntegrationConvertPrometheusEndpoints_Conflict(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
runTest := func(t *testing.T, enableLokiPaths bool) {
testinfra.SQLiteIntegrationTest(t)
// Setup Grafana and its Database
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"alertingConversionAPI"},
})
// Setup Grafana and its Database
dir, gpath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"alertingConversionAPI"},
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, gpath)
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, gpath)
// Create users to make authenticated requests
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "admin",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
// Create users to make authenticated requests
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "admin",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
apiClient.prometheusConversionUseLokiPaths = enableLokiPaths
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleViewer),
Password: "password",
Login: "viewer",
})
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleViewer),
Password: "password",
Login: "viewer",
})
namespace1 := "test-namespace-1"
namespace1 := "test-namespace-1"
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
t.Run("cannot overwrite a rule group with different provenance", func(t *testing.T) {
// Create a rule group using the provisioning API and then try to overwrite it
// using the Prometheus Conversion API. It should fail because the provenance
// we set for rules in these two APIs is different and we check that when updating.
provisionedRuleGroup := apimodels.AlertRuleGroup{
Title: promGroup1.Name,
Interval: 60,
FolderUID: namespace1,
Rules: []apimodels.ProvisionedAlertRule{
{
Title: "Rule1",
OrgID: 1,
RuleGroup: promGroup1.Name,
Condition: "A",
NoDataState: apimodels.Alerting,
ExecErrState: apimodels.AlertingErrState,
For: prommodel.Duration(time.Duration(60) * time.Second),
Data: []apimodels.AlertQuery{
{
RefID: "A",
RelativeTimeRange: apimodels.RelativeTimeRange{
From: apimodels.Duration(time.Duration(5) * time.Hour),
To: apimodels.Duration(time.Duration(3) * time.Hour),
t.Run("cannot overwrite a rule group with different provenance", func(t *testing.T) {
// Create a rule group using the provisioning API and then try to overwrite it
// using the Prometheus Conversion API. It should fail because the provenance
// we set for rules in these two APIs is different and we check that when updating.
provisionedRuleGroup := apimodels.AlertRuleGroup{
Title: promGroup1.Name,
Interval: 60,
FolderUID: namespace1,
Rules: []apimodels.ProvisionedAlertRule{
{
Title: "Rule1",
OrgID: 1,
RuleGroup: promGroup1.Name,
Condition: "A",
NoDataState: apimodels.Alerting,
ExecErrState: apimodels.AlertingErrState,
For: prommodel.Duration(time.Duration(60) * time.Second),
Data: []apimodels.AlertQuery{
{
RefID: "A",
RelativeTimeRange: apimodels.RelativeTimeRange{
From: apimodels.Duration(time.Duration(5) * time.Hour),
To: apimodels.Duration(time.Duration(3) * time.Hour),
},
DatasourceUID: expr.DatasourceUID,
Model: json.RawMessage([]byte(`{"type":"math","expression":"2 + 3 \u003e 1"}`)),
},
DatasourceUID: expr.DatasourceUID,
Model: json.RawMessage([]byte(`{"type":"math","expression":"2 + 3 \u003e 1"}`)),
},
},
},
},
}
}
// Create the folder
apiClient.CreateFolder(t, namespace1, namespace1)
// Create rule in the root folder using another API
_, status, response := apiClient.CreateOrUpdateRuleGroupProvisioning(t, provisionedRuleGroup)
require.Equalf(t, http.StatusOK, status, response)
// Create the folder
apiClient.CreateFolder(t, namespace1, namespace1)
// Create rule in the root folder using another API
_, status, response := apiClient.CreateOrUpdateRuleGroupProvisioning(t, provisionedRuleGroup)
require.Equalf(t, http.StatusOK, status, response)
// Should fail to post the group
_, status, body := apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusConflict, status, body)
// Should fail to post the group
_, status, body := apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, nil)
requireStatusCode(t, http.StatusConflict, status, body)
})
}
t.Run("with the mimirtool paths", func(t *testing.T) {
runTest(t, false)
})
t.Run("with the cortextool Loki paths", func(t *testing.T) {
runTest(t, true)
})
}
func TestIntegrationConvertPrometheusEndpoints_CreatePausedRules(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
runTest := func(t *testing.T, enableLokiPaths bool) {
testinfra.SQLiteIntegrationTest(t)
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"alertingConversionAPI"},
})
// Setup Grafana and its Database
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"alertingConversionAPI"},
})
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path)
// Create users to make authenticated requests
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "admin",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
// Create users to make authenticated requests
createUser(t, env.SQLStore, env.Cfg, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "admin",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "password")
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
ds := apiClient.CreateDatasource(t, datasources.DS_PROMETHEUS)
namespace1 := "test-namespace-1"
namespace1 := "test-namespace-1"
namespace1UID := util.GenerateShortUID()
apiClient.CreateFolder(t, namespace1UID, namespace1)
namespace1UID := util.GenerateShortUID()
apiClient.CreateFolder(t, namespace1UID, namespace1)
t.Run("when pausing header is set, rules should be paused", func(t *testing.T) {
tests := []struct {
name string
recordingPaused bool
alertPaused bool
}{
{
name: "do not pause rules",
recordingPaused: false,
alertPaused: false,
},
{
name: "pause recording rules",
recordingPaused: true,
alertPaused: false,
},
{
name: "pause alert rules",
recordingPaused: false,
alertPaused: true,
},
{
name: "pause both recording and alert rules",
recordingPaused: true,
alertPaused: true,
},
}
t.Run("when pausing header is set, rules should be paused", func(t *testing.T) {
tests := []struct {
name string
recordingPaused bool
alertPaused bool
}{
{
name: "do not pause rules",
recordingPaused: false,
alertPaused: false,
},
{
name: "pause recording rules",
recordingPaused: true,
alertPaused: false,
},
{
name: "pause alert rules",
recordingPaused: false,
alertPaused: true,
},
{
name: "pause both recording and alert rules",
recordingPaused: true,
alertPaused: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
headers := map[string]string{}
if tc.recordingPaused {
headers["X-Grafana-Alerting-Recording-Rules-Paused"] = "true"
}
if tc.alertPaused {
headers["X-Grafana-Alerting-Alert-Rules-Paused"] = "true"
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
headers := map[string]string{}
if tc.recordingPaused {
headers["X-Grafana-Alerting-Recording-Rules-Paused"] = "true"
}
if tc.alertPaused {
headers["X-Grafana-Alerting-Alert-Rules-Paused"] = "true"
}
apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, headers)
apiClient.ConvertPrometheusPostRuleGroup(t, namespace1, ds.Body.Datasource.UID, promGroup1, headers)
gr, _, _ := apiClient.GetRulesGroupWithStatus(t, namespace1UID, promGroup1.Name)
gr, _, _ := apiClient.GetRulesGroupWithStatus(t, namespace1UID, promGroup1.Name)
require.Len(t, gr.Rules, 3)
require.Len(t, gr.Rules, 3)
pausedRecordingRules := 0
pausedAlertRules := 0
pausedRecordingRules := 0
pausedAlertRules := 0
for _, rule := range gr.Rules {
if rule.GrafanaManagedAlert.IsPaused {
if rule.GrafanaManagedAlert.Record != nil {
pausedRecordingRules++
} else {
pausedAlertRules++
for _, rule := range gr.Rules {
if rule.GrafanaManagedAlert.IsPaused {
if rule.GrafanaManagedAlert.Record != nil {
pausedRecordingRules++
} else {
pausedAlertRules++
}
}
}
}
if tc.recordingPaused {
require.Equal(t, 1, pausedRecordingRules)
} else {
require.Equal(t, 0, pausedRecordingRules)
}
if tc.recordingPaused {
require.Equal(t, 1, pausedRecordingRules)
} else {
require.Equal(t, 0, pausedRecordingRules)
}
if tc.alertPaused {
require.Equal(t, 2, pausedAlertRules)
} else {
require.Equal(t, 0, pausedAlertRules)
}
})
}
if tc.alertPaused {
require.Equal(t, 2, pausedAlertRules)
} else {
require.Equal(t, 0, pausedAlertRules)
}
})
}
})
}
t.Run("with the mimirtool paths", func(t *testing.T) {
runTest(t, false)
})
t.Run("with the cortextool Loki paths", func(t *testing.T) {
runTest(t, true)
})
}