Alerting: Enable recording rules by default (#105603)

This commit is contained in:
Alexander Akhmetov
2025-06-02 10:56:05 +02:00
committed by GitHub
parent 4cde79e802
commit e256f2d5e2
26 changed files with 113 additions and 347 deletions
-8
View File
@@ -1554,14 +1554,6 @@ var (
Owner: grafanaAlertingSquad,
Expression: "true", // enabled by default
},
{
Name: "grafanaManagedRecordingRulesDatasources",
Description: "Enables writing to data sources for Grafana-managed recording rules.",
Stage: FeatureStageGeneralAvailability,
Owner: grafanaAlertingSquad,
AllowSelfServe: false,
Expression: "false",
},
{
Name: "infinityRunQueriesInParallel",
Description: "Enables running Infinity queries in parallel",
-1
View File
@@ -204,7 +204,6 @@ newShareReportDrawer,experimental,@grafana/sharing-squad,false,false,false
rendererDisableAppPluginsPreload,experimental,@grafana/sharing-squad,false,false,true
assetSriChecks,experimental,@grafana/frontend-ops,false,false,true
alertRuleRestore,preview,@grafana/alerting-squad,false,false,false
grafanaManagedRecordingRulesDatasources,GA,@grafana/alerting-squad,false,false,false
infinityRunQueriesInParallel,privatePreview,@grafana/oss-big-tent,false,false,false
inviteUserExperimental,experimental,@grafana/sharing-squad,false,false,true
alertingMigrationUI,GA,@grafana/alerting-squad,false,false,true
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
204 rendererDisableAppPluginsPreload experimental @grafana/sharing-squad false false true
205 assetSriChecks experimental @grafana/frontend-ops false false true
206 alertRuleRestore preview @grafana/alerting-squad false false false
grafanaManagedRecordingRulesDatasources GA @grafana/alerting-squad false false false
207 infinityRunQueriesInParallel privatePreview @grafana/oss-big-tent false false false
208 inviteUserExperimental experimental @grafana/sharing-squad false false true
209 alertingMigrationUI GA @grafana/alerting-squad false false true
-4
View File
@@ -827,10 +827,6 @@ const (
// Enables the alert rule restore feature
FlagAlertRuleRestore = "alertRuleRestore"
// FlagGrafanaManagedRecordingRulesDatasources
// Enables writing to data sources for Grafana-managed recording rules.
FlagGrafanaManagedRecordingRulesDatasources = "grafanaManagedRecordingRulesDatasources"
// FlagInfinityRunQueriesInParallel
// Enables running Infinity queries in parallel
FlagInfinityRunQueriesInParallel = "infinityRunQueriesInParallel"
@@ -1449,6 +1449,7 @@
"name": "grafanaManagedRecordingRulesDatasources",
"resourceVersion": "1746998568254",
"creationTimestamp": "2025-03-07T13:30:40Z",
"deletionTimestamp": "2025-05-19T10:18:04Z",
"annotations": {
"grafana.app/updatedTimestamp": "2025-05-11 21:22:48.254378 +0000 UTC"
}
@@ -62,11 +62,6 @@ var (
"alerting.recordingRulesNotEnabled",
errutil.WithPublicMessage("Cannot import recording rules: Feature not enabled."),
).Errorf("recording rules not enabled")
errRecordingRulesDatasourcesNotEnabled = errutil.ValidationFailed(
"alerting.recordingRulesDatasourcesNotEnabled",
errutil.WithPublicMessage("Cannot import recording rules: Configuration of target datasources not enabled."),
).Errorf("recording rules target datasources configuration not enabled")
)
func errInvalidHeaderValue(header string, err error) error {
@@ -399,11 +394,6 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroups(c *context
logger.Error("Cannot import recording rules", "error", errRecordingRulesNotEnabled)
return errorToResponse(errRecordingRulesNotEnabled)
}
if !srv.featureToggles.IsEnabledGlobally(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources) {
logger.Error("Cannot import recording rules", "error", errRecordingRulesDatasourcesNotEnabled)
return errorToResponse(errRecordingRulesDatasourcesNotEnabled)
}
}
grafanaGroup, err := srv.convertToGrafanaRuleGroup(
@@ -295,45 +295,25 @@ func TestRouteConvertPrometheusPostRuleGroup(t *testing.T) {
t.Run("with disabled recording rules", func(t *testing.T) {
testCases := []struct {
name string
recordingRules bool
recordingRulesTargetDS bool
expectedStatus int
name string
recordingRules bool
expectedStatus int
}{
{
name: "when recording rules are enabled",
recordingRules: true,
recordingRulesTargetDS: true,
expectedStatus: http.StatusAccepted,
name: "when recording rules are enabled",
recordingRules: true,
expectedStatus: http.StatusAccepted,
},
{
name: "when recording rules are disabled",
recordingRules: false,
recordingRulesTargetDS: true,
expectedStatus: http.StatusBadRequest,
},
{
name: "when target datasources for recording rules are disabled",
recordingRules: true,
recordingRulesTargetDS: false,
expectedStatus: http.StatusBadRequest,
},
{
name: "when both recording rules and target datasources are disabled",
recordingRules: false,
recordingRulesTargetDS: false,
expectedStatus: http.StatusBadRequest,
name: "when recording rules are disabled",
recordingRules: false,
expectedStatus: http.StatusBadRequest,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
var features featuremgmt.FeatureToggles
if tc.recordingRulesTargetDS {
features = featuremgmt.WithFeatures(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources)
} else {
features = featuremgmt.WithFeatures()
}
features := featuremgmt.WithFeatures()
srv, _, _, _ := createConvertPrometheusSrv(t, withFeatureToggles(features))
srv.cfg.RecordingRules.Enabled = tc.recordingRules
@@ -1317,7 +1297,6 @@ func createConvertPrometheusSrv(t *testing.T, opts ...convertPrometheusSrvOption
provenanceStore: fakes.NewFakeProvisioningStore(),
fakeAccessControlRuleService: &acfakes.FakeRuleService{},
quotaChecker: quotas,
featureToggles: featuremgmt.WithFeatures(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources),
}
for _, opt := range opts {
+11 -16
View File
@@ -317,7 +317,7 @@ func (ng *AlertNG) init() error {
evalFactory := eval.NewEvaluatorFactory(ng.Cfg.UnifiedAlerting, ng.DataSourceCache, ng.ExpressionService)
conditionValidator := eval.NewConditionValidator(ng.DataSourceCache, ng.ExpressionService, ng.pluginsStore)
recordingWriter, err := createRecordingWriter(ng.FeatureToggles, ng.Cfg.UnifiedAlerting.RecordingRules, ng.httpClientProvider, ng.DataSourceService, clk, ng.Metrics.GetRemoteWriterMetrics())
recordingWriter, err := createRecordingWriter(ng.Cfg.UnifiedAlerting.RecordingRules, ng.httpClientProvider, ng.DataSourceService, clk, ng.Metrics.GetRemoteWriterMetrics())
if err != nil {
return fmt.Errorf("failed to initialize recording writer: %w", err)
}
@@ -649,25 +649,20 @@ func createRemoteAlertmanager(ctx context.Context, cfg remote.AlertmanagerConfig
return remote.NewAlertmanager(ctx, cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
}
func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings, httpClientProvider httpclient.Provider, datasourceService datasources.DataSourceService, clock clock.Clock, m *metrics.RemoteWriter) (schedule.RecordingWriter, error) {
func createRecordingWriter(settings setting.RecordingRuleSettings, httpClientProvider httpclient.Provider, datasourceService datasources.DataSourceService, clock clock.Clock, m *metrics.RemoteWriter) (schedule.RecordingWriter, error) {
logger := log.New("ngalert.writer")
if settings.Enabled {
if featureToggles.IsEnabledGlobally(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources) {
cfg := writer.DatasourceWriterConfig{
Timeout: settings.Timeout,
CustomHeaders: settings.CustomHeaders,
DefaultDatasourceUID: settings.DefaultDatasourceUID,
}
logger.Info("Setting up remote write using data sources",
"timeout", cfg.Timeout, "default_datasource_uid", cfg.DefaultDatasourceUID)
return writer.NewDatasourceWriter(cfg, datasourceService, httpClientProvider, clock, logger, m), nil
} else {
logger.Info("Setting up remote write using static configuration")
return writer.NewPrometheusWriterWithSettings(settings, httpClientProvider, clock, logger, m)
cfg := writer.DatasourceWriterConfig{
Timeout: settings.Timeout,
CustomHeaders: settings.CustomHeaders,
DefaultDatasourceUID: settings.DefaultDatasourceUID,
}
logger.Info("Setting up remote write using data sources",
"timeout", cfg.Timeout, "default_datasource_uid", cfg.DefaultDatasourceUID)
return writer.NewDatasourceWriter(cfg, datasourceService, httpClientProvider, clock, logger, m), nil
}
return writer.NoopWriter{}, nil
@@ -177,13 +177,6 @@ func blankRecordingRuleForTests(ctx context.Context) *recordingRule {
}
func TestRecordingRule_Integration(t *testing.T) {
t.Run("with prometheus writer", func(t *testing.T) {
writeTarget := writer.NewTestRemoteWriteTarget(t)
defer writeTarget.Close()
writerReg := prometheus.NewPedanticRegistry()
writer := setupPrometheusWriter(t, writeTarget, writerReg)
testRecordingRule_Integration(t, writeTarget, writer, writerReg, "")
})
t.Run("with datasource writer", func(t *testing.T) {
writeTarget := writer.NewTestRemoteWriteTarget(t)
defer writeTarget.Close()
@@ -798,14 +791,6 @@ func withQueryForHealth(health string) models.AlertRuleMutator {
}
}
func setupPrometheusWriter(t *testing.T, target *writer.TestRemoteWriteTarget, reg prometheus.Registerer) *writer.PrometheusWriter {
provider := testClientProvider{}
m := metrics.NewNGAlert(reg)
wr, err := writer.NewPrometheusWriterWithSettings(target.ClientSettings(), provider, clock.NewMock(), log.NewNopLogger(), m.GetRemoteWriterMetrics())
require.NoError(t, err)
return wr
}
func setupDatasourceWriter(t *testing.T, target *writer.TestRemoteWriteTarget, reg prometheus.Registerer, dsUID string) *writer.DatasourceWriter {
provider := testClientProvider{}
m := metrics.NewNGAlert(reg)
-58
View File
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
@@ -16,7 +15,6 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana-plugin-sdk-go/data"
@@ -250,34 +248,6 @@ type PrometheusWriterConfig struct {
Timeout time.Duration
}
func NewPrometheusWriterWithSettings(
settings setting.RecordingRuleSettings,
httpClientProvider HttpClientProvider,
clock clock.Clock,
l log.Logger,
metrics *metrics.RemoteWriter,
) (*PrometheusWriter, error) {
if err := validateSettings(settings); err != nil {
return nil, err
}
headers := make(http.Header)
for k, v := range settings.CustomHeaders {
headers.Add(k, v)
}
cfg := PrometheusWriterConfig{
URL: settings.URL,
HTTPOptions: httpclient.Options{
BasicAuth: createAuthOpts(settings.BasicAuthUsername, settings.BasicAuthPassword),
Header: headers,
},
Timeout: settings.Timeout,
}
return NewPrometheusWriter(cfg, httpClientProvider, clock, l, metrics)
}
func NewPrometheusWriter(
cfg PrometheusWriterConfig,
httpClientProvider HttpClientProvider,
@@ -310,34 +280,6 @@ func NewPrometheusWriter(
}, nil
}
func validateSettings(settings setting.RecordingRuleSettings) error {
if settings.BasicAuthUsername != "" && settings.BasicAuthPassword == "" {
return fmt.Errorf("basic auth password is required if username is set")
}
if _, err := url.Parse(settings.URL); err != nil {
return fmt.Errorf("invalid URL: %w", err)
}
if settings.Timeout <= 0 {
return fmt.Errorf("timeout must be greater than 0")
}
return nil
}
func createAuthOpts(username, password string) *httpclient.BasicAuthOptions {
// If username is empty, do not use basic auth and ignore password.
if username == "" {
return nil
}
return &httpclient.BasicAuthOptions{
User: username,
Password: password,
}
}
// Write writes the given frames to the Prometheus remote write endpoint.
func (w PrometheusWriter) WriteDatasource(ctx context.Context, dsUID string, name string, t time.Time, frames data.Frames, orgID int64, extraLabels map[string]string) error {
l := w.logger.FromContext(ctx)
-62
View File
@@ -19,74 +19,12 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
)
func TestValidateSettings(t *testing.T) {
for _, tc := range []struct {
name string
settings setting.RecordingRuleSettings
err bool
}{
{
name: "invalid url",
settings: setting.RecordingRuleSettings{
URL: "invalid url",
},
err: true,
},
{
name: "missing password",
settings: setting.RecordingRuleSettings{
URL: "http://localhost:9090",
BasicAuthUsername: "user",
},
err: true,
},
{
name: "timeout is 0",
settings: setting.RecordingRuleSettings{
URL: "http://localhost:9090",
BasicAuthUsername: "user",
BasicAuthPassword: "password",
Timeout: 0,
},
err: true,
},
{
name: "valid settings w/ auth",
settings: setting.RecordingRuleSettings{
URL: "http://localhost:9090",
BasicAuthUsername: "user",
BasicAuthPassword: "password",
Timeout: 10,
},
err: false,
},
{
name: "valid settings w/o auth",
settings: setting.RecordingRuleSettings{
URL: "http://localhost:9090",
Timeout: 10,
},
err: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := validateSettings(tc.settings)
if tc.err {
require.Error(t, err)
} else {
require.NoError(t, err)
}
})
}
}
func TestPointsFromFrames(t *testing.T) {
extraLabels := map[string]string{"extra": "label"}
+1 -4
View File
@@ -71,10 +71,7 @@ func (s *TestRemoteWriteTarget) DatasourceURL() string {
func (s *TestRemoteWriteTarget) ClientSettings() setting.RecordingRuleSettings {
return setting.RecordingRuleSettings{
URL: s.srv.URL + RemoteWriteEndpoint,
Timeout: 1 * time.Second,
BasicAuthUsername: "",
BasicAuthPassword: "",
Timeout: 1 * time.Second,
}
}
+1 -7
View File
@@ -138,9 +138,6 @@ type UnifiedAlertingSettings struct {
type RecordingRuleSettings struct {
Enabled bool
URL string
BasicAuthUsername string
BasicAuthPassword string
CustomHeaders map[string]string
Timeout time.Duration
DefaultDatasourceUID string
@@ -455,10 +452,7 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
rr := iniFile.Section("recording_rules")
uaCfgRecordingRules := RecordingRuleSettings{
Enabled: rr.Key("enabled").MustBool(false),
URL: rr.Key("url").MustString(""),
BasicAuthUsername: rr.Key("basic_auth_username").MustString(""),
BasicAuthPassword: rr.Key("basic_auth_password").MustString(""),
Enabled: rr.Key("enabled").MustBool(true),
Timeout: rr.Key("timeout").MustDuration(defaultRecordingRequestTimeout),
DefaultDatasourceUID: rr.Key("default_datasource_uid").MustString(""),
}
@@ -28,7 +28,6 @@ func TestIntegrationConvertPrometheusNotificationSettings(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -113,7 +113,6 @@ func TestIntegrationConvertPrometheusEndpoints_RecordingRuleTargetDatasource(t *
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -181,7 +180,6 @@ func TestIntegrationConvertPrometheusEndpoints(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -387,7 +385,6 @@ func TestIntegrationConvertPrometheusEndpoints_UpdateRule(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -468,7 +465,6 @@ func TestIntegrationConvertPrometheusEndpoints_Conflict(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -550,7 +546,6 @@ func TestIntegrationConvertPrometheusEndpoints_CreatePausedRules(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -658,7 +653,6 @@ func TestIntegrationConvertPrometheusEndpoints_FolderUIDHeader(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -755,7 +749,6 @@ func TestIntegrationConvertPrometheusEndpoints_Provenance(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})
@@ -865,7 +858,6 @@ func TestIntegrationConvertPrometheusEndpoints_Delete(t *testing.T) {
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
EnableRecordingRules: true,
})