Alerting: Allow selection of recording rule write target on per-rule basis. (#101778)

* Alerting: Allow selection of recording rule write target on per-rule basis.

Introduces a new feature flag (`grafanaManagedRecordingRulesDatasources`),
disabled by default, to enable the ability to write recording rules data using
data source settings, and selecting the data source to use on a per-rule basis.

To cope with the scenario of users upgrading, a configuration file option
allows setting the default data source to use, if none is specified in the rule,
emulating the behaviour of recording rules without the flag enabled.

* Lint

* Update conf/sample.ini

Co-authored-by: Alexander Akhmetov <me@alx.cx>

---------

Co-authored-by: Alexander Akhmetov <me@alx.cx>

conf/defaults.ini

@@ -1523,6 +1523,13 @@ basic_auth_password =
 # Request timeout for recording rule writes.
 timeout = 10s
 
+# Default data source UID to write to if not specified in the rule definition.
+# Only has effect if the grafanaManagedRecordRulesDatasources feature toggle is enabled.
+default_datasource_uid = 
+
+# Suffix to apply to the data source URL for remote write requests.
+remote_write_path_suffix = /push
+
 # Optional custom headers to include in recording rule write requests.
 [recording_rules.custom_headers]
 # exampleHeader = exampleValue

conf/sample.ini

@@ -1505,6 +1505,13 @@ basic_auth_password =
 # Request timeout for recording rule writes.
 timeout = 30s
 
+# Default data source UID to write to if not specified in the rule definition.
+# Only has effect if the grafanaManagedRecordRulesDatasources feature toggle is enabled.
+default_datasource_uid = 
+
+# Suffix to apply to the data source URL for remote write requests.
+remote_write_path_suffix = /push
+
 # Optional custom headers to include in recording rule write requests.
 [recording_rules.custom_headers]
 # exampleHeader = exampleValue

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -256,4 +256,5 @@ export interface FeatureToggles {
   rendererDisableAppPluginsPreload?: boolean;
   assetSriChecks?: boolean;
   alertRuleRestore?: boolean;
+  grafanaManagedRecordingRulesDatasources?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -1790,6 +1790,15 @@ var (
 			Owner:       grafanaAlertingSquad,
 			Expression:  "true", // enabled by default
 		},
+		{
+			Name:              "grafanaManagedRecordingRulesDatasources",
+			Description:       "Enables writing to data sources for Grafana-managed recording rules.",
+			Stage:             FeatureStageExperimental,
+			Owner:             grafanaAlertingSquad,
+			AllowSelfServe:    false,
+			HideFromAdminPage: true,
+			HideFromDocs:      true,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -237,3 +237,4 @@ newShareReportDrawer,experimental,@grafana/sharing-squad,false,false,false
 rendererDisableAppPluginsPreload,experimental,@grafana/sharing-squad,false,false,true
 assetSriChecks,experimental,@grafana/frontend-ops,false,false,true
 alertRuleRestore,preview,@grafana/alerting-squad,false,false,false
+grafanaManagedRecordingRulesDatasources,experimental,@grafana/alerting-squad,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -958,4 +958,8 @@ const (
 	// FlagAlertRuleRestore
 	// Enables the alert rule restore feature
 	FlagAlertRuleRestore = "alertRuleRestore"
+
+	// FlagGrafanaManagedRecordingRulesDatasources
+	// Enables writing to data sources for Grafana-managed recording rules.
+	FlagGrafanaManagedRecordingRulesDatasources = "grafanaManagedRecordingRulesDatasources"
 )

pkg/services/featuremgmt/toggles_gen.json

@@ -1964,6 +1964,20 @@
         "hideFromDocs": true
       }
     },
+    {
+      "metadata": {
+        "name": "grafanaManagedRecordingRulesDatasources",
+        "resourceVersion": "1741291902441",
+        "creationTimestamp": "2025-03-06T20:11:42Z"
+      },
+      "spec": {
+        "description": "Enables writing to data sources for Grafana-managed recording rules.",
+        "stage": "experimental",
+        "codeowner": "@grafana/alerting-squad",
+        "hideFromAdminPage": true,
+        "hideFromDocs": true
+      }
+    },
     {
       "metadata": {
         "name": "grafanaconThemes",

pkg/services/ngalert/ngalert.go

@@ -374,7 +374,7 @@ func (ng *AlertNG) init() error {
 		// Force-disable the feature if the feature toggle is not on - sets us up for feature toggle removal.
 		ng.Cfg.UnifiedAlerting.RecordingRules.Enabled = false
 	}
-	recordingWriter, err := createRecordingWriter(ng.FeatureToggles, ng.Cfg.UnifiedAlerting.RecordingRules, ng.httpClientProvider, clk, ng.Metrics.GetRemoteWriterMetrics())
+	recordingWriter, err := createRecordingWriter(ng.FeatureToggles, ng.Cfg.UnifiedAlerting.RecordingRules, ng.httpClientProvider, ng.DataSourceService, clk, ng.Metrics.GetRemoteWriterMetrics())
 	if err != nil {
 		return fmt.Errorf("failed to initialize recording writer: %w", err)
 	}
@@ -754,11 +754,26 @@ func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVS
 	return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
 }
 
-func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings, httpClientProvider httpclient.Provider, clock clock.Clock, m *metrics.RemoteWriter) (schedule.RecordingWriter, error) {
+func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings, httpClientProvider httpclient.Provider, datasourceService datasources.DataSourceService, clock clock.Clock, m *metrics.RemoteWriter) (schedule.RecordingWriter, error) {
 	logger := log.New("ngalert.writer")
 
 	if settings.Enabled {
-		return writer.NewPrometheusWriterWithSettings(settings, httpClientProvider, clock, logger, m)
+		if featureToggles.IsEnabledGlobally(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources) {
+			cfg := writer.DatasourceWriterConfig{
+				Timeout:               settings.Timeout,
+				DefaultDatasourceUID:  settings.DefaultDatasourceUID,
+				RemoteWritePathSuffix: settings.RemoteWritePathSuffix,
+			}
+
+			logger.Info("Setting up remote write using data sources",
+				"timeout", cfg.Timeout, "default_datasource_uid", cfg.DefaultDatasourceUID,
+				"remote_write_path_suffix", cfg.RemoteWritePathSuffix)
+
+			return writer.NewDatasourceWriter(cfg, datasourceService, httpClientProvider, clock, logger, m), nil
+		} else {
+			logger.Info("Setting up remote write using static configuration")
+			return writer.NewPrometheusWriterWithSettings(settings, httpClientProvider, clock, logger, m)
+		}
 	}
 
 	return writer.NoopWriter{}, nil

pkg/setting/setting_unified_alerting.go

@@ -132,12 +132,14 @@ type UnifiedAlertingSettings struct {
 }
 
 type RecordingRuleSettings struct {
-	Enabled           bool
-	URL               string
-	BasicAuthUsername string
-	BasicAuthPassword string
-	CustomHeaders     map[string]string
-	Timeout           time.Duration
+	Enabled               bool
+	URL                   string
+	BasicAuthUsername     string
+	BasicAuthPassword     string
+	CustomHeaders         map[string]string
+	Timeout               time.Duration
+	DefaultDatasourceUID  string
+	RemoteWritePathSuffix string
 }
 
 // RemoteAlertmanagerSettings contains the configuration needed
@@ -435,11 +437,13 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 
 	rr := iniFile.Section("recording_rules")
 	uaCfgRecordingRules := RecordingRuleSettings{
-		Enabled:           rr.Key("enabled").MustBool(false),
-		URL:               rr.Key("url").MustString(""),
-		BasicAuthUsername: rr.Key("basic_auth_username").MustString(""),
-		BasicAuthPassword: rr.Key("basic_auth_password").MustString(""),
-		Timeout:           rr.Key("timeout").MustDuration(defaultRecordingRequestTimeout),
+		Enabled:               rr.Key("enabled").MustBool(false),
+		URL:                   rr.Key("url").MustString(""),
+		BasicAuthUsername:     rr.Key("basic_auth_username").MustString(""),
+		BasicAuthPassword:     rr.Key("basic_auth_password").MustString(""),
+		Timeout:               rr.Key("timeout").MustDuration(defaultRecordingRequestTimeout),
+		DefaultDatasourceUID:  rr.Key("default_datasource_uid").MustString(""),
+		RemoteWritePathSuffix: rr.Key("remote_write_path_suffix").MustString("/push"),
 	}
 
 	rrHeaders := iniFile.Section("recording_rules.custom_headers")