Alerting: Enable recording rules by default (#105603)
This commit is contained in:
@@ -1554,14 +1554,6 @@ var (
|
||||
Owner: grafanaAlertingSquad,
|
||||
Expression: "true", // enabled by default
|
||||
},
|
||||
{
|
||||
Name: "grafanaManagedRecordingRulesDatasources",
|
||||
Description: "Enables writing to data sources for Grafana-managed recording rules.",
|
||||
Stage: FeatureStageGeneralAvailability,
|
||||
Owner: grafanaAlertingSquad,
|
||||
AllowSelfServe: false,
|
||||
Expression: "false",
|
||||
},
|
||||
{
|
||||
Name: "infinityRunQueriesInParallel",
|
||||
Description: "Enables running Infinity queries in parallel",
|
||||
|
||||
@@ -204,7 +204,6 @@ newShareReportDrawer,experimental,@grafana/sharing-squad,false,false,false
|
||||
rendererDisableAppPluginsPreload,experimental,@grafana/sharing-squad,false,false,true
|
||||
assetSriChecks,experimental,@grafana/frontend-ops,false,false,true
|
||||
alertRuleRestore,preview,@grafana/alerting-squad,false,false,false
|
||||
grafanaManagedRecordingRulesDatasources,GA,@grafana/alerting-squad,false,false,false
|
||||
infinityRunQueriesInParallel,privatePreview,@grafana/oss-big-tent,false,false,false
|
||||
inviteUserExperimental,experimental,@grafana/sharing-squad,false,false,true
|
||||
alertingMigrationUI,GA,@grafana/alerting-squad,false,false,true
|
||||
|
||||
|
@@ -827,10 +827,6 @@ const (
|
||||
// Enables the alert rule restore feature
|
||||
FlagAlertRuleRestore = "alertRuleRestore"
|
||||
|
||||
// FlagGrafanaManagedRecordingRulesDatasources
|
||||
// Enables writing to data sources for Grafana-managed recording rules.
|
||||
FlagGrafanaManagedRecordingRulesDatasources = "grafanaManagedRecordingRulesDatasources"
|
||||
|
||||
// FlagInfinityRunQueriesInParallel
|
||||
// Enables running Infinity queries in parallel
|
||||
FlagInfinityRunQueriesInParallel = "infinityRunQueriesInParallel"
|
||||
|
||||
@@ -1449,6 +1449,7 @@
|
||||
"name": "grafanaManagedRecordingRulesDatasources",
|
||||
"resourceVersion": "1746998568254",
|
||||
"creationTimestamp": "2025-03-07T13:30:40Z",
|
||||
"deletionTimestamp": "2025-05-19T10:18:04Z",
|
||||
"annotations": {
|
||||
"grafana.app/updatedTimestamp": "2025-05-11 21:22:48.254378 +0000 UTC"
|
||||
}
|
||||
|
||||
@@ -62,11 +62,6 @@ var (
|
||||
"alerting.recordingRulesNotEnabled",
|
||||
errutil.WithPublicMessage("Cannot import recording rules: Feature not enabled."),
|
||||
).Errorf("recording rules not enabled")
|
||||
|
||||
errRecordingRulesDatasourcesNotEnabled = errutil.ValidationFailed(
|
||||
"alerting.recordingRulesDatasourcesNotEnabled",
|
||||
errutil.WithPublicMessage("Cannot import recording rules: Configuration of target datasources not enabled."),
|
||||
).Errorf("recording rules target datasources configuration not enabled")
|
||||
)
|
||||
|
||||
func errInvalidHeaderValue(header string, err error) error {
|
||||
@@ -399,11 +394,6 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroups(c *context
|
||||
logger.Error("Cannot import recording rules", "error", errRecordingRulesNotEnabled)
|
||||
return errorToResponse(errRecordingRulesNotEnabled)
|
||||
}
|
||||
|
||||
if !srv.featureToggles.IsEnabledGlobally(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources) {
|
||||
logger.Error("Cannot import recording rules", "error", errRecordingRulesDatasourcesNotEnabled)
|
||||
return errorToResponse(errRecordingRulesDatasourcesNotEnabled)
|
||||
}
|
||||
}
|
||||
|
||||
grafanaGroup, err := srv.convertToGrafanaRuleGroup(
|
||||
|
||||
@@ -295,45 +295,25 @@ func TestRouteConvertPrometheusPostRuleGroup(t *testing.T) {
|
||||
|
||||
t.Run("with disabled recording rules", func(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
recordingRules bool
|
||||
recordingRulesTargetDS bool
|
||||
expectedStatus int
|
||||
name string
|
||||
recordingRules bool
|
||||
expectedStatus int
|
||||
}{
|
||||
{
|
||||
name: "when recording rules are enabled",
|
||||
recordingRules: true,
|
||||
recordingRulesTargetDS: true,
|
||||
expectedStatus: http.StatusAccepted,
|
||||
name: "when recording rules are enabled",
|
||||
recordingRules: true,
|
||||
expectedStatus: http.StatusAccepted,
|
||||
},
|
||||
{
|
||||
name: "when recording rules are disabled",
|
||||
recordingRules: false,
|
||||
recordingRulesTargetDS: true,
|
||||
expectedStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "when target datasources for recording rules are disabled",
|
||||
recordingRules: true,
|
||||
recordingRulesTargetDS: false,
|
||||
expectedStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "when both recording rules and target datasources are disabled",
|
||||
recordingRules: false,
|
||||
recordingRulesTargetDS: false,
|
||||
expectedStatus: http.StatusBadRequest,
|
||||
name: "when recording rules are disabled",
|
||||
recordingRules: false,
|
||||
expectedStatus: http.StatusBadRequest,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var features featuremgmt.FeatureToggles
|
||||
if tc.recordingRulesTargetDS {
|
||||
features = featuremgmt.WithFeatures(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources)
|
||||
} else {
|
||||
features = featuremgmt.WithFeatures()
|
||||
}
|
||||
features := featuremgmt.WithFeatures()
|
||||
|
||||
srv, _, _, _ := createConvertPrometheusSrv(t, withFeatureToggles(features))
|
||||
srv.cfg.RecordingRules.Enabled = tc.recordingRules
|
||||
@@ -1317,7 +1297,6 @@ func createConvertPrometheusSrv(t *testing.T, opts ...convertPrometheusSrvOption
|
||||
provenanceStore: fakes.NewFakeProvisioningStore(),
|
||||
fakeAccessControlRuleService: &acfakes.FakeRuleService{},
|
||||
quotaChecker: quotas,
|
||||
featureToggles: featuremgmt.WithFeatures(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources),
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
|
||||
@@ -317,7 +317,7 @@ func (ng *AlertNG) init() error {
|
||||
evalFactory := eval.NewEvaluatorFactory(ng.Cfg.UnifiedAlerting, ng.DataSourceCache, ng.ExpressionService)
|
||||
conditionValidator := eval.NewConditionValidator(ng.DataSourceCache, ng.ExpressionService, ng.pluginsStore)
|
||||
|
||||
recordingWriter, err := createRecordingWriter(ng.FeatureToggles, ng.Cfg.UnifiedAlerting.RecordingRules, ng.httpClientProvider, ng.DataSourceService, clk, ng.Metrics.GetRemoteWriterMetrics())
|
||||
recordingWriter, err := createRecordingWriter(ng.Cfg.UnifiedAlerting.RecordingRules, ng.httpClientProvider, ng.DataSourceService, clk, ng.Metrics.GetRemoteWriterMetrics())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize recording writer: %w", err)
|
||||
}
|
||||
@@ -649,25 +649,20 @@ func createRemoteAlertmanager(ctx context.Context, cfg remote.AlertmanagerConfig
|
||||
return remote.NewAlertmanager(ctx, cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
|
||||
}
|
||||
|
||||
func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings, httpClientProvider httpclient.Provider, datasourceService datasources.DataSourceService, clock clock.Clock, m *metrics.RemoteWriter) (schedule.RecordingWriter, error) {
|
||||
func createRecordingWriter(settings setting.RecordingRuleSettings, httpClientProvider httpclient.Provider, datasourceService datasources.DataSourceService, clock clock.Clock, m *metrics.RemoteWriter) (schedule.RecordingWriter, error) {
|
||||
logger := log.New("ngalert.writer")
|
||||
|
||||
if settings.Enabled {
|
||||
if featureToggles.IsEnabledGlobally(featuremgmt.FlagGrafanaManagedRecordingRulesDatasources) {
|
||||
cfg := writer.DatasourceWriterConfig{
|
||||
Timeout: settings.Timeout,
|
||||
CustomHeaders: settings.CustomHeaders,
|
||||
DefaultDatasourceUID: settings.DefaultDatasourceUID,
|
||||
}
|
||||
|
||||
logger.Info("Setting up remote write using data sources",
|
||||
"timeout", cfg.Timeout, "default_datasource_uid", cfg.DefaultDatasourceUID)
|
||||
|
||||
return writer.NewDatasourceWriter(cfg, datasourceService, httpClientProvider, clock, logger, m), nil
|
||||
} else {
|
||||
logger.Info("Setting up remote write using static configuration")
|
||||
return writer.NewPrometheusWriterWithSettings(settings, httpClientProvider, clock, logger, m)
|
||||
cfg := writer.DatasourceWriterConfig{
|
||||
Timeout: settings.Timeout,
|
||||
CustomHeaders: settings.CustomHeaders,
|
||||
DefaultDatasourceUID: settings.DefaultDatasourceUID,
|
||||
}
|
||||
|
||||
logger.Info("Setting up remote write using data sources",
|
||||
"timeout", cfg.Timeout, "default_datasource_uid", cfg.DefaultDatasourceUID)
|
||||
|
||||
return writer.NewDatasourceWriter(cfg, datasourceService, httpClientProvider, clock, logger, m), nil
|
||||
}
|
||||
|
||||
return writer.NoopWriter{}, nil
|
||||
|
||||
@@ -177,13 +177,6 @@ func blankRecordingRuleForTests(ctx context.Context) *recordingRule {
|
||||
}
|
||||
|
||||
func TestRecordingRule_Integration(t *testing.T) {
|
||||
t.Run("with prometheus writer", func(t *testing.T) {
|
||||
writeTarget := writer.NewTestRemoteWriteTarget(t)
|
||||
defer writeTarget.Close()
|
||||
writerReg := prometheus.NewPedanticRegistry()
|
||||
writer := setupPrometheusWriter(t, writeTarget, writerReg)
|
||||
testRecordingRule_Integration(t, writeTarget, writer, writerReg, "")
|
||||
})
|
||||
t.Run("with datasource writer", func(t *testing.T) {
|
||||
writeTarget := writer.NewTestRemoteWriteTarget(t)
|
||||
defer writeTarget.Close()
|
||||
@@ -798,14 +791,6 @@ func withQueryForHealth(health string) models.AlertRuleMutator {
|
||||
}
|
||||
}
|
||||
|
||||
func setupPrometheusWriter(t *testing.T, target *writer.TestRemoteWriteTarget, reg prometheus.Registerer) *writer.PrometheusWriter {
|
||||
provider := testClientProvider{}
|
||||
m := metrics.NewNGAlert(reg)
|
||||
wr, err := writer.NewPrometheusWriterWithSettings(target.ClientSettings(), provider, clock.NewMock(), log.NewNopLogger(), m.GetRemoteWriterMetrics())
|
||||
require.NoError(t, err)
|
||||
return wr
|
||||
}
|
||||
|
||||
func setupDatasourceWriter(t *testing.T, target *writer.TestRemoteWriteTarget, reg prometheus.Registerer, dsUID string) *writer.DatasourceWriter {
|
||||
provider := testClientProvider{}
|
||||
m := metrics.NewNGAlert(reg)
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data"
|
||||
@@ -250,34 +248,6 @@ type PrometheusWriterConfig struct {
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func NewPrometheusWriterWithSettings(
|
||||
settings setting.RecordingRuleSettings,
|
||||
httpClientProvider HttpClientProvider,
|
||||
clock clock.Clock,
|
||||
l log.Logger,
|
||||
metrics *metrics.RemoteWriter,
|
||||
) (*PrometheusWriter, error) {
|
||||
if err := validateSettings(settings); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
headers := make(http.Header)
|
||||
for k, v := range settings.CustomHeaders {
|
||||
headers.Add(k, v)
|
||||
}
|
||||
|
||||
cfg := PrometheusWriterConfig{
|
||||
URL: settings.URL,
|
||||
HTTPOptions: httpclient.Options{
|
||||
BasicAuth: createAuthOpts(settings.BasicAuthUsername, settings.BasicAuthPassword),
|
||||
Header: headers,
|
||||
},
|
||||
Timeout: settings.Timeout,
|
||||
}
|
||||
|
||||
return NewPrometheusWriter(cfg, httpClientProvider, clock, l, metrics)
|
||||
}
|
||||
|
||||
func NewPrometheusWriter(
|
||||
cfg PrometheusWriterConfig,
|
||||
httpClientProvider HttpClientProvider,
|
||||
@@ -310,34 +280,6 @@ func NewPrometheusWriter(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateSettings(settings setting.RecordingRuleSettings) error {
|
||||
if settings.BasicAuthUsername != "" && settings.BasicAuthPassword == "" {
|
||||
return fmt.Errorf("basic auth password is required if username is set")
|
||||
}
|
||||
|
||||
if _, err := url.Parse(settings.URL); err != nil {
|
||||
return fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
if settings.Timeout <= 0 {
|
||||
return fmt.Errorf("timeout must be greater than 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createAuthOpts(username, password string) *httpclient.BasicAuthOptions {
|
||||
// If username is empty, do not use basic auth and ignore password.
|
||||
if username == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &httpclient.BasicAuthOptions{
|
||||
User: username,
|
||||
Password: password,
|
||||
}
|
||||
}
|
||||
|
||||
// Write writes the given frames to the Prometheus remote write endpoint.
|
||||
func (w PrometheusWriter) WriteDatasource(ctx context.Context, dsUID string, name string, t time.Time, frames data.Frames, orgID int64, extraLabels map[string]string) error {
|
||||
l := w.logger.FromContext(ctx)
|
||||
|
||||
@@ -19,74 +19,12 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
||||
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
)
|
||||
|
||||
func TestValidateSettings(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
settings setting.RecordingRuleSettings
|
||||
err bool
|
||||
}{
|
||||
{
|
||||
name: "invalid url",
|
||||
settings: setting.RecordingRuleSettings{
|
||||
URL: "invalid url",
|
||||
},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "missing password",
|
||||
settings: setting.RecordingRuleSettings{
|
||||
URL: "http://localhost:9090",
|
||||
BasicAuthUsername: "user",
|
||||
},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "timeout is 0",
|
||||
settings: setting.RecordingRuleSettings{
|
||||
URL: "http://localhost:9090",
|
||||
BasicAuthUsername: "user",
|
||||
BasicAuthPassword: "password",
|
||||
Timeout: 0,
|
||||
},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "valid settings w/ auth",
|
||||
settings: setting.RecordingRuleSettings{
|
||||
URL: "http://localhost:9090",
|
||||
BasicAuthUsername: "user",
|
||||
BasicAuthPassword: "password",
|
||||
Timeout: 10,
|
||||
},
|
||||
err: false,
|
||||
},
|
||||
{
|
||||
name: "valid settings w/o auth",
|
||||
settings: setting.RecordingRuleSettings{
|
||||
URL: "http://localhost:9090",
|
||||
Timeout: 10,
|
||||
},
|
||||
err: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := validateSettings(tc.settings)
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPointsFromFrames(t *testing.T) {
|
||||
extraLabels := map[string]string{"extra": "label"}
|
||||
|
||||
|
||||
@@ -71,10 +71,7 @@ func (s *TestRemoteWriteTarget) DatasourceURL() string {
|
||||
|
||||
func (s *TestRemoteWriteTarget) ClientSettings() setting.RecordingRuleSettings {
|
||||
return setting.RecordingRuleSettings{
|
||||
URL: s.srv.URL + RemoteWriteEndpoint,
|
||||
Timeout: 1 * time.Second,
|
||||
BasicAuthUsername: "",
|
||||
BasicAuthPassword: "",
|
||||
Timeout: 1 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -138,9 +138,6 @@ type UnifiedAlertingSettings struct {
|
||||
|
||||
type RecordingRuleSettings struct {
|
||||
Enabled bool
|
||||
URL string
|
||||
BasicAuthUsername string
|
||||
BasicAuthPassword string
|
||||
CustomHeaders map[string]string
|
||||
Timeout time.Duration
|
||||
DefaultDatasourceUID string
|
||||
@@ -455,10 +452,7 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
|
||||
|
||||
rr := iniFile.Section("recording_rules")
|
||||
uaCfgRecordingRules := RecordingRuleSettings{
|
||||
Enabled: rr.Key("enabled").MustBool(false),
|
||||
URL: rr.Key("url").MustString(""),
|
||||
BasicAuthUsername: rr.Key("basic_auth_username").MustString(""),
|
||||
BasicAuthPassword: rr.Key("basic_auth_password").MustString(""),
|
||||
Enabled: rr.Key("enabled").MustBool(true),
|
||||
Timeout: rr.Key("timeout").MustDuration(defaultRecordingRequestTimeout),
|
||||
DefaultDatasourceUID: rr.Key("default_datasource_uid").MustString(""),
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ func TestIntegrationConvertPrometheusNotificationSettings(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
|
||||
@@ -113,7 +113,6 @@ func TestIntegrationConvertPrometheusEndpoints_RecordingRuleTargetDatasource(t *
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -181,7 +180,6 @@ func TestIntegrationConvertPrometheusEndpoints(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -387,7 +385,6 @@ func TestIntegrationConvertPrometheusEndpoints_UpdateRule(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -468,7 +465,6 @@ func TestIntegrationConvertPrometheusEndpoints_Conflict(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -550,7 +546,6 @@ func TestIntegrationConvertPrometheusEndpoints_CreatePausedRules(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -658,7 +653,6 @@ func TestIntegrationConvertPrometheusEndpoints_FolderUIDHeader(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -755,7 +749,6 @@ func TestIntegrationConvertPrometheusEndpoints_Provenance(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
@@ -865,7 +858,6 @@ func TestIntegrationConvertPrometheusEndpoints_Delete(t *testing.T) {
|
||||
EnableUnifiedAlerting: true,
|
||||
DisableAnonymous: true,
|
||||
AppModeProduction: true,
|
||||
EnableFeatureToggles: []string{"grafanaManagedRecordingRulesDatasources"},
|
||||
EnableRecordingRules: true,
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user