diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 44428027f11..d0f88f50b63 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -100,6 +100,9 @@ Alpha features might be changed or removed without prior notice. | `logsContextDatasourceUi` | Allow datasource to provide custom UI for context view | | `prometheusMetricEncyclopedia` | Replaces the Prometheus query builder metric select option with a paginated and filterable component | | `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy | +| `alertStateHistoryLokiSecondary` | Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations. | +| `alertStateHistoryLokiPrimary` | Enable a remote Loki instance as the primary source for state history reads. | +| `alertStateHistoryLokiOnly` | Disable Grafana alerts from emitting annotations when a remote Loki instance is available. | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 4a1167c8c3a..17bd4880dee 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -93,4 +93,7 @@ export interface FeatureToggles { logsContextDatasourceUi?: boolean; prometheusMetricEncyclopedia?: boolean; influxdbBackendMigration?: boolean; + alertStateHistoryLokiSecondary?: boolean; + alertStateHistoryLokiPrimary?: boolean; + alertStateHistoryLokiOnly?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 5c5ce4fa256..1bdec29efa9 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -435,5 +435,20 @@ var ( State: FeatureStateAlpha, FrontendOnly: true, }, + { + Name: "alertStateHistoryLokiSecondary", + Description: "Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations.", + State: FeatureStateAlpha, + }, + { + Name: "alertStateHistoryLokiPrimary", + Description: "Enable a remote Loki instance as the primary source for state history reads.", + State: FeatureStateAlpha, + }, + { + Name: "alertStateHistoryLokiOnly", + Description: "Disable Grafana alerts from emitting annotations when a remote Loki instance is available.", + State: FeatureStateAlpha, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 052c3f17381..ccbcb11c149 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -314,4 +314,16 @@ const ( // FlagInfluxdbBackendMigration // Query InfluxDB InfluxQL without the proxy FlagInfluxdbBackendMigration = "influxdbBackendMigration" + + // FlagAlertStateHistoryLokiSecondary + // Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations. + FlagAlertStateHistoryLokiSecondary = "alertStateHistoryLokiSecondary" + + // FlagAlertStateHistoryLokiPrimary + // Enable a remote Loki instance as the primary source for state history reads. + FlagAlertStateHistoryLokiPrimary = "alertStateHistoryLokiPrimary" + + // FlagAlertStateHistoryLokiOnly + // Disable Grafana alerts from emitting annotations when a remote Loki instance is available. + FlagAlertStateHistoryLokiOnly = "alertStateHistoryLokiOnly" ) diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 4bafe8cd657..b3ab964b17b 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -210,6 +210,9 @@ func (ng *AlertNG) init() error { Tracer: ng.tracer, } + // There are a set of feature toggles available that act as short-circuits for common configurations. + // If any are set, override the config accordingly. + applyStateHistoryFeatureToggles(&ng.Cfg.UnifiedAlerting.StateHistory, ng.FeatureToggles, ng.Log) history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log) if err != nil { return err @@ -438,3 +441,48 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS return nil, fmt.Errorf("unrecognized state history backend: %s", backend) } + +// applyStateHistoryFeatureToggles edits state history configuration to comply with currently active feature toggles. +func applyStateHistoryFeatureToggles(cfg *setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, logger log.Logger) { + backend, _ := historian.ParseBackendType(cfg.Backend) + // These feature toggles represent specific, common backend configurations. + // If all toggles are enabled, we listen to the state history config as written. + // If any of them are disabled, we ignore the configured backend and treat the toggles as an override. + // If multiple toggles are disabled, we go with the most "restrictive" one. + if !ft.IsEnabled(featuremgmt.FlagAlertStateHistoryLokiSecondary) { + // If we cannot even treat Loki as a secondary, we must use annotations only. + if backend == historian.BackendTypeMultiple || backend == historian.BackendTypeLoki { + logger.Info("Forcing Annotation backend due to state history feature toggles") + cfg.Backend = historian.BackendTypeAnnotations.String() + cfg.MultiPrimary = "" + cfg.MultiSecondaries = make([]string, 0) + } + return + } + if !ft.IsEnabled(featuremgmt.FlagAlertStateHistoryLokiPrimary) { + // If we're using multiple backends, Loki must be the secondary. + if backend == historian.BackendTypeMultiple { + logger.Info("Coercing Loki to a secondary backend due to state history feature toggles") + cfg.MultiPrimary = historian.BackendTypeAnnotations.String() + cfg.MultiSecondaries = []string{historian.BackendTypeLoki.String()} + } + // If we're using loki, we are only allowed to use it as a secondary. Dual write to it, plus annotations. + if backend == historian.BackendTypeLoki { + logger.Info("Coercing Loki to dual writes with a secondary backend due to state history feature toggles") + cfg.Backend = historian.BackendTypeMultiple.String() + cfg.MultiPrimary = historian.BackendTypeAnnotations.String() + cfg.MultiSecondaries = []string{historian.BackendTypeLoki.String()} + } + return + } + if !ft.IsEnabled(featuremgmt.FlagAlertStateHistoryLokiOnly) { + // If we're not allowed to use Loki only, make it the primary but keep the annotation writes. + if backend == historian.BackendTypeLoki { + logger.Info("Forcing dual writes to Loki and Annotations due to state history feature toggles") + cfg.Backend = historian.BackendTypeMultiple.String() + cfg.MultiPrimary = historian.BackendTypeLoki.String() + cfg.MultiSecondaries = []string{historian.BackendTypeAnnotations.String()} + } + return + } +} diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go index c368d4e0d2c..bf80548abbf 100644 --- a/pkg/setting/setting_unified_alerting.go +++ b/pkg/setting/setting_unified_alerting.go @@ -137,7 +137,7 @@ func (cfg *Cfg) readUnifiedAlertingEnabledSetting(section *ini.Section) (*bool, // than disable it. This issue can be found here hasEnabled := section.Key("enabled").Value() != "" if !hasEnabled { - // TODO: Remove in Grafana v9 + // TODO: Remove in Grafana v10 if cfg.IsFeatureToggleEnabled("ngalert") { cfg.Logger.Warn("ngalert feature flag is deprecated: use unified alerting enabled setting instead") // feature flag overrides the legacy alerting setting