public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Remove feature toggles relating to Loki Alert State History (#103540)

Browse Source 
* Remove feature toggles relating to Loki Alert State History
This commit is contained in:
William Wernert
2025-04-08 09:50:27 -04:00
committed by GitHub
parent a9ef8bcced
commit a8f60de620
12 changed files with 19 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/sources/alerting/set-up/configure-alert-state-history/index.md
-3
View File
@@ -47,9 +47,6 @@ The example below instructs Grafana to write alert state history to a local Loki
enabled = true
backend = "loki"
loki_remote_url = "http://localhost:3100"
[feature_toggles]
enable = alertStateHistoryLokiSecondary, alertStateHistoryLokiPrimary, alertStateHistoryLokiOnly
```
<!-- TODO can we add some more info here about the feature flags and the various different supported setups with Loki as Primary / Secondary, etc? -->
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
-3
View File
@@ -139,9 +139,6 @@ Experimental features might be changed or removed without prior notice.
| `influxqlStreamingParser` | Enable streaming JSON parser for InfluxDB datasource InfluxQL query language |
| `lokiLogsDataplane` | Changes logs responses from Loki to be compliant with the dataplane specification. |
| `disableSSEDataplane` | Disables dataplane specific processing in server side expressions. |
| `alertStateHistoryLokiSecondary` | Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations. |
| `alertStateHistoryLokiPrimary` | Enable a remote Loki instance as the primary source for state history reads. |
| `alertStateHistoryLokiOnly` | Disable Grafana alerts from emitting annotations when a remote Loki instance is available. |
| `extraThemes` | Enables extra themes |
| `lokiPredefinedOperations` | Adds predefined query operations to Loki query editor |
| `awsDatasourcesTempCredentials` | Support temporary security credentials in AWS plugins for Grafana Cloud customers |
packages/grafana-data/src/types/featureToggles.gen.ts
-12
View File
@@ -150,18 +150,6 @@ export interface FeatureToggles {
*/
disableSSEDataplane?: boolean;
/**
* Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations.
*/
alertStateHistoryLokiSecondary?: boolean;
/**
* Enable a remote Loki instance as the primary source for state history reads.
*/
alertStateHistoryLokiPrimary?: boolean;
/**
* Disable Grafana alerts from emitting annotations when a remote Loki instance is available.
*/
alertStateHistoryLokiOnly?: boolean;
/**
* Writes error logs to the request logger
* @default true
*/
pkg/services/annotations/annotationsimpl/annotations.go
+1 -1
View File
@@ -41,7 +41,7 @@ func ProvideService(
write := xormStore
var read readStore
historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, ruleStore, log.New("annotations.loki"), tracer)
historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, db, ruleStore, log.New("annotations.loki"), tracer)
if historianStore != nil {
l.Debug("Using composite read store")
read = NewCompositeStore(log.New("annotations.composite"), xormStore, historianStore)
pkg/services/annotations/annotationsimpl/loki/historian_store.go
+4 -12
View File
@@ -12,8 +12,6 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/annotations/accesscontrol"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
@@ -58,8 +56,8 @@ type LokiHistorianStore struct {
ruleStore RuleStore
}
func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, ruleStore RuleStore, log log.Logger, tracer tracing.Tracer) *LokiHistorianStore {
if !useStore(cfg, ft) {
func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, db db.DB, ruleStore RuleStore, log log.Logger, tracer tracing.Tracer) *LokiHistorianStore {
if !useStore(cfg) {
return nil
}
lokiCfg, err := historian.NewLokiConfig(cfg)
@@ -294,22 +292,16 @@ func buildHistoryQuery(query *annotations.ItemQuery, dashboards map[string]int64
return historyQuery
}
func useStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles) bool {
func useStore(cfg setting.UnifiedAlertingStateHistorySettings) bool {
if !cfg.Enabled {
return false
}
// Override config based on feature toggles.
// We pass in a no-op logger here since this function is also called during ngalert init,
// and we don't want to log the same info twice.
ngalert.ApplyStateHistoryFeatureToggles(&cfg, ft, log.NewNopLogger())
backend, err := historian.ParseBackendType(cfg.Backend)
if err != nil {
return false
}
// We should only query Loki if annotations do not exist in the database.
// To be doubly sure, ensure that the feature toggle to only use Loki is enabled.
return backend == historian.BackendTypeLoki && ft.IsEnabledGlobally(featuremgmt.FlagAlertStateHistoryLokiOnly)
return backend == historian.BackendTypeLoki
}
pkg/services/annotations/annotationsimpl/loki/historian_store_test.go
+8 -13
View File
@@ -872,7 +872,7 @@ func TestUseStore(t *testing.T) {
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: false,
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
@@ -882,7 +882,7 @@ func TestUseStore(t *testing.T) {
Enabled: true,
Backend: "invalid-backend",
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
@@ -892,7 +892,7 @@ func TestUseStore(t *testing.T) {
Backend: "multiple",
MultiPrimary: "invalid-backend",
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
@@ -903,7 +903,7 @@ func TestUseStore(t *testing.T) {
MultiPrimary: "annotations",
MultiSecondaries: []string{"annotations", "invalid-backend"},
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
})
@@ -913,7 +913,7 @@ func TestUseStore(t *testing.T) {
Enabled: true,
Backend: "annotations",
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
@@ -924,7 +924,7 @@ func TestUseStore(t *testing.T) {
Backend: "multiple",
MultiPrimary: "loki",
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
@@ -935,7 +935,7 @@ func TestUseStore(t *testing.T) {
MultiPrimary: "annotations",
MultiSecondaries: []string{"loki"},
}
use := useStore(cfg, featuremgmt.WithFeatures())
use := useStore(cfg)
require.False(t, use)
})
})
@@ -946,12 +946,7 @@ func TestUseStore(t *testing.T) {
Enabled: true,
Backend: "loki",
}
features := featuremgmt.WithFeatures(
featuremgmt.FlagAlertStateHistoryLokiOnly,
featuremgmt.FlagAlertStateHistoryLokiPrimary,
featuremgmt.FlagAlertStateHistoryLokiSecondary,
)
use := useStore(cfg, features)
use := useStore(cfg)
require.True(t, use)
})
})
pkg/services/featuremgmt/registry.go
-18
View File
@@ -235,24 +235,6 @@ var (
Stage: FeatureStageExperimental,
Owner: grafanaObservabilityMetricsSquad,
},
{
Name: "alertStateHistoryLokiSecondary",
Description: "Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations.",
Stage: FeatureStageExperimental,
Owner: grafanaAlertingSquad,
},
{
Name: "alertStateHistoryLokiPrimary",
Description: "Enable a remote Loki instance as the primary source for state history reads.",
Stage: FeatureStageExperimental,
Owner: grafanaAlertingSquad,
},
{
Name: "alertStateHistoryLokiOnly",
Description: "Disable Grafana alerts from emitting annotations when a remote Loki instance is available.",
Stage: FeatureStageExperimental,
Owner: grafanaAlertingSquad,
},
{
Name: "unifiedRequestLog",
Description: "Writes error logs to the request logger",
pkg/services/featuremgmt/toggles_gen.csv
-3
View File
@@ -29,9 +29,6 @@ prometheusRunQueriesInParallel,GA,@grafana/oss-big-tent,false,false,false
lokiLogsDataplane,experimental,@grafana/observability-logs,false,false,false
dataplaneFrontendFallback,GA,@grafana/observability-metrics,false,false,true
disableSSEDataplane,experimental,@grafana/observability-metrics,false,false,false
alertStateHistoryLokiSecondary,experimental,@grafana/alerting-squad,false,false,false
alertStateHistoryLokiPrimary,experimental,@grafana/alerting-squad,false,false,false
alertStateHistoryLokiOnly,experimental,@grafana/alerting-squad,false,false,false
unifiedRequestLog,GA,@grafana/grafana-backend-group,false,false,false
renderAuthJWT,preview,@grafana/grafana-as-code,false,false,false
refactorVariablesTimeRange,preview,@grafana/dashboards-squad,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
29 lokiLogsDataplane experimental @grafana/observability-logs false false false
30 dataplaneFrontendFallback GA @grafana/observability-metrics false false true
31 disableSSEDataplane experimental @grafana/observability-metrics false false false
alertStateHistoryLokiSecondary experimental @grafana/alerting-squad false false false
alertStateHistoryLokiPrimary experimental @grafana/alerting-squad false false false
alertStateHistoryLokiOnly experimental @grafana/alerting-squad false false false
32 unifiedRequestLog GA @grafana/grafana-backend-group false false false
33 renderAuthJWT preview @grafana/grafana-as-code false false false
34 refactorVariablesTimeRange preview @grafana/dashboards-squad false false false
pkg/services/featuremgmt/toggles_gen.go
-12
View File
@@ -127,18 +127,6 @@ const (
// Disables dataplane specific processing in server side expressions.
FlagDisableSSEDataplane = "disableSSEDataplane"
// FlagAlertStateHistoryLokiSecondary
// Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations.
FlagAlertStateHistoryLokiSecondary = "alertStateHistoryLokiSecondary"
// FlagAlertStateHistoryLokiPrimary
// Enable a remote Loki instance as the primary source for state history reads.
FlagAlertStateHistoryLokiPrimary = "alertStateHistoryLokiPrimary"
// FlagAlertStateHistoryLokiOnly
// Disable Grafana alerts from emitting annotations when a remote Loki instance is available.
FlagAlertStateHistoryLokiOnly = "alertStateHistoryLokiOnly"
// FlagUnifiedRequestLog
// Writes error logs to the request logger
FlagUnifiedRequestLog = "unifiedRequestLog"
pkg/services/featuremgmt/toggles_gen.json
+6 -3
View File
@@ -75,7 +75,8 @@
"metadata": {
"name": "alertStateHistoryLokiOnly",
"resourceVersion": "1743693517832",
"creationTimestamp": "2023-03-30T18:53:21Z"
"creationTimestamp": "2023-03-30T18:53:21Z",
"deletionTimestamp": "2025-04-07T14:33:12Z"
},
"spec": {
"description": "Disable Grafana alerts from emitting annotations when a remote Loki instance is available.",
@@ -87,7 +88,8 @@
"metadata": {
"name": "alertStateHistoryLokiPrimary",
"resourceVersion": "1743693517832",
"creationTimestamp": "2023-03-30T18:53:21Z"
"creationTimestamp": "2023-03-30T18:53:21Z",
"deletionTimestamp": "2025-04-07T14:33:12Z"
},
"spec": {
"description": "Enable a remote Loki instance as the primary source for state history reads.",
@@ -99,7 +101,8 @@
"metadata": {
"name": "alertStateHistoryLokiSecondary",
"resourceVersion": "1743693517832",
"creationTimestamp": "2023-03-30T18:53:21Z"
"creationTimestamp": "2023-03-30T18:53:21Z",
"deletionTimestamp": "2025-04-07T14:33:12Z"
},
"spec": {
"description": "Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations.",
pkg/services/ngalert/ngalert.go
-48
View File
@@ -398,9 +398,6 @@ func (ng *AlertNG) init() error {
RecordingWriter: ng.RecordingWriter,
}
// There are a set of feature toggles available that act as short-circuits for common configurations.
// If any are set, override the config accordingly.
ApplyStateHistoryFeatureToggles(&ng.Cfg.UnifiedAlerting.StateHistory, ng.FeatureToggles, ng.Log)
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log, ng.tracer, ac.NewRuleService(ng.accesscontrol))
if err != nil {
return err
@@ -704,51 +701,6 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
return nil, fmt.Errorf("unrecognized state history backend: %s", backend)
}
// ApplyStateHistoryFeatureToggles edits state history configuration to comply with currently active feature toggles.
func ApplyStateHistoryFeatureToggles(cfg *setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, logger log.Logger) {
backend, _ := historian.ParseBackendType(cfg.Backend)
// These feature toggles represent specific, common backend configurations.
// If all toggles are enabled, we listen to the state history config as written.
// If any of them are disabled, we ignore the configured backend and treat the toggles as an override.
// If multiple toggles are disabled, we go with the most "restrictive" one.
if !ft.IsEnabledGlobally(featuremgmt.FlagAlertStateHistoryLokiSecondary) {
// If we cannot even treat Loki as a secondary, we must use annotations only.
if backend == historian.BackendTypeMultiple || backend == historian.BackendTypeLoki {
logger.Info("Forcing Annotation backend due to state history feature toggles")
cfg.Backend = historian.BackendTypeAnnotations.String()
cfg.MultiPrimary = ""
cfg.MultiSecondaries = make([]string, 0)
}
return
}
if !ft.IsEnabledGlobally(featuremgmt.FlagAlertStateHistoryLokiPrimary) {
// If we're using multiple backends, Loki must be the secondary.
if backend == historian.BackendTypeMultiple {
logger.Info("Coercing Loki to a secondary backend due to state history feature toggles")
cfg.MultiPrimary = historian.BackendTypeAnnotations.String()
cfg.MultiSecondaries = []string{historian.BackendTypeLoki.String()}
}
// If we're using loki, we are only allowed to use it as a secondary. Dual write to it, plus annotations.
if backend == historian.BackendTypeLoki {
logger.Info("Coercing Loki to dual writes with a secondary backend due to state history feature toggles")
cfg.Backend = historian.BackendTypeMultiple.String()
cfg.MultiPrimary = historian.BackendTypeAnnotations.String()
cfg.MultiSecondaries = []string{historian.BackendTypeLoki.String()}
}
return
}
if !ft.IsEnabledGlobally(featuremgmt.FlagAlertStateHistoryLokiOnly) {
// If we're not allowed to use Loki only, make it the primary but keep the annotation writes.
if backend == historian.BackendTypeLoki {
logger.Info("Forcing dual writes to Loki and Annotations due to state history feature toggles")
cfg.Backend = historian.BackendTypeMultiple.String()
cfg.MultiPrimary = historian.BackendTypeLoki.String()
cfg.MultiSecondaries = []string{historian.BackendTypeAnnotations.String()}
}
return
}
}
func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*remote.Alertmanager, error) {
return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
}
pkg/tests/alertmanager/alertmanager_scenario.go
-3
View File
@@ -269,9 +269,6 @@ func (s *AlertmanagerScenario) NewGrafanaService(name string, peers []string, pe
flags := map[string]string{}
ft := []string{
"alertStateHistoryLokiSecondary",
"alertStateHistoryLokiPrimary",
"alertStateHistoryLokiOnly",
"alertingAlertmanagerExtraDedupStage",
}
if stopOnExtraDedup {