public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add support for Redis Sentinel for Alerting HA (#106322)

Browse Source 
* Alerting: Add support for Redis Sentinel

* docs

* docs

* Use minisentinel in test

* Apply suggestions from code review

Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com>
Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com>

* "address(es)" -> "address or addresses"

* make update-workspace

* make lint-go-diff

---------

Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com>
Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com>
This commit is contained in:
Vadim Stepanov
2025-06-05 15:02:40 +01:00
committed by GitHub
co-authored by Johnny Kartheiser Fayzal Ghantiwala
parent 941162ca79
commit 5137995830
11 changed files with 292 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+14 -10
View File
@@ -179,16 +179,20 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error {
// Redis setup.
if cfg.UnifiedAlerting.HARedisAddr != "" {
redisPeer, err := newRedisPeer(redisConfig{
addr: cfg.UnifiedAlerting.HARedisAddr,
name: cfg.UnifiedAlerting.HARedisPeerName,
prefix: cfg.UnifiedAlerting.HARedisPrefix,
password: cfg.UnifiedAlerting.HARedisPassword,
username: cfg.UnifiedAlerting.HARedisUsername,
db: cfg.UnifiedAlerting.HARedisDB,
maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled,
tls: cfg.UnifiedAlerting.HARedisTLSConfig,
clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled,
addr: cfg.UnifiedAlerting.HARedisAddr,
name: cfg.UnifiedAlerting.HARedisPeerName,
prefix: cfg.UnifiedAlerting.HARedisPrefix,
password: cfg.UnifiedAlerting.HARedisPassword,
username: cfg.UnifiedAlerting.HARedisUsername,
db: cfg.UnifiedAlerting.HARedisDB,
maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled,
tls: cfg.UnifiedAlerting.HARedisTLSConfig,
clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled,
sentinelMode: cfg.UnifiedAlerting.HARedisSentinelModeEnabled,
masterName: cfg.UnifiedAlerting.HARedisSentinelMasterName,
sentinelUsername: cfg.UnifiedAlerting.HARedisSentinelUsername,
sentinelPassword: cfg.UnifiedAlerting.HARedisSentinelPassword,
}, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval)
if err != nil {
return fmt.Errorf("unable to initialize redis: %w", err)
pkg/services/ngalert/notifier/redis_peer.go
+18 -16
View File
@@ -23,14 +23,18 @@ import (
)
type redisConfig struct {
addr string
username string
password string
db int
name string
prefix string
maxConns int
clusterMode bool
addr string
username string
password string
db int
name string
prefix string
maxConns int
clusterMode bool
sentinelMode bool
masterName string
sentinelUsername string
sentinelPassword string
tlsEnabled bool
tls dstls.ClientConfig
@@ -111,21 +115,19 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer,
}
}
opts := &redis.UniversalOptions{
rdb := redis.NewUniversalClient(&redis.UniversalOptions{
Addrs: addrs,
Username: cfg.username,
Password: cfg.password,
DB: cfg.db,
PoolSize: poolSize,
TLSConfig: tlsClientConfig,
}
var rdb redis.UniversalClient
if cfg.clusterMode {
rdb = redis.NewClusterClient(opts.Cluster())
} else {
rdb = redis.NewClient(opts.Simple())
}
// Options specific to Sentinel mode.
MasterName: cfg.masterName,
SentinelUsername: cfg.sentinelUsername,
SentinelPassword: cfg.sentinelPassword,
})
cmd := rdb.Ping(context.Background())
if cmd.Err() != nil {
pkg/services/ngalert/notifier/redis_peer_test.go
+23
View File
@@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/Bose/minisentinel"
"github.com/alicebob/miniredis/v2"
dstls "github.com/grafana/dskit/crypto/tls"
"github.com/grafana/grafana/pkg/infra/log"
@@ -45,6 +46,28 @@ func TestNewRedisPeerClusterMode(t *testing.T) {
require.NoError(t, ping.Err())
}
func TestNewRedisPeerSentinelMode(t *testing.T) {
// Can't use RunTLS here because minisentinel does not support TLS.
mr, err := miniredis.Run()
require.NoError(t, err)
defer mr.Close()
ms := minisentinel.NewSentinel(mr, minisentinel.WithReplica(mr))
err = ms.Start()
require.NoError(t, err)
defer ms.Close()
redisPeer, err := newRedisPeer(redisConfig{
sentinelMode: true,
masterName: ms.MasterInfo().Name,
addr: ms.Addr(),
}, log.NewNopLogger(), prometheus.NewRegistry(), time.Second*60)
require.NoError(t, err)
ping := redisPeer.redis.Ping(context.Background())
require.NoError(t, ping.Err())
}
func TestNewRedisPeerWithTLS(t *testing.T) {
// Write client and server certificates/keys to tempDir, both issued by the same CA
certPaths := createX509TestDir(t)
pkg/setting/setting_unified_alerting.go
+19
View File
@@ -69,6 +69,11 @@ const (
lokiDefaultMaxQuerySize = 65536 // 64kb
)
var (
errHARedisBothClusterAndSentinel = fmt.Errorf("'ha_redis_cluster_mode_enabled' and 'ha_redis_sentinel_mode_enabled' are mutually exclusive")
errHARedisSentinelMasterNameRequired = fmt.Errorf("'ha_redis_sentinel_master_name' is required when 'ha_redis_sentinel_mode_enabled' is true")
)
type UnifiedAlertingSettings struct {
AdminConfigPollInterval time.Duration
AlertmanagerConfigPollInterval time.Duration
@@ -83,6 +88,10 @@ type UnifiedAlertingSettings struct {
HAPushPullInterval time.Duration
HALabel string
HARedisClusterModeEnabled bool
HARedisSentinelModeEnabled bool
HARedisSentinelMasterName string
HARedisSentinelUsername string
HARedisSentinelPassword string
HARedisAddr string
HARedisPeerName string
HARedisPrefix string
@@ -276,6 +285,16 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
uaCfg.HAAdvertiseAddr = ua.Key("ha_advertise_address").MustString("")
uaCfg.HALabel = ua.Key("ha_label").MustString("")
uaCfg.HARedisClusterModeEnabled = ua.Key("ha_redis_cluster_mode_enabled").MustBool(false)
uaCfg.HARedisSentinelModeEnabled = ua.Key("ha_redis_sentinel_mode_enabled").MustBool(false)
if uaCfg.HARedisClusterModeEnabled && uaCfg.HARedisSentinelModeEnabled {
return errHARedisBothClusterAndSentinel
}
uaCfg.HARedisSentinelMasterName = ua.Key("ha_redis_sentinel_master_name").MustString("")
if uaCfg.HARedisSentinelModeEnabled && uaCfg.HARedisSentinelMasterName == "" {
return errHARedisSentinelMasterNameRequired
}
uaCfg.HARedisSentinelUsername = ua.Key("ha_redis_sentinel_username").MustString("")
uaCfg.HARedisSentinelPassword = ua.Key("ha_redis_sentinel_password").MustString("")
uaCfg.HARedisAddr = ua.Key("ha_redis_address").MustString("")
uaCfg.HARedisPeerName = ua.Key("ha_redis_peer_name").MustString("")
uaCfg.HARedisPrefix = ua.Key("ha_redis_prefix").MustString("")
pkg/setting/setting_unified_alerting_test.go
+69
View File
@@ -350,3 +350,72 @@ func TestHARedisTLSSettings(t *testing.T) {
require.Equal(t, cipherSuites, cfg.UnifiedAlerting.HARedisTLSConfig.CipherSuites)
require.Equal(t, minVersion, cfg.UnifiedAlerting.HARedisTLSConfig.MinVersion)
}
func TestHARedisSentinelModeSettings(t *testing.T) {
testCases := []struct {
desc string
haRedisSentinelModeEnabled bool
haRedisClusterModeEnabled bool
haRedisSentinelMasterName string
haRedisSentinelUsername string
haRedisSentinelPassword string
expectedErr error
}{
{
desc: "should not fail when Sentinel mode is enabled and master name is set",
haRedisSentinelModeEnabled: true,
haRedisSentinelMasterName: "exampleMasterName",
},
{
desc: "should not fail when Sentinel mode is enabled, master name is set, and Sentinel username and password are provided",
haRedisSentinelModeEnabled: true,
haRedisSentinelMasterName: "exampleMasterName",
haRedisSentinelUsername: "exampleSentinelUsername",
haRedisSentinelPassword: "exampleSentinelPassword",
},
{
desc: "should fail when Sentinel mode is enabled but master name is not set",
haRedisSentinelModeEnabled: true,
expectedErr: errHARedisSentinelMasterNameRequired,
},
{
desc: "should fail when both Sentinel mode and Cluster mode are enabled",
haRedisSentinelModeEnabled: true,
haRedisClusterModeEnabled: true,
expectedErr: errHARedisBothClusterAndSentinel,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
f := ini.Empty()
section, err := f.NewSection("unified_alerting")
require.NoError(t, err)
_, err = section.NewKey("ha_redis_sentinel_mode_enabled", strconv.FormatBool(tc.haRedisSentinelModeEnabled))
require.NoError(t, err)
_, err = section.NewKey("ha_redis_cluster_mode_enabled", strconv.FormatBool(tc.haRedisClusterModeEnabled))
require.NoError(t, err)
_, err = section.NewKey("ha_redis_sentinel_master_name", tc.haRedisSentinelMasterName)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_sentinel_username", tc.haRedisSentinelUsername)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_sentinel_password", tc.haRedisSentinelPassword)
require.NoError(t, err)
cfg := NewCfg()
err = cfg.ReadUnifiedAlertingSettings(f)
if tc.expectedErr == nil {
require.NoError(t, err)
} else {
require.ErrorIs(t, err, tc.expectedErr)
return
}
require.Equal(t, tc.haRedisSentinelModeEnabled, cfg.UnifiedAlerting.HARedisSentinelModeEnabled)
require.Equal(t, tc.haRedisSentinelMasterName, cfg.UnifiedAlerting.HARedisSentinelMasterName)
require.Equal(t, tc.haRedisSentinelUsername, cfg.UnifiedAlerting.HARedisSentinelUsername)
require.Equal(t, tc.haRedisSentinelPassword, cfg.UnifiedAlerting.HARedisSentinelPassword)
})
}
}