Alerting: Add support for Redis Sentinel for Alerting HA (#106322)

* Alerting: Add support for Redis Sentinel

* docs

* docs

* Use minisentinel in test

* Apply suggestions from code review

Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com>
Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com>

* "address(es)" -> "address or addresses"

* make update-workspace

* make lint-go-diff

---------

Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com>
Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com>

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -179,16 +179,20 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error {
 	// Redis setup.
 	if cfg.UnifiedAlerting.HARedisAddr != "" {
 		redisPeer, err := newRedisPeer(redisConfig{
-			addr:        cfg.UnifiedAlerting.HARedisAddr,
-			name:        cfg.UnifiedAlerting.HARedisPeerName,
-			prefix:      cfg.UnifiedAlerting.HARedisPrefix,
-			password:    cfg.UnifiedAlerting.HARedisPassword,
-			username:    cfg.UnifiedAlerting.HARedisUsername,
-			db:          cfg.UnifiedAlerting.HARedisDB,
-			maxConns:    cfg.UnifiedAlerting.HARedisMaxConns,
-			tlsEnabled:  cfg.UnifiedAlerting.HARedisTLSEnabled,
-			tls:         cfg.UnifiedAlerting.HARedisTLSConfig,
-			clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled,
+			addr:             cfg.UnifiedAlerting.HARedisAddr,
+			name:             cfg.UnifiedAlerting.HARedisPeerName,
+			prefix:           cfg.UnifiedAlerting.HARedisPrefix,
+			password:         cfg.UnifiedAlerting.HARedisPassword,
+			username:         cfg.UnifiedAlerting.HARedisUsername,
+			db:               cfg.UnifiedAlerting.HARedisDB,
+			maxConns:         cfg.UnifiedAlerting.HARedisMaxConns,
+			tlsEnabled:       cfg.UnifiedAlerting.HARedisTLSEnabled,
+			tls:              cfg.UnifiedAlerting.HARedisTLSConfig,
+			clusterMode:      cfg.UnifiedAlerting.HARedisClusterModeEnabled,
+			sentinelMode:     cfg.UnifiedAlerting.HARedisSentinelModeEnabled,
+			masterName:       cfg.UnifiedAlerting.HARedisSentinelMasterName,
+			sentinelUsername: cfg.UnifiedAlerting.HARedisSentinelUsername,
+			sentinelPassword: cfg.UnifiedAlerting.HARedisSentinelPassword,
 		}, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval)
 		if err != nil {
 			return fmt.Errorf("unable to initialize redis: %w", err)

pkg/services/ngalert/notifier/redis_peer.go

@@ -23,14 +23,18 @@ import (
 )
 
 type redisConfig struct {
-	addr        string
-	username    string
-	password    string
-	db          int
-	name        string
-	prefix      string
-	maxConns    int
-	clusterMode bool
+	addr             string
+	username         string
+	password         string
+	db               int
+	name             string
+	prefix           string
+	maxConns         int
+	clusterMode      bool
+	sentinelMode     bool
+	masterName       string
+	sentinelUsername string
+	sentinelPassword string
 
 	tlsEnabled bool
 	tls        dstls.ClientConfig
@@ -111,21 +115,19 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer,
 		}
 	}
 
-	opts := &redis.UniversalOptions{
+	rdb := redis.NewUniversalClient(&redis.UniversalOptions{
 		Addrs:     addrs,
 		Username:  cfg.username,
 		Password:  cfg.password,
 		DB:        cfg.db,
 		PoolSize:  poolSize,
 		TLSConfig: tlsClientConfig,
-	}
 
-	var rdb redis.UniversalClient
-	if cfg.clusterMode {
-		rdb = redis.NewClusterClient(opts.Cluster())
-	} else {
-		rdb = redis.NewClient(opts.Simple())
-	}
+		// Options specific to Sentinel mode.
+		MasterName:       cfg.masterName,
+		SentinelUsername: cfg.sentinelUsername,
+		SentinelPassword: cfg.sentinelPassword,
+	})
 
 	cmd := rdb.Ping(context.Background())
 	if cmd.Err() != nil {

pkg/services/ngalert/notifier/redis_peer_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/Bose/minisentinel"
 	"github.com/alicebob/miniredis/v2"
 	dstls "github.com/grafana/dskit/crypto/tls"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -45,6 +46,28 @@ func TestNewRedisPeerClusterMode(t *testing.T) {
 	require.NoError(t, ping.Err())
 }
 
+func TestNewRedisPeerSentinelMode(t *testing.T) {
+	// Can't use RunTLS here because minisentinel does not support TLS.
+	mr, err := miniredis.Run()
+	require.NoError(t, err)
+	defer mr.Close()
+
+	ms := minisentinel.NewSentinel(mr, minisentinel.WithReplica(mr))
+	err = ms.Start()
+	require.NoError(t, err)
+	defer ms.Close()
+
+	redisPeer, err := newRedisPeer(redisConfig{
+		sentinelMode: true,
+		masterName:   ms.MasterInfo().Name,
+		addr:         ms.Addr(),
+	}, log.NewNopLogger(), prometheus.NewRegistry(), time.Second*60)
+	require.NoError(t, err)
+
+	ping := redisPeer.redis.Ping(context.Background())
+	require.NoError(t, ping.Err())
+}
+
 func TestNewRedisPeerWithTLS(t *testing.T) {
 	// Write client and server certificates/keys to tempDir, both issued by the same CA
 	certPaths := createX509TestDir(t)