public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add options to configure TLS for HA using Redis (#87567)

Browse Source 
* Add Alerting HA Redis Client TLS configs

* Add test to ping miniredis with mTLS

* Update .ini files and docs

* Add tests for unified alerting ha redis TLS settings

* Fix malformed go.sum

* Add modowner

* Fix lint error

* Update docs and use dstls config
This commit is contained in:
Fayzal Ghantiwala
2024-05-14 14:21:42 +01:00
committed by GitHub
parent e39658097f
commit 7a2fbad0c8
11 changed files with 242 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/notifier/multiorg_alertmanager.go
+9 -7
View File
@@ -169,13 +169,15 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error {
// Redis setup.
if cfg.UnifiedAlerting.HARedisAddr != "" {
redisPeer, err := newRedisPeer(redisConfig{
addr: cfg.UnifiedAlerting.HARedisAddr,
name: cfg.UnifiedAlerting.HARedisPeerName,
prefix: cfg.UnifiedAlerting.HARedisPrefix,
password: cfg.UnifiedAlerting.HARedisPassword,
username: cfg.UnifiedAlerting.HARedisUsername,
db: cfg.UnifiedAlerting.HARedisDB,
maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
addr: cfg.UnifiedAlerting.HARedisAddr,
name: cfg.UnifiedAlerting.HARedisPeerName,
prefix: cfg.UnifiedAlerting.HARedisPrefix,
password: cfg.UnifiedAlerting.HARedisPassword,
username: cfg.UnifiedAlerting.HARedisUsername,
db: cfg.UnifiedAlerting.HARedisDB,
maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled,
tls: cfg.UnifiedAlerting.HARedisTLSConfig,
}, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval)
if err != nil {
return fmt.Errorf("unable to initialize redis: %w", err)
pkg/services/ngalert/notifier/redis_peer.go
+19 -2
View File
@@ -12,6 +12,7 @@ import (
"github.com/google/uuid"
alertingCluster "github.com/grafana/alerting/cluster"
alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb"
dstls "github.com/grafana/dskit/crypto/tls"
"github.com/prometheus/client_golang/prometheus"
"github.com/redis/go-redis/v9"
@@ -27,6 +28,9 @@ type redisConfig struct {
name string
prefix string
maxConns int
tlsEnabled bool
tls dstls.ClientConfig
}
const (
@@ -90,13 +94,26 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer,
if cfg.maxConns >= 0 {
poolSize = cfg.maxConns
}
rdb := redis.NewClient(&redis.Options{
opts := &redis.Options{
Addr: cfg.addr,
Username: cfg.username,
Password: cfg.password,
DB: cfg.db,
PoolSize: poolSize,
})
}
if cfg.tlsEnabled {
tlsClientConfig, err := cfg.tls.GetTLSConfig()
if err != nil {
logger.Error("Failed to get TLS config", "err", err)
return nil, err
} else {
opts.TLSConfig = tlsClientConfig
}
}
rdb := redis.NewClient(opts)
cmd := rdb.Ping(context.Background())
if cmd.Err() != nil {
logger.Error("Failed to ping redis - redis-based alertmanager clustering may not be available", "err", cmd.Err())
pkg/services/ngalert/notifier/redis_peer_test.go
+89
View File
@@ -0,0 +1,89 @@
package notifier
import (
"context"
"crypto/tls"
"crypto/x509"
"os"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
dstls "github.com/grafana/dskit/crypto/tls"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/madflojo/testcerts"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
)
func TestNewRedisPeerWithTLS(t *testing.T) {
// Write client and server certificates/keys to tempDir, both issues by the same CA
certPaths := createX509TestDir(t)
// Set up tls.Config and start miniredis with server-side TLS
x509Cert, err := tls.LoadX509KeyPair(certPaths.serverCert, certPaths.serverKey)
require.NoError(t, err)
clientCAPool := x509.NewCertPool()
clientCAFile, err := os.ReadFile(certPaths.ca)
require.NoError(t, err)
clientCAPool.AppendCertsFromPEM(clientCAFile)
mr, err := miniredis.RunTLS(&tls.Config{
Certificates: []tls.Certificate{x509Cert},
ClientCAs: clientCAPool,
})
require.NoError(t, err)
defer mr.Close()
// Create redis peer with client-side TLS
redisPeer, err := newRedisPeer(redisConfig{
addr: mr.Addr(),
tlsEnabled: true,
tls: dstls.ClientConfig{
CertPath: certPaths.clientCert,
KeyPath: certPaths.clientKey,
CAPath: certPaths.ca,
ServerName: "localhost",
}}, log.NewNopLogger(), prometheus.DefaultRegisterer, time.Second*60)
require.NoError(t, err)
ping := redisPeer.redis.Ping(context.Background())
require.NoError(t, ping.Err())
}
type certPaths struct {
clientCert string
clientKey string
serverCert string
serverKey string
ca string
}
func createX509TestDir(t *testing.T) certPaths {
t.Helper()
tmpDir := t.TempDir()
ca := testcerts.NewCA()
caCertFile, _, err := ca.ToTempFile(tmpDir)
require.NoError(t, err)
serverKp, err := ca.NewKeyPair("localhost")
require.NoError(t, err)
serverCertFile, serverKeyFile, err := serverKp.ToTempFile(tmpDir)
require.NoError(t, err)
clientKp, err := ca.NewKeyPair()
require.NoError(t, err)
clientCertFile, clientKeyFile, err := clientKp.ToTempFile(tmpDir)
require.NoError(t, err)
return certPaths{
clientCert: clientCertFile.Name(),
clientKey: clientKeyFile.Name(),
serverCert: serverCertFile.Name(),
serverKey: serverKeyFile.Name(),
ca: caCertFile.Name(),
}
}
pkg/setting/setting_unified_alerting.go
+11
View File
@@ -7,6 +7,7 @@ import (
"time"
alertingCluster "github.com/grafana/alerting/cluster"
dstls "github.com/grafana/dskit/crypto/tls"
"github.com/grafana/grafana-plugin-sdk-go/backend/gtime"
"gopkg.in/ini.v1"
@@ -79,6 +80,8 @@ type UnifiedAlertingSettings struct {
HARedisPassword string
HARedisDB int
HARedisMaxConns int
HARedisTLSEnabled bool
HARedisTLSConfig dstls.ClientConfig
MaxAttempts int64
MinInterval time.Duration
EvaluationTimeout time.Duration
@@ -234,6 +237,14 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
uaCfg.HAPeers = append(uaCfg.HAPeers, peer)
}
}
uaCfg.HARedisTLSEnabled = ua.Key("ha_redis_tls_enabled").MustBool(false)
uaCfg.HARedisTLSConfig.CertPath = ua.Key("ha_redis_tls_cert_path").MustString("")
uaCfg.HARedisTLSConfig.KeyPath = ua.Key("ha_redis_tls_key_path").MustString("")
uaCfg.HARedisTLSConfig.CAPath = ua.Key("ha_redis_tls_ca_path").MustString("")
uaCfg.HARedisTLSConfig.ServerName = ua.Key("ha_redis_tls_server_name").MustString("")
uaCfg.HARedisTLSConfig.InsecureSkipVerify = ua.Key("ha_redis_tls_insecure_skip_verify").MustBool(false)
uaCfg.HARedisTLSConfig.CipherSuites = ua.Key("ha_redis_tls_cipher_suites").MustString("")
uaCfg.HARedisTLSConfig.MinVersion = ua.Key("ha_redis_tls_min_version").MustString("")
// TODO load from ini file
uaCfg.DefaultConfiguration = alertmanagerDefaultConfiguration
pkg/setting/setting_unified_alerting_test.go
+47
View File
@@ -298,3 +298,50 @@ func TestMinInterval(t *testing.T) {
})
}
}
func TestHARedisTLSSettings(t *testing.T) {
// Initialize .ini file with new HA Redis TLS Settings
f := ini.Empty()
section, err := f.NewSection("unified_alerting")
require.NoError(t, err)
const (
tlsEnabled = true
certPath = "path/to/cert"
keyPath = "path/to/key"
caPath = "path/to/ca"
serverName = "server_name"
insecureSkipVerify = true
cipherSuites = "TLS_AES_128_GCM_SHA256"
minVersion = "VersionTLS13"
)
_, err = section.NewKey("ha_redis_tls_enabled", strconv.FormatBool(tlsEnabled))
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_cert_path", certPath)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_key_path", keyPath)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_ca_path", caPath)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_server_name", serverName)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_insecure_skip_verify", strconv.FormatBool(insecureSkipVerify))
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_cipher_suites", cipherSuites)
require.NoError(t, err)
_, err = section.NewKey("ha_redis_tls_min_version", minVersion)
require.NoError(t, err)
cfg := NewCfg()
err = cfg.ReadUnifiedAlertingSettings(f)
require.Nil(t, err)
require.Equal(t, tlsEnabled, cfg.UnifiedAlerting.HARedisTLSEnabled)
require.Equal(t, certPath, cfg.UnifiedAlerting.HARedisTLSConfig.CertPath)
require.Equal(t, keyPath, cfg.UnifiedAlerting.HARedisTLSConfig.KeyPath)
require.Equal(t, caPath, cfg.UnifiedAlerting.HARedisTLSConfig.CAPath)
require.Equal(t, serverName, cfg.UnifiedAlerting.HARedisTLSConfig.ServerName)
require.Equal(t, insecureSkipVerify, cfg.UnifiedAlerting.HARedisTLSConfig.InsecureSkipVerify)
require.Equal(t, cipherSuites, cfg.UnifiedAlerting.HARedisTLSConfig.CipherSuites)
require.Equal(t, minVersion, cfg.UnifiedAlerting.HARedisTLSConfig.MinVersion)
}