Alerting: Add support for Redis Sentinel for Alerting HA (#106322)
* Alerting: Add support for Redis Sentinel * docs * docs * Use minisentinel in test * Apply suggestions from code review Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com> Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com> * "address(es)" -> "address or addresses" * make update-workspace * make lint-go-diff --------- Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com> Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com>
This commit is contained in:
Vadim Stepanov
co-authored by
Johnny Kartheiser
Fayzal Ghantiwala
Johnny Kartheiser
Fayzal Ghantiwala
parent
941162ca79
commit
5137995830
+30
-18
@@ -1288,39 +1288,51 @@ alertmanager_max_silences_count =
|
||||
# Maximum silence size in bytes. Default: 0 (no limit).
|
||||
alertmanager_max_silence_size_bytes =
|
||||
|
||||
# Set to true when using redis in cluster mode.
|
||||
ha_redis_cluster_mode_enabled = false
|
||||
|
||||
# The redis server address(es) that should be connected to.
|
||||
# Can either be a single address, or if using redis in cluster mode,
|
||||
# the cluster configuration address or a comma-separated list of addresses.
|
||||
# Redis server address or addresses. It can be a single Redis address if using Redis standalone,
|
||||
# or a list of comma-separated addresses if using Redis Cluster/Sentinel.
|
||||
ha_redis_address =
|
||||
|
||||
# The username that should be used to authenticate with the redis server.
|
||||
# Set to true when using Redis in Cluster mode. Mutually exclusive with ha_redis_sentinel_mode_enabled.
|
||||
ha_redis_cluster_mode_enabled = false
|
||||
|
||||
# Set to true when using Redis in Sentinel mode. Mutually exclusive with ha_redis_cluster_mode_enabled.
|
||||
ha_redis_sentinel_mode_enabled = false
|
||||
|
||||
# Redis Sentinel master name. Only applicable when ha_redis_sentinel_mode_enabled is set to true.
|
||||
ha_redis_sentinel_master_name =
|
||||
|
||||
# The username that should be used to authenticate with Redis.
|
||||
ha_redis_username =
|
||||
|
||||
# The password that should be used to authenticate with the redis server.
|
||||
# The password that should be used to authenticate with Redis.
|
||||
ha_redis_password =
|
||||
|
||||
# The redis database, by default it's 0.
|
||||
# The username that should be used to authenticate with Redis Sentinel.
|
||||
# Only applicable when ha_redis_sentinel_mode_enabled is set to true.
|
||||
ha_redis_sentinel_username =
|
||||
|
||||
# The password that should be used to authenticate with Redis Sentinel.
|
||||
# Only applicable when ha_redis_sentinel_mode_enabled is set to true.
|
||||
ha_redis_sentinel_password =
|
||||
|
||||
# The Redis database. The default value is 0.
|
||||
ha_redis_db =
|
||||
|
||||
# A prefix that is used for every key or channel that is created on the redis server
|
||||
# as part of HA for alerting.
|
||||
# A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting.
|
||||
# Useful if you plan to share Redis with multiple Grafana instances.
|
||||
ha_redis_prefix =
|
||||
|
||||
# The name of the cluster peer that will be used as identifier. If none is
|
||||
# provided, a random one will be generated.
|
||||
# The name of the cluster peer to use as an identifier. If none is provided, a random one is generated.
|
||||
ha_redis_peer_name =
|
||||
|
||||
# The maximum number of simultaneous redis connections.
|
||||
# The maximum number of simultaneous Redis connections.
|
||||
ha_redis_max_conns = 5
|
||||
|
||||
# Enable TLS on the client used to communicate with the redis server. This should be set to true
|
||||
# Enable TLS on the client used to communicate with the Redis server. This should be set to true
|
||||
# if using any of the other ha_redis_tls_* fields.
|
||||
ha_redis_tls_enabled = false
|
||||
|
||||
# Path to the PEM-encoded TLS client certificate file used to authenticate with the redis server.
|
||||
# Path to the PEM-encoded TLS client certificate file used to authenticate with the Redis server.
|
||||
# Required if using Mutual TLS.
|
||||
ha_redis_tls_cert_path =
|
||||
|
||||
@@ -1331,10 +1343,10 @@ ha_redis_tls_key_path =
|
||||
# Path to the PEM-encoded CA certificates file. If not set, the host's root CA certificates are used.
|
||||
ha_redis_tls_ca_path =
|
||||
|
||||
# Overrides the expected name of the redis server certificate.
|
||||
# Overrides the expected name of the Redis server certificate.
|
||||
ha_redis_tls_server_name =
|
||||
|
||||
# Skips validating the redis server certificate.
|
||||
# Skips validating the Redis server certificate.
|
||||
ha_redis_tls_insecure_skip_verify =
|
||||
|
||||
# Overrides the default TLS cipher suite list.
|
||||
|
||||
+39
-28
@@ -1261,68 +1261,79 @@
|
||||
# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
|
||||
;alertmanager_config_poll_interval = 60s
|
||||
|
||||
|
||||
# Maximum number of active and pending silences that a tenant can have at once. Default: 0 (no limit).
|
||||
;alertmanager_max_silences_count =
|
||||
|
||||
# Maximum silence size in bytes. Default: 0 (no limit).
|
||||
;alertmanager_max_silence_size_bytes =
|
||||
|
||||
# Set to true when using redis in cluster mode.
|
||||
;ha_redis_cluster_mode_enabled = false
|
||||
|
||||
# The redis server address(es) that should be connected to.
|
||||
# Can either be a single address, or if using redis in cluster mode,
|
||||
# the cluster configuration address or a comma-separated list of addresses.
|
||||
# Redis server address or addresses. It can be a single Redis address if using Redis standalone,
|
||||
# or a list of comma-separated addresses if using Redis Cluster/Sentinel.
|
||||
;ha_redis_address =
|
||||
|
||||
# The username that should be used to authenticate with the redis server.
|
||||
# Set to true when using Redis in Cluster mode. Mutually exclusive with ha_redis_sentinel_mode_enabled.
|
||||
;ha_redis_cluster_mode_enabled = false
|
||||
|
||||
# Set to true when using Redis in Sentinel mode. Mutually exclusive with ha_redis_cluster_mode_enabled.
|
||||
;ha_redis_sentinel_mode_enabled = false
|
||||
|
||||
# Redis Sentinel master name. Only applicable when ha_redis_sentinel_mode_enabled is set to true.
|
||||
;ha_redis_sentinel_master_name =
|
||||
|
||||
# The username that should be used to authenticate with Redis.
|
||||
;ha_redis_username =
|
||||
|
||||
# The password that should be used to authenticate with the redis server.
|
||||
# The password that should be used to authenticate with Redis.
|
||||
;ha_redis_password =
|
||||
|
||||
# The redis database, by default it's 0.
|
||||
# The username that should be used to authenticate with Redis Sentinel.
|
||||
# Only applicable when ha_redis_sentinel_mode_enabled is set to true.
|
||||
;ha_redis_sentinel_username =
|
||||
|
||||
# The password that should be used to authenticate with Redis Sentinel.
|
||||
# Only applicable when ha_redis_sentinel_mode_enabled is set to true.
|
||||
;ha_redis_sentinel_password =
|
||||
|
||||
# The Redis database. The default value is 0.
|
||||
;ha_redis_db =
|
||||
|
||||
# A prefix that is used for every key or channel that is created on the redis server
|
||||
# as part of HA for alerting.
|
||||
# A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting.
|
||||
# Useful if you plan to share Redis with multiple Grafana instances.
|
||||
;ha_redis_prefix =
|
||||
|
||||
# The name of the cluster peer that will be used as identifier. If none is
|
||||
# provided, a random one will be generated.
|
||||
# The name of the cluster peer to use as an identifier. If none is provided, a random one is generated.
|
||||
;ha_redis_peer_name =
|
||||
|
||||
# The maximum number of simultaneous redis connections.
|
||||
# ha_redis_max_conns = 5
|
||||
# The maximum number of simultaneous Redis connections.
|
||||
;ha_redis_max_conns = 5
|
||||
|
||||
# Enable TLS on the client used to communicate with the redis server. This should be set to true
|
||||
# Enable TLS on the client used to communicate with the Redis server. This should be set to true
|
||||
# if using any of the other ha_redis_tls_* fields.
|
||||
# ha_redis_tls_enabled = false
|
||||
;ha_redis_tls_enabled = false
|
||||
|
||||
# Path to the PEM-encoded TLS client certificate file used to authenticate with the redis server.
|
||||
# Path to the PEM-encoded TLS client certificate file used to authenticate with the Redis server.
|
||||
# Required if using Mutual TLS.
|
||||
# ha_redis_tls_cert_path =
|
||||
;ha_redis_tls_cert_path =
|
||||
|
||||
# Path to the PEM-encoded TLS private key file. Also requires the client certificate to be configured.
|
||||
# Required if using Mutual TLS.
|
||||
# ha_redis_tls_key_path =
|
||||
;ha_redis_tls_key_path =
|
||||
|
||||
# Path to the PEM-encoded CA certificates file. If not set, the host's root CA certificates are used.
|
||||
# ha_redis_tls_ca_path =
|
||||
;ha_redis_tls_ca_path =
|
||||
|
||||
# Overrides the expected name of the redis server certificate.
|
||||
# ha_redis_tls_server_name =
|
||||
# Overrides the expected name of the Redis server certificate.
|
||||
;ha_redis_tls_server_name =
|
||||
|
||||
# Skips validating the redis server certificate.
|
||||
# ha_redis_tls_insecure_skip_verify =
|
||||
# Skips validating the Redis server certificate.
|
||||
;ha_redis_tls_insecure_skip_verify =
|
||||
|
||||
# Overrides the default TLS cipher suite list.
|
||||
# ha_redis_tls_cipher_suites =
|
||||
;ha_redis_tls_cipher_suites =
|
||||
|
||||
# Overrides the default minimum TLS version.
|
||||
# Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13
|
||||
# ha_redis_tls_min_version =
|
||||
;ha_redis_tls_min_version =
|
||||
|
||||
# Listen address/hostname and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. The default value is `0.0.0.0:9094`.
|
||||
;ha_listen_address = "0.0.0.0:9094"
|
||||
|
||||
@@ -67,7 +67,7 @@ For a demo, see this [example using Docker Compose](https://github.com/grafana/a
|
||||
|
||||
## Enable alerting high availability using Redis
|
||||
|
||||
As an alternative to Memberlist, you can configure Redis to enable high availability. Only **Redis Server** and **Redis Cluster** modes are supported.
|
||||
As an alternative to Memberlist, you can configure Redis to enable high availability. Redis standalone, Redis Cluster and Redis Sentinel modes are supported.
|
||||
|
||||
{{% admonition type="note" %}}
|
||||
|
||||
@@ -77,8 +77,11 @@ Memberlist is the preferred option for high availability. Use Redis only in envi
|
||||
|
||||
1. Make sure you have a Redis server that supports pub/sub. If you use a proxy in front of your Redis cluster, make sure the proxy supports pub/sub.
|
||||
1. In your custom configuration file ($WORKING_DIR/conf/custom.ini), go to the `[unified_alerting]` section.
|
||||
1. Set `ha_redis_address` to the Redis server address Grafana should connect to.
|
||||
1. Set `ha_redis_address` to the Redis server address or addresses Grafana should connect to. It can be a single Redis address if using Redis standalone, or a list of comma-separated addresses if using Redis Cluster or Sentinel.
|
||||
1. Optional: Set `ha_redis_cluster_mode_enabled` to `true` if you are using Redis Cluster.
|
||||
1. Optional: Set `ha_redis_sentinel_mode_enabled` to `true` if you are using Redis Sentinel. Also set `ha_redis_sentinel_master_name` to the Redis Sentinel master name.
|
||||
1. Optional: Set the username and password if authentication is enabled on the Redis server using `ha_redis_username` and `ha_redis_password`.
|
||||
1. Optional: Set the username and password if authentication is enabled on Redis Sentinel using `ha_redis_sentinel_username` and `ha_redis_sentinel_password`.
|
||||
1. Optional: Set `ha_redis_prefix` to something unique if you plan to share the Redis server with multiple Grafana instances.
|
||||
1. Optional: Set `ha_redis_tls_enabled` to `true` and configure the corresponding `ha_redis_tls_*` fields to secure communications between Grafana and Redis with Transport Layer Security (TLS).
|
||||
1. Set `[ha_advertise_address]` to `ha_advertise_address = "${POD_IP}:9094"` This is required if the instance doesn't have an IP address that is part of RFC 6890 with a default route.
|
||||
|
||||
@@ -1771,19 +1771,40 @@ The interval string is a possibly signed sequence of decimal numbers, followed b
|
||||
|
||||
#### `ha_redis_address`
|
||||
|
||||
The Redis server address that should be connected to.
|
||||
Redis server address or addresses. It can be a single Redis address if using Redis standalone,
|
||||
or a list of comma-separated addresses if using Redis Cluster/Sentinel.
|
||||
|
||||
{{< admonition type="note" >}}
|
||||
For more information on Redis, refer to [Enable alerting high availability using Redis](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-high-availability/#enable-alerting-high-availability-using-redis).
|
||||
{{< /admonition >}}
|
||||
|
||||
#### `ha_redis_cluster_mode_enabled`
|
||||
|
||||
Set to `true` when using Redis in Cluster mode. Mutually exclusive with `ha_redis_sentinel_mode_enabled`.
|
||||
|
||||
#### `ha_redis_sentinel_mode_enabled`
|
||||
|
||||
Set to `true` when using Redis in Sentinel mode. Mutually exclusive with `ha_redis_cluster_mode_enabled`.
|
||||
|
||||
#### `ha_redis_sentinel_master_name`
|
||||
|
||||
Redis Sentinel master name. Only applicable when `ha_redis_sentinel_mode_enabled` is set to `true`.
|
||||
|
||||
#### `ha_redis_username`
|
||||
|
||||
The username that should be used to authenticate with the Redis server.
|
||||
The username that should be used to authenticate with Redis.
|
||||
|
||||
#### `ha_redis_password`
|
||||
|
||||
The password that should be used to authenticate with the Redis server.
|
||||
The password that should be used to authenticate with Redis.
|
||||
|
||||
#### `ha_redis_sentinel_username`
|
||||
|
||||
The username that should be used to authenticate with Redis Sentinel. Only applicable when `ha_redis_sentinel_mode_enabled` is set to `true`.
|
||||
|
||||
#### `ha_redis_sentinel_password`
|
||||
|
||||
The password that should be used to authenticate with Redis Sentinel. Only applicable when `ha_redis_sentinel_mode_enabled` is set to `true`.
|
||||
|
||||
#### `ha_redis_db`
|
||||
|
||||
@@ -1791,7 +1812,7 @@ The Redis database. The default value is `0`.
|
||||
|
||||
#### `ha_redis_prefix`
|
||||
|
||||
A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting.
|
||||
A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting. Useful if you plan to share Redis with multiple Grafana instances.
|
||||
|
||||
#### `ha_redis_peer_name`
|
||||
|
||||
@@ -1801,6 +1822,38 @@ The name of the cluster peer to use as an identifier. If none is provided, a ran
|
||||
|
||||
The maximum number of simultaneous Redis connections.
|
||||
|
||||
#### `ha_redis_tls_enabled`
|
||||
|
||||
Enable TLS on the client used to communicate with the Redis server. This should be set to `true` if using any of the other `ha_redis_tls_*` fields.
|
||||
|
||||
#### `ha_redis_tls_cert_path`
|
||||
|
||||
Path to the PEM-encoded TLS client certificate file used to authenticate with the Redis server. Required if using Mutual TLS.
|
||||
|
||||
#### `ha_redis_tls_key_path`
|
||||
|
||||
Path to the PEM-encoded TLS private key file. Also requires the client certificate to be configured. Required if using Mutual TLS.
|
||||
|
||||
#### `ha_redis_tls_ca_path`
|
||||
|
||||
Path to the PEM-encoded CA certificates file. If not set, the host's root CA certificates are used.
|
||||
|
||||
#### `ha_redis_tls_server_name`
|
||||
|
||||
Overrides the expected name of the Redis server certificate.
|
||||
|
||||
#### `ha_redis_tls_insecure_skip_verify`
|
||||
|
||||
Skips validating the Redis server certificate.
|
||||
|
||||
#### `ha_redis_tls_cipher_suites`
|
||||
|
||||
Overrides the default TLS cipher suite list.
|
||||
|
||||
#### `ha_redis_tls_min_version`
|
||||
|
||||
Overrides the default minimum TLS version. Allowed values: `VersionTLS10`, `VersionTLS11`, `VersionTLS12`, `VersionTLS13`
|
||||
|
||||
#### `ha_listen_address`
|
||||
|
||||
Listen IP address and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. The default value is `0.0.0.0:9094`.
|
||||
|
||||
@@ -18,6 +18,7 @@ require (
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0 // @grafana/grafana-backend-group
|
||||
github.com/Azure/go-autorest/autorest v0.11.29 // @grafana/grafana-backend-group
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.24 // @grafana/grafana-backend-group
|
||||
github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb // @grafana/alerting-backend
|
||||
github.com/BurntSushi/toml v1.5.0 // @grafana/identity-access-team
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2 // @grafana/grafana-search-and-storage
|
||||
github.com/Masterminds/semver v1.5.0 // @grafana/grafana-backend-group
|
||||
@@ -389,6 +390,7 @@ require (
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
@@ -426,6 +428,7 @@ require (
|
||||
github.com/lestrrat-go/strftime v1.0.4 // indirect
|
||||
github.com/magefile/mage v1.15.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/matryer/is v1.4.1 // indirect
|
||||
github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38 // indirect
|
||||
github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
|
||||
github.com/mattetti/filebuffer v1.0.1 // indirect
|
||||
@@ -567,8 +570,6 @@ require (
|
||||
sigs.k8s.io/yaml v1.4.0 // indirect
|
||||
)
|
||||
|
||||
require github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
|
||||
|
||||
// Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream
|
||||
replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56
|
||||
|
||||
|
||||
@@ -702,6 +702,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb h1:ZVN4Iat3runWOFLaBCDVU5a9X/XikSRBosye++6gojw=
|
||||
github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb/go.mod h1:WsAABbY4HQBgd3mGuG4KMNTbHJCPvx9IVBHzysbknss=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
|
||||
@@ -715,6 +717,8 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
|
||||
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/FZambia/eagle v0.2.0 h1:1kQaZpJvbkvAXFRE/9K2ucBMuVqo+E29EMLYB74hIis=
|
||||
github.com/FZambia/eagle v0.2.0/go.mod h1:LKMYBwGYhao5sJI0TppvQ4SvvldFj9gITxrl8NvGwG0=
|
||||
github.com/FZambia/sentinel v1.0.0 h1:KJ0ryjKTZk5WMp0dXvSdNqp3lFaW1fNFuEYfrkLOYIc=
|
||||
github.com/FZambia/sentinel v1.0.0/go.mod h1:ytL1Am/RLlAoAXG6Kj5LNuw/TRRQrv2rt2FT26vP5gI=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU=
|
||||
@@ -790,8 +794,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
|
||||
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
|
||||
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
|
||||
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
|
||||
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis/v2 v2.11.1/go.mod h1:UA48pmi7aSazcGAvcdKcBB49z521IC9VjTTRz2nIaJE=
|
||||
github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0=
|
||||
github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
@@ -1427,6 +1433,9 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
|
||||
github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
|
||||
@@ -1919,6 +1928,9 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
|
||||
github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
|
||||
github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38 h1:hQWBtNqRYrI7CWIaUSXXtNKR90KzcUA5uiuxFVWw7sU=
|
||||
github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
|
||||
github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU=
|
||||
@@ -2467,6 +2479,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
|
||||
github.com/yuin/gopher-lua v0.0.0-20191213034115-f46add6fdb5c/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0=
|
||||
@@ -2845,6 +2859,7 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@@ -179,16 +179,20 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error {
|
||||
// Redis setup.
|
||||
if cfg.UnifiedAlerting.HARedisAddr != "" {
|
||||
redisPeer, err := newRedisPeer(redisConfig{
|
||||
addr: cfg.UnifiedAlerting.HARedisAddr,
|
||||
name: cfg.UnifiedAlerting.HARedisPeerName,
|
||||
prefix: cfg.UnifiedAlerting.HARedisPrefix,
|
||||
password: cfg.UnifiedAlerting.HARedisPassword,
|
||||
username: cfg.UnifiedAlerting.HARedisUsername,
|
||||
db: cfg.UnifiedAlerting.HARedisDB,
|
||||
maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
|
||||
tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled,
|
||||
tls: cfg.UnifiedAlerting.HARedisTLSConfig,
|
||||
clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled,
|
||||
addr: cfg.UnifiedAlerting.HARedisAddr,
|
||||
name: cfg.UnifiedAlerting.HARedisPeerName,
|
||||
prefix: cfg.UnifiedAlerting.HARedisPrefix,
|
||||
password: cfg.UnifiedAlerting.HARedisPassword,
|
||||
username: cfg.UnifiedAlerting.HARedisUsername,
|
||||
db: cfg.UnifiedAlerting.HARedisDB,
|
||||
maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
|
||||
tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled,
|
||||
tls: cfg.UnifiedAlerting.HARedisTLSConfig,
|
||||
clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled,
|
||||
sentinelMode: cfg.UnifiedAlerting.HARedisSentinelModeEnabled,
|
||||
masterName: cfg.UnifiedAlerting.HARedisSentinelMasterName,
|
||||
sentinelUsername: cfg.UnifiedAlerting.HARedisSentinelUsername,
|
||||
sentinelPassword: cfg.UnifiedAlerting.HARedisSentinelPassword,
|
||||
}, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to initialize redis: %w", err)
|
||||
|
||||
@@ -23,14 +23,18 @@ import (
|
||||
)
|
||||
|
||||
type redisConfig struct {
|
||||
addr string
|
||||
username string
|
||||
password string
|
||||
db int
|
||||
name string
|
||||
prefix string
|
||||
maxConns int
|
||||
clusterMode bool
|
||||
addr string
|
||||
username string
|
||||
password string
|
||||
db int
|
||||
name string
|
||||
prefix string
|
||||
maxConns int
|
||||
clusterMode bool
|
||||
sentinelMode bool
|
||||
masterName string
|
||||
sentinelUsername string
|
||||
sentinelPassword string
|
||||
|
||||
tlsEnabled bool
|
||||
tls dstls.ClientConfig
|
||||
@@ -111,21 +115,19 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer,
|
||||
}
|
||||
}
|
||||
|
||||
opts := &redis.UniversalOptions{
|
||||
rdb := redis.NewUniversalClient(&redis.UniversalOptions{
|
||||
Addrs: addrs,
|
||||
Username: cfg.username,
|
||||
Password: cfg.password,
|
||||
DB: cfg.db,
|
||||
PoolSize: poolSize,
|
||||
TLSConfig: tlsClientConfig,
|
||||
}
|
||||
|
||||
var rdb redis.UniversalClient
|
||||
if cfg.clusterMode {
|
||||
rdb = redis.NewClusterClient(opts.Cluster())
|
||||
} else {
|
||||
rdb = redis.NewClient(opts.Simple())
|
||||
}
|
||||
// Options specific to Sentinel mode.
|
||||
MasterName: cfg.masterName,
|
||||
SentinelUsername: cfg.sentinelUsername,
|
||||
SentinelPassword: cfg.sentinelPassword,
|
||||
})
|
||||
|
||||
cmd := rdb.Ping(context.Background())
|
||||
if cmd.Err() != nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Bose/minisentinel"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
dstls "github.com/grafana/dskit/crypto/tls"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
@@ -45,6 +46,28 @@ func TestNewRedisPeerClusterMode(t *testing.T) {
|
||||
require.NoError(t, ping.Err())
|
||||
}
|
||||
|
||||
func TestNewRedisPeerSentinelMode(t *testing.T) {
|
||||
// Can't use RunTLS here because minisentinel does not support TLS.
|
||||
mr, err := miniredis.Run()
|
||||
require.NoError(t, err)
|
||||
defer mr.Close()
|
||||
|
||||
ms := minisentinel.NewSentinel(mr, minisentinel.WithReplica(mr))
|
||||
err = ms.Start()
|
||||
require.NoError(t, err)
|
||||
defer ms.Close()
|
||||
|
||||
redisPeer, err := newRedisPeer(redisConfig{
|
||||
sentinelMode: true,
|
||||
masterName: ms.MasterInfo().Name,
|
||||
addr: ms.Addr(),
|
||||
}, log.NewNopLogger(), prometheus.NewRegistry(), time.Second*60)
|
||||
require.NoError(t, err)
|
||||
|
||||
ping := redisPeer.redis.Ping(context.Background())
|
||||
require.NoError(t, ping.Err())
|
||||
}
|
||||
|
||||
func TestNewRedisPeerWithTLS(t *testing.T) {
|
||||
// Write client and server certificates/keys to tempDir, both issued by the same CA
|
||||
certPaths := createX509TestDir(t)
|
||||
|
||||
@@ -69,6 +69,11 @@ const (
|
||||
lokiDefaultMaxQuerySize = 65536 // 64kb
|
||||
)
|
||||
|
||||
var (
|
||||
errHARedisBothClusterAndSentinel = fmt.Errorf("'ha_redis_cluster_mode_enabled' and 'ha_redis_sentinel_mode_enabled' are mutually exclusive")
|
||||
errHARedisSentinelMasterNameRequired = fmt.Errorf("'ha_redis_sentinel_master_name' is required when 'ha_redis_sentinel_mode_enabled' is true")
|
||||
)
|
||||
|
||||
type UnifiedAlertingSettings struct {
|
||||
AdminConfigPollInterval time.Duration
|
||||
AlertmanagerConfigPollInterval time.Duration
|
||||
@@ -83,6 +88,10 @@ type UnifiedAlertingSettings struct {
|
||||
HAPushPullInterval time.Duration
|
||||
HALabel string
|
||||
HARedisClusterModeEnabled bool
|
||||
HARedisSentinelModeEnabled bool
|
||||
HARedisSentinelMasterName string
|
||||
HARedisSentinelUsername string
|
||||
HARedisSentinelPassword string
|
||||
HARedisAddr string
|
||||
HARedisPeerName string
|
||||
HARedisPrefix string
|
||||
@@ -276,6 +285,16 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
|
||||
uaCfg.HAAdvertiseAddr = ua.Key("ha_advertise_address").MustString("")
|
||||
uaCfg.HALabel = ua.Key("ha_label").MustString("")
|
||||
uaCfg.HARedisClusterModeEnabled = ua.Key("ha_redis_cluster_mode_enabled").MustBool(false)
|
||||
uaCfg.HARedisSentinelModeEnabled = ua.Key("ha_redis_sentinel_mode_enabled").MustBool(false)
|
||||
if uaCfg.HARedisClusterModeEnabled && uaCfg.HARedisSentinelModeEnabled {
|
||||
return errHARedisBothClusterAndSentinel
|
||||
}
|
||||
uaCfg.HARedisSentinelMasterName = ua.Key("ha_redis_sentinel_master_name").MustString("")
|
||||
if uaCfg.HARedisSentinelModeEnabled && uaCfg.HARedisSentinelMasterName == "" {
|
||||
return errHARedisSentinelMasterNameRequired
|
||||
}
|
||||
uaCfg.HARedisSentinelUsername = ua.Key("ha_redis_sentinel_username").MustString("")
|
||||
uaCfg.HARedisSentinelPassword = ua.Key("ha_redis_sentinel_password").MustString("")
|
||||
uaCfg.HARedisAddr = ua.Key("ha_redis_address").MustString("")
|
||||
uaCfg.HARedisPeerName = ua.Key("ha_redis_peer_name").MustString("")
|
||||
uaCfg.HARedisPrefix = ua.Key("ha_redis_prefix").MustString("")
|
||||
|
||||
@@ -350,3 +350,72 @@ func TestHARedisTLSSettings(t *testing.T) {
|
||||
require.Equal(t, cipherSuites, cfg.UnifiedAlerting.HARedisTLSConfig.CipherSuites)
|
||||
require.Equal(t, minVersion, cfg.UnifiedAlerting.HARedisTLSConfig.MinVersion)
|
||||
}
|
||||
|
||||
func TestHARedisSentinelModeSettings(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
haRedisSentinelModeEnabled bool
|
||||
haRedisClusterModeEnabled bool
|
||||
haRedisSentinelMasterName string
|
||||
haRedisSentinelUsername string
|
||||
haRedisSentinelPassword string
|
||||
expectedErr error
|
||||
}{
|
||||
{
|
||||
desc: "should not fail when Sentinel mode is enabled and master name is set",
|
||||
haRedisSentinelModeEnabled: true,
|
||||
haRedisSentinelMasterName: "exampleMasterName",
|
||||
},
|
||||
{
|
||||
desc: "should not fail when Sentinel mode is enabled, master name is set, and Sentinel username and password are provided",
|
||||
haRedisSentinelModeEnabled: true,
|
||||
haRedisSentinelMasterName: "exampleMasterName",
|
||||
haRedisSentinelUsername: "exampleSentinelUsername",
|
||||
haRedisSentinelPassword: "exampleSentinelPassword",
|
||||
},
|
||||
{
|
||||
desc: "should fail when Sentinel mode is enabled but master name is not set",
|
||||
haRedisSentinelModeEnabled: true,
|
||||
expectedErr: errHARedisSentinelMasterNameRequired,
|
||||
},
|
||||
{
|
||||
desc: "should fail when both Sentinel mode and Cluster mode are enabled",
|
||||
haRedisSentinelModeEnabled: true,
|
||||
haRedisClusterModeEnabled: true,
|
||||
expectedErr: errHARedisBothClusterAndSentinel,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
f := ini.Empty()
|
||||
section, err := f.NewSection("unified_alerting")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = section.NewKey("ha_redis_sentinel_mode_enabled", strconv.FormatBool(tc.haRedisSentinelModeEnabled))
|
||||
require.NoError(t, err)
|
||||
_, err = section.NewKey("ha_redis_cluster_mode_enabled", strconv.FormatBool(tc.haRedisClusterModeEnabled))
|
||||
require.NoError(t, err)
|
||||
_, err = section.NewKey("ha_redis_sentinel_master_name", tc.haRedisSentinelMasterName)
|
||||
require.NoError(t, err)
|
||||
_, err = section.NewKey("ha_redis_sentinel_username", tc.haRedisSentinelUsername)
|
||||
require.NoError(t, err)
|
||||
_, err = section.NewKey("ha_redis_sentinel_password", tc.haRedisSentinelPassword)
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg := NewCfg()
|
||||
err = cfg.ReadUnifiedAlertingSettings(f)
|
||||
if tc.expectedErr == nil {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.ErrorIs(t, err, tc.expectedErr)
|
||||
return
|
||||
}
|
||||
|
||||
require.Equal(t, tc.haRedisSentinelModeEnabled, cfg.UnifiedAlerting.HARedisSentinelModeEnabled)
|
||||
require.Equal(t, tc.haRedisSentinelMasterName, cfg.UnifiedAlerting.HARedisSentinelMasterName)
|
||||
require.Equal(t, tc.haRedisSentinelUsername, cfg.UnifiedAlerting.HARedisSentinelUsername)
|
||||
require.Equal(t, tc.haRedisSentinelPassword, cfg.UnifiedAlerting.HARedisSentinelPassword)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user