public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Support Unified Alerting with Grafana HA (#37920)

Browse Source 
* Alerting: Support Unified Alerting in Grafana's HA mode.
This commit is contained in:
gotjosh
2021-09-16 15:33:51 +01:00
committed by GitHub
parent 92209f1011
commit 7db97097c9
25 changed files with 6377 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devenv/docker/ha-test-unified-alerting/.gitignore
+1
View File
@@ -0,0 +1 @@
grafana/provisioning/dashboards/alerts/alert-*
devenv/docker/ha-test-unified-alerting/README.md
+66
View File
@@ -0,0 +1,66 @@
# Grafana Unified Alerting High Availability (HA) test setup
A set of docker compose services which together creates a Grafana HA test setup for unified alerting.
Included services
- Grafana
- Mysql - Grafana configuration database, exporter for metrics and session storage
- Prometheus - Monitoring of Grafana and used as data source
- Nginx - Reverse proxy for Grafana and Prometheus. Enables browsing Grafana/Prometheus UI using a hostname
## Prerequisites
### Build grafana docker container
Build a Grafana docker container from current branch and commit and tag it as grafana/grafana:dev.
```bash
$ cd <grafana repo>
$ make build-docker-full
```
### Virtual host names
#### Alternative 1 - Use dnsmasq
```bash
$ sudo apt-get install dnsmasq
$ echo 'address=/loc/127.0.0.1' | sudo tee /etc/dnsmasq.d/dnsmasq-loc.conf > /dev/null
$ sudo /etc/init.d/dnsmasq restart
$ ping whatever.loc
PING whatever.loc (127.0.0.1) 56(84) bytes of data.
64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.076 ms
--- whatever.loc ping statistics ---
1 packet transmitted, 1 received, 0% packet loss, time 1998ms
```
#### Alternative 2 - Manually update /etc/hosts
Update your `/etc/hosts` to be able to access Grafana and/or Prometheus UI using a hostname.
```bash
$ cat /etc/hosts
127.0.0.1 grafana.loc
127.0.0.1 prometheus.loc
```
## Start services
```bash
$ docker-compose up -d
```
Browse
- http://grafana.loc/
- http://prometheus.loc/
## Test alerting
### Create contact points
TBD
### Create alerts
TBD
### Create silences
TBD
devenv/docker/ha-test-unified-alerting/docker-compose.yaml
+90
View File
@@ -0,0 +1,90 @@
version: "2.1"
services:
db:
image: mysql:5.6
platform: linux/x86_64
environment:
MYSQL_ROOT_PASSWORD: rootpass
MYSQL_DATABASE: grafana
MYSQL_USER: grafana
MYSQL_PASSWORD: password
command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all, --max-connections=1001]
ports:
- 3306
healthcheck:
test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
timeout: 10s
retries: 10
mysqld-exporter:
image: prom/mysqld-exporter
environment:
- DATA_SOURCE_NAME=root:rootpass@(db:3306)/
ports:
- 9104
depends_on:
db:
condition: service_healthy
prometheus:
image: prom/prometheus:v2.4.2
volumes:
- ./prometheus/:/etc/prometheus/
environment:
- VIRTUAL_HOST=prometheus.loc
ports:
- 909
nginx-proxy:
image: jwilder/nginx-proxy
ports:
- "80:80"
volumes:
- /var/run/docker.sock:/tmp/docker.sock:ro
depends_on:
db:
condition: service_healthy
grafana1:
image: grafana/grafana:dev
volumes:
- ./grafana/provisioning/:/etc/grafana/provisioning/
environment:
- VIRTUAL_HOST=grafana.loc
- GF_FEATURE_TOGGLES_ENABLE=ngalert
- GF_UNIFIED_ALERTING_HA_PEERS=ha-test-unified-alerting_grafana2_1:9094,ha-test-unified-alerting_grafana1_1:9094
- GF_SERVER_ROOT_URL=http://grafana.loc
- GF_DATABASE_NAME=grafana
- GF_DATABASE_USER=grafana
- GF_DATABASE_PASSWORD=password
- GF_DATABASE_TYPE=mysql
- GF_DATABASE_HOST=db:3306
- GF_DATABASE_MAX_OPEN_CONN=300
- GF_SESSION_PROVIDER=mysql
- GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(db:3306)/grafana?allowNativePasswords=true
ports:
- 3010:3000
depends_on:
db:
condition: service_healthy
grafana2:
image: grafana/grafana:dev
volumes:
- ./grafana/provisioning/:/etc/grafana/provisioning/
environment:
- VIRTUAL_HOST=grafana.loc
- GF_FEATURE_TOGGLES_ENABLE=ngalert
- GF_UNIFIED_ALERTING_HA_PEERS=ha-test-unified-alerting_grafana2_1:9094,ha-test-unified-alerting_grafana1_1:9094
- GF_SERVER_ROOT_URL=http://grafana.loc
- GF_DATABASE_NAME=grafana
- GF_DATABASE_USER=grafana
- GF_DATABASE_PASSWORD=password
- GF_DATABASE_TYPE=mysql
- GF_DATABASE_HOST=db:3306
- GF_DATABASE_MAX_OPEN_CONN=300
- GF_SESSION_PROVIDER=mysql
- GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(db:3306)/grafana?allowNativePasswords=true
ports:
- 3020:3000
depends_on:
db:
condition: service_healthy
devenv/docker/ha-test-unified-alerting/grafana/provisioning/alerts.jsonnet
+203
View File
@@ -0,0 +1,203 @@
local numAlerts = std.extVar('alerts');
local condition = std.extVar('condition');
local arr = std.range(1, numAlerts);
local alertDashboardTemplate = {
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
65
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"for": "1m",
"name": "bulk alerting",
"noDataState": "no_data",
"notifications": [
{
"id": 2
}
]
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:117",
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 50
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel Title",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "New dashboard",
"uid": null,
"version": 0
};
{
['alert-' + std.toString(x) + '.json']:
alertDashboardTemplate + {
panels: [
alertDashboardTemplate.panels[0] +
{
alert+: {
name: 'Alert rule ' + x,
conditions: [
alertDashboardTemplate.panels[0].alert.conditions[0] +
{
evaluator+: {
params: [condition]
}
},
],
},
},
],
uid: 'alert-' + x,
title: 'Alert ' + x
},
for x in arr
}
devenv/docker/ha-test-unified-alerting/grafana/provisioning/dashboards/alerts/overview.json
+172
View File
@@ -0,0 +1,172 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"links": [],
"panels": [
{
"aliasColors": {
"Active alerts": "#bf1b00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"interval": "",
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Active grafana instances",
"dashes": true,
"fill": 0
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(grafana_alerting_notification_sent_total[1m])) by(job)",
"format": "time_series",
"instant": false,
"interval": "1m",
"intervalFactor": 1,
"legendFormat": "Notifications sent",
"refId": "A"
},
{
"expr": "min(grafana_alerting_active_alerts) without(instance)",
"format": "time_series",
"interval": "1m",
"intervalFactor": 1,
"legendFormat": "Active alerts",
"refId": "B"
},
{
"expr": "count(up{job=\"grafana\"})",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "Active grafana instances",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Notifications sent vs active alerts",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": 3
}
}
],
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Overview",
"uid": "xHy7-hAik",
"version": 6
}
devenv/docker/ha-test-unified-alerting/grafana/provisioning/dashboards/dashboards.yaml
+14
View File
@@ -0,0 +1,14 @@
apiVersion: 1
providers:
- name: 'Alerts'
folder: 'Alerts'
type: file
options:
path: /etc/grafana/provisioning/dashboards/alerts
- name: 'MySQL'
folder: 'MySQL'
type: file
options:
path: /etc/grafana/provisioning/dashboards/mysql
devenv/docker/ha-test-unified-alerting/grafana/provisioning/dashboards/mysql/overview.json
+5397
View File
File diff suppressed because it is too large Load Diff
devenv/docker/ha-test-unified-alerting/grafana/provisioning/datasources/datasources.yaml
+16
View File
@@ -0,0 +1,16 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
jsonData:
timeInterval: 10s
queryTimeout: 30s
httpMethod: POST
- name: Loki
type: loki
access: proxy
url: http://loki:3100
devenv/docker/ha-test-unified-alerting/prometheus/prometheus.yml
+47
View File
@@ -0,0 +1,47 @@
# my global config
global:
scrape_interval: 10s # By default, scrape targets every 15 seconds.
evaluation_interval: 10s # By default, scrape targets every 15 seconds.
# scrape_timeout is set to the global default (10s).
# Load and evaluate rules in this file every 'evaluation_interval' seconds.
#rule_files:
# - "alert.rules"
# - "first.rules"
# - "second.rules"
# alerting:
# alertmanagers:
# - scheme: http
# static_configs:
# - targets:
# - "127.0.0.1:9093"
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'grafana'
dns_sd_configs:
- names:
- 'grafana'
type: 'A'
port: 3000
refresh_interval: 10s
- job_name: 'mysql'
dns_sd_configs:
- names:
- 'mysqld-exporter'
type: 'A'
port: 9104
refresh_interval: 10s
- job_name: 'loki'
dns_sd_configs:
- names:
- 'loki'
type: 'A'
port: 3100
refresh_interval: 10s