public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting/metrics (#33547)

Browse Source 
* moves alerting metrics to their own pkg

* adds grafana_alerting_alerts (by state) metric

* alerts_received_{total,invalid}

* embed alertmanager alerting struct in ng metrics & remove duplicated notification metrics (already embed alertmanager notifier metrics)

* use silence metrics from alertmanager lib

* fix - manager has metrics

* updates ngalert tests

* comment lint
Signed-off-by: Owen Diehl <ow.diehl@gmail.com>

* cleaner prom registry code

* removes ngalert global metrics

* new registry use in all tests

* ngalert metrics impl service, hack testinfra code to prevent duplicate metric registrations

* nilmetrics unexported
This commit is contained in:
Owen Diehl
2021-04-30 12:28:06 -04:00
committed by GitHub
parent b45120b999
commit 5e48b54549
15 changed files with 165 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/api.go
+6 -11
View File
@@ -3,8 +3,8 @@ package api
import (
"time"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/prometheus/client_golang/prometheus"
"github.com/go-macaron/binding"
@@ -24,11 +24,6 @@ import (
// timeNow makes it possible to test usage of time
var timeNow = time.Now
// metrics are a globally registered metric suite for alerting.
// TODO: refactor testware to allow these to be created without
// panicking on duplicate registration, thus enabling non-global vars.
var metrics = NewMetrics(prometheus.DefaultRegisterer)
type Alertmanager interface {
// Configuration
SaveAndApplyConfig(config *apimodels.PostableUserConfig) error
@@ -60,7 +55,7 @@ type API struct {
}
// RegisterAPIEndpoints registers API handlers
func (api *API) RegisterAPIEndpoints() {
func (api *API) RegisterAPIEndpoints(m *metrics.Metrics) {
logger := log.New("ngalert.api")
proxy := &AlertingProxy{
DataProxy: api.DataProxy,
@@ -71,26 +66,26 @@ func (api *API) RegisterAPIEndpoints() {
api.DatasourceCache,
NewLotexAM(proxy, logger),
AlertmanagerSrv{store: api.AlertingStore, am: api.Alertmanager, log: logger},
), metrics)
), m)
// Register endpoints for proxing to Prometheus-compatible backends.
api.RegisterPrometheusApiEndpoints(NewForkedProm(
api.DatasourceCache,
NewLotexProm(proxy, logger),
PrometheusSrv{log: logger, manager: api.StateManager, store: api.RuleStore},
), metrics)
), m)
// Register endpoints for proxing to Cortex Ruler-compatible backends.
api.RegisterRulerApiEndpoints(NewForkedRuler(
api.DatasourceCache,
NewLotexRuler(proxy, logger),
RulerSrv{DatasourceCache: api.DatasourceCache, store: api.RuleStore, log: logger},
), metrics)
), m)
api.RegisterTestingApiEndpoints(TestingApiSrv{
AlertingProxy: proxy,
Cfg: api.Cfg,
DataService: api.DataService,
DatasourceCache: api.DatasourceCache,
log: logger,
}, metrics)
}, m)
// Legacy routes; they will be removed in v8
api.RouteRegister.Group("/api/alert-definitions", func(alertDefinitions routing.RouteRegister) {
pkg/services/ngalert/api/generated_base_api_alertmanager.go
+22 -21
View File
@@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
)
type AlertmanagerApiService interface {
@@ -32,99 +33,99 @@ type AlertmanagerApiService interface {
RoutePostAlertingConfig(*models.ReqContext, apimodels.PostableUserConfig) response.Response
}
func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, metrics *Metrics) {
func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, m *metrics.Metrics) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Post(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silences"),
binding.Bind(apimodels.PostableSilence{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/alertmanager/{Recipient}/api/v2/silences",
srv.RouteCreateSilence,
metrics,
m,
),
)
group.Delete(
toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"),
Instrument(
metrics.Instrument(
http.MethodDelete,
"/api/alertmanager/{Recipient}/config/api/v1/alerts",
srv.RouteDeleteAlertingConfig,
metrics,
m,
),
)
group.Delete(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"),
Instrument(
metrics.Instrument(
http.MethodDelete,
"/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}",
srv.RouteDeleteSilence,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts/groups"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/alertmanager/{Recipient}/api/v2/alerts/groups",
srv.RouteGetAMAlertGroups,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/alertmanager/{Recipient}/api/v2/alerts",
srv.RouteGetAMAlerts,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/alertmanager/{Recipient}/config/api/v1/alerts",
srv.RouteGetAlertingConfig,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}",
srv.RouteGetSilence,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silences"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/alertmanager/{Recipient}/api/v2/silences",
srv.RouteGetSilences,
metrics,
m,
),
)
group.Post(
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts"),
binding.Bind(apimodels.PostableAlerts{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/alertmanager/{Recipient}/api/v2/alerts",
srv.RoutePostAMAlerts,
metrics,
m,
),
)
group.Post(
toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"),
binding.Bind(apimodels.PostableUserConfig{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/alertmanager/{Recipient}/config/api/v1/alerts",
srv.RoutePostAlertingConfig,
metrics,
m,
),
)
}, middleware.ReqSignedIn)
pkg/services/ngalert/api/generated_base_api_prometheus.go
+6 -5
View File
@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
)
type PrometheusApiService interface {
@@ -21,24 +22,24 @@ type PrometheusApiService interface {
RouteGetRuleStatuses(*models.ReqContext) response.Response
}
func (api *API) RegisterPrometheusApiEndpoints(srv PrometheusApiService, metrics *Metrics) {
func (api *API) RegisterPrometheusApiEndpoints(srv PrometheusApiService, m *metrics.Metrics) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Get(
toMacaronPath("/api/prometheus/{Recipient}/api/v1/alerts"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/prometheus/{Recipient}/api/v1/alerts",
srv.RouteGetAlertStatuses,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/prometheus/{Recipient}/api/v1/rules"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/prometheus/{Recipient}/api/v1/rules",
srv.RouteGetRuleStatuses,
metrics,
m,
),
)
}, middleware.ReqSignedIn)
pkg/services/ngalert/api/generated_base_api_ruler.go
+14 -13
View File
@@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
)
type RulerApiService interface {
@@ -28,61 +29,61 @@ type RulerApiService interface {
RoutePostNameRulesConfig(*models.ReqContext, apimodels.PostableRuleGroupConfig) response.Response
}
func (api *API) RegisterRulerApiEndpoints(srv RulerApiService, metrics *Metrics) {
func (api *API) RegisterRulerApiEndpoints(srv RulerApiService, m *metrics.Metrics) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Delete(
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"),
Instrument(
metrics.Instrument(
http.MethodDelete,
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}",
srv.RouteDeleteNamespaceRulesConfig,
metrics,
m,
),
)
group.Delete(
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}"),
Instrument(
metrics.Instrument(
http.MethodDelete,
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}",
srv.RouteDeleteRuleGroupConfig,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}",
srv.RouteGetNamespaceRulesConfig,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}",
srv.RouteGetRulegGroupConfig,
metrics,
m,
),
)
group.Get(
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules"),
Instrument(
metrics.Instrument(
http.MethodGet,
"/api/ruler/{Recipient}/api/v1/rules",
srv.RouteGetRulesConfig,
metrics,
m,
),
)
group.Post(
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"),
binding.Bind(apimodels.PostableRuleGroupConfig{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}",
srv.RoutePostNameRulesConfig,
metrics,
m,
),
)
}, middleware.ReqSignedIn)
pkg/services/ngalert/api/generated_base_api_testing.go
+8 -7
View File
@@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
)
type TestingApiService interface {
@@ -25,36 +26,36 @@ type TestingApiService interface {
RouteTestRuleConfig(*models.ReqContext, apimodels.TestRulePayload) response.Response
}
func (api *API) RegisterTestingApiEndpoints(srv TestingApiService, metrics *Metrics) {
func (api *API) RegisterTestingApiEndpoints(srv TestingApiService, m *metrics.Metrics) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Post(
toMacaronPath("/api/v1/eval"),
binding.Bind(apimodels.EvalQueriesPayload{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/v1/eval",
srv.RouteEvalQueries,
metrics,
m,
),
)
group.Post(
toMacaronPath("/api/v1/receiver/test/{Recipient}"),
binding.Bind(apimodels.ExtendedReceiver{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/v1/receiver/test/{Recipient}",
srv.RouteTestReceiverConfig,
metrics,
m,
),
)
group.Post(
toMacaronPath("/api/v1/rule/test/{Recipient}"),
binding.Bind(apimodels.TestRulePayload{}),
Instrument(
metrics.Instrument(
http.MethodPost,
"/api/v1/rule/test/{Recipient}",
srv.RouteTestRuleConfig,
metrics,
m,
),
)
}, middleware.ReqSignedIn)
pkg/services/ngalert/api/metrics.go
-149
View File
@@ -1,149 +0,0 @@
package api
import (
"fmt"
"regexp"
"strings"
"time"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"gopkg.in/macaron.v1"
)
const (
GrafanaBackend = "grafana"
ProxyBackend = "proxy"
)
type Metrics struct {
alerts *prometheus.GaugeVec
alertsInvalid prometheus.Counter
alertsReceived prometheus.Counter
notificationLatency prometheus.Histogram
notifications *prometheus.CounterVec
notificationsFailed *prometheus.CounterVec
requestDuration *prometheus.HistogramVec
silences *prometheus.GaugeVec
}
func NewMetrics(r prometheus.Registerer) *Metrics {
return &Metrics{
alerts: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "alerts",
Help: "How many alerts by state.",
}, []string{"state"}),
alertsInvalid: promauto.With(r).NewCounter(prometheus.CounterOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "alerts_invalid_total",
Help: "The total number of invalid received alerts.",
}),
alertsReceived: promauto.With(r).NewCounter(prometheus.CounterOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "alerts_received_total",
Help: "The total number of received alerts.",
}),
notificationLatency: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "notification_latency_seconds",
Help: "Histogram of notification deliveries",
Buckets: prometheus.DefBuckets,
}),
notifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "notifications_total",
Help: "The total number of attempted notfications by integration.",
}, []string{"integration"}),
notificationsFailed: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "notifications_failed_total",
Help: "The total number of failed notfications by integration.",
}, []string{"integration"}),
requestDuration: promauto.With(r).NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "request_duration_seconds",
Help: "Histogram of requests to the Alerting API",
Buckets: prometheus.DefBuckets,
},
[]string{"method", "route", "status_code", "backend"},
),
silences: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
Namespace: "grafana",
Subsystem: "alerting",
Name: "silences",
Help: "The total number of silences by state.",
}, []string{"state"}),
}
}
// Instrument wraps a middleware, instrumenting the request latencies.
func Instrument(
method,
path string,
action interface{},
metrics *Metrics,
) macaron.Handler {
normalizedPath := MakeLabelValue(path)
return func(c *models.ReqContext) {
start := time.Now()
var res response.Response
val, err := c.Invoke(action)
if err == nil && val != nil && len(val) > 0 {
res = val[0].Interface().(response.Response)
} else {
res = routing.ServerError(err)
}
// TODO: We could look up the datasource type via our datasource service
var backend string
recipient := c.Params("Recipient")
if recipient == apimodels.GrafanaBackend.String() || recipient == "" {
backend = GrafanaBackend
} else {
backend = ProxyBackend
}
ls := prometheus.Labels{
"method": method,
"route": normalizedPath,
"status_code": fmt.Sprint(res.Status()),
"backend": backend,
}
res.WriteTo(c)
metrics.requestDuration.With(ls).Observe(time.Since(start).Seconds())
}
}
var invalidChars = regexp.MustCompile(`[^a-zA-Z0-9]+`)
// MakeLabelValue normalizes a path template
func MakeLabelValue(path string) string {
// Convert non-alnums to underscores.
result := invalidChars.ReplaceAllString(path, "_")
// Trim leading and trailing underscores.
result = strings.Trim(result, "_")
// Make it all lowercase
result = strings.ToLower(result)
// Special case.
if result == "" {
result = "root"
}
return result
}
pkg/services/ngalert/api/tooling/swagger-codegen/templates/controller-api.mustache
+4 -3
View File
@@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/middleware"
)
@@ -16,16 +17,16 @@ type {{classname}}Service interface { {{#operation}}
{{nickname}}(*models.ReqContext{{#bodyParams}}, apimodels.{{dataType}}{{/bodyParams}}) response.Response{{/operation}}
}
func (api *API) Register{{classname}}Endpoints(srv {{classname}}Service, metrics *Metrics) {
func (api *API) Register{{classname}}Endpoints(srv {{classname}}Service, m *metrics.Metrics) {
api.RouteRegister.Group("", func(group routing.RouteRegister){ {{#operations}}{{#operation}}
group.{{httpMethod}}(
toMacaronPath("{{{path}}}"){{#bodyParams}},
binding.Bind(apimodels.{{dataType}}{}){{/bodyParams}},
Instrument(
metrics.Instrument(
http.Method{{httpMethod}},
"{{{path}}}",
srv.{{nickname}},
metrics,
m,
),
){{/operation}}{{/operations}}
}, middleware.ReqSignedIn)