diff --git a/go.sum b/go.sum index 5769a898fb1..a744b70e413 100644 --- a/go.sum +++ b/go.sum @@ -1028,6 +1028,7 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc= github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index 3a2c470460b..47de6e7dbed 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -4,6 +4,7 @@ import ( "time" "github.com/grafana/grafana/pkg/services/ngalert/state" + "github.com/prometheus/client_golang/prometheus" "github.com/go-macaron/binding" @@ -59,6 +60,14 @@ func (api *API) RegisterAPIEndpoints() { proxy := &AlertingProxy{ DataProxy: api.DataProxy, } + + var reg prometheus.Registerer + // hack, this just assumes that if this histogram is enabled, we should enable others + // TODO(owen-d): expose this as a config option (alerting-instrumentation or similar) + if api.Cfg.IsHTTPRequestHistogramEnabled() { + reg = prometheus.DefaultRegisterer + } + // Register endpoints for proxing to Alertmanager-compatible backends. api.RegisterAlertmanagerApiEndpoints(NewForkedAM( api.DatasourceCache, @@ -76,6 +85,7 @@ func (api *API) RegisterAPIEndpoints() { api.DatasourceCache, NewLotexRuler(proxy, logger), RulerSrv{store: api.RuleStore, log: logger}, + reg, )) api.RegisterTestingApiEndpoints(TestingApiSrv{ AlertingProxy: proxy, diff --git a/pkg/services/ngalert/api/fork_ruler.go b/pkg/services/ngalert/api/fork_ruler.go index b698eda8be2..317d0d8bfbe 100644 --- a/pkg/services/ngalert/api/fork_ruler.go +++ b/pkg/services/ngalert/api/fork_ruler.go @@ -2,26 +2,57 @@ package api import ( "fmt" + "time" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/datasources" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + GrafanaBackend = "grafana" + ProxyBackend = "proxy" // metric cardinality is too high to enumerate all non-grafana backends ) // ForkedRuler will validate and proxy requests to the correct backend type depending on the datasource. type ForkedRuler struct { LotexRuler, GrafanaRuler RulerApiService DatasourceCache datasources.CacheService + + // metrics + duration *prometheus.HistogramVec } // NewForkedRuler implements a set of routes that proxy to various Cortex Ruler-compatible backends. -func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService) *ForkedRuler { - return &ForkedRuler{ +func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService, reg prometheus.Registerer) *ForkedRuler { + r := &ForkedRuler{ LotexRuler: lotex, GrafanaRuler: grafana, DatasourceCache: datasourceCache, + duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "grafana", + Name: "alerting_ruler_api_duration_seconds", + Help: "Histogram of latencies affecting the Unified Alerting Ruler API", + Buckets: prometheus.DefBuckets, + }, []string{"backend", "status"}), } + if reg != nil { + reg.MustRegister( + r.duration, + ) + } + return r +} + +func (r *ForkedRuler) instrument(backend string, fn func() response.Response) response.Response { + start := time.Now() + resp := fn() + r.duration. + WithLabelValues(backend, fmt.Sprint(resp.Status())). + Observe(time.Since(start).Seconds()) + return resp } func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) response.Response { @@ -31,9 +62,9 @@ func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) re } switch t { case apimodels.GrafanaBackend: - return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx) + return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx) }) case apimodels.LoTexRulerBackend: - return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx) + return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx) }) default: return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil) } @@ -46,9 +77,9 @@ func (r *ForkedRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) respons } switch t { case apimodels.GrafanaBackend: - return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx) + return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx) }) case apimodels.LoTexRulerBackend: - return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx) + return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx) }) default: return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil) } @@ -61,9 +92,9 @@ func (r *ForkedRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) respo } switch t { case apimodels.GrafanaBackend: - return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx) + return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx) }) case apimodels.LoTexRulerBackend: - return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx) + return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx) }) default: return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil) } @@ -76,9 +107,9 @@ func (r *ForkedRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response. } switch t { case apimodels.GrafanaBackend: - return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx) + return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx) }) case apimodels.LoTexRulerBackend: - return r.LotexRuler.RouteGetRulegGroupConfig(ctx) + return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulegGroupConfig(ctx) }) default: return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil) } @@ -91,9 +122,9 @@ func (r *ForkedRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Respo } switch t { case apimodels.GrafanaBackend: - return r.GrafanaRuler.RouteGetRulesConfig(ctx) + return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulesConfig(ctx) }) case apimodels.LoTexRulerBackend: - return r.LotexRuler.RouteGetRulesConfig(ctx) + return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulesConfig(ctx) }) default: return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil) } @@ -120,9 +151,9 @@ func (r *ForkedRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apim switch backendType { case apimodels.GrafanaBackend: - return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf) + return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf) }) case apimodels.LoTexRulerBackend: - return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf) + return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf) }) default: return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", backendType), nil) }