diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index ee6c4a24c5f..022836826de 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -13,6 +13,7 @@ import ( "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" + "github.com/prometheus/client_golang/prometheus" "xorm.io/xorm" "github.com/grafana/grafana/pkg/bus" @@ -53,6 +54,17 @@ type SQLStore struct { skipEnsureDefaultOrgAndUser bool migrations registry.DatabaseMigrator tracer tracing.Tracer + metrics struct { + maxOpenConnections prometheus.Gauge + openConnections prometheus.Gauge + inUse prometheus.Gauge + idle prometheus.Gauge + waitCount prometheus.Counter + waitDuration prometheus.Counter + maxIdleClosed prometheus.Counter + maxIdleTimeClosed prometheus.Counter + maxLifetimeClosed prometheus.Counter + } } func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, migrations registry.DatabaseMigrator, bus bus.Bus, tracer tracing.Tracer) (*SQLStore, error) { @@ -73,6 +85,11 @@ func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, mig return nil, err } s.tracer = tracer + + s.initMetrics() + + prometheus.MustRegister(s) + return s, nil } @@ -425,6 +442,118 @@ func (ss *SQLStore) readConfig() error { return nil } +// initMetrics initializes the database connection metrics +func (ss *SQLStore) initMetrics() { + namespace := "grafana" + subsystem := "database" + + ss.metrics.maxOpenConnections = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_max_open", + Help: "Maximum number of open connections to the database", + }) + + ss.metrics.openConnections = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_open", + Help: "The number of established connections both in use and idle", + }) + + ss.metrics.inUse = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_in_use", + Help: "The number of connections currently in use", + }) + + ss.metrics.idle = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_idle", + Help: "The number of idle connections", + }) + + ss.metrics.waitCount = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_wait_count_total", + Help: "The total number of connections waited for", + }) + + ss.metrics.waitDuration = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_wait_duration_seconds", + Help: "The total time blocked waiting for a new connection", + }) + + ss.metrics.maxIdleClosed = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_max_idle_closed_total", + Help: "The total number of connections closed due to SetMaxIdleConns", + }) + + ss.metrics.maxIdleTimeClosed = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_max_idle_closed_seconds", + Help: "The total number of connections closed due to SetConnMaxIdleTime", + }) + + ss.metrics.maxLifetimeClosed = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "conn_max_lifetime_closed_total", + Help: "The total number of connections closed due to SetConnMaxLifetime", + }) +} + +// collectDBStats instruments connections stats from the database. +func (ss *SQLStore) collectDBstats() { + dbstats := ss.engine.DB().Stats() + ss.metrics.maxOpenConnections.Set(float64(dbstats.MaxOpenConnections)) + ss.metrics.openConnections.Set(float64(dbstats.MaxOpenConnections)) + ss.metrics.inUse.Set(float64(dbstats.InUse)) + ss.metrics.idle.Set(float64(dbstats.Idle)) + + ss.metrics.waitCount.Add(float64(dbstats.WaitCount)) + ss.metrics.waitDuration.Add(float64(dbstats.WaitDuration / time.Second)) + ss.metrics.maxIdleClosed.Add(float64(dbstats.MaxIdleClosed)) + ss.metrics.maxIdleTimeClosed.Add(float64(dbstats.MaxIdleTimeClosed)) + ss.metrics.maxLifetimeClosed.Add(float64(dbstats.MaxLifetimeClosed)) +} + +// Collect implements Prometheus.Collector. +func (ss *SQLStore) Collect(ch chan<- prometheus.Metric) { + ss.collectDBstats() + + ss.metrics.maxOpenConnections.Collect(ch) + ss.metrics.openConnections.Collect(ch) + ss.metrics.inUse.Collect(ch) + ss.metrics.idle.Collect(ch) + ss.metrics.waitCount.Collect(ch) + ss.metrics.waitDuration.Collect(ch) + ss.metrics.maxIdleClosed.Collect(ch) + ss.metrics.maxIdleTimeClosed.Collect(ch) + ss.metrics.maxLifetimeClosed.Collect(ch) +} + +// Describe implements Prometheus.Collector. +func (ss *SQLStore) Describe(ch chan<- *prometheus.Desc) { + ss.metrics.maxOpenConnections.Describe(ch) + ss.metrics.openConnections.Describe(ch) + ss.metrics.inUse.Describe(ch) + ss.metrics.idle.Describe(ch) + ss.metrics.waitCount.Describe(ch) + ss.metrics.waitDuration.Describe(ch) + ss.metrics.maxIdleClosed.Describe(ch) + ss.metrics.maxIdleTimeClosed.Describe(ch) + ss.metrics.maxLifetimeClosed.Describe(ch) +} + // ITestDB is an interface of arguments for testing db type ITestDB interface { Helper()