From 713075c4945bb8b86e73c187a0c9b7b5797adbe8 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Wed, 31 Aug 2022 08:54:32 -0400 Subject: [PATCH] Metrics: Fixed `grafana_database_conn_*` metrics, and added new `go_sql_stats_*` metrics as eventual replacement (#54405) * metrics: Fix broken DBStats metrics Signed-off-by: Dave Henderson * Register the sqlstats metrics by default Signed-off-by: Dave Henderson Signed-off-by: Dave Henderson --- go.mod | 7 +- go.sum | 22 +-- pkg/services/sqlstore/sqlstore.go | 132 +--------------- pkg/services/sqlstore/sqlstore_metrics.go | 144 ++++++++++++++++++ .../sqlstore/sqlstore_metrics_test.go | 68 +++++++++ 5 files changed, 224 insertions(+), 149 deletions(-) create mode 100644 pkg/services/sqlstore/sqlstore_metrics.go create mode 100644 pkg/services/sqlstore/sqlstore_metrics_test.go diff --git a/go.mod b/go.mod index cfe8dec31f6..e2cd8ca6b5f 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/go-stack/stack v1.8.1 github.com/gobwas/glob v0.2.3 - github.com/gofrs/uuid v4.2.0+incompatible // indirect + github.com/gofrs/uuid v4.2.0+incompatible github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/golang/snappy v0.0.4 @@ -89,7 +89,6 @@ require ( github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f github.com/uber/jaeger-client-go v2.29.1+incompatible - github.com/unknwon/com v1.0.1 github.com/urfave/cli/v2 v2.3.0 github.com/vectordotdev/go-datemath v0.1.1-0.20220323213446-f3954d0b18ae github.com/weaveworks/common v0.0.0-20210913144402-035033b78a78 // indirect @@ -245,12 +244,14 @@ require ( github.com/armon/go-radix v1.0.0 github.com/blugelabs/bluge v0.1.9 github.com/blugelabs/bluge_segment_api v0.2.0 + github.com/dlmiddlecote/sqlstats v1.0.2 github.com/getkin/kin-openapi v0.94.0 github.com/golang-migrate/migrate/v4 v4.7.0 github.com/google/go-github/v45 v45.2.0 github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f github.com/jmoiron/sqlx v1.3.5 go.etcd.io/etcd/api/v3 v3.5.4 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.31.0 go.opentelemetry.io/contrib/propagators/jaeger v1.6.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3 @@ -269,7 +270,6 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.31.0 // indirect ) require ( @@ -318,7 +318,6 @@ require ( github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/segmentio/asm v1.1.1 // indirect - github.com/smartystreets/goconvey v1.7.2 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect github.com/wk8/go-ordered-map v1.0.0 github.com/xanzy/ssh-agent v0.3.0 // indirect diff --git a/go.sum b/go.sum index 732f1f6290c..a43ad44eb67 100644 --- a/go.sum +++ b/go.sum @@ -728,6 +728,8 @@ github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlmiddlecote/sqlstats v1.0.2 h1:gSU11YN23D/iY50A2zVYwgXgy072khatTsIW6UPjUtI= +github.com/dlmiddlecote/sqlstats v1.0.2/go.mod h1:0CWaIh/Th+z2aI6Q9Jpfg/o21zmGxWhbByHgQSCUQvY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= @@ -1316,8 +1318,6 @@ github.com/gophercloud/gophercloud v0.18.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075 github.com/gophercloud/gophercloud v0.20.0 h1:1+4jrsjVhdX5omlAo4jkmFc6ftLbuXLzgFo4i6lH+Gk= github.com/gophercloud/gophercloud v0.20.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de h1:F7WD09S8QB4LrkEpka0dFPLSotH11HRpCsLIbIcJ7sU= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -1356,10 +1356,6 @@ github.com/grafana/grafana-plugin-sdk-go v0.139.0 h1:2RQKM2QpSaWTtaGN6sK+R7LO7zy github.com/grafana/grafana-plugin-sdk-go v0.139.0/go.mod h1:Y+Ps2sesZ62AyCnX+hzrYnyDQYe/ZZl+A8yKLOBm12c= github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc h1:1PY8n+rXuBNr3r1JQhoytWDCpc+pq+BibxV0SZv+Cr4= github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4= -github.com/grafana/thema v0.0.0-20220816214754-af057f99a2dd h1:OukQ1Nu4PSreZTAaOfXyYhM9jYBs4UflVfOSAIG8JzM= -github.com/grafana/thema v0.0.0-20220816214754-af057f99a2dd/go.mod h1:fCV1rqv6XRQg2GfIQ7pU9zdxd5fLRcEBCnrDVwlK+ZY= -github.com/grafana/thema v0.0.0-20220816215847-acc0b0aca0f0 h1:jp0SAT7/Xo9NMND8zEwPo7urvSx0EhgNKXOQ0x4s2PE= -github.com/grafana/thema v0.0.0-20220816215847-acc0b0aca0f0/go.mod h1:fCV1rqv6XRQg2GfIQ7pU9zdxd5fLRcEBCnrDVwlK+ZY= github.com/grafana/thema v0.0.0-20220817114012-ebeee841c104 h1:dYpwFYIChrMfpq3wDa/ZBxAbUGSW5NYmYBeSezhaoao= github.com/grafana/thema v0.0.0-20220817114012-ebeee841c104/go.mod h1:fCV1rqv6XRQg2GfIQ7pU9zdxd5fLRcEBCnrDVwlK+ZY= github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6 h1:I9dh1MXGX0wGyxdV/Sl7+ugnki4Dfsy8lv2s5Yf887o= @@ -1640,8 +1636,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jsternberg/zap-logfmt v1.2.0/go.mod h1:kz+1CUmCutPWABnNkOu9hOHKdT2q3TDYCcsFy9hpqb0= -github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -2105,7 +2099,6 @@ github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0 github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= @@ -2143,7 +2136,6 @@ github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= @@ -2173,7 +2165,6 @@ github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= @@ -2297,15 +2288,9 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/snowflakedb/gosnowflake v1.3.4/go.mod h1:NsRq2QeiMUuoNUJhp5Q6xGC4uBrsS9g6LwZVEkTWgsE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= @@ -2425,8 +2410,6 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= -github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= -github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -3460,7 +3443,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 4f3571381ce..b87a447a779 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/dlmiddlecote/sqlstats" "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" @@ -57,17 +58,6 @@ type SQLStore struct { skipEnsureDefaultOrgAndUser bool migrations registry.DatabaseMigrator tracer tracing.Tracer - metrics struct { - maxOpenConnections prometheus.Gauge - openConnections prometheus.Gauge - inUse prometheus.Gauge - idle prometheus.Gauge - waitCount prometheus.Counter - waitDuration prometheus.Counter - maxIdleClosed prometheus.Counter - maxIdleTimeClosed prometheus.Counter - maxLifetimeClosed prometheus.Counter - } } func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, migrations registry.DatabaseMigrator, bus bus.Bus, tracer tracing.Tracer) (*SQLStore, error) { @@ -89,9 +79,13 @@ func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, mig } s.tracer = tracer - s.initMetrics() + // initialize and register metrics wrapper around the *sql.DB + db := s.engine.DB().DB - prometheus.MustRegister(s) + // register the go_sql_stats_connections_* metrics + prometheus.MustRegister(sqlstats.NewStatsCollector("grafana", db)) + // TODO: deprecate/remove these metrics + prometheus.MustRegister(newSQLStoreMetrics(db)) return s, nil } @@ -464,118 +458,6 @@ func (ss *SQLStore) readConfig() error { return nil } -// initMetrics initializes the database connection metrics -func (ss *SQLStore) initMetrics() { - namespace := "grafana" - subsystem := "database" - - ss.metrics.maxOpenConnections = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_max_open", - Help: "Maximum number of open connections to the database", - }) - - ss.metrics.openConnections = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_open", - Help: "The number of established connections both in use and idle", - }) - - ss.metrics.inUse = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_in_use", - Help: "The number of connections currently in use", - }) - - ss.metrics.idle = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_idle", - Help: "The number of idle connections", - }) - - ss.metrics.waitCount = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_wait_count_total", - Help: "The total number of connections waited for", - }) - - ss.metrics.waitDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_wait_duration_seconds", - Help: "The total time blocked waiting for a new connection", - }) - - ss.metrics.maxIdleClosed = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_max_idle_closed_total", - Help: "The total number of connections closed due to SetMaxIdleConns", - }) - - ss.metrics.maxIdleTimeClosed = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_max_idle_closed_seconds", - Help: "The total number of connections closed due to SetConnMaxIdleTime", - }) - - ss.metrics.maxLifetimeClosed = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "conn_max_lifetime_closed_total", - Help: "The total number of connections closed due to SetConnMaxLifetime", - }) -} - -// collectDBStats instruments connections stats from the database. -func (ss *SQLStore) collectDBstats() { - dbstats := ss.engine.DB().Stats() - ss.metrics.maxOpenConnections.Set(float64(dbstats.MaxOpenConnections)) - ss.metrics.openConnections.Set(float64(dbstats.MaxOpenConnections)) - ss.metrics.inUse.Set(float64(dbstats.InUse)) - ss.metrics.idle.Set(float64(dbstats.Idle)) - - ss.metrics.waitCount.Add(float64(dbstats.WaitCount)) - ss.metrics.waitDuration.Add(float64(dbstats.WaitDuration / time.Second)) - ss.metrics.maxIdleClosed.Add(float64(dbstats.MaxIdleClosed)) - ss.metrics.maxIdleTimeClosed.Add(float64(dbstats.MaxIdleTimeClosed)) - ss.metrics.maxLifetimeClosed.Add(float64(dbstats.MaxLifetimeClosed)) -} - -// Collect implements Prometheus.Collector. -func (ss *SQLStore) Collect(ch chan<- prometheus.Metric) { - ss.collectDBstats() - - ss.metrics.maxOpenConnections.Collect(ch) - ss.metrics.openConnections.Collect(ch) - ss.metrics.inUse.Collect(ch) - ss.metrics.idle.Collect(ch) - ss.metrics.waitCount.Collect(ch) - ss.metrics.waitDuration.Collect(ch) - ss.metrics.maxIdleClosed.Collect(ch) - ss.metrics.maxIdleTimeClosed.Collect(ch) - ss.metrics.maxLifetimeClosed.Collect(ch) -} - -// Describe implements Prometheus.Collector. -func (ss *SQLStore) Describe(ch chan<- *prometheus.Desc) { - ss.metrics.maxOpenConnections.Describe(ch) - ss.metrics.openConnections.Describe(ch) - ss.metrics.inUse.Describe(ch) - ss.metrics.idle.Describe(ch) - ss.metrics.waitCount.Describe(ch) - ss.metrics.waitDuration.Describe(ch) - ss.metrics.maxIdleClosed.Describe(ch) - ss.metrics.maxIdleTimeClosed.Describe(ch) - ss.metrics.maxLifetimeClosed.Describe(ch) -} - // ITestDB is an interface of arguments for testing db type ITestDB interface { Helper() diff --git a/pkg/services/sqlstore/sqlstore_metrics.go b/pkg/services/sqlstore/sqlstore_metrics.go new file mode 100644 index 00000000000..17621189e29 --- /dev/null +++ b/pkg/services/sqlstore/sqlstore_metrics.go @@ -0,0 +1,144 @@ +package sqlstore + +import ( + "github.com/dlmiddlecote/sqlstats" + "github.com/prometheus/client_golang/prometheus" +) + +type sqlStoreMetrics struct { + db sqlstats.StatsGetter + + // gauges + maxOpenConnections *prometheus.Desc + openConnections *prometheus.Desc + inUse *prometheus.Desc + idle *prometheus.Desc + + // counters + waitCount *prometheus.Desc + waitDuration *prometheus.Desc + maxIdleClosed *prometheus.Desc + maxIdleTimeClosed *prometheus.Desc + maxLifetimeClosed *prometheus.Desc +} + +func newSQLStoreMetrics(db sqlstats.StatsGetter) *sqlStoreMetrics { + ns := "grafana" + sub := "database" + + return &sqlStoreMetrics{ + db: db, + maxOpenConnections: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_max_open"), + "Maximum number of open connections to the database", + nil, nil, + ), + openConnections: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_open"), + "The number of established connections both in use and idle", + nil, nil, + ), + inUse: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_in_use"), + "The number of connections currently in use", + nil, nil, + ), + idle: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_idle"), + "The number of idle connections", + nil, nil, + ), + + waitCount: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_wait_count_total"), + "The total number of connections waited for", + nil, nil, + ), + waitDuration: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_wait_duration_seconds"), + "The total time blocked waiting for a new connection", + nil, nil, + ), + maxIdleClosed: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_max_idle_closed_total"), + "The total number of connections closed due to SetMaxIdleConns", + nil, nil, + ), + maxIdleTimeClosed: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_max_idle_closed_seconds"), + "The total number of connections closed due to SetConnMaxIdleTime", + nil, nil, + ), + maxLifetimeClosed: prometheus.NewDesc( + prometheus.BuildFQName(ns, sub, "conn_max_lifetime_closed_total"), + "The total number of connections closed due to SetConnMaxLifetime", + nil, nil, + ), + } +} + +// Collect implements Prometheus.Collector. +func (m *sqlStoreMetrics) Collect(ch chan<- prometheus.Metric) { + stats := m.db.Stats() + + ch <- prometheus.MustNewConstMetric( + m.maxOpenConnections, + prometheus.GaugeValue, + float64(stats.MaxOpenConnections), + ) + ch <- prometheus.MustNewConstMetric( + m.openConnections, + prometheus.GaugeValue, + float64(stats.OpenConnections), + ) + ch <- prometheus.MustNewConstMetric( + m.inUse, + prometheus.GaugeValue, + float64(stats.InUse), + ) + ch <- prometheus.MustNewConstMetric( + m.idle, + prometheus.GaugeValue, + float64(stats.Idle), + ) + + ch <- prometheus.MustNewConstMetric( + m.waitCount, + prometheus.CounterValue, + float64(stats.WaitCount), + ) + ch <- prometheus.MustNewConstMetric( + m.waitDuration, + prometheus.CounterValue, + stats.WaitDuration.Seconds(), + ) + ch <- prometheus.MustNewConstMetric( + m.maxIdleClosed, + prometheus.CounterValue, + float64(stats.MaxIdleClosed), + ) + ch <- prometheus.MustNewConstMetric( + m.maxIdleTimeClosed, + prometheus.CounterValue, + float64(stats.MaxIdleTimeClosed), + ) + ch <- prometheus.MustNewConstMetric( + m.maxLifetimeClosed, + prometheus.CounterValue, + float64(stats.MaxLifetimeClosed), + ) +} + +// Describe implements Prometheus.Collector. +func (m *sqlStoreMetrics) Describe(ch chan<- *prometheus.Desc) { + ch <- m.maxOpenConnections + ch <- m.openConnections + ch <- m.inUse + ch <- m.idle + + ch <- m.waitCount + ch <- m.waitDuration + ch <- m.maxIdleClosed + ch <- m.maxIdleTimeClosed + ch <- m.maxLifetimeClosed +} diff --git a/pkg/services/sqlstore/sqlstore_metrics_test.go b/pkg/services/sqlstore/sqlstore_metrics_test.go new file mode 100644 index 00000000000..60d604a1d5c --- /dev/null +++ b/pkg/services/sqlstore/sqlstore_metrics_test.go @@ -0,0 +1,68 @@ +package sqlstore + +import ( + "database/sql" + "strings" + "testing" + "time" + + "github.com/dlmiddlecote/sqlstats" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestSQLStore_Metrics(t *testing.T) { + stats := sql.DBStats{ + MaxOpenConnections: 9, + OpenConnections: 8, + InUse: 4, + Idle: 4, + WaitCount: 5, + WaitDuration: 6 * time.Second, + MaxIdleClosed: 7, + MaxIdleTimeClosed: 8, + MaxLifetimeClosed: 9, + } + + m := newSQLStoreMetrics(&fakeStatsGetter{stats: stats}) + + require.NoError(t, testutil.CollectAndCompare(m, strings.NewReader(` + # HELP grafana_database_conn_idle The number of idle connections + # TYPE grafana_database_conn_idle gauge + grafana_database_conn_idle 4 + # HELP grafana_database_conn_in_use The number of connections currently in use + # TYPE grafana_database_conn_in_use gauge + grafana_database_conn_in_use 4 + # HELP grafana_database_conn_max_idle_closed_seconds The total number of connections closed due to SetConnMaxIdleTime + # TYPE grafana_database_conn_max_idle_closed_seconds counter + grafana_database_conn_max_idle_closed_seconds 8 + # HELP grafana_database_conn_max_idle_closed_total The total number of connections closed due to SetMaxIdleConns + # TYPE grafana_database_conn_max_idle_closed_total counter + grafana_database_conn_max_idle_closed_total 7 + # HELP grafana_database_conn_max_lifetime_closed_total The total number of connections closed due to SetConnMaxLifetime + # TYPE grafana_database_conn_max_lifetime_closed_total counter + grafana_database_conn_max_lifetime_closed_total 9 + # HELP grafana_database_conn_max_open Maximum number of open connections to the database + # TYPE grafana_database_conn_max_open gauge + grafana_database_conn_max_open 9 + # HELP grafana_database_conn_open The number of established connections both in use and idle + # TYPE grafana_database_conn_open gauge + grafana_database_conn_open 8 + # HELP grafana_database_conn_wait_count_total The total number of connections waited for + # TYPE grafana_database_conn_wait_count_total counter + grafana_database_conn_wait_count_total 5 + # HELP grafana_database_conn_wait_duration_seconds The total time blocked waiting for a new connection + # TYPE grafana_database_conn_wait_duration_seconds counter + grafana_database_conn_wait_duration_seconds 6 + `))) +} + +type fakeStatsGetter struct { + stats sql.DBStats +} + +var _ sqlstats.StatsGetter = (*fakeStatsGetter)(nil) + +func (f *fakeStatsGetter) Stats() sql.DBStats { + return f.stats +}