From e35f06b938b90750f162849f3831a905831d6168 Mon Sep 17 00:00:00 2001 From: Axel Pirek Date: Mon, 24 Sep 2018 14:33:45 +0200 Subject: [PATCH 1/6] Run all sql data source queries for one panel concurrently --- pkg/tsdb/sql_engine.go | 50 ++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/pkg/tsdb/sql_engine.go b/pkg/tsdb/sql_engine.go index 18e02e328d1..8dcf35a70bc 100644 --- a/pkg/tsdb/sql_engine.go +++ b/pkg/tsdb/sql_engine.go @@ -116,9 +116,7 @@ func (e *sqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, Results: make(map[string]*QueryResult), } - session := e.engine.NewSession() - defer session.Close() - db := session.DB() + var wg sync.WaitGroup for _, query := range tsdbQuery.Queries { rawSQL := query.Model.Get("rawSql").MustString() @@ -145,31 +143,41 @@ func (e *sqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, queryResult.Meta.Set("sql", rawSQL) - rows, err := db.Query(rawSQL) - if err != nil { - queryResult.Error = err - continue - } + wg.Add(1) - defer rows.Close() + go func(rawSQL string, query *Query, queryResult *QueryResult) { + defer wg.Done() + session := e.engine.NewSession() + defer session.Close() + db := session.DB() - format := query.Model.Get("format").MustString("time_series") - - switch format { - case "time_series": - err := e.transformToTimeSeries(query, rows, queryResult, tsdbQuery) + rows, err := db.Query(rawSQL) if err != nil { queryResult.Error = err - continue + return } - case "table": - err := e.transformToTable(query, rows, queryResult, tsdbQuery) - if err != nil { - queryResult.Error = err - continue + + defer rows.Close() + + format := query.Model.Get("format").MustString("time_series") + + switch format { + case "time_series": + err := e.transformToTimeSeries(query, rows, queryResult, tsdbQuery) + if err != nil { + queryResult.Error = err + return + } + case "table": + err := e.transformToTable(query, rows, queryResult, tsdbQuery) + if err != nil { + queryResult.Error = err + return + } } - } + }(rawSQL, query, queryResult) } + wg.Wait() return result, nil } From dd09ece8c9d6afe063149e2b5358e7ef538b6d67 Mon Sep 17 00:00:00 2001 From: Axel Pirek Date: Mon, 24 Sep 2018 14:35:13 +0200 Subject: [PATCH 2/6] Make max open, max idle connections and connection max life time configurable --- pkg/tsdb/sql_engine.go | 8 ++++++-- .../datasource/mssql/partials/config.html | 18 ++++++++++++++++++ .../datasource/mysql/partials/config.html | 18 ++++++++++++++++++ .../datasource/postgres/partials/config.html | 18 ++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/sql_engine.go b/pkg/tsdb/sql_engine.go index 8dcf35a70bc..60b1e68ebd8 100644 --- a/pkg/tsdb/sql_engine.go +++ b/pkg/tsdb/sql_engine.go @@ -98,8 +98,12 @@ var NewSqlQueryEndpoint = func(config *SqlQueryEndpointConfiguration, rowTransfo return nil, err } - engine.SetMaxOpenConns(10) - engine.SetMaxIdleConns(10) + maxOpenConns := config.Datasource.JsonData.Get("maxOpenConns").MustInt(0) + engine.SetMaxOpenConns(maxOpenConns) + maxIdleConns := config.Datasource.JsonData.Get("maxIdleConns").MustInt(2) + engine.SetMaxIdleConns(maxIdleConns) + connMaxLifetime := config.Datasource.JsonData.Get("connMaxLifetime").MustInt(14400) + engine.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) engineCache.versions[config.Datasource.Id] = config.Datasource.Version engineCache.cache[config.Datasource.Id] = engine diff --git a/public/app/plugins/datasource/mssql/partials/config.html b/public/app/plugins/datasource/mssql/partials/config.html index f8a36502009..f59bf6ae267 100644 --- a/public/app/plugins/datasource/mssql/partials/config.html +++ b/public/app/plugins/datasource/mssql/partials/config.html @@ -29,6 +29,24 @@ +
+
+ Max open connections + +
+
+ Max idle connections + +
+
+ Connection lifetime + + + The connection lifetime in seconds. + +
+
+

MSSQL details

diff --git a/public/app/plugins/datasource/mysql/partials/config.html b/public/app/plugins/datasource/mysql/partials/config.html index 6bc9cceb8f1..8ddad1f47cc 100644 --- a/public/app/plugins/datasource/mysql/partials/config.html +++ b/public/app/plugins/datasource/mysql/partials/config.html @@ -24,6 +24,24 @@
+
+
+ Max open connections + +
+
+ Max idle connections + +
+
+ Connection lifetime + + + The connection lifetime in seconds. + +
+
+

MySQL details

diff --git a/public/app/plugins/datasource/postgres/partials/config.html b/public/app/plugins/datasource/postgres/partials/config.html index c8b551c2aa8..b0012faacbd 100644 --- a/public/app/plugins/datasource/postgres/partials/config.html +++ b/public/app/plugins/datasource/postgres/partials/config.html @@ -38,6 +38,24 @@
+
+
+ Max open connections + +
+
+ Max idle connections + +
+
+ Connection lifetime + + + The connection lifetime in seconds. + +
+
+

PostgreSQL details

From 12230dfa71892da2fcaf8426195edf611ba72e91 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 2 Oct 2018 17:43:15 +0200 Subject: [PATCH 3/6] ux: put connection limits under own section --- .../datasource/mssql/partials/config.html | 43 ++++++++++++------- .../datasource/mysql/partials/config.html | 43 ++++++++++++------- .../datasource/postgres/partials/config.html | 43 ++++++++++++------- 3 files changed, 84 insertions(+), 45 deletions(-) diff --git a/public/app/plugins/datasource/mssql/partials/config.html b/public/app/plugins/datasource/mssql/partials/config.html index f59bf6ae267..4cb42e90f7c 100644 --- a/public/app/plugins/datasource/mssql/partials/config.html +++ b/public/app/plugins/datasource/mssql/partials/config.html @@ -29,22 +29,35 @@
+Connection limits +
-
- Max open connections - -
-
- Max idle connections - -
-
- Connection lifetime - - - The connection lifetime in seconds. - -
+
+ Max open + + + The maximum number of open connections to the database. If Max idle connections is greater than 0 and the + Max open connections is less than Max idle connections, then Max idle connections will be + reduced to match the Max open connections limit. If set to 0, there is no limit on the number of open + connections. + +
+
+ Max idle + + + The maximum number of connections in the idle connection pool. If Max open connections is greater than 0 but + less than the Max idle connections, then the Max idle connections will be reduced to match the + Max open connections limit. If set to 0, no idle connections are retained. + +
+
+ Max lifetime + + + The maximum amount of time in seconds a connection may be reused. If set to 0, connections are reused forever. + +

MSSQL details

diff --git a/public/app/plugins/datasource/mysql/partials/config.html b/public/app/plugins/datasource/mysql/partials/config.html index 8ddad1f47cc..3d27dd38bf5 100644 --- a/public/app/plugins/datasource/mysql/partials/config.html +++ b/public/app/plugins/datasource/mysql/partials/config.html @@ -24,22 +24,35 @@ +Connection limits +
-
- Max open connections - -
-
- Max idle connections - -
-
- Connection lifetime - - - The connection lifetime in seconds. - -
+
+ Max open + + + The maximum number of open connections to the database. If Max idle connections is greater than 0 and the + Max open connections is less than Max idle connections, then Max idle connections will be + reduced to match the Max open connections limit. If set to 0, there is no limit on the number of open + connections. + +
+
+ Max idle + + + The maximum number of connections in the idle connection pool. If Max open connections is greater than 0 but + less than the Max idle connections, then the Max idle connections will be reduced to match the + Max open connections limit. If set to 0, no idle connections are retained. + +
+
+ Max lifetime + + + The maximum amount of time in seconds a connection may be reused. If set to 0, connections are reused forever. + +

MySQL details

diff --git a/public/app/plugins/datasource/postgres/partials/config.html b/public/app/plugins/datasource/postgres/partials/config.html index b0012faacbd..13b74d6b20e 100644 --- a/public/app/plugins/datasource/postgres/partials/config.html +++ b/public/app/plugins/datasource/postgres/partials/config.html @@ -38,22 +38,35 @@ +Connection limits +
-
- Max open connections - -
-
- Max idle connections - -
-
- Connection lifetime - - - The connection lifetime in seconds. - -
+
+ Max open + + + The maximum number of open connections to the database. If Max idle connections is greater than 0 and the + Max open connections is less than Max idle connections, then Max idle connections will be + reduced to match the Max open connections limit. If set to 0, there is no limit on the number of open + connections. + +
+
+ Max idle + + + The maximum number of connections in the idle connection pool. If Max open connections is greater than 0 but + less than the Max idle connections, then the Max idle connections will be reduced to match the + Max open connections limit. If set to 0, no idle connections are retained. + +
+
+ Max lifetime + + + The maximum amount of time in seconds a connection may be reused. If set to 0, connections are reused forever. + +

PostgreSQL details

From e5c376aeadd230f70d9b1f7f38ad023c31fea7bb Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 2 Oct 2018 17:43:54 +0200 Subject: [PATCH 4/6] docs: connection limits for sql datasources --- docs/sources/administration/provisioning.md | 3 +++ docs/sources/features/datasources/mssql.md | 7 +++++++ docs/sources/features/datasources/mysql.md | 7 +++++++ docs/sources/features/datasources/postgres.md | 6 ++++++ 4 files changed, 23 insertions(+) diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index a026d1ec0cd..587e1411c26 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -168,6 +168,9 @@ Since not all datasources have the same configuration settings we only have the | sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | | postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 | | timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension | +| maxOpenConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of open connections to the database | +| maxIdleConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of connections in the idle connection pool | +| connMaxLifetime | number | MySQL, PostgreSQL & MSSQL | Maximum amount of time in seconds a connection may be reused | #### Secure Json Data diff --git a/docs/sources/features/datasources/mssql.md b/docs/sources/features/datasources/mssql.md index debf771ffb0..a3e350ab47b 100644 --- a/docs/sources/features/datasources/mssql.md +++ b/docs/sources/features/datasources/mssql.md @@ -32,6 +32,9 @@ Name | Description *Database* | Name of your MSSQL database. *User* | Database user's login/username *Password* | Database user's password +*Max open* | The maximum number of open connections to the database, default `unlimited`. +*Max idle* | The maximum number of connections in the idle connection pool, default `2`. +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400` (4 hours). ### Min time interval @@ -585,6 +588,10 @@ datasources: url: localhost:1433 database: grafana user: grafana + jsonData: + maxOpenConns: 0 + maxIdleConns: 2 + connMaxLifetime: 14400 secureJsonData: password: "Password!" diff --git a/docs/sources/features/datasources/mysql.md b/docs/sources/features/datasources/mysql.md index d713a4b42b7..8cb6dea785d 100644 --- a/docs/sources/features/datasources/mysql.md +++ b/docs/sources/features/datasources/mysql.md @@ -35,6 +35,9 @@ Name | Description *Database* | Name of your MySQL database. *User* | Database user's login/username *Password* | Database user's password +*Max open* | The maximum number of open connections to the database, default `unlimited`. +*Max idle* | The maximum number of connections in the idle connection pool, default `2`. +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400` (4 hours). ### Min time interval @@ -316,4 +319,8 @@ datasources: database: grafana user: grafana password: password + jsonData: + maxOpenConns: 0 + maxIdleConns: 2 + connMaxLifetime: 14400 ``` diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index 7076ff033b3..07413ad6510 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -31,6 +31,9 @@ Name | Description *User* | Database user's login/username *Password* | Database user's password *SSL Mode* | This option determines whether or with what priority a secure SSL TCP/IP connection will be negotiated with the server. +*Max open* | The maximum number of open connections to the database, default `unlimited`. +*Max idle* | The maximum number of connections in the idle connection pool, default `2`. +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400` (4 hours). *Version* | This option determines which functions are available in the query builder (only available in Grafana 5.3+). *TimescaleDB* | TimescaleDB is a time-series database built as a PostgreSQL extension. If enabled, Grafana will use `time_bucket` in the `$__timeGroup` macro and display TimescaleDB specific aggregate functions in the query builder (only available in Grafana 5.3+). @@ -374,6 +377,9 @@ datasources: password: "Password!" jsonData: sslmode: "disable" # disable/require/verify-ca/verify-full + maxOpenConns: 0 + maxIdleConns: 2 + connMaxLifetime: 14400 postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 timescaledb: false ``` From 79929c1f001f3d59477a19625b722fcf39f2ec23 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 4 Oct 2018 17:42:34 +0200 Subject: [PATCH 5/6] mysql: note about connection max lifetime and wait_timeout --- public/app/plugins/datasource/mysql/partials/config.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/mysql/partials/config.html b/public/app/plugins/datasource/mysql/partials/config.html index 3d27dd38bf5..a35633c626a 100644 --- a/public/app/plugins/datasource/mysql/partials/config.html +++ b/public/app/plugins/datasource/mysql/partials/config.html @@ -50,7 +50,8 @@ Max lifetime - The maximum amount of time in seconds a connection may be reused. If set to 0, connections are reused forever. + The maximum amount of time in seconds a connection may be reused. If set to 0, connections are reused forever.

+ This should always be lower than configured wait_timeout in MySQL. From 4757ca2242fdf941bf4aac678837128838d2275e Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 10 Oct 2018 18:59:09 +0200 Subject: [PATCH 6/6] docs: add version notes --- docs/sources/administration/provisioning.md | 6 +++--- docs/sources/features/datasources/mssql.md | 12 ++++++------ docs/sources/features/datasources/mysql.md | 12 ++++++------ docs/sources/features/datasources/postgres.md | 12 ++++++------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index 0860531b528..b44d8277675 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -168,9 +168,9 @@ Since not all datasources have the same configuration settings we only have the | sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | | postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 | | timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension | -| maxOpenConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of open connections to the database | -| maxIdleConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of connections in the idle connection pool | -| connMaxLifetime | number | MySQL, PostgreSQL & MSSQL | Maximum amount of time in seconds a connection may be reused | +| maxOpenConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of open connections to the database (Grafana v5.4+) | +| maxIdleConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of connections in the idle connection pool (Grafana v5.4+) | +| connMaxLifetime | number | MySQL, PostgreSQL & MSSQL | Maximum amount of time in seconds a connection may be reused (Grafana v5.4+) | #### Secure Json Data diff --git a/docs/sources/features/datasources/mssql.md b/docs/sources/features/datasources/mssql.md index a3e350ab47b..c0cc1b801e8 100644 --- a/docs/sources/features/datasources/mssql.md +++ b/docs/sources/features/datasources/mssql.md @@ -32,9 +32,9 @@ Name | Description *Database* | Name of your MSSQL database. *User* | Database user's login/username *Password* | Database user's password -*Max open* | The maximum number of open connections to the database, default `unlimited`. -*Max idle* | The maximum number of connections in the idle connection pool, default `2`. -*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400` (4 hours). +*Max open* | The maximum number of open connections to the database, default `unlimited` (Grafana v5.4+). +*Max idle* | The maximum number of connections in the idle connection pool, default `2` (Grafana v5.4+). +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400`/4 hours (Grafana v5.4+). ### Min time interval @@ -589,9 +589,9 @@ datasources: database: grafana user: grafana jsonData: - maxOpenConns: 0 - maxIdleConns: 2 - connMaxLifetime: 14400 + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ secureJsonData: password: "Password!" diff --git a/docs/sources/features/datasources/mysql.md b/docs/sources/features/datasources/mysql.md index 8cb6dea785d..ce607a115d3 100644 --- a/docs/sources/features/datasources/mysql.md +++ b/docs/sources/features/datasources/mysql.md @@ -35,9 +35,9 @@ Name | Description *Database* | Name of your MySQL database. *User* | Database user's login/username *Password* | Database user's password -*Max open* | The maximum number of open connections to the database, default `unlimited`. -*Max idle* | The maximum number of connections in the idle connection pool, default `2`. -*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400` (4 hours). +*Max open* | The maximum number of open connections to the database, default `unlimited` (Grafana v5.4+). +*Max idle* | The maximum number of connections in the idle connection pool, default `2` (Grafana v5.4+). +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400`/4 hours. This should always be lower than configured [wait_timeout](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_wait_timeout) in MySQL (Grafana v5.4+). ### Min time interval @@ -320,7 +320,7 @@ datasources: user: grafana password: password jsonData: - maxOpenConns: 0 - maxIdleConns: 2 - connMaxLifetime: 14400 + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ ``` diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index 07413ad6510..52f8804f27f 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -31,9 +31,9 @@ Name | Description *User* | Database user's login/username *Password* | Database user's password *SSL Mode* | This option determines whether or with what priority a secure SSL TCP/IP connection will be negotiated with the server. -*Max open* | The maximum number of open connections to the database, default `unlimited`. -*Max idle* | The maximum number of connections in the idle connection pool, default `2`. -*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400` (4 hours). +*Max open* | The maximum number of open connections to the database, default `unlimited` (Grafana v5.4+). +*Max idle* | The maximum number of connections in the idle connection pool, default `2` (Grafana v5.4+). +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400`/4 hours (Grafana v5.4+). *Version* | This option determines which functions are available in the query builder (only available in Grafana 5.3+). *TimescaleDB* | TimescaleDB is a time-series database built as a PostgreSQL extension. If enabled, Grafana will use `time_bucket` in the `$__timeGroup` macro and display TimescaleDB specific aggregate functions in the query builder (only available in Grafana 5.3+). @@ -377,9 +377,9 @@ datasources: password: "Password!" jsonData: sslmode: "disable" # disable/require/verify-ca/verify-full - maxOpenConns: 0 - maxIdleConns: 2 - connMaxLifetime: 14400 + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 timescaledb: false ```