From 92c2a6c23985fdc8fe71b213b1ea1f2ddf61572c Mon Sep 17 00:00:00 2001 From: Kyle Hinton Date: Mon, 12 Oct 2020 04:36:47 -0400 Subject: [PATCH] Fix: Add additional settings for dataproxy to help with network proxy timeouts (#27841) * adding additional settings for datasource cache transport * added documentation for the new changes * fixing small typo in defaults.ini comment * fixing small typo in configuration.md comment * Update conf/defaults.ini keepalive comment per review Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> * Update conf/defaults.ini idle conn comment per review Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> * Update conf/defaults.ini anon user comment per review Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> * Update docs/sources/administration/configuration.md idle conn comment per review Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> * adding suggestions from papagian * fixing configuration.md * fixing configuration.md typo * Apply suggestions from code review aknuds1 Co-authored-by: Arve Knudsen * updating sample.ini * Apply suggestions for docs from code review papagian Co-authored-by: Sofia Papagiannaki * Update docs/sources/administration/configuration.md fix typo Co-authored-by: Sofia Papagiannaki Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> Co-authored-by: Arve Knudsen Co-authored-by: Sofia Papagiannaki --- conf/defaults.ini | 20 +++++++++++- conf/sample.ini | 18 +++++++++++ docs/sources/administration/configuration.md | 20 ++++++++++++ pkg/models/datasource_cache.go | 10 +++--- pkg/setting/setting.go | 34 +++++++++++++------- 5 files changed, 84 insertions(+), 18 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 925bf1acc23..88f556f0d9c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -130,7 +130,25 @@ logging = false # This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. timeout = 30 -# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +# How many seconds the data proxy waits before sending a keepalive request. +keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +expect_continue_timeout_seconds = 1 + +# The maximum number of idle connections that Grafana will keep alive. +max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request. send_user_header = false #################################### Analytics ########################### diff --git a/conf/sample.ini b/conf/sample.ini index 3fd5dd58e5e..7def40604ef 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -131,6 +131,24 @@ # This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. ;timeout = 30 +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# The maximum number of idle connections that Grafana will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + # If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. ;send_user_header = false diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md index 000903e4f40..04643e76f9e 100644 --- a/docs/sources/administration/configuration.md +++ b/docs/sources/administration/configuration.md @@ -389,6 +389,26 @@ How long the data proxy should wait before timing out. Default is 30 seconds. This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +### keep_alive_seconds + +Interval between keep-alive probes. Default is `30` seconds. For more details check the [Dialer.KeepAlive](https://golang.org/pkg/net/#Dialer.KeepAlive) documentation. + +### tls_handshake_timeout_seconds + +The length of time that Grafana will wait for a successful TLS handshake with the datasource. Default is `10` seconds. For more details check the [Transport.TLSHandshakeTimeout](https://golang.org/pkg/net/http/#Transport.TLSHandshakeTimeout) documentation. + +### expect_continue_timeout_seconds + +The length of time that Grafana will wait for a datasource’s first response headers after fully writing the request headers, if the request has an “Expect: 100-continue” header. A value of `0` will result in the body being sent immediately. Default is `1` second. For more details check the [Transport.ExpectContinueTimeout](https://golang.org/pkg/net/http/#Transport.ExpectContinueTimeout) documentation. + +### max_idle_connections + +The maximum number of idle connections that Grafana will maintain. Default is `100`. For more details check the [Transport.MaxIdleConns](https://golang.org/pkg/net/http/#Transport.MaxIdleConns) documentation. + +### idle_conn_timeout_seconds + +The length of time that Grafana maintains idle connections before closing them. Default is `90` seconds. For more details check the [Transport.IdleConnTimeout](https://golang.org/pkg/net/http/#Transport.IdleConnTimeout) documentation. + ### send_user_header If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request. Default is `false`. diff --git a/pkg/models/datasource_cache.go b/pkg/models/datasource_cache.go index f2728ad0742..0d3ac2bcecb 100644 --- a/pkg/models/datasource_cache.go +++ b/pkg/models/datasource_cache.go @@ -157,12 +157,12 @@ func (ds *DataSource) GetHttpTransport() (*dataSourceTransport, error) { Proxy: http.ProxyFromEnvironment, Dial: (&net.Dialer{ Timeout: time.Duration(setting.DataProxyTimeout) * time.Second, - KeepAlive: 30 * time.Second, + KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second, }).Dial, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second, + ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second, + MaxIdleConns: setting.DataProxyMaxIdleConns, + IdleConnTimeout: time.Duration(setting.DataProxyIdleConnTimeout) * time.Second, } // Set default next round tripper to the default transport diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 1b8a06e54f4..fcc6a2c625b 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -78,18 +78,23 @@ var ( LogConfigs []util.DynMap // Http server options - Protocol Scheme - Domain string - HttpAddr, HttpPort string - SshPort int - CertFile, KeyFile string - SocketPath string - RouterLogging bool - DataProxyLogging bool - DataProxyTimeout int - StaticRootPath string - EnableGzip bool - EnforceDomain bool + Protocol Scheme + Domain string + HttpAddr, HttpPort string + SshPort int + CertFile, KeyFile string + SocketPath string + RouterLogging bool + DataProxyLogging bool + DataProxyTimeout int + DataProxyTLSHandshakeTimeout int + DataProxyExpectContinueTimeout int + DataProxyMaxIdleConns int + DataProxyKeepAlive int + DataProxyIdleConnTimeout int + StaticRootPath string + EnableGzip bool + EnforceDomain bool // Security settings. SecretKey string @@ -684,6 +689,11 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { dataproxy := iniFile.Section("dataproxy") DataProxyLogging = dataproxy.Key("logging").MustBool(false) DataProxyTimeout = dataproxy.Key("timeout").MustInt(30) + DataProxyKeepAlive = dataproxy.Key("keep_alive_seconds").MustInt(30) + DataProxyTLSHandshakeTimeout = dataproxy.Key("tls_handshake_timeout_seconds").MustInt(10) + DataProxyExpectContinueTimeout = dataproxy.Key("expect_continue_timeout_seconds").MustInt(1) + DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt(100) + DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90) cfg.SendUserHeader = dataproxy.Key("send_user_header").MustBool(false) if err := readSecuritySettings(iniFile, cfg); err != nil {