From d03977ac0ec2b3280604366c8b7c8a86128fc0e0 Mon Sep 17 00:00:00 2001 From: Ricky Niemi Date: Mon, 9 Jan 2017 22:59:43 -0800 Subject: [PATCH 1/9] Add initial audit logging to data proxy --- pkg/api/dataproxy.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index db4c5166feb..b044f8956db 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -1,6 +1,9 @@ package api import ( + "bytes" + "fmt" + "io/ioutil" "net/http" "net/http/httputil" "net/url" @@ -8,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/api/cloudwatch" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" @@ -115,12 +119,31 @@ func ProxyDataSourceRequest(c *middleware.Context) { } } + outputToAuditLog(ds.Type, c) + proxy := NewReverseProxy(ds, proxyPath, targetUrl) proxy.Transport, err = ds.GetHttpTransport() if err != nil { c.JsonApiErr(400, "Unable to load TLS certificate", err) return } + proxy.ServeHTTP(c.Resp, c.Req.Request) c.Resp.Header().Del("Set-Cookie") } + +func outputToAuditLog(dataSourceType string, c *middleware.Context) { + auditLogger := log.New("data-proxy-audit", "userId", c.UserId, "orgId", c.OrgId, "uname", c.Login) + + bodyString := "" + if c.Req.Request.Body != nil { + reqBody := c.Req.Request.Body + buffer, _ := ioutil.ReadAll(reqBody) + + c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buffer)) + bodyString = string(buffer) + } + + logFormat := "Datasource: %s, URI: %s, Method: %s, Body: %s" + auditLogger.Info(fmt.Sprintf(logFormat, dataSourceType, c.Req.RequestURI, c.Req.Request.Method, bodyString)) +} From d9cbb994b8ad9f90953391a3d7932361e1c67a1d Mon Sep 17 00:00:00 2001 From: Ricky Niemi Date: Tue, 10 Jan 2017 11:42:03 -0800 Subject: [PATCH 2/9] Clean up audit log formatting and code --- pkg/api/dataproxy.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index b044f8956db..b40a7d8c23b 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -2,7 +2,6 @@ package api import ( "bytes" - "fmt" "io/ioutil" "net/http" "net/http/httputil" @@ -133,17 +132,14 @@ func ProxyDataSourceRequest(c *middleware.Context) { } func outputToAuditLog(dataSourceType string, c *middleware.Context) { - auditLogger := log.New("data-proxy-audit", "userId", c.UserId, "orgId", c.OrgId, "uname", c.Login) + auditLogger := log.New("data-proxy-audit", "userid", c.UserId, "orgid", c.OrgId, "username", c.Login) - bodyString := "" + var body string if c.Req.Request.Body != nil { - reqBody := c.Req.Request.Body - buffer, _ := ioutil.ReadAll(reqBody) - + buffer, _ := ioutil.ReadAll(c.Req.Request.Body) c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buffer)) - bodyString = string(buffer) + body = string(buffer) } - logFormat := "Datasource: %s, URI: %s, Method: %s, Body: %s" - auditLogger.Info(fmt.Sprintf(logFormat, dataSourceType, c.Req.RequestURI, c.Req.Request.Method, bodyString)) + auditLogger.Info("Proxying incoming request", "datasource", dataSourceType, "uri", c.Req.RequestURI, "method", c.Req.Request.Method, "body", body) } From 0fee7c863af4df5ef4ac9ae8f29133041ef34447 Mon Sep 17 00:00:00 2001 From: Ricky Niemi Date: Wed, 11 Jan 2017 07:22:57 -0800 Subject: [PATCH 3/9] Gate data proxy audit logging behind audit_logging server setting --- conf/defaults.ini | 3 +++ conf/sample.ini | 3 +++ docs/sources/http_api/admin.md | 1 + pkg/api/dataproxy.go | 24 +++++++++++++----------- pkg/setting/setting.go | 2 ++ 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 81d6c8d8050..d1a4ee19681 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -47,6 +47,9 @@ root_url = %(protocol)s://%(domain)s:%(http_port)s/ # Log web requests router_logging = false +# This enables query request audit logging, output at warn level, default is false +audit_logging = false + # the path relative working path static_root_path = public diff --git a/conf/sample.ini b/conf/sample.ini index 6a8aa623f85..2d4ce660dfc 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -49,6 +49,9 @@ # Log web requests ;router_logging = false +# This enables query request audit logging, output at warn level, default is false +audit_logging = false + # the path relative working path ;static_root_path = public diff --git a/docs/sources/http_api/admin.md b/docs/sources/http_api/admin.md index 2d15b19454c..118265edddb 100644 --- a/docs/sources/http_api/admin.md +++ b/docs/sources/http_api/admin.md @@ -143,6 +143,7 @@ with Grafana admin permission. "protocol":"http", "root_url":"%(protocol)s://%(domain)s:%(http_port)s/", "router_logging":"true", + "audit_logging":"true", "static_root_path":"public" }, "session":{ diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index b40a7d8c23b..92c85af3370 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -118,8 +118,6 @@ func ProxyDataSourceRequest(c *middleware.Context) { } } - outputToAuditLog(ds.Type, c) - proxy := NewReverseProxy(ds, proxyPath, targetUrl) proxy.Transport, err = ds.GetHttpTransport() if err != nil { @@ -127,19 +125,23 @@ func ProxyDataSourceRequest(c *middleware.Context) { return } + auditLog(ds.Type, c) + proxy.ServeHTTP(c.Resp, c.Req.Request) c.Resp.Header().Del("Set-Cookie") } -func outputToAuditLog(dataSourceType string, c *middleware.Context) { - auditLogger := log.New("data-proxy-audit", "userid", c.UserId, "orgid", c.OrgId, "username", c.Login) +func auditLog(dataSourceType string, c *middleware.Context) { + if setting.AuditLogging { + auditLogger := log.New("data-proxy-audit", "userid", c.UserId, "orgid", c.OrgId, "username", c.Login) - var body string - if c.Req.Request.Body != nil { - buffer, _ := ioutil.ReadAll(c.Req.Request.Body) - c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buffer)) - body = string(buffer) + var body string + if c.Req.Request.Body != nil { + buffer, _ := ioutil.ReadAll(c.Req.Request.Body) + c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buffer)) + body = string(buffer) + } + + auditLogger.Warn("Proxying incoming request", "datasource", dataSourceType, "uri", c.Req.RequestURI, "method", c.Req.Request.Method, "body", body) } - - auditLogger.Info("Proxying incoming request", "datasource", dataSourceType, "uri", c.Req.RequestURI, "method", c.Req.Request.Method, "body", body) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 0de7b87b164..9159058b6e1 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -65,6 +65,7 @@ var ( SshPort int CertFile, KeyFile string RouterLogging bool + AuditLogging bool StaticRootPath string EnableGzip bool EnforceDomain bool @@ -490,6 +491,7 @@ func NewConfigContext(args *CommandLineArgs) error { HttpAddr = server.Key("http_addr").MustString(DEFAULT_HTTP_ADDR) HttpPort = server.Key("http_port").MustString("3000") RouterLogging = server.Key("router_logging").MustBool(false) + AuditLogging = server.Key("audit_logging").MustBool(false) EnableGzip = server.Key("enable_gzip").MustBool(false) EnforceDomain = server.Key("enforce_domain").MustBool(false) StaticRootPath = makeAbsolute(server.Key("static_root_path").String(), HomePath) From 269d16301a99b3d6c3c82e70ec3bc2fc34029b70 Mon Sep 17 00:00:00 2001 From: Ricky Niemi Date: Wed, 11 Jan 2017 07:40:06 -0800 Subject: [PATCH 4/9] Rename to audit logging to data proxy logging --- conf/defaults.ini | 4 ++-- conf/sample.ini | 2 +- docs/sources/http_api/admin.md | 2 +- pkg/api/dataproxy.go | 10 +++++----- pkg/setting/setting.go | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index d1a4ee19681..8fcd8ddcf12 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -47,8 +47,8 @@ root_url = %(protocol)s://%(domain)s:%(http_port)s/ # Log web requests router_logging = false -# This enables query request audit logging, output at warn level, default is false -audit_logging = false +# This enables data proxy logging, default is false +data_proxy_logging = false # the path relative working path static_root_path = public diff --git a/conf/sample.ini b/conf/sample.ini index 2d4ce660dfc..5e8c1216dca 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -50,7 +50,7 @@ ;router_logging = false # This enables query request audit logging, output at warn level, default is false -audit_logging = false +;data_proxy_logging = false # the path relative working path ;static_root_path = public diff --git a/docs/sources/http_api/admin.md b/docs/sources/http_api/admin.md index 118265edddb..356a8569d55 100644 --- a/docs/sources/http_api/admin.md +++ b/docs/sources/http_api/admin.md @@ -143,7 +143,7 @@ with Grafana admin permission. "protocol":"http", "root_url":"%(protocol)s://%(domain)s:%(http_port)s/", "router_logging":"true", - "audit_logging":"true", + "data_proxy_logging":"true", "static_root_path":"public" }, "session":{ diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index 92c85af3370..2c1355154bc 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -125,15 +125,15 @@ func ProxyDataSourceRequest(c *middleware.Context) { return } - auditLog(ds.Type, c) + proxyLog(ds.Type, c) proxy.ServeHTTP(c.Resp, c.Req.Request) c.Resp.Header().Del("Set-Cookie") } -func auditLog(dataSourceType string, c *middleware.Context) { - if setting.AuditLogging { - auditLogger := log.New("data-proxy-audit", "userid", c.UserId, "orgid", c.OrgId, "username", c.Login) +func proxyLog(dataSourceType string, c *middleware.Context) { + if setting.DataProxyLogging { + auditLogger := log.New("data-proxy-log", "userid", c.UserId, "orgid", c.OrgId, "username", c.Login) var body string if c.Req.Request.Body != nil { @@ -142,6 +142,6 @@ func auditLog(dataSourceType string, c *middleware.Context) { body = string(buffer) } - auditLogger.Warn("Proxying incoming request", "datasource", dataSourceType, "uri", c.Req.RequestURI, "method", c.Req.Request.Method, "body", body) + auditLogger.Info("Proxying incoming request", "datasource", dataSourceType, "uri", c.Req.RequestURI, "method", c.Req.Request.Method, "body", body) } } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 9159058b6e1..b532ad487cb 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -65,7 +65,7 @@ var ( SshPort int CertFile, KeyFile string RouterLogging bool - AuditLogging bool + DataProxyLogging bool StaticRootPath string EnableGzip bool EnforceDomain bool @@ -491,7 +491,7 @@ func NewConfigContext(args *CommandLineArgs) error { HttpAddr = server.Key("http_addr").MustString(DEFAULT_HTTP_ADDR) HttpPort = server.Key("http_port").MustString("3000") RouterLogging = server.Key("router_logging").MustBool(false) - AuditLogging = server.Key("audit_logging").MustBool(false) + DataProxyLogging = server.Key("data_proxy_logging").MustBool(false) EnableGzip = server.Key("enable_gzip").MustBool(false) EnforceDomain = server.Key("enforce_domain").MustBool(false) StaticRootPath = makeAbsolute(server.Key("static_root_path").String(), HomePath) From 0b48e48e3dbedec769ba6d8bd389833d644cb290 Mon Sep 17 00:00:00 2001 From: Ricky Niemi Date: Wed, 11 Jan 2017 07:51:46 -0800 Subject: [PATCH 5/9] Small whitespace fix on setting file --- pkg/setting/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index b532ad487cb..e82553c1f09 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -65,7 +65,7 @@ var ( SshPort int CertFile, KeyFile string RouterLogging bool - DataProxyLogging bool + DataProxyLogging bool StaticRootPath string EnableGzip bool EnforceDomain bool From 51bca7d84bd26926a6f41fca53e8fa4bc0ea6dc9 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 16 Jan 2017 12:16:41 +0100 Subject: [PATCH 6/9] tech(dataproxy): moves all parameters to Info call --- pkg/api/dataproxy.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index 2c1355154bc..32352c79ea2 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -18,6 +18,10 @@ import ( "github.com/grafana/grafana/pkg/util" ) +var ( + auditLogger log.Logger = log.New("data-proxy-log") +) + func NewReverseProxy(ds *m.DataSource, proxyPath string, targetUrl *url.URL) *httputil.ReverseProxy { director := func(req *http.Request) { req.URL.Scheme = targetUrl.Scheme @@ -133,7 +137,6 @@ func ProxyDataSourceRequest(c *middleware.Context) { func proxyLog(dataSourceType string, c *middleware.Context) { if setting.DataProxyLogging { - auditLogger := log.New("data-proxy-log", "userid", c.UserId, "orgid", c.OrgId, "username", c.Login) var body string if c.Req.Request.Body != nil { @@ -142,6 +145,13 @@ func proxyLog(dataSourceType string, c *middleware.Context) { body = string(buffer) } - auditLogger.Info("Proxying incoming request", "datasource", dataSourceType, "uri", c.Req.RequestURI, "method", c.Req.Request.Method, "body", body) + auditLogger.Info("Proxying incoming request", + "userid", c.UserId, + "orgid", c.OrgId, + "username", c.Login, + "datasource", dataSourceType, + "uri", c.Req.RequestURI, + "method", c.Req.Request.Method, + "body", body) } } From dd84fb563a3506fa281a1b42c456fd4029554f5e Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 16 Jan 2017 12:23:49 +0100 Subject: [PATCH 7/9] style(dataproxy): renames log functions --- pkg/api/dataproxy.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index 32352c79ea2..a5c047d95ff 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -19,7 +19,7 @@ import ( ) var ( - auditLogger log.Logger = log.New("data-proxy-log") + dataproxyLogger log.Logger = log.New("data-proxy-log") ) func NewReverseProxy(ds *m.DataSource, proxyPath string, targetUrl *url.URL) *httputil.ReverseProxy { @@ -129,14 +129,15 @@ func ProxyDataSourceRequest(c *middleware.Context) { return } - proxyLog(ds.Type, c) - + logProxyRequest(ds.Type, c) proxy.ServeHTTP(c.Resp, c.Req.Request) c.Resp.Header().Del("Set-Cookie") } -func proxyLog(dataSourceType string, c *middleware.Context) { - if setting.DataProxyLogging { +func logProxyRequest(dataSourceType string, c *middleware.Context) { + if !setting.DataProxyLogging { + return + } var body string if c.Req.Request.Body != nil { @@ -144,14 +145,14 @@ func proxyLog(dataSourceType string, c *middleware.Context) { c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buffer)) body = string(buffer) } - - auditLogger.Info("Proxying incoming request", - "userid", c.UserId, - "orgid", c.OrgId, - "username", c.Login, - "datasource", dataSourceType, - "uri", c.Req.RequestURI, - "method", c.Req.Request.Method, - "body", body) } + + dataproxyLogger.Info("Proxying incoming request", + "userid", c.UserId, + "orgid", c.OrgId, + "username", c.Login, + "datasource", dataSourceType, + "uri", c.Req.RequestURI, + "method", c.Req.Request.Method, + "body", body) } From d4a96b9741e569568900da80e5aa96b8fa387f8d Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 16 Jan 2017 12:24:08 +0100 Subject: [PATCH 8/9] tech(dataproxy): make the code a little bit more defensive --- pkg/api/dataproxy.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index a5c047d95ff..dfdc867d4a4 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -139,9 +139,10 @@ func logProxyRequest(dataSourceType string, c *middleware.Context) { return } - var body string - if c.Req.Request.Body != nil { - buffer, _ := ioutil.ReadAll(c.Req.Request.Body) + var body string + if c.Req.Request.Body != nil { + buffer, err := ioutil.ReadAll(c.Req.Request.Body) + if err == nil { c.Req.Request.Body = ioutil.NopCloser(bytes.NewBuffer(buffer)) body = string(buffer) } From 06440ef57b3a7d868de93dae694ec542218db092 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 16 Jan 2017 12:43:59 +0100 Subject: [PATCH 9/9] tech(dataproxy): moves cfg to [dataproxy] --- conf/defaults.ini | 9 ++++++--- conf/sample.ini | 7 +++++++ pkg/setting/setting.go | 6 +++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 5f63dbc2b45..3bce7dc511b 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -47,9 +47,6 @@ root_url = %(protocol)s://%(domain)s:%(http_port)s/ # Log web requests router_logging = false -# This enables data proxy logging, default is false -data_proxy_logging = false - # the path relative working path static_root_path = public @@ -116,6 +113,12 @@ cookie_secure = false session_life_time = 86400 gc_interval_time = 86400 +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +logging = false + #################################### Analytics ########################### [analytics] # Server reporting, sends usage counters to stats.grafana.org every 24 hours. diff --git a/conf/sample.ini b/conf/sample.ini index 0d2adb55a9f..08d87c799ea 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -107,6 +107,13 @@ # Session life time, default is 86400 ;session_life_time = 86400 +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + + #################################### Analytics #################################### [analytics] # Server reporting, sends usage counters to stats.grafana.org every 24 hours. diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 6e3a2e2085f..9a302b10996 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -492,7 +492,7 @@ func NewConfigContext(args *CommandLineArgs) error { HttpAddr = server.Key("http_addr").MustString(DEFAULT_HTTP_ADDR) HttpPort = server.Key("http_port").MustString("3000") RouterLogging = server.Key("router_logging").MustBool(false) - DataProxyLogging = server.Key("data_proxy_logging").MustBool(false) + EnableGzip = server.Key("enable_gzip").MustBool(false) EnforceDomain = server.Key("enforce_domain").MustBool(false) StaticRootPath = makeAbsolute(server.Key("static_root_path").String(), HomePath) @@ -501,6 +501,10 @@ func NewConfigContext(args *CommandLineArgs) error { return err } + // read data proxy settings + dataproxy := Cfg.Section("dataproxy") + DataProxyLogging = dataproxy.Key("logging").MustBool(false) + // read security settings security := Cfg.Section("security") SecretKey = security.Key("secret_key").String()