From b16cb92b32e96e69e99785374f7c21489d6ccb85 Mon Sep 17 00:00:00 2001 From: Sofia Papagiannaki Date: Mon, 25 Nov 2019 10:40:10 +0200 Subject: [PATCH] Chore: Improve grafana-server profiling and tracing (#20593) Profiling and tracing can now be enabled/disabled separately. Adds argument for trace file path. Support overriding profiling and tracing argument using environment variables. Update docs. Fixes #20576 --- docs/sources/installation/troubleshooting.md | 54 +++++++++++++- pkg/cmd/grafana-server/diagnostics.go | 78 ++++++++++++++++++++ pkg/cmd/grafana-server/diagnostics_test.go | 73 ++++++++++++++++++ pkg/cmd/grafana-server/main.go | 26 ++++++- 4 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 pkg/cmd/grafana-server/diagnostics.go create mode 100644 pkg/cmd/grafana-server/diagnostics_test.go diff --git a/docs/sources/installation/troubleshooting.md b/docs/sources/installation/troubleshooting.md index fad6e0f4dca..348c194bddf 100644 --- a/docs/sources/installation/troubleshooting.md +++ b/docs/sources/installation/troubleshooting.md @@ -30,7 +30,59 @@ If you encounter an error or problem it is a good idea to check the grafana serv located at `/var/log/grafana/grafana.log` on Unix systems or in `/data/log` on other platforms and manual installs. -You can enable more logging by changing log level in you grafana configuration file. +You can enable more logging by changing log level in your grafana configuration file. + +## Diagnostics + +The `grafana-server` process can be instructued to enable certain diagnostics when it starts. This can be helpful +when experiencing/investigating certain performance problems. It's `not` recommended to have these enabled per default. + +### Profiling + +The `grafana-server` can be started with the arguments `-profile` to enable profiling and `-profile-port` to override +the default HTTP port (`6060`) where the pprof debugging endpoints will be available, e.g. + +```bash +./grafana-server -profile -profile-port=8080 +``` + +Note that pprof debugging endpoints are served on a different port than the Grafana HTTP server. + +You can configure/override profiling settings using environment variables: + +```bash +export GF_DIAGNOSTICS_PROFILING_ENABLED=true +export GF_DIAGNOSTICS_PROFILING_PORT=8080 +``` + +See [Go command pprof](https://golang.org/cmd/pprof/) for more information about how to collect and analyze profiling data. + +### Tracing + +The `grafana-server` can be started with the arguments `-tracing` to enable tracing and `-tracing-file` to +override the default trace file (`trace.out`) where trace result will be written to, e.g. + +```bash +./grafana-server -tracing -tracing-file=/tmp/trace.out +``` + +You can configure/override profiling settings using environment variables: + +```bash +export GF_DIAGNOSTICS_TRACING_ENABLED=true +export GF_DIAGNOSTICS_TRACING_FILE=/tmp/trace.out +``` + +View the trace in a web browser (Go required to be installed): + +```bash +go tool trace +2019/11/24 22:20:42 Parsing trace... +2019/11/24 22:20:42 Splitting trace... +2019/11/24 22:20:42 Opening browser. Trace viewer is listening on http://127.0.0.1:39735 +``` + +See [Go command trace](https://golang.org/cmd/trace/) for more information about how to analyze trace files. ## FAQ diff --git a/pkg/cmd/grafana-server/diagnostics.go b/pkg/cmd/grafana-server/diagnostics.go new file mode 100644 index 00000000000..776a05b42a6 --- /dev/null +++ b/pkg/cmd/grafana-server/diagnostics.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "os" + "strconv" +) + +const ( + profilingEnabledEnvName = "GF_DIAGNOSTICS_PROFILING_ENABLED" + profilingPortEnvName = "GF_DIAGNOSTICS_PROFILING_PORT" + tracingEnabledEnvName = "GF_DIAGNOSTICS_TRACING_ENABLED" + tracingFileEnvName = "GF_DIAGNOSTICS_TRACING_FILE" +) + +type profilingDiagnostics struct { + enabled bool + port uint +} + +func newProfilingDiagnostics(enabled bool, port uint) *profilingDiagnostics { + return &profilingDiagnostics{ + enabled: enabled, + port: port, + } +} + +func (pd *profilingDiagnostics) overrideWithEnv() error { + enabledEnv := os.Getenv(profilingEnabledEnvName) + if enabledEnv != "" { + enabled, err := strconv.ParseBool(enabledEnv) + if err != nil { + return fmt.Errorf("Failed to parse %s environment variable as bool", profilingEnabledEnvName) + } + pd.enabled = enabled + } + + portEnv := os.Getenv(profilingPortEnvName) + if portEnv != "" { + port, parseErr := strconv.ParseUint(portEnv, 0, 64) + if parseErr != nil { + return fmt.Errorf("Failed to parse %s enviroment variable to unsigned integer", profilingPortEnvName) + } + pd.port = uint(port) + } + + return nil +} + +type tracingDiagnostics struct { + enabled bool + file string +} + +func newTracingDiagnostics(enabled bool, file string) *tracingDiagnostics { + return &tracingDiagnostics{ + enabled: enabled, + file: file, + } +} + +func (td *tracingDiagnostics) overrideWithEnv() error { + enabledEnv := os.Getenv(tracingEnabledEnvName) + if enabledEnv != "" { + enabled, err := strconv.ParseBool(enabledEnv) + if err != nil { + return fmt.Errorf("Failed to parse %s environment variable as bool", tracingEnabledEnvName) + } + td.enabled = enabled + } + + fileEnv := os.Getenv(tracingFileEnvName) + if fileEnv != "" { + td.file = fileEnv + } + + return nil +} diff --git a/pkg/cmd/grafana-server/diagnostics_test.go b/pkg/cmd/grafana-server/diagnostics_test.go new file mode 100644 index 00000000000..f7e7b9874aa --- /dev/null +++ b/pkg/cmd/grafana-server/diagnostics_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProfilingDiagnostics(t *testing.T) { + tcs := []struct { + defaults *profilingDiagnostics + enabledEnv string + portEnv string + expected *profilingDiagnostics + }{ + {defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "", portEnv: "", expected: newProfilingDiagnostics(false, 6060)}, + {defaults: newProfilingDiagnostics(true, 8080), enabledEnv: "", portEnv: "", expected: newProfilingDiagnostics(true, 8080)}, + {defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "false", portEnv: "8080", expected: newProfilingDiagnostics(false, 8080)}, + {defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "true", portEnv: "8080", expected: newProfilingDiagnostics(true, 8080)}, + {defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "true", portEnv: "", expected: newProfilingDiagnostics(true, 6060)}, + } + + for i, tc := range tcs { + t.Run(fmt.Sprintf("testcase %d", i), func(t *testing.T) { + os.Clearenv() + if tc.enabledEnv != "" { + err := os.Setenv(profilingEnabledEnvName, tc.enabledEnv) + assert.NoError(t, err) + } + if tc.portEnv != "" { + err := os.Setenv(profilingPortEnvName, tc.portEnv) + assert.NoError(t, err) + } + err := tc.defaults.overrideWithEnv() + assert.NoError(t, err) + assert.Exactly(t, tc.expected, tc.defaults) + }) + } +} + +func TestTracingDiagnostics(t *testing.T) { + tcs := []struct { + defaults *tracingDiagnostics + enabledEnv string + fileEnv string + expected *tracingDiagnostics + }{ + {defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "", fileEnv: "", expected: newTracingDiagnostics(false, "trace.out")}, + {defaults: newTracingDiagnostics(true, "/tmp/trace.out"), enabledEnv: "", fileEnv: "", expected: newTracingDiagnostics(true, "/tmp/trace.out")}, + {defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "false", fileEnv: "/tmp/trace.out", expected: newTracingDiagnostics(false, "/tmp/trace.out")}, + {defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "true", fileEnv: "/tmp/trace.out", expected: newTracingDiagnostics(true, "/tmp/trace.out")}, + {defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "true", fileEnv: "", expected: newTracingDiagnostics(true, "trace.out")}, + } + + for i, tc := range tcs { + t.Run(fmt.Sprintf("testcase %d", i), func(t *testing.T) { + os.Clearenv() + if tc.enabledEnv != "" { + err := os.Setenv(tracingEnabledEnvName, tc.enabledEnv) + assert.NoError(t, err) + } + if tc.fileEnv != "" { + err := os.Setenv(tracingFileEnvName, tc.fileEnv) + assert.NoError(t, err) + } + err := tc.defaults.overrideWithEnv() + assert.NoError(t, err) + assert.Exactly(t, tc.expected, tc.defaults) + }) + } +} diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index 232500ab9b7..14d6036e826 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -46,7 +46,9 @@ func main() { v = flag.Bool("v", false, "prints current version and exits") profile = flag.Bool("profile", false, "Turn on pprof profiling") - profilePort = flag.Int("profile-port", 6060, "Define custom port for profiling") + profilePort = flag.Uint("profile-port", 6060, "Define custom port for profiling") + tracing = flag.Bool("tracing", false, "Turn on tracing") + tracingFile = flag.String("tracing-file", "trace.out", "Define tracing output file") ) flag.Parse() @@ -56,16 +58,32 @@ func main() { os.Exit(0) } - if *profile { + profileDiagnostics := newProfilingDiagnostics(*profile, *profilePort) + if err := profileDiagnostics.overrideWithEnv(); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + traceDiagnostics := newTracingDiagnostics(*tracing, *tracingFile) + if err := traceDiagnostics.overrideWithEnv(); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + if profileDiagnostics.enabled { + fmt.Println("diagnostics: pprof profiling enabled", "port", profileDiagnostics.port) runtime.SetBlockProfileRate(1) go func() { - err := http.ListenAndServe(fmt.Sprintf("localhost:%d", *profilePort), nil) + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", profileDiagnostics.port), nil) if err != nil { panic(err) } }() + } - f, err := os.Create("trace.out") + if traceDiagnostics.enabled { + fmt.Println("diagnostics: tracing enabled", "file", traceDiagnostics.file) + f, err := os.Create(traceDiagnostics.file) if err != nil { panic(err) }