From 7e0f1a57afde587d98e8f750559a2891e32c0928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 3 Mar 2015 10:18:24 +0100 Subject: [PATCH] Progress on deb and rpm packaging, renamed config files, added file logging, #1476 --- Gruntfile.js | 16 -- build.go | 13 +- conf/custom.ini | 12 + conf/{grafana.ini => defaults.ini} | 6 +- conf/{grafana.dev.ini => dev.ini} | 0 conf/sample.ini | 43 ++++ pkg/cmd/common.go | 8 + pkg/cmd/web.go | 4 - pkg/log/file.go | 243 ++++++++++++++++++ pkg/setting/data/log/grafana.log | 6 + pkg/setting/setting.go | 108 ++++++-- scripts/init.sh | 2 +- .../{require.backend.js => require.config.js} | 0 src/app/components/require.standalone.js | 99 ------- tasks/build_task.js | 65 ++--- tasks/options/compress.js | 4 +- tasks/options/requirejs.js | 15 +- tasks/options/uglify.js | 4 +- 18 files changed, 446 insertions(+), 202 deletions(-) create mode 100644 conf/custom.ini rename conf/{grafana.ini => defaults.ini} (96%) rename conf/{grafana.dev.ini => dev.ini} (100%) create mode 100644 conf/sample.ini create mode 100644 pkg/log/file.go create mode 100644 pkg/setting/data/log/grafana.log rename src/app/components/{require.backend.js => require.config.js} (100%) delete mode 100644 src/app/components/require.standalone.js diff --git a/Gruntfile.js b/Gruntfile.js index 5065398642b..461edc91578 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -11,22 +11,6 @@ module.exports = function (grunt) { docsDir: 'docs/' }; - config.mode = grunt.option('mode') || 'backend'; - config.modeOptions = { - zipSuffix: '', - requirejs: { - paths: { config: '../config.sample' }, - excludeConfig: true, - } - }; - - if (config.mode === 'backend') { - grunt.log.writeln('Setting backend build mode'); - config.modeOptions.zipSuffix = '-backend'; - config.modeOptions.requirejs.paths = {}; - config.modeOptions.requirejs.excludeConfig = false; - } - // load plugins require('load-grunt-tasks')(grunt); diff --git a/build.go b/build.go index 97ef4faaf00..3dc4f6fe284 100644 --- a/build.go +++ b/build.go @@ -30,7 +30,7 @@ var ( workingDir string installRoot = "/opt/grafana" - configRoot = "/etc/grafana" + configRoot = "/etc/opt/grafana" grafanaLogDir = "/var/log/grafana" ) @@ -115,9 +115,14 @@ func createRpmAndDeb() { postInstallScriptPath, _ := ioutil.TempFile("", "postinstall") versionFolder := filepath.Join(packageRoot, installRoot, "versions", version) - runError("mkdir", "-p", versionFolder) - runError("mkdir", "-p", filepath.Join(packageRoot, configRoot)) + configDir := filepath.Join(packageRoot, configRoot) + runError("mkdir", "-p", versionFolder) + runError("mkdir", "-p", configDir) + + // copy sample ini file to /etc/opt/grafana + runError("cp", "conf/sample.ini", filepath.Join(configDir, "grafana.ini")) + // copy release files runError("cp", "-a", filepath.Join(workingDir, "tmp")+"/.", versionFolder) fmt.Printf("PackageDir: %v\n", versionFolder) @@ -268,7 +273,7 @@ func clean() { rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin") rmr("dist") rmr("tmp") - rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafan", goos, goarch))) + rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafana", goos, goarch))) } func setBuildEnv() { diff --git a/conf/custom.ini b/conf/custom.ini new file mode 100644 index 00000000000..9de647dccd0 --- /dev/null +++ b/conf/custom.ini @@ -0,0 +1,12 @@ +app_mode = development + +[server] +router_logging = false +static_root_path = src + +[log] +level = Trace +mode = console, file + + + diff --git a/conf/grafana.ini b/conf/defaults.ini similarity index 96% rename from conf/grafana.ini rename to conf/defaults.ini index 491dfd1063b..1658bc7244e 100644 --- a/conf/grafana.ini +++ b/conf/defaults.ini @@ -30,7 +30,7 @@ ssl_mode = disable path = data/grafana.db [session] -; Either "memory", "file", default is "memory" +; Either "memory", "file", "redis", "mysql", default is "memory" provider = file ; Provider config options ; memory: not have any config yet @@ -100,8 +100,8 @@ auth_url = https://accounts.google.com/o/oauth2/auth token_url = https://accounts.google.com/o/oauth2/token [log] -root_path = -; Either "console", "file", "conn", "smtp" or "database", default is "console" +root_path = data/log +; Either "console", "file", default is "console" ; Use comma to separate multiple modes, e.g. "console, file" mode = console ; Buffer length of channel, keep it as it is if you don't know what it is. diff --git a/conf/grafana.dev.ini b/conf/dev.ini similarity index 100% rename from conf/grafana.dev.ini rename to conf/dev.ini diff --git a/conf/sample.ini b/conf/sample.ini new file mode 100644 index 00000000000..fa5c43a6e6a --- /dev/null +++ b/conf/sample.ini @@ -0,0 +1,43 @@ +# Sample grafana config +# You only need to specify overrides here +# Defaults are in the /opt/grafana/current/conf/defaults.ini file +# This file is never ovewritten when upgrading grafana via deb or rpm package + +app_mode = production + +[server] +; protocol (http or https) +protocol = http +; the ip address to bind to, empty will bind to all interfaces +http_addr = +; the http port to use +http_port = 3000 +; The public facing domain name used to access grafana from a browser +domain = localhost +; the full public facing url +root_url = %(protocol)s://%(domain)s:%(http_port)s/ +router_logging = false +; the path relative to grafana process working directory +static_root_path = public +enable_gzip = false + +[database] +; Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = root +password = +; For "postgres" only, either "disable", "require" or "verify-full" +ssl_mode = disable +; For "sqlite3" only +path = /opt/grafana/data/grafana.db + +[log] +level = Trace +mode = console, file +root_path = /var/log/grafana + + + + diff --git a/pkg/cmd/common.go b/pkg/cmd/common.go index 338baa36aac..3caa8350700 100644 --- a/pkg/cmd/common.go +++ b/pkg/cmd/common.go @@ -1,13 +1,21 @@ package cmd import ( + "time" + "github.com/codegangsta/cli" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" ) func initRuntime(c *cli.Context) { setting.NewConfigContext(c.GlobalString("config")) + + log.Info("Starting Grafana") + log.Info("Version: %v, Commit: %v, Build date: %v", setting.BuildVersion, setting.BuildCommit, time.Unix(setting.BuildStamp, 0)) + setting.LogLoadedConfigFiles() + sqlstore.NewEngine() sqlstore.EnsureAdminUser() } diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index f5c4b6d9d17..0a4424446f7 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "path" - "time" "github.com/Unknwon/macaron" "github.com/codegangsta/cli" @@ -68,9 +67,6 @@ func mapStatic(m *macaron.Macaron, dir string, prefix string) { } func runWeb(c *cli.Context) { - log.Info("Starting Grafana") - log.Info("Version: %v, Commit: %v, Build date: %v", setting.BuildVersion, setting.BuildCommit, time.Unix(setting.BuildStamp, 0)) - initRuntime(c) social.NewOAuthService() diff --git a/pkg/log/file.go b/pkg/log/file.go new file mode 100644 index 00000000000..e9402815f0b --- /dev/null +++ b/pkg/log/file.go @@ -0,0 +1,243 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package log + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// FileLogWriter implements LoggerInterface. +// It writes messages by lines limit, file size limit, or time frequency. +type FileLogWriter struct { + *log.Logger + mw *MuxWriter + // The opened file + Filename string `json:"filename"` + + Maxlines int `json:"maxlines"` + maxlines_curlines int + + // Rotate at size + Maxsize int `json:"maxsize"` + maxsize_cursize int + + // Rotate daily + Daily bool `json:"daily"` + Maxdays int64 `json:"maxdays"` + daily_opendate int + + Rotate bool `json:"rotate"` + + startLock sync.Mutex // Only one log can write to the file + + Level int `json:"level"` +} + +// an *os.File writer with locker. +type MuxWriter struct { + sync.Mutex + fd *os.File +} + +// write to os.File. +func (l *MuxWriter) Write(b []byte) (int, error) { + l.Lock() + defer l.Unlock() + return l.fd.Write(b) +} + +// set os.File in writer. +func (l *MuxWriter) SetFd(fd *os.File) { + if l.fd != nil { + l.fd.Close() + } + l.fd = fd +} + +// create a FileLogWriter returning as LoggerInterface. +func NewFileWriter() LoggerInterface { + w := &FileLogWriter{ + Filename: "", + Maxlines: 1000000, + Maxsize: 1 << 28, //256 MB + Daily: true, + Maxdays: 7, + Rotate: true, + Level: TRACE, + } + // use MuxWriter instead direct use os.File for lock write when rotate + w.mw = new(MuxWriter) + // set MuxWriter as Logger's io.Writer + w.Logger = log.New(w.mw, "", log.Ldate|log.Ltime) + return w +} + +// Init file logger with json config. +// config like: +// { +// "filename":"log/gogs.log", +// "maxlines":10000, +// "maxsize":1<<30, +// "daily":true, +// "maxdays":15, +// "rotate":true +// } +func (w *FileLogWriter) Init(config string) error { + if err := json.Unmarshal([]byte(config), w); err != nil { + return err + } + if len(w.Filename) == 0 { + return errors.New("config must have filename") + } + return w.StartLogger() +} + +// start file logger. create log file and set to locker-inside file writer. +func (w *FileLogWriter) StartLogger() error { + fd, err := w.createLogFile() + if err != nil { + return err + } + w.mw.SetFd(fd) + if err = w.initFd(); err != nil { + return err + } + return nil +} + +func (w *FileLogWriter) docheck(size int) { + w.startLock.Lock() + defer w.startLock.Unlock() + if w.Rotate && ((w.Maxlines > 0 && w.maxlines_curlines >= w.Maxlines) || + (w.Maxsize > 0 && w.maxsize_cursize >= w.Maxsize) || + (w.Daily && time.Now().Day() != w.daily_opendate)) { + if err := w.DoRotate(); err != nil { + fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) + return + } + } + w.maxlines_curlines++ + w.maxsize_cursize += size +} + +// write logger message into file. +func (w *FileLogWriter) WriteMsg(msg string, skip, level int) error { + if level < w.Level { + return nil + } + n := 24 + len(msg) // 24 stand for the length "2013/06/23 21:00:22 [T] " + w.docheck(n) + w.Logger.Println(msg) + return nil +} + +func (w *FileLogWriter) createLogFile() (*os.File, error) { + // Open the log file + return os.OpenFile(w.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0660) +} + +func (w *FileLogWriter) initFd() error { + fd := w.mw.fd + finfo, err := fd.Stat() + if err != nil { + return fmt.Errorf("get stat: %s\n", err) + } + w.maxsize_cursize = int(finfo.Size()) + w.daily_opendate = time.Now().Day() + if finfo.Size() > 0 { + content, err := ioutil.ReadFile(w.Filename) + if err != nil { + return err + } + w.maxlines_curlines = len(strings.Split(string(content), "\n")) + } else { + w.maxlines_curlines = 0 + } + return nil +} + +// DoRotate means it need to write file in new file. +// new file name like xx.log.2013-01-01.2 +func (w *FileLogWriter) DoRotate() error { + _, err := os.Lstat(w.Filename) + if err == nil { // file exists + // Find the next available number + num := 1 + fname := "" + for ; err == nil && num <= 999; num++ { + fname = w.Filename + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), num) + _, err = os.Lstat(fname) + } + // return error if the last file checked still existed + if err == nil { + return fmt.Errorf("rotate: cannot find free log number to rename %s\n", w.Filename) + } + + // block Logger's io.Writer + w.mw.Lock() + defer w.mw.Unlock() + + fd := w.mw.fd + fd.Close() + + // close fd before rename + // Rename the file to its newfound home + if err = os.Rename(w.Filename, fname); err != nil { + return fmt.Errorf("Rotate: %s\n", err) + } + + // re-start logger + if err = w.StartLogger(); err != nil { + return fmt.Errorf("Rotate StartLogger: %s\n", err) + } + + go w.deleteOldLog() + } + + return nil +} + +func (w *FileLogWriter) deleteOldLog() { + dir := filepath.Dir(w.Filename) + filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { + defer func() { + if r := recover(); r != nil { + returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r) + } + }() + + if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*w.Maxdays) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(w.Filename)) { + os.Remove(path) + } + } + return returnErr + }) +} + +// destroy file logger, close file writer. +func (w *FileLogWriter) Destroy() { + w.mw.fd.Close() +} + +// flush file logger. +// there are no buffering messages in file logger in memory. +// flush file means sync file from disk. +func (w *FileLogWriter) Flush() { + w.mw.fd.Sync() +} + +func init() { + Register("file", NewFileWriter) +} diff --git a/pkg/setting/data/log/grafana.log b/pkg/setting/data/log/grafana.log new file mode 100644 index 00000000000..448df51d36b --- /dev/null +++ b/pkg/setting/data/log/grafana.log @@ -0,0 +1,6 @@ +2015/03/03 09:52:42 [I] Setting: ENV override found: GF_SECURITY_ADMIN_USER +2015/03/03 09:59:43 [I] Setting: ENV override found: GF_SECURITY_ADMIN_USER +2015/03/03 10:03:13 [I] Setting: ENV override found: GF_SECURITY_ADMIN_USER +2015/03/03 10:09:13 [I] Setting: ENV override found: GF_SECURITY_ADMIN_USER +2015/03/03 10:09:34 [I] Setting: ENV override found: GF_SECURITY_ADMIN_USER +2015/03/03 10:15:20 [I] Setting: ENV override found: GF_SECURITY_ADMIN_USER diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 36a7d7fcf1c..d5bca4e3984 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -94,6 +94,8 @@ var ( // PhantomJs Rendering ImagesDir string PhantomDir string + + configFiles []string ) func init() { @@ -102,30 +104,32 @@ func init() { WorkDir, _ = filepath.Abs(".") } -func findConfigFiles() []string { +func findConfigFiles(customConfigFile string) { ConfRootPath = path.Join(WorkDir, "conf") - filenames := make([]string, 0) + configFiles = make([]string, 0) - configFile := path.Join(ConfRootPath, "grafana.ini") + configFile := path.Join(ConfRootPath, "defaults.ini") if com.IsFile(configFile) { - filenames = append(filenames, configFile) + configFiles = append(configFiles, configFile) } - configFile = path.Join(ConfRootPath, "grafana.dev.ini") + configFile = path.Join(ConfRootPath, "dev.ini") if com.IsFile(configFile) { - filenames = append(filenames, configFile) + configFiles = append(configFiles, configFile) } - configFile = path.Join(ConfRootPath, "grafana.custom.ini") + configFile = path.Join(ConfRootPath, "custom.ini") if com.IsFile(configFile) { - filenames = append(filenames, configFile) + configFiles = append(configFiles, configFile) } - if len(filenames) == 0 { + if customConfigFile != "" { + configFiles = append(configFiles, customConfigFile) + } + + if len(configFiles) == 0 { log.Fatal(3, "Could not find any config file") } - - return filenames } func parseAppUrlAndSubUrl(section *ini.Section) (string, string) { @@ -165,11 +169,7 @@ func loadEnvVariableOverrides() { } func NewConfigContext(config string) { - configFiles := findConfigFiles() - - if config != "" { - configFiles = append(configFiles, config) - } + findConfigFiles(config) var err error @@ -186,6 +186,7 @@ func NewConfigContext(config string) { } loadEnvVariableOverrides() + initLogging() AppName = Cfg.Section("").Key("app_name").MustString("Grafana") Env = Cfg.Section("").Key("app_mode").MustString("development") @@ -233,8 +234,6 @@ func NewConfigContext(config string) { ImagesDir = "data/png" PhantomDir = "vendor/phantomjs" - LogRootPath = Cfg.Section("log").Key("root_path").MustString(path.Join(WorkDir, "/data/log")) - readSessionConfig() } @@ -253,3 +252,76 @@ func readSessionConfig() { os.MkdirAll(path.Dir(SessionOptions.ProviderConfig), os.ModePerm) } } + +var logLevels = map[string]string{ + "Trace": "0", + "Debug": "1", + "Info": "2", + "Warn": "3", + "Error": "4", + "Critical": "5", +} + +func initLogging() { + // Get and check log mode. + LogModes = strings.Split(Cfg.Section("log").Key("mode").MustString("console"), ",") + LogRootPath = Cfg.Section("log").Key("root_path").MustString(path.Join(WorkDir, "/data/log")) + LogConfigs = make([]string, len(LogModes)) + for i, mode := range LogModes { + mode = strings.TrimSpace(mode) + sec, err := Cfg.GetSection("log." + mode) + if err != nil { + log.Fatal(4, "Unknown log mode: %s", mode) + } + + // Log level. + levelName := Cfg.Section("log."+mode).Key("level").In("Trace", + []string{"Trace", "Debug", "Info", "Warn", "Error", "Critical"}) + level, ok := logLevels[levelName] + if !ok { + log.Fatal(4, "Unknown log level: %s", levelName) + } + + // Generate log configuration. + switch mode { + case "console": + LogConfigs[i] = fmt.Sprintf(`{"level":%s}`, level) + case "file": + logPath := sec.Key("file_name").MustString(path.Join(LogRootPath, "grafana.log")) + os.MkdirAll(path.Dir(logPath), os.ModePerm) + LogConfigs[i] = fmt.Sprintf( + `{"level":%s,"filename":"%s","rotate":%v,"maxlines":%d,"maxsize":%d,"daily":%v,"maxdays":%d}`, level, + logPath, + sec.Key("log_rotate").MustBool(true), + sec.Key("max_lines").MustInt(1000000), + 1<', - src: '**/*', - dest: '<%= tempDir %>/public/', - }); - grunt.config('clean.dest_dir', ['<%= destDir %>']); - grunt.config('copy.backend_bin', { - cwd: 'bin', - expand: true, - src: ['grafana'], - options: { mode: true}, - dest: '<%= tempDir %>' - }); - grunt.config('copy.backend_files', { - expand: true, - src: ['conf/grafana.ini', 'vendor/**/*', 'scripts/*'], - options: { mode: true}, - dest: '<%= tempDir %>' - }); - grunt.task.run('copy:dist_to_tmp'); - grunt.task.run('clean:dest_dir'); - grunt.task.run('copy:backend_bin'); - grunt.task.run('copy:backend_files'); - } - }); - - grunt.registerTask('build:grafanaVersion', function() { - grunt.config('string-replace.config', { - files: { - '<%= tempDir %>/app/app.js': '<%= tempDir %>/app/app.js' - }, - options: { - replacements: [{ pattern: /@grafanaVersion@/g, replacement: '<%= pkg.version %>' }] - } + grunt.config('copy.dist_to_tmp', { + expand: true, + cwd: '<%= destDir %>', + src: '**/*', + dest: '<%= tempDir %>/public/', }); - grunt.task.run('string-replace:config'); + grunt.config('clean.dest_dir', ['<%= destDir %>']); + grunt.config('copy.backend_bin', { + cwd: 'bin', + expand: true, + src: ['grafana'], + options: { mode: true}, + dest: '<%= tempDir %>' + }); + grunt.config('copy.backend_files', { + expand: true, + src: ['conf/defaults.ini', 'vendor/**/*', 'scripts/*'], + options: { mode: true}, + dest: '<%= tempDir %>' + }); + + grunt.task.run('copy:dist_to_tmp'); + grunt.task.run('clean:dest_dir'); + grunt.task.run('copy:backend_bin'); + grunt.task.run('copy:backend_files'); }); }; diff --git a/tasks/options/compress.js b/tasks/options/compress.js index 4ad42ef41b8..66038af6a4f 100644 --- a/tasks/options/compress.js +++ b/tasks/options/compress.js @@ -38,7 +38,7 @@ module.exports = function(config) { }, zip_release: { options: { - archive: '<%= destDir %>/<%= pkg.name %><%= modeOptions.zipSuffix %>-<%= pkg.version %>.zip' + archive: '<%= destDir %>/<%= pkg.name %>-<%= pkg.version %>.zip' }, files : [ { @@ -56,7 +56,7 @@ module.exports = function(config) { }, tgz_release: { options: { - archive: '<%= destDir %>/<%= pkg.name %><%= modeOptions.zipSuffix %>-<%= pkg.version %>.tar.gz' + archive: '<%= destDir %>/<%= pkg.name %>-<%= pkg.version %>.tar.gz' }, files : [ { diff --git a/tasks/options/requirejs.js b/tasks/options/requirejs.js index fd7fa2975b5..947553f1266 100644 --- a/tasks/options/requirejs.js +++ b/tasks/options/requirejs.js @@ -6,9 +6,8 @@ module.exports = function(config,grunt) { var options = { appDir: '<%= tempDir %>', dir: '<%= destDir %>', - mainConfigFile: '<%= tempDir %>/app/components/require.<%= mode %>.js', + mainConfigFile: '<%= tempDir %>/app/components/require.config.js', baseUrl: './app', - paths: config.modeOptions.requirejs.paths, modules: [], // populated below, @@ -60,8 +59,7 @@ module.exports = function(config,grunt) { 'directives/all', 'filters/all', 'controllers/all', - 'routes/standalone/all', - 'routes/backend/all', + 'routes/all', 'components/partials', ] } @@ -76,15 +74,6 @@ module.exports = function(config,grunt) { requireModules[0].include.push('text!panels/'+panelName+'/module.html'); }); - if (config.modeOptions.requirejs.excludeConfig) { - // exclude the literal config definition from all modules - requireModules - .forEach(function (module) { - module.excludeShallow = module.excludeShallow || []; - module.excludeShallow.push('config'); - }); - } - return { options: options }; } diff --git a/tasks/options/uglify.js b/tasks/options/uglify.js index 1ca15b06957..ab97b756c0b 100644 --- a/tasks/options/uglify.js +++ b/tasks/options/uglify.js @@ -2,7 +2,7 @@ module.exports = function(config) { return { dest: { expand: true, - src: ['**/*.js', '!config.sample.js', '!app/dashboards/*.js', '!app/dashboards/**/*.js',], + src: ['**/*.js', '!dashboards/*.js'], dest: '<%= destDir %>', cwd: '<%= destDir %>', options: { @@ -13,4 +13,4 @@ module.exports = function(config) { } } }; -}; \ No newline at end of file +};