From d1767144a88afe42acafa9cddd7cc8e27665356a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 9 Apr 2015 12:16:59 +0200 Subject: [PATCH] Reworking configuration loading and overriding --- .bra.toml | 8 +- build.go | 2 +- conf/defaults.ini | 17 +- main.go | 11 +- pkg/services/sqlstore/sqlstore.go | 2 +- pkg/setting/setting.go | 249 ++++++++++++++++++++---------- pkg/setting/setting_test.go | 60 ++++++- tasks/build_task.js | 2 +- tests/config-files/override.ini | 2 + 9 files changed, 241 insertions(+), 112 deletions(-) create mode 100644 tests/config-files/override.ini diff --git a/.bra.toml b/.bra.toml index 4b5566c51a7..e832e9beee5 100644 --- a/.bra.toml +++ b/.bra.toml @@ -1,7 +1,7 @@ [run] init_cmds = [ - ["go", "build", "-o", "./bin/grafana"], - ["./bin/grafana", "web"] + ["go", "build", "-o", "./bin/grafana-server"], + ["./bin/grafana-server"] ] watch_all = true watch_dirs = [ @@ -12,6 +12,6 @@ watch_dirs = [ watch_exts = [".go", ".ini"] build_delay = 1500 cmds = [ - ["go", "build", "-o", "./bin/grafana"], - ["./bin/grafana", "web"] + ["go", "build", "-o", "./bin/grafana-server"], + ["./bin/grafana-server"] ] diff --git a/build.go b/build.go index ff5e48364a5..c3d61362c2a 100644 --- a/build.go +++ b/build.go @@ -209,7 +209,7 @@ func test(pkg string) { } func build(pkg string, tags []string) { - binary := "./bin/grafana" + binary := "./bin/grafana-server" if goos == "windows" { binary += ".exe" } diff --git a/conf/defaults.ini b/conf/defaults.ini index d9f84196fde..8dddace471c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1,13 +1,13 @@ app_name = Grafana app_mode = production +[paths] ; data_path ; where rendered png images are temporarily stored ; file based sessions are stored here (if file based session is configured below) ; the database is stored here if sqlite3 database is used -; can be overriden from command line --data-path -; defaults to `data` path relative to working directory -data_path = +data = data +logs = data/log [server] ; protocol (http or https) @@ -21,7 +21,7 @@ domain = localhost ; the full public facing url root_url = %(protocol)s://%(domain)s:%(http_port)s/ router_logging = false -; the path relative to the binary where the static (html/js/css) files are placed +; the path relative home path where frontend assets are located static_root_path = public ; enable gzip enable_gzip = false @@ -47,7 +47,7 @@ user = root password = ; For "postgres" only, either "disable", "require" or "verify-full" ssl_mode = disable -; For "sqlite3" only, path relative to data_dir setting +; For "sqlite3" only, path relative to data_path setting path = grafana.db [session] @@ -55,7 +55,7 @@ path = grafana.db provider = file ; Provider config options ; memory: not have any config yet -; file: session dir path, is relative to grafana data_dir +; file: session dir path, is relative to grafana data_path ; redis: config like redis server addr, poolSize, password, e.g. `127.0.0.1:6379,100,grafana` ; mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1)/database_name` provider_config = sessions @@ -117,11 +117,6 @@ token_url = https://accounts.google.com/o/oauth2/token ; allowed_domains = mycompany.com othercompany.com [log] -; root_path -; for deb or rpm package installs this is specified via command line -; change it in /etc/default/grafana, it defaults to /var/log/grafana -; for non package installs (running manually) defaults to `log` dir under data_dir -root_path = ; Either "console", "file", default is "console" ; Use comma to separate multiple modes, e.g. "console, file" mode = console, file diff --git a/main.go b/main.go index 243e124b022..a7e731bc84c 100644 --- a/main.go +++ b/main.go @@ -64,15 +64,14 @@ func main() { } func initRuntime() { - setting.NewConfigContext(&setting.CommandLineArgs{}) + setting.NewConfigContext(&setting.CommandLineArgs{ + Config: *configFile, + Args: flag.Args(), + }) log.Info("Starting Grafana") log.Info("Version: %v, Commit: %v, Build date: %v", setting.BuildVersion, setting.BuildCommit, time.Unix(setting.BuildStamp, 0)) - setting.LogLoadedConfigFiles() - - log.Info("Working Path: %s", setting.WorkPath) - log.Info("Data Path: %s", setting.DataPath) - log.Info("Log Path: %s", setting.LogRootPath) + setting.LogConfigurationInfo() sqlstore.NewEngine() sqlstore.EnsureAdminUser() diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 40e54a827e5..2dd483651c0 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -86,7 +86,7 @@ func SetEngine(engine *xorm.Engine, enableLog bool) (err error) { } if enableLog { - logPath := path.Join(setting.LogRootPath, "xorm.log") + logPath := path.Join(setting.LogsPath, "xorm.log") os.MkdirAll(path.Dir(logPath), os.ModePerm) f, err := os.Create(logPath) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index b12bd6ee0c4..d6e6d0f4e64 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -4,15 +4,16 @@ package setting import ( + "bytes" "fmt" "net/url" "os" "path" "path/filepath" + "regexp" "runtime" "strings" - "github.com/Unknwon/com" "github.com/macaron-contrib/session" "gopkg.in/ini.v1" @@ -44,10 +45,14 @@ var ( BuildCommit string BuildStamp int64 + // Paths + LogsPath string + HomePath string + DataPath string + // Log settings. - LogRootPath string - LogModes []string - LogConfigs []string + LogModes []string + LogConfigs []string // Http server options Protocol Scheme @@ -83,8 +88,6 @@ var ( SessionOptions session.Options // Global setting objects. - DataPath string - WorkPath string Cfg *ini.File ConfRootPath string IsWindows bool @@ -93,50 +96,24 @@ var ( ImagesDir string PhantomDir string - configFiles []string + // for logging purposes + configFiles []string + appliedCommandLineProperties []string + appliedEnvOverrides []string ReportingEnabled bool GoogleAnalyticsId string ) type CommandLineArgs struct { - DefaultDataPath string - DefaultLogPath string - Config string + Config string + Args []string } func init() { IsWindows = runtime.GOOS == "windows" log.NewLogger(0, "console", `{"level": 0}`) - WorkPath, _ = filepath.Abs(".") -} - -func findConfigFiles(customConfigFile string) { - ConfRootPath = path.Join(WorkPath, "conf") - configFiles = make([]string, 0) - - configFile := path.Join(ConfRootPath, "defaults.ini") - if com.IsFile(configFile) { - configFiles = append(configFiles, configFile) - } - - configFile = path.Join(ConfRootPath, "dev.ini") - if com.IsFile(configFile) { - configFiles = append(configFiles, configFile) - } - - configFile = path.Join(ConfRootPath, "custom.ini") - if com.IsFile(configFile) { - configFiles = append(configFiles, configFile) - } - - if customConfigFile != "" { - configFiles = append(configFiles, customConfigFile) - } - - if len(configFiles) == 0 { - log.Fatal(3, "Could not find any config file") - } + HomePath, _ = filepath.Abs(".") } func parseAppUrlAndSubUrl(section *ini.Section) (string, string) { @@ -159,7 +136,8 @@ func ToAbsUrl(relativeUrl string) string { return AppUrl + relativeUrl } -func loadEnvVariableOverrides() { +func applyEnvVariableOverrides() { + appliedEnvOverrides = make([]string, 0) for _, section := range Cfg.Sections() { for _, key := range section.Keys() { sectionName := strings.ToUpper(strings.Replace(section.Name(), ".", "_", -1)) @@ -168,46 +146,131 @@ func loadEnvVariableOverrides() { envValue := os.Getenv(envKey) if len(envValue) > 0 { - log.Info("Setting: ENV override found: %s", envKey) key.SetValue(envValue) + appliedEnvOverrides = append(appliedEnvOverrides, fmt.Sprintf("%s=%s", envKey, envValue)) } } } } -func evalutePathConfig(value, commandLineVal, def string) string { - if value == "" { - if commandLineVal != "" { - return commandLineVal - } else { - return def +func applyCommandLineDefaultProperties(props map[string]string) { + appliedCommandLineProperties = make([]string, 0) + for _, section := range Cfg.Sections() { + for _, key := range section.Keys() { + keyString := fmt.Sprintf("default.%s.%s", section.Name(), key.Name()) + value, exists := props[keyString] + if exists { + key.SetValue(value) + appliedCommandLineProperties = append(appliedCommandLineProperties, fmt.Sprintf("%s=%s", keyString, value)) + } } } - return value +} + +func applyCommandLineProperties(props map[string]string) { + for _, section := range Cfg.Sections() { + for _, key := range section.Keys() { + keyString := fmt.Sprintf("%s.%s", section.Name(), key.Name()) + value, exists := props[keyString] + if exists { + key.SetValue(value) + appliedCommandLineProperties = append(appliedCommandLineProperties, fmt.Sprintf("%s=%s", keyString, value)) + } + } + } +} + +func getCommandLineProperties(args []string) map[string]string { + props := make(map[string]string) + + for _, arg := range args { + if !strings.HasPrefix(arg, "cfg:") { + continue + } + + trimmed := strings.TrimPrefix(arg, "cfg:") + parts := strings.Split(trimmed, "=") + if len(parts) != 2 { + log.Fatal(3, "Invalid command line argument", arg) + return nil + } + + props[parts[0]] = parts[1] + } + return props +} + +func makeAbsolute(path string, root string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(root, path) +} + +func evalEnvVarExpression(value string) string { + regex := regexp.MustCompile(`\${(\w+)}`) + return regex.ReplaceAllStringFunc(value, func(envVar string) string { + envVar = strings.TrimPrefix(envVar, "${") + envVar = strings.TrimSuffix(envVar, "}") + envValue := os.Getenv(envVar) + return envValue + }) +} + +func evalConfigValues() { + for _, section := range Cfg.Sections() { + for _, key := range section.Keys() { + key.SetValue(evalEnvVarExpression(key.Value())) + } + } +} + +func loadConfiguration(args *CommandLineArgs) { + var err error + + args.Config = evalEnvVarExpression(args.Config) + + // load config defaults + defaultConfigFile := path.Join(HomePath, "conf/defaults.ini") + configFiles = append(configFiles, defaultConfigFile) + + Cfg, err = ini.Load(defaultConfigFile) + Cfg.BlockMode = true + + if err != nil { + log.Fatal(3, "Failed to parse defaults.ini, %v", err) + } + + // command line props + commandLineProps := getCommandLineProperties(args.Args) + + // load default overrides + applyCommandLineDefaultProperties(commandLineProps) + + // load specified config file + if args.Config != "" { + err = Cfg.Append(args.Config) + if err != nil { + log.Fatal(3, "Failed to parse %v, %v", args.Config, err) + } + configFiles = append(configFiles, args.Config) + appliedCommandLineProperties = append(appliedCommandLineProperties, "config="+args.Config) + } + + // apply environment overrides + applyEnvVariableOverrides() + + // apply command line overrides + applyCommandLineProperties(commandLineProps) + + // evaluate config values containing environment variables + evalConfigValues() } func NewConfigContext(args *CommandLineArgs) { - findConfigFiles(args.Config) + loadConfiguration(args) - var err error - - for i, file := range configFiles { - if i == 0 { - Cfg, err = ini.Load(configFiles[i]) - Cfg.BlockMode = false - } else { - err = Cfg.Append(configFiles[i]) - } - - if err != nil { - log.Fatal(4, "Fail to parse config file: %v, error: %v", file, err) - } - } - - loadEnvVariableOverrides() - - DataPath = Cfg.Section("").Key("data_path").String() - DataPath = evalutePathConfig(DataPath, args.DefaultDataPath, filepath.Join(WorkPath, "data")) + DataPath = makeAbsolute(Cfg.Section("paths").Key("data").String(), HomePath) initLogging(args) @@ -228,7 +291,7 @@ func NewConfigContext(args *CommandLineArgs) { HttpAddr = server.Key("http_addr").MustString("0.0.0.0") HttpPort = server.Key("http_port").MustString("3000") - StaticRootPath = server.Key("static_root_path").MustString(path.Join(WorkPath, "public")) + StaticRootPath = server.Key("static_root_path").MustString(path.Join(HomePath, "public")) RouterLogging = server.Key("router_logging").MustBool(false) EnableGzip = server.Key("enable_gzip").MustBool(false) @@ -254,7 +317,7 @@ func NewConfigContext(args *CommandLineArgs) { // PhantomJS rendering ImagesDir = filepath.Join(DataPath, "png") - PhantomDir = filepath.Join(WorkPath, "vendor/phantomjs") + PhantomDir = filepath.Join(HomePath, "vendor/phantomjs") analytics := Cfg.Section("analytics") ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true) @@ -276,9 +339,7 @@ func readSessionConfig() { SessionOptions.IDLength = 16 if SessionOptions.Provider == "file" { - if !filepath.IsAbs(SessionOptions.ProviderConfig) { - SessionOptions.ProviderConfig = filepath.Join(DataPath, SessionOptions.ProviderConfig) - } + SessionOptions.ProviderConfig = makeAbsolute(SessionOptions.ProviderConfig, DataPath) os.MkdirAll(path.Dir(SessionOptions.ProviderConfig), os.ModePerm) } @@ -299,14 +360,7 @@ var logLevels = map[string]string{ func initLogging(args *CommandLineArgs) { // Get and check log mode. LogModes = strings.Split(Cfg.Section("log").Key("mode").MustString("console"), ",") - LogRootPath = Cfg.Section("log").Key("root_path").String() - if LogRootPath == "" { - if args.DefaultLogPath != "" { - LogRootPath = args.DefaultLogPath - } else { - LogRootPath = filepath.Join(DataPath, "log") - } - } + LogsPath = makeAbsolute(Cfg.Section("paths").Key("logs").String(), HomePath) LogConfigs = make([]string, len(LogModes)) for i, mode := range LogModes { @@ -329,7 +383,7 @@ func initLogging(args *CommandLineArgs) { case "console": LogConfigs[i] = fmt.Sprintf(`{"level":%s}`, level) case "file": - logPath := sec.Key("file_name").MustString(path.Join(LogRootPath, "grafana.log")) + logPath := sec.Key("file_name").MustString(path.Join(LogsPath, "grafana.log")) os.MkdirAll(path.Dir(logPath), os.ModePerm) LogConfigs[i] = fmt.Sprintf( `{"level":%s,"filename":"%s","rotate":%v,"maxlines":%d,"maxsize":%d,"daily":%v,"maxdays":%d}`, level, @@ -362,8 +416,33 @@ func initLogging(args *CommandLineArgs) { } } -func LogLoadedConfigFiles() { - for _, file := range configFiles { - log.Info("Config: Loaded from %s", file) +func LogConfigurationInfo() { + var text bytes.Buffer + text.WriteString("Configuration Info\n") + + text.WriteString("Config files:\n") + for i, file := range configFiles { + text.WriteString(fmt.Sprintf(" [%d]: %s\n", i, file)) } + + if len(appliedCommandLineProperties) > 0 { + text.WriteString("Command lines overrides:\n") + for i, prop := range appliedCommandLineProperties { + text.WriteString(fmt.Sprintf(" [%d]: %s\n", i, prop)) + } + } + + if len(appliedEnvOverrides) > 0 { + text.WriteString("\tEnvironment variables used:\n") + for i, prop := range appliedCommandLineProperties { + text.WriteString(fmt.Sprintf(" [%d]: %s\n", i, prop)) + } + } + + text.WriteString("Paths:\n") + text.WriteString(fmt.Sprintf(" home: %s\n", HomePath)) + text.WriteString(fmt.Sprintf(" data: %s\n", DataPath)) + text.WriteString(fmt.Sprintf(" logs: %s\n", LogsPath)) + + log.Info(text.String()) } diff --git a/pkg/setting/setting_test.go b/pkg/setting/setting_test.go index 253eb94f123..ad97939ca1b 100644 --- a/pkg/setting/setting_test.go +++ b/pkg/setting/setting_test.go @@ -10,12 +10,12 @@ import ( func TestLoadingSettings(t *testing.T) { - WorkDir, _ = filepath.Abs("../../") + HomePath, _ = filepath.Abs("../../") Convey("Testing loading settings from ini file", t, func() { Convey("Given the default ini files", func() { - NewConfigContext("") + NewConfigContext(&CommandLineArgs{}) So(AppName, ShouldEqual, "Grafana") So(AdminUser, ShouldEqual, "admin") @@ -23,9 +23,63 @@ func TestLoadingSettings(t *testing.T) { Convey("Should be able to override via environment variables", func() { os.Setenv("GF_SECURITY_ADMIN_USER", "superduper") - NewConfigContext("") + NewConfigContext(&CommandLineArgs{}) So(AdminUser, ShouldEqual, "superduper") + So(DataPath, ShouldEqual, filepath.Join(HomePath, "data")) + So(LogsPath, ShouldEqual, filepath.Join(DataPath, "log")) + }) + + Convey("Should get property map from command line args array", func() { + props := getCommandLineProperties([]string{"cfg:test=value", "cfg:map.test=1"}) + + So(len(props), ShouldEqual, 2) + So(props["test"], ShouldEqual, "value") + So(props["map.test"], ShouldEqual, "1") + }) + + Convey("Should be able to override via command line", func() { + NewConfigContext(&CommandLineArgs{ + Args: []string{"cfg:paths.data=/tmp/data", "cfg:paths.logs=/tmp/logs"}, + }) + + So(DataPath, ShouldEqual, "/tmp/data") + So(LogsPath, ShouldEqual, "/tmp/logs") + }) + + Convey("Should be able to override defaults via command line", func() { + NewConfigContext(&CommandLineArgs{ + Args: []string{"cfg:default.paths.data=/tmp/data"}, + }) + + So(DataPath, ShouldEqual, "/tmp/data") + }) + + Convey("Defaults can be overriden in specified config file", func() { + NewConfigContext(&CommandLineArgs{ + Args: []string{"cfg:default.paths.data=/tmp/data"}, + Config: filepath.Join(HomePath, "tests/config-files/override.ini"), + }) + + So(DataPath, ShouldEqual, "/tmp/override") + }) + + Convey("Command line overrides specified config file", func() { + NewConfigContext(&CommandLineArgs{ + Args: []string{"cfg:paths.data=/tmp/data"}, + Config: filepath.Join(HomePath, "tests/config-files/override.ini"), + }) + + So(DataPath, ShouldEqual, "/tmp/data") + }) + + Convey("Can use environment variables in config values", func() { + os.Setenv("GF_DATA_PATH", "/tmp/env_override") + NewConfigContext(&CommandLineArgs{ + Args: []string{"cfg:paths.data=${GF_DATA_PATH}"}, + }) + + So(DataPath, ShouldEqual, "/tmp/env_override") }) }) diff --git a/tasks/build_task.js b/tasks/build_task.js index 7a3627afd92..6db49cb1ef3 100644 --- a/tasks/build_task.js +++ b/tasks/build_task.js @@ -55,7 +55,7 @@ module.exports = function(grunt) { grunt.config('copy.backend_bin', { cwd: 'bin', expand: true, - src: ['grafana'], + src: ['grafana-server'], options: { mode: true}, dest: '<%= tempDir %>/bin/' }); diff --git a/tests/config-files/override.ini b/tests/config-files/override.ini new file mode 100644 index 00000000000..4bcb5574973 --- /dev/null +++ b/tests/config-files/override.ini @@ -0,0 +1,2 @@ +[paths] +data = /tmp/override