From e8b5f2330df2a37bfb2541e5bd3dbbc46a8fb687 Mon Sep 17 00:00:00 2001 From: Emil Tullstedt Date: Wed, 10 Jun 2020 14:58:42 +0200 Subject: [PATCH] Settings: Expand variables in configuration (#25075) --- pkg/services/ldap/settings.go | 5 +- pkg/setting/expanders.go | 147 ++++++++++++++++++++++++++++++++++ pkg/setting/expanders_test.go | 96 ++++++++++++++++++++++ pkg/setting/setting.go | 30 +------ 4 files changed, 251 insertions(+), 27 deletions(-) create mode 100644 pkg/setting/expanders.go create mode 100644 pkg/setting/expanders_test.go diff --git a/pkg/services/ldap/settings.go b/pkg/services/ldap/settings.go index 98116c1dc5f..8ecd93674da 100644 --- a/pkg/services/ldap/settings.go +++ b/pkg/services/ldap/settings.go @@ -125,7 +125,10 @@ func readConfig(configFile string) (*Config, error) { } // interpolate full toml string (it can contain ENV variables) - stringContent := setting.EvalEnvVarExpression(string(fileBytes)) + stringContent, err := setting.ExpandVar(string(fileBytes)) + if err != nil { + return nil, errutil.Wrap("Failed to expand variables", err) + } _, err = toml.Decode(stringContent, result) if err != nil { diff --git a/pkg/setting/expanders.go b/pkg/setting/expanders.go new file mode 100644 index 00000000000..021722ba998 --- /dev/null +++ b/pkg/setting/expanders.go @@ -0,0 +1,147 @@ +package setting + +import ( + "fmt" + "io/ioutil" + "os" + "regexp" + "sort" + "strings" + + "gopkg.in/ini.v1" +) + +type Expander interface { + SetupExpander(file *ini.File) error + Expand(string) (string, error) +} + +type registeredExpander struct { + name string + priority int64 + expander Expander +} + +var expanders = []registeredExpander{ + { + name: "env", + priority: -10, + expander: envExpander{}, + }, + { + name: "file", + priority: -5, + expander: fileExpander{}, + }, +} + +func AddExpander(name string, priority int64, e Expander) { + expanders = append(expanders, registeredExpander{ + name: name, + priority: priority, + expander: e, + }) +} + +var regex = regexp.MustCompile(`\$(|__\w+){([^}]+)}`) + +func expandConfig(file *ini.File) error { + sort.Slice(expanders, func(i, j int) bool { + return expanders[i].priority < expanders[j].priority + }) + + for _, expander := range expanders { + err := expander.expander.SetupExpander(file) + if err != nil { + return fmt.Errorf("got error during initilazation of expander '%s': %w", expander.name, err) + } + + for _, section := range file.Sections() { + for _, key := range section.Keys() { + updated, err := applyExpander(key.Value(), expander) + if err != nil { + return fmt.Errorf("got error while expanding %s.%s with expander '%s': %w", + section.Name(), + key.Name(), + expander.name, + err) + } + + key.SetValue(updated) + } + } + } + return nil +} + +func ExpandVar(s string) (string, error) { + for _, expander := range expanders { + var err error + s, err = applyExpander(s, expander) + if err != nil { + return "", fmt.Errorf("got error while expanding expander %s: %w", expander.name, err) + } + } + return s, nil +} + +func applyExpander(s string, e registeredExpander) (string, error) { + matches := regex.FindAllStringSubmatch(s, -1) + + for _, match := range matches { + if len(match) < 3 { + return "", fmt.Errorf("regex error, got %d results back for match, expected 3", len(match)) + } + + _, isEnv := e.expander.(envExpander) + if match[1] == "__"+e.name || (match[1] == "" && isEnv) { + updated, err := e.expander.Expand(match[2]) + if err != nil { + return "", err + } + + s = strings.Replace(s, match[0], updated, 1) + } + } + + return s, nil +} + +type envExpander struct { +} + +func (e envExpander) SetupExpander(file *ini.File) error { + return nil +} + +func (e envExpander) Expand(s string) (string, error) { + envValue := os.Getenv(s) + + // if env variable is hostname and it is empty use os.Hostname as default + if s == "HOSTNAME" && envValue == "" { + return os.Hostname() + } + + return os.Getenv(s), nil +} + +type fileExpander struct { +} + +func (e fileExpander) SetupExpander(file *ini.File) error { + return nil +} + +func (e fileExpander) Expand(s string) (string, error) { + _, err := os.Stat(s) + if err != nil { + return "", err + } + + f, err := ioutil.ReadFile(s) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(f)), nil +} diff --git a/pkg/setting/expanders_test.go b/pkg/setting/expanders_test.go new file mode 100644 index 00000000000..645a0b02919 --- /dev/null +++ b/pkg/setting/expanders_test.go @@ -0,0 +1,96 @@ +package setting + +import ( + "errors" + "fmt" + "io/ioutil" + "math/rand" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" +) + +func TestExpandVar_EnvSuccessful(t *testing.T) { + const key = "GF_TEST_SETTING_EXPANDER_ENV" + const expected = "aurora borealis" + os.Setenv(key, expected) + + // expanded format + { + got, err := ExpandVar(fmt.Sprintf("$__env{%s}", key)) + assert.NoError(t, err) + assert.Equal(t, expected, got) + } + + // short format + { + got, err := ExpandVar(fmt.Sprintf("${%s}", key)) + assert.NoError(t, err) + assert.Equal(t, expected, got) + } +} + +func TestExpandVar_FileSuccessful(t *testing.T) { + f, err := ioutil.TempFile(os.TempDir(), "file expansion *") + require.NoError(t, err) + file := f.Name() + + defer func() { + os.Remove(file) + }() + + _, err = f.WriteString("hello, world") + require.NoError(t, err) + f.Close() + + got, err := ExpandVar(fmt.Sprintf("$__file{%s}", file)) + assert.NoError(t, err) + assert.Equal(t, "hello, world", got) +} + +func TestExpandVar_FileDoesNotExist(t *testing.T) { + got, err := ExpandVar( + fmt.Sprintf("$__file{%s%sthisisnotarealfile_%d}", + os.TempDir(), + string(os.PathSeparator), + rand.Int63()), + ) + assert.Error(t, err) + assert.True(t, errors.Is(err, os.ErrNotExist)) + assert.Empty(t, got) +} + +func TestExpanderRegex(t *testing.T) { + tests := map[string][][]string{ + // we should not expand variables where there are none + "smoketest": {}, + "Pa$$word{0}": {}, + "$_almost{but not quite a variable}": {}, + // args are required + "$__file{}": {}, + + // base cases + "${ENV}": {{"", "ENV"}}, + "$__env{ENV}": {{"__env", "ENV"}}, + "$__file{/dev/null}": {{"__file", "/dev/null"}}, + "$__vault{item}": {{"__vault", "item"}}, + // contains a space in the argument + "$__file{C:\\Program Files\\grafana\\something}": {{"__file", "C:\\Program Files\\grafana\\something"}}, + + // complex cases + "get variable from $__env{ENV}ironment": {{"__env", "ENV"}}, + "$__env{VAR1} $__file{/var/lib/grafana/secrets/var1}": {{"__env", "VAR1"}, {"__file", "/var/lib/grafana/secrets/var1"}}, + "$__env{$__file{this is invalid}}": {{"__env", "$__file{this is invalid"}}, + } + + for input, expected := range tests { + output := regex.FindAllStringSubmatch(input, -1) + require.Len(t, output, len(expected)) + for i, variable := range output { + assert.Equal(t, expected[i], variable[1:]) + } + } +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index edcf599ab24..b8d568894a7 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -12,7 +12,6 @@ import ( "os" "path" "path/filepath" - "regexp" "runtime" "strings" "time" @@ -440,30 +439,6 @@ func makeAbsolute(path string, root string) string { return filepath.Join(root, path) } -func EvalEnvVarExpression(value string) string { - regex := regexp.MustCompile(`\${(\w+)}`) - return regex.ReplaceAllStringFunc(value, func(envVar string) string { - envVar = strings.TrimPrefix(envVar, "${") - envVar = strings.TrimSuffix(envVar, "}") - envValue := os.Getenv(envVar) - - // if env variable is hostname and it is empty use os.Hostname as default - if envVar == "HOSTNAME" && envValue == "" { - envValue, _ = os.Hostname() - } - - return envValue - }) -} - -func evalConfigValues(file *ini.File) { - for _, section := range file.Sections() { - for _, key := range section.Keys() { - key.SetValue(EvalEnvVarExpression(key.Value())) - } - } -} - func loadSpecifiedConfigFile(configFile string, masterFile *ini.File) error { if configFile == "" { configFile = filepath.Join(HomePath, CustomInitPath) @@ -550,7 +525,10 @@ func (cfg *Cfg) loadConfiguration(args *CommandLineArgs) (*ini.File, error) { applyCommandLineProperties(commandLineProps, parsedFile) // evaluate config values containing environment variables - evalConfigValues(parsedFile) + err = expandConfig(parsedFile) + if err != nil { + return nil, err + } // update data path and logging config dataPath, err := valueAsString(parsedFile.Section("paths"), "data", "")