Provisioning: Interpolate env vars in provisioning files (#16499)
* Add value types with custom unmarshalling logic * Add env support for notifications config * Use env vars in json data tests for values * Add some more complexities to value tests * Update comment with example usage * Set env directly in the tests, removing patching * Update documentation * Add env var to the file reader tests * Add raw value * Post merge fixes * Add comment
This commit is contained in:
Andrej Ocenas
@@ -0,0 +1,206 @@
|
||||
// A set of value types to use in provisioning. They add custom unmarshaling logic that puts the string values
|
||||
// through os.ExpandEnv.
|
||||
// Usage:
|
||||
// type Data struct {
|
||||
// Field StringValue `yaml:"field"` // Instead of string
|
||||
// }
|
||||
// d := &Data{}
|
||||
// // unmarshal into d
|
||||
// d.Field.Value() // returns the final interpolated value from the yaml file
|
||||
//
|
||||
package values
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type IntValue struct {
|
||||
value int
|
||||
Raw string
|
||||
}
|
||||
|
||||
func (val *IntValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
interpolated, err := getInterpolated(unmarshal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(interpolated.value) == 0 {
|
||||
// To keep the same behaviour as the yaml lib which just does not set the value if it is empty.
|
||||
return nil
|
||||
}
|
||||
val.Raw = interpolated.raw
|
||||
val.value, err = strconv.Atoi(interpolated.value)
|
||||
return errors.Wrap(err, "cannot convert value int")
|
||||
}
|
||||
|
||||
func (val *IntValue) Value() int {
|
||||
return val.value
|
||||
}
|
||||
|
||||
type Int64Value struct {
|
||||
value int64
|
||||
Raw string
|
||||
}
|
||||
|
||||
func (val *Int64Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
interpolated, err := getInterpolated(unmarshal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(interpolated.value) == 0 {
|
||||
// To keep the same behaviour as the yaml lib which just does not set the value if it is empty.
|
||||
return nil
|
||||
}
|
||||
val.Raw = interpolated.raw
|
||||
val.value, err = strconv.ParseInt(interpolated.value, 10, 64)
|
||||
return err
|
||||
}
|
||||
|
||||
func (val *Int64Value) Value() int64 {
|
||||
return val.value
|
||||
}
|
||||
|
||||
type StringValue struct {
|
||||
value string
|
||||
Raw string
|
||||
}
|
||||
|
||||
func (val *StringValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
interpolated, err := getInterpolated(unmarshal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
val.Raw = interpolated.raw
|
||||
val.value = interpolated.value
|
||||
return err
|
||||
}
|
||||
|
||||
func (val *StringValue) Value() string {
|
||||
return val.value
|
||||
}
|
||||
|
||||
type BoolValue struct {
|
||||
value bool
|
||||
Raw string
|
||||
}
|
||||
|
||||
func (val *BoolValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
interpolated, err := getInterpolated(unmarshal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
val.Raw = interpolated.raw
|
||||
val.value, err = strconv.ParseBool(interpolated.value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (val *BoolValue) Value() bool {
|
||||
return val.value
|
||||
}
|
||||
|
||||
type JSONValue struct {
|
||||
value map[string]interface{}
|
||||
Raw map[string]interface{}
|
||||
}
|
||||
|
||||
func (val *JSONValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
unmarshaled := make(map[string]interface{})
|
||||
err := unmarshal(unmarshaled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
val.Raw = unmarshaled
|
||||
interpolated := make(map[string]interface{})
|
||||
for key, val := range unmarshaled {
|
||||
interpolated[key] = tranformInterface(val)
|
||||
}
|
||||
val.value = interpolated
|
||||
return err
|
||||
}
|
||||
|
||||
func (val *JSONValue) Value() map[string]interface{} {
|
||||
return val.value
|
||||
}
|
||||
|
||||
type StringMapValue struct {
|
||||
value map[string]string
|
||||
Raw map[string]string
|
||||
}
|
||||
|
||||
func (val *StringMapValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
unmarshaled := make(map[string]string)
|
||||
err := unmarshal(unmarshaled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
val.Raw = unmarshaled
|
||||
interpolated := make(map[string]string)
|
||||
for key, val := range unmarshaled {
|
||||
interpolated[key] = interpolateValue(val)
|
||||
}
|
||||
val.value = interpolated
|
||||
return err
|
||||
}
|
||||
|
||||
func (val *StringMapValue) Value() map[string]string {
|
||||
return val.value
|
||||
}
|
||||
|
||||
// tranformInterface tries to transform any interface type into proper value with env expansion. It travers maps and
|
||||
// slices and the actual interpolation is done on all simple string values in the structure. It returns a copy of any
|
||||
// map or slice value instead of modifying them in place.
|
||||
func tranformInterface(i interface{}) interface{} {
|
||||
switch reflect.TypeOf(i).Kind() {
|
||||
case reflect.Slice:
|
||||
return transformSlice(i.([]interface{}))
|
||||
case reflect.Map:
|
||||
return transformMap(i.(map[interface{}]interface{}))
|
||||
case reflect.String:
|
||||
return interpolateValue(i.(string))
|
||||
default:
|
||||
// Was int, float or some other value that we do not need to do any transform on.
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
func transformSlice(i []interface{}) interface{} {
|
||||
var transformed []interface{}
|
||||
for _, val := range i {
|
||||
transformed = append(transformed, tranformInterface(val))
|
||||
}
|
||||
return transformed
|
||||
}
|
||||
|
||||
func transformMap(i map[interface{}]interface{}) interface{} {
|
||||
transformed := make(map[interface{}]interface{})
|
||||
for key, val := range i {
|
||||
transformed[key] = tranformInterface(val)
|
||||
}
|
||||
return transformed
|
||||
}
|
||||
|
||||
// interpolateValue returns final value after interpolation. At the moment only env var interpolation is done
|
||||
// here but in the future something like interpolation from file could be also done here.
|
||||
func interpolateValue(val string) string {
|
||||
return os.ExpandEnv(val)
|
||||
}
|
||||
|
||||
type interpolated struct {
|
||||
value string
|
||||
raw string
|
||||
}
|
||||
|
||||
// getInterpolated unmarshals the value as string and runs interpolation on it. It is the responsibility of each
|
||||
// value type to convert this string value to appropriate type.
|
||||
func getInterpolated(unmarshal func(interface{}) error) (*interpolated, error) {
|
||||
var raw string
|
||||
err := unmarshal(&raw)
|
||||
if err != nil {
|
||||
return &interpolated{}, err
|
||||
}
|
||||
value := interpolateValue(raw)
|
||||
return &interpolated{raw: raw, value: value}, nil
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package values
|
||||
|
||||
import (
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"gopkg.in/yaml.v2"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValues(t *testing.T) {
|
||||
Convey("Values", t, func() {
|
||||
os.Setenv("INT", "1")
|
||||
os.Setenv("STRING", "test")
|
||||
os.Setenv("BOOL", "true")
|
||||
|
||||
Convey("IntValue", func() {
|
||||
type Data struct {
|
||||
Val IntValue `yaml:"val"`
|
||||
}
|
||||
d := &Data{}
|
||||
|
||||
Convey("Should unmarshal simple number", func() {
|
||||
unmarshalingTest(`val: 1`, d)
|
||||
So(d.Val.Value(), ShouldEqual, 1)
|
||||
So(d.Val.Raw, ShouldEqual, "1")
|
||||
})
|
||||
|
||||
Convey("Should unmarshal env var", func() {
|
||||
unmarshalingTest(`val: $INT`, d)
|
||||
So(d.Val.Value(), ShouldEqual, 1)
|
||||
So(d.Val.Raw, ShouldEqual, "$INT")
|
||||
})
|
||||
|
||||
Convey("Should ignore empty value", func() {
|
||||
unmarshalingTest(`val: `, d)
|
||||
So(d.Val.Value(), ShouldEqual, 0)
|
||||
So(d.Val.Raw, ShouldEqual, "")
|
||||
})
|
||||
})
|
||||
|
||||
Convey("StringValue", func() {
|
||||
type Data struct {
|
||||
Val StringValue `yaml:"val"`
|
||||
}
|
||||
d := &Data{}
|
||||
|
||||
Convey("Should unmarshal simple string", func() {
|
||||
unmarshalingTest(`val: test`, d)
|
||||
So(d.Val.Value(), ShouldEqual, "test")
|
||||
So(d.Val.Raw, ShouldEqual, "test")
|
||||
})
|
||||
|
||||
Convey("Should unmarshal env var", func() {
|
||||
unmarshalingTest(`val: $STRING`, d)
|
||||
So(d.Val.Value(), ShouldEqual, "test")
|
||||
So(d.Val.Raw, ShouldEqual, "$STRING")
|
||||
})
|
||||
|
||||
Convey("Should ignore empty value", func() {
|
||||
unmarshalingTest(`val: `, d)
|
||||
So(d.Val.Value(), ShouldEqual, "")
|
||||
So(d.Val.Raw, ShouldEqual, "")
|
||||
})
|
||||
})
|
||||
|
||||
Convey("BoolValue", func() {
|
||||
type Data struct {
|
||||
Val BoolValue `yaml:"val"`
|
||||
}
|
||||
d := &Data{}
|
||||
|
||||
Convey("Should unmarshal bool value", func() {
|
||||
unmarshalingTest(`val: true`, d)
|
||||
So(d.Val.Value(), ShouldBeTrue)
|
||||
So(d.Val.Raw, ShouldEqual, "true")
|
||||
})
|
||||
|
||||
Convey("Should unmarshal explicit string", func() {
|
||||
unmarshalingTest(`val: "true"`, d)
|
||||
So(d.Val.Value(), ShouldBeTrue)
|
||||
So(d.Val.Raw, ShouldEqual, "true")
|
||||
})
|
||||
|
||||
Convey("Should unmarshal env var", func() {
|
||||
unmarshalingTest(`val: $BOOL`, d)
|
||||
So(d.Val.Value(), ShouldBeTrue)
|
||||
So(d.Val.Raw, ShouldEqual, "$BOOL")
|
||||
})
|
||||
|
||||
Convey("Should ignore empty value", func() {
|
||||
unmarshalingTest(`val: `, d)
|
||||
So(d.Val.Value(), ShouldBeFalse)
|
||||
So(d.Val.Raw, ShouldEqual, "")
|
||||
})
|
||||
})
|
||||
|
||||
Convey("JSONValue", func() {
|
||||
|
||||
type Data struct {
|
||||
Val JSONValue `yaml:"val"`
|
||||
}
|
||||
d := &Data{}
|
||||
|
||||
Convey("Should unmarshal variable nesting", func() {
|
||||
doc := `
|
||||
val:
|
||||
one: 1
|
||||
two: $STRING
|
||||
three:
|
||||
- 1
|
||||
- two
|
||||
- three:
|
||||
inside: $STRING
|
||||
four:
|
||||
nested:
|
||||
onemore: $INT
|
||||
multiline: >
|
||||
Some text with $STRING
|
||||
anchor: &label $INT
|
||||
anchored: *label
|
||||
`
|
||||
unmarshalingTest(doc, d)
|
||||
|
||||
type anyMap = map[interface{}]interface{}
|
||||
So(d.Val.Value(), ShouldResemble, map[string]interface{}{
|
||||
"one": 1,
|
||||
"two": "test",
|
||||
"three": []interface{}{
|
||||
1, "two", anyMap{
|
||||
"three": anyMap{
|
||||
"inside": "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
"four": anyMap{
|
||||
"nested": anyMap{
|
||||
"onemore": "1",
|
||||
},
|
||||
},
|
||||
"multiline": "Some text with test\n",
|
||||
"anchor": "1",
|
||||
"anchored": "1",
|
||||
})
|
||||
|
||||
So(d.Val.Raw, ShouldResemble, map[string]interface{}{
|
||||
"one": 1,
|
||||
"two": "$STRING",
|
||||
"three": []interface{}{
|
||||
1, "two", anyMap{
|
||||
"three": anyMap{
|
||||
"inside": "$STRING",
|
||||
},
|
||||
},
|
||||
},
|
||||
"four": anyMap{
|
||||
"nested": anyMap{
|
||||
"onemore": "$INT",
|
||||
},
|
||||
},
|
||||
"multiline": "Some text with $STRING\n",
|
||||
"anchor": "$INT",
|
||||
"anchored": "$INT",
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Convey("StringMapValue", func() {
|
||||
type Data struct {
|
||||
Val StringMapValue `yaml:"val"`
|
||||
}
|
||||
d := &Data{}
|
||||
|
||||
Convey("Should unmarshal mapping", func() {
|
||||
doc := `
|
||||
val:
|
||||
one: 1
|
||||
two: "test string"
|
||||
three: $STRING
|
||||
four: true
|
||||
`
|
||||
unmarshalingTest(doc, d)
|
||||
So(d.Val.Value(), ShouldResemble, map[string]string{
|
||||
"one": "1",
|
||||
"two": "test string",
|
||||
"three": "test",
|
||||
"four": "true",
|
||||
})
|
||||
|
||||
So(d.Val.Raw, ShouldResemble, map[string]string{
|
||||
"one": "1",
|
||||
"two": "test string",
|
||||
"three": "$STRING",
|
||||
"four": "true",
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
Reset(func() {
|
||||
os.Unsetenv("INT")
|
||||
os.Unsetenv("STRING")
|
||||
os.Unsetenv("BOOL")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func unmarshalingTest(document string, out interface{}) {
|
||||
err := yaml.Unmarshal([]byte(document), out)
|
||||
So(err, ShouldBeNil)
|
||||
}
|
||||
Reference in New Issue
Block a user