Merge branch 'master' into feature/add_or_compose_decode_hook_func

Author: mitchellh

CHANGELOG.md

@@ -1,6 +1,26 @@
+## 1.4.4
+
+* New option `IgnoreUntaggedFields` to ignore decoding to any fields
+  without `mapstructure` (or the configured tag name) set [GH-277]
+* New option `ErrorUnset` which makes it an error if any fields
+  in a target struct are not set by the decoding process. [GH-225]
+* Decoding to slice from array no longer crashes [GH-265]
+* Decode nested struct pointers to map [GH-271]
+* Fix issue where `,squash` was ignored if `Squash` option was set. [GH-280]
+
+## 1.4.3
+
+* Fix cases where `json.Number` didn't decode properly [GH-261]
+
+## 1.4.2
+
+* Custom name matchers to support any sort of casing, formatting, etc. for
+  field names. [GH-250]
+* Fix possible panic in ComposeDecodeHookFunc [GH-251]
+
 ## 1.4.1
 
-* Fix regression where `*time.Time` value would be set to empty and not be sent 
+* Fix regression where `*time.Time` value would be set to empty and not be sent
   to decode hooks properly [GH-232]
 
 ## 1.4.0

decode_hooks.go

@@ -62,7 +62,8 @@ func DecodeHookExec(
 func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc {
 	return func(f reflect.Value, t reflect.Value) (interface{}, error) {
 		var err error
-		var data interface{}
+		data := f.Interface()
+
 		newFrom := f
 		for _, f1 := range fs {
 			data, err = DecodeHookExec(f1, newFrom, t)

decode_hooks_test.go

@@ -172,6 +172,38 @@ func TestOrComposeDecodeHookFunc_err(t *testing.T) {
 	}
 }
 
+func TestComposeDecodeHookFunc_safe_nofuncs(t *testing.T) {
+	f := ComposeDecodeHookFunc()
+	type myStruct2 struct {
+		MyInt int
+	}
+
+	type myStruct1 struct {
+		Blah map[string]myStruct2
+	}
+
+	src := &myStruct1{Blah: map[string]myStruct2{
+		"test": {
+			MyInt: 1,
+		},
+	}}
+
+	dst := &myStruct1{}
+	dConf := &DecoderConfig{
+		Result:      dst,
+		ErrorUnused: true,
+		DecodeHook:  f,
+	}
+	d, err := NewDecoder(dConf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = d.Decode(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestStringToSliceHookFunc(t *testing.T) {
 	f := StringToSliceHookFunc(",")
 

mapstructure.go

@@ -122,7 +122,7 @@
 // field value is zero and a numeric type, the field is empty, and it won't
 // be encoded into the destination type.
 //
-//     type Source {
+//     type Source struct {
 //         Age int `mapstructure:",omitempty"`
 //     }
 //
@@ -192,7 +192,7 @@ type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface
 // source and target types.
 type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error)
 
-// DecodeHookFuncRaw is a DecodeHookFunc which has complete access to both the source and target
+// DecodeHookFuncValue is a DecodeHookFunc which has complete access to both the source and target
 // values.
 type DecodeHookFuncValue func(from reflect.Value, to reflect.Value) (interface{}, error)
 
@@ -215,6 +215,12 @@ type DecoderConfig struct {
 	// (extra keys).
 	ErrorUnused bool
 
+	// If ErrorUnset is true, then it is an error for there to exist
+	// fields in the result that were not set in the decoding process
+	// (extra fields). This only applies to decoding to a struct. This
+	// will affect all nested structs as well.
+	ErrorUnset bool
+
 	// ZeroFields, if set to true, will zero fields before writing them.
 	// For example, a map will be emptied before decoded values are put in
 	// it. If this is false, a map will be merged.
@@ -258,6 +264,15 @@ type DecoderConfig struct {
 	// The tag name that mapstructure reads for field names. This
 	// defaults to "mapstructure"
 	TagName string
+
+	// IgnoreUntaggedFields ignores all struct fields without explicit
+	// TagName, comparable to `mapstructure:"-"` as default behaviour.
+	IgnoreUntaggedFields bool
+
+	// MatchName is the function used to match the map key to the struct
+	// field name or tag. Defaults to `strings.EqualFold`. This can be used
+	// to implement case-sensitive tag values, support snake casing, etc.
+	MatchName func(mapKey, fieldName string) bool
 }
 
 // A Decoder takes a raw interface value and turns it into structured
@@ -279,6 +294,11 @@ type Metadata struct {
 	// Unused is a slice of keys that were found in the raw value but
 	// weren't decoded since there was no matching field in the result interface
 	Unused []string
+
+	// Unset is a slice of field names that were found in the result interface
+	// but weren't set in the decoding process since there was no matching value
+	// in the input
+	Unset []string
 }
 
 // Decode takes an input structure and uses reflection to translate it to
@@ -370,12 +390,20 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) {
 		if config.Metadata.Unused == nil {
 			config.Metadata.Unused = make([]string, 0)
 		}
+
+		if config.Metadata.Unset == nil {
+			config.Metadata.Unset = make([]string, 0)
+		}
 	}
 
 	if config.TagName == "" {
 		config.TagName = "mapstructure"
 	}
 
+	if config.MatchName == nil {
+		config.MatchName = strings.EqualFold
+	}
+
 	result := &Decoder{
 		config: config,
 	}
@@ -675,16 +703,12 @@ func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) e
 		}
 	case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number":
 		jn := data.(json.Number)
-		i, err := jn.Int64()
+		i, err := strconv.ParseUint(string(jn), 0, 64)
 		if err != nil {
 			return fmt.Errorf(
 				"error decoding json.Number into %s: %s", name, err)
 		}
-		if i < 0 && !d.config.WeaklyTypedInput {
-			return fmt.Errorf("cannot parse '%s', %d overflows uint",
-				name, i)
-		}
-		val.SetUint(uint64(i))
+		val.SetUint(i)
 	default:
 		return fmt.Errorf(
 			"'%s' expected type '%s', got unconvertible type '%s', value: '%v'",
@@ -901,9 +925,15 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re
 		tagValue := f.Tag.Get(d.config.TagName)
 		keyName := f.Name
 
+		if tagValue == "" && d.config.IgnoreUntaggedFields {
+			continue
+		}
+
 		// If Squash is set in the config, we squash the field down.
 		squash := d.config.Squash && v.Kind() == reflect.Struct && f.Anonymous
 
+		v = dereferencePtrToStructIfNeeded(v, d.config.TagName)
+
 		// Determine the name of the key in the map
 		if index := strings.Index(tagValue, ","); index != -1 {
 			if tagValue[:index] == "-" {
@@ -915,7 +945,7 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re
 			}
 
 			// If "squash" is specified in the tag, we squash the field down.
-			squash = !squash && strings.Index(tagValue[index+1:], "squash") != -1
+			squash = squash || strings.Index(tagValue[index+1:], "squash") != -1
 			if squash {
 				// When squashing, the embedded type can be a pointer to a struct.
 				if v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.Struct {
@@ -1083,7 +1113,7 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value)
 	}
 
 	// If the input value is nil, then don't allocate since empty != nil
-	if dataVal.IsNil() {
+	if dataValKind != reflect.Array && dataVal.IsNil() {
 		return nil
 	}
 
@@ -1245,6 +1275,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
 		dataValKeysUnused[dataValKey.Interface()] = struct{}{}
 	}
 
+	targetValKeysUnused := make(map[interface{}]struct{})
 	errors := make([]string, 0)
 
 	// This slice will keep track of all the structs we'll be decoding.
@@ -1340,7 +1371,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
 					continue
 				}
 
-				if strings.EqualFold(mK, fieldName) {
+				if d.config.MatchName(mK, fieldName) {
 					rawMapKey = dataValKey
 					rawMapVal = dataVal.MapIndex(dataValKey)
 					break
@@ -1349,7 +1380,8 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
 
 			if !rawMapVal.IsValid() {
 				// There was no matching key in the map for the value in
-				// the struct. Just ignore.
+				// the struct. Remember it for potential errors and metadata.
+				targetValKeysUnused[fieldName] = struct{}{}
 				continue
 			}
 		}
@@ -1409,6 +1441,17 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
 		errors = appendErrors(errors, err)
 	}
 
+	if d.config.ErrorUnset && len(targetValKeysUnused) > 0 {
+		keys := make([]string, 0, len(targetValKeysUnused))
+		for rawKey := range targetValKeysUnused {
+			keys = append(keys, rawKey.(string))
+		}
+		sort.Strings(keys)
+
+		err := fmt.Errorf("'%s' has unset fields: %s", name, strings.Join(keys, ", "))
+		errors = appendErrors(errors, err)
+	}
+
 	if len(errors) > 0 {
 		return &Error{errors}
 	}
@@ -1423,6 +1466,14 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e
 
 			d.config.Metadata.Unused = append(d.config.Metadata.Unused, key)
 		}
+		for rawKey := range targetValKeysUnused {
+			key := rawKey.(string)
+			if name != "" {
+				key = name + "." + key
+			}
+
+			d.config.Metadata.Unset = append(d.config.Metadata.Unset, key)
+		}
 	}
 
 	return nil
@@ -1460,3 +1511,28 @@ func getKind(val reflect.Value) reflect.Kind {
 		return kind
 	}
 }
+
+func isStructTypeConvertibleToMap(typ reflect.Type, checkMapstructureTags bool, tagName string) bool {
+	for i := 0; i < typ.NumField(); i++ {
+		f := typ.Field(i)
+		if f.PkgPath == "" && !checkMapstructureTags { // check for unexported fields
+			return true
+		}
+		if checkMapstructureTags && f.Tag.Get(tagName) != "" { // check for mapstructure tags inside
+			return true
+		}
+	}
+	return false
+}
+
+func dereferencePtrToStructIfNeeded(v reflect.Value, tagName string) reflect.Value {
+	if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
+		return v
+	}
+	deref := v.Elem()
+	derefT := deref.Type()
+	if isStructTypeConvertibleToMap(derefT, true, tagName) {
+		return deref
+	}
+	return v
+}

mapstructure_bugs_test.go

@@ -479,7 +479,7 @@ func TestDecodeBadDataTypeInSlice(t *testing.T) {
 
 // #202 Ensure that intermediate maps in the struct -> struct decode process are settable
 // and not just the elements within them.
-func TestDecodeIntermeidateMapsSettable(t *testing.T) {
+func TestDecodeIntermediateMapsSettable(t *testing.T) {
 	type Timestamp struct {
 		Seconds int64
 		Nanos   int32