Merge pull request #79 from aeneasr/fix-memory-keyparts

Reduce memory consumption for populateKeyParts

Author: knadh

koanf.go

@@ -6,7 +6,6 @@ import (
 	"github.com/mitchellh/copystructure"
 	"sort"
 	"strconv"
-	"strings"
 
 	"github.com/knadh/koanf/maps"
 	"github.com/knadh/koanf/providers/confmap"
@@ -69,7 +68,7 @@ type UnmarshalConf struct {
 // or a / for `parent/child/key`.
 func New(delim string) *Koanf {
 	return NewWithConf(Conf{
-		Delim: delim,
+		Delim:       delim,
 		StrictMerge: false,
 	})
 }
@@ -80,7 +79,7 @@ func NewWithConf(conf Conf) *Koanf {
 		confMap:     make(map[string]interface{}),
 		confMapFlat: make(map[string]interface{}),
 		keyMap:      make(KeyMap),
-		conf: conf,
+		conf:        conf,
 	}
 }
 
@@ -385,7 +384,7 @@ func (ko *Koanf) MapKeys(path string) []string {
 	return out
 }
 
-func (ko *Koanf) merge(c map[string]interface{}) error{
+func (ko *Koanf) merge(c map[string]interface{}) error {
 	maps.IntfaceKeysToStrings(c)
 	if ko.conf.StrictMerge {
 		if err := maps.MergeStrict(c, ko.confMap); err != nil {
@@ -469,10 +468,19 @@ func toBool(v interface{}) (bool, error) {
 // traversal paths. For instance, `parent.child.key` generates
 // `parent`, and `parent.child`.
 func populateKeyParts(m KeyMap, delim string) KeyMap {
-	out := make(KeyMap)
+	out := make(KeyMap, len(m)) // The size of the result is at very least same to KeyMap
 	for _, parts := range m {
+		// parts is a slice of [parent, child, key]
+		var nk string
+
 		for i := range parts {
-			nk := strings.Join(parts[0:i+1], delim)
+			if i == 0 {
+				// On first iteration only use first part
+				nk = parts[i]
+			} else {
+				// If nk already contains a part (e.g. `parent`) append delim + `child`
+				nk += delim + parts[i]
+			}
 			if _, ok := out[nk]; ok {
 				continue
 			}

koanf_test.go

@@ -279,6 +279,21 @@ func init() {
 	}
 }
 
+func BenchmarkLoadFile(b *testing.B) {
+	k := koanf.New(delim)
+
+	// Don't use TOML here because it distorts memory benchmarks due to high memory use
+	providers := []*file.File{file.Provider(mockJSON), file.Provider(mockYAML)}
+	parsers := []koanf.Parser{json.Parser(), yaml.Parser()}
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		if err := k.Load(providers[n%2], parsers[n%2]); err != nil {
+			b.Fatalf("Unexpected error: %+v", k)
+		}
+	}
+}
+
 func TestLoadFile(t *testing.T) {
 	// Load a non-existent file.
 	_, err := file.Provider("does-not-exist").ReadBytes()
@@ -996,7 +1011,7 @@ func TestGetTypes(t *testing.T) {
 		assert.Equal([]string{"red", "blue", "orange"}, c.koanf.Strings("orphan"))
 
 		assert.Equal(map[string]string{"key1": "val1", "key2": "val2", "key3": "val3"}, c.koanf.StringMap("parent1.strmap"))
-		assert.Equal(map[string][]string{"key1": []string{"val1", "val2", "val3"}, "key2": []string{"val4", "val5"}}, c.koanf.StringsMap("parent1.strsmap"))
+		assert.Equal(map[string][]string{"key1": {"val1", "val2", "val3"}, "key2": {"val4", "val5"}}, c.koanf.StringsMap("parent1.strsmap"))
 		assert.Equal(map[string]string{}, c.koanf.StringMap("xxxx"))
 		assert.Equal(map[string]string{}, c.koanf.StringMap("parent1.intmap"))
 
@@ -1084,7 +1099,7 @@ func TestMustGetTypes(t *testing.T) {
 		assert.Panics(func() { c.koanf.MustStringMap("xxxx") })
 		assert.Panics(func() { c.koanf.MustStringMap("parent1.intmap") })
 		assert.Equal(map[string]string{"key1": "val1", "key2": "val2", "key3": "val3"}, c.koanf.MustStringMap("parent1.strmap"))
-		assert.Equal(map[string][]string{"key1": []string{"val1", "val2", "val3"}, "key2": []string{"val4", "val5"}}, c.koanf.MustStringsMap("parent1.strsmap"))
+		assert.Equal(map[string][]string{"key1": {"val1", "val2", "val3"}, "key2": {"val4", "val5"}}, c.koanf.MustStringsMap("parent1.strsmap"))
 
 		// // Bools.
 		assert.Panics(func() { c.koanf.MustBools("xxxx") })