feat(config): ipfs add and Import options for controling UnixFS DAG Width

config/import.go

@@ -1,11 +1,18 @@
 package config
 
+import (
+	"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
+	"github.com/ipfs/boxo/ipld/unixfs/io"
+)
+
 const (
 	DefaultCidVersion      = 0
 	DefaultUnixFSRawLeaves = false
 	DefaultUnixFSChunker   = "size-262144"
 	DefaultHashFunction    = "sha2-256"
 
+	DefaultUnixFSHAMTDirectorySizeThreshold = "256KiB" // https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L26
+
 	// DefaultBatchMaxNodes controls the maximum number of nodes in a
 	// write-batch. The total size of the batch is limited by
 	// BatchMaxnodes and BatchMaxSize.
@@ -14,15 +21,26 @@ const (
 	// write-batch. The total size of the batch is limited by
 	// BatchMaxnodes and BatchMaxSize.
 	DefaultBatchMaxSize = 100 << 20 // 20MiB
+
+)
+
+var (
+	DefaultUnixFSFileMaxLinks           = int64(helpers.DefaultLinksPerBlock)
+	DefaultUnixFSDirectoryMaxLinks      = int64(0)
+	DefaultUnixFSHAMTDirectoryMaxFanout = int64(io.DefaultShardWidth)
 )
 
 // Import configures the default options for ingesting data. This affects commands
 // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
 type Import struct {
-	CidVersion      OptionalInteger
-	UnixFSRawLeaves Flag
-	UnixFSChunker   OptionalString
-	HashFunction    OptionalString
-	BatchMaxNodes   OptionalInteger
-	BatchMaxSize    OptionalInteger
+	CidVersion                       OptionalInteger
+	UnixFSRawLeaves                  Flag
+	UnixFSChunker                    OptionalString
+	HashFunction                     OptionalString
+	UnixFSFileMaxLinks               OptionalInteger
+	UnixFSDirectoryMaxLinks          OptionalInteger
+	UnixFSHAMTDirectoryMaxFanout     OptionalInteger
+	UnixFSHAMTDirectorySizeThreshold OptionalString
+	BatchMaxNodes                    OptionalInteger
+	BatchMaxSize                     OptionalInteger
 }

config/internal.go

@@ -3,7 +3,7 @@ package config
 type Internal struct {
 	// All marked as omitempty since we are expecting to make changes to all subcomponents of Internal
 	Bitswap                     *InternalBitswap  `json:",omitempty"`
-	UnixFSShardingSizeThreshold *OptionalString   `json:",omitempty"`
+	UnixFSShardingSizeThreshold *OptionalString   `json:",omitempty"` // moved to Import.UnixFSHAMTDirectorySizeThreshold
 	Libp2pForceReachability     *OptionalString   `json:",omitempty"`
 	BackupBootstrapInterval     *OptionalDuration `json:",omitempty"`
 }

config/profile.go

@@ -266,24 +266,44 @@ fetching may be degraded.
 		},
 	},
 	"legacy-cid-v0": {
-		Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`,
-
+		Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. This is likely the least optimal preset, use only if legacy behavior is required.`,
 		Transform: func(c *Config) error {
 			c.Import.CidVersion = *NewOptionalInteger(0)
 			c.Import.UnixFSRawLeaves = False
 			c.Import.UnixFSChunker = *NewOptionalString("size-262144")
 			c.Import.HashFunction = *NewOptionalString("sha2-256")
+			c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174)
+			c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0)
+			c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
+			c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalString("256KiB")
 			return nil
 		},
 	},
 	"test-cid-v1": {
-		Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`,
-
+		Description: `Makes UnixFS import produce CIDv1 with raw leaves, sha2-256 and 1 MiB chunks (max 174 links per file, 256 per HAMT node, switch dir to HAMT above 256KiB).`,
 		Transform: func(c *Config) error {
 			c.Import.CidVersion = *NewOptionalInteger(1)
 			c.Import.UnixFSRawLeaves = True
 			c.Import.UnixFSChunker = *NewOptionalString("size-1048576")
 			c.Import.HashFunction = *NewOptionalString("sha2-256")
+			c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(174)
+			c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0)
+			c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256)
+			c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalString("256KiB")
+			return nil
+		},
+	},
+	"test-cid-v1-wide": {
+		Description: `Makes UnixFS import produce CIDv1 with raw leaves, sha2-256 and 1MiB chunks and wider file DAGs (max 1024 links per every node type, switch dir to HAMT above 1MiB).`,
+		Transform: func(c *Config) error {
+			c.Import.CidVersion = *NewOptionalInteger(1)
+			c.Import.UnixFSRawLeaves = True
+			c.Import.UnixFSChunker = *NewOptionalString("size-1048576") // 1MiB
+			c.Import.HashFunction = *NewOptionalString("sha2-256")
+			c.Import.UnixFSFileMaxLinks = *NewOptionalInteger(1024)
+			c.Import.UnixFSDirectoryMaxLinks = *NewOptionalInteger(0) // no limit here, use size-based Import.UnixFSHAMTDirectorySizeThreshold instead
+			c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(1024)
+			c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalString("1MiB") // 1MiB
 			return nil
 		},
 	},

core/commands/add.go

@@ -37,23 +37,26 @@ type AddEvent struct {
 }
 
 const (
-	quietOptionName       = "quiet"
-	quieterOptionName     = "quieter"
-	silentOptionName      = "silent"
-	progressOptionName    = "progress"
-	trickleOptionName     = "trickle"
-	wrapOptionName        = "wrap-with-directory"
-	onlyHashOptionName    = "only-hash"
-	chunkerOptionName     = "chunker"
-	pinOptionName         = "pin"
-	rawLeavesOptionName   = "raw-leaves"
-	noCopyOptionName      = "nocopy"
-	fstoreCacheOptionName = "fscache"
-	cidVersionOptionName  = "cid-version"
-	hashOptionName        = "hash"
-	inlineOptionName      = "inline"
-	inlineLimitOptionName = "inline-limit"
-	toFilesOptionName     = "to-files"
+	quietOptionName             = "quiet"
+	quieterOptionName           = "quieter"
+	silentOptionName            = "silent"
+	progressOptionName          = "progress"
+	trickleOptionName           = "trickle"
+	wrapOptionName              = "wrap-with-directory"
+	onlyHashOptionName          = "only-hash"
+	chunkerOptionName           = "chunker"
+	pinOptionName               = "pin"
+	rawLeavesOptionName         = "raw-leaves"
+	maxFileLinksOptionName      = "max-file-links"
+	maxDirectoryLinksOptionName = "max-directory-links"
+	maxHAMTFanoutOptionName     = "max-hamt-fanout"
+	noCopyOptionName            = "nocopy"
+	fstoreCacheOptionName       = "fscache"
+	cidVersionOptionName        = "cid-version"
+	hashOptionName              = "hash"
+	inlineOptionName            = "inline"
+	inlineLimitOptionName       = "inline-limit"
+	toFilesOptionName           = "to-files"
 
 	preserveModeOptionName  = "preserve-mode"
 	preserveMtimeOptionName = "preserve-mtime"
@@ -143,6 +146,9 @@ new flags may be added in the future. It is not guaranteed for the implicit
 defaults of 'ipfs add' to remain the same in future Kubo releases, or for other
 IPFS software to use the same import parameters as Kubo.
 
+Use Import.* configuration options to override global implicit defaults:
+https://github.com/ipfs/kubo/blob/master/docs/config.md#import
+
 If you need to back up or transport content-addressed data using a non-IPFS
 medium, CID can be preserved with CAR files.
 See 'dag export' and 'dag import' for more information.
@@ -166,12 +172,15 @@ See 'dag export' and 'dag import' for more information.
 		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
 		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
 		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
-		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
-		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
+		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash. Default: Import.UnixFSChunker"),
+		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. Default: Import.UnixFSRawLeaves"),
+		cmds.IntOption(maxFileLinksOptionName, "Limit the maximum number of links in UnixFS file nodes to this value. (experimental) Default: Import.UnixFSFileMaxLinks"),
+		cmds.IntOption(maxDirectoryLinksOptionName, "Limit the maximum number of links in UnixFS basic directory nodes to this value. Default: Import.UnixFSDirectoryMaxLinks. WARNING: experimental, Import.UnixFSHAMTThreshold is a safer alternative."),
+		cmds.IntOption(maxHAMTFanoutOptionName, "Limit the maximum number of links of a UnixFS HAMT directory node to this (power of 2, multiple of 8). Default: Import.UnixFSHAMTDirectoryMaxFanout WARNING: experimental, see Import.UnixFSHAMTDirectorySizeThreshold as well."),
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
-		cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
-		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
+		cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true. Default: Import.CidVersion"),
+		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. Default: Import.HashFunction"),
 		cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
 		cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
 		cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
@@ -222,6 +231,9 @@ See 'dag export' and 'dag import' for more information.
 		chunker, _ := req.Options[chunkerOptionName].(string)
 		dopin, _ := req.Options[pinOptionName].(bool)
 		rawblks, rbset := req.Options[rawLeavesOptionName].(bool)
+		maxFileLinks, maxFileLinksSet := req.Options[maxFileLinksOptionName].(int)
+		maxDirectoryLinks, maxDirectoryLinksSet := req.Options[maxDirectoryLinksOptionName].(int)
+		maxHAMTFanout, maxHAMTFanoutSet := req.Options[maxHAMTFanoutOptionName].(int)
 		nocopy, _ := req.Options[noCopyOptionName].(bool)
 		fscache, _ := req.Options[fstoreCacheOptionName].(bool)
 		cidVer, cidVerSet := req.Options[cidVersionOptionName].(int)
@@ -253,6 +265,21 @@ See 'dag export' and 'dag import' for more information.
 			rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
 		}
 
+		if !maxFileLinksSet && !cfg.Import.UnixFSFileMaxLinks.IsDefault() {
+			maxFileLinksSet = true
+			maxFileLinks = int(cfg.Import.UnixFSFileMaxLinks.WithDefault(config.DefaultUnixFSFileMaxLinks))
+		}
+
+		if !maxDirectoryLinksSet && !cfg.Import.UnixFSDirectoryMaxLinks.IsDefault() {
+			maxDirectoryLinksSet = true
+			maxDirectoryLinks = int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks))
+		}
+
+		if !maxHAMTFanoutSet && !cfg.Import.UnixFSHAMTDirectoryMaxFanout.IsDefault() {
+			maxHAMTFanoutSet = true
+			maxHAMTFanout = int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout))
+		}
+
 		// Storing optional mode or mtime (UnixFS 1.5) requires root block
 		// to always be 'dag-pb' and not 'raw'. Below adjusts raw-leaves setting, if possible.
 		if preserveMode || preserveMtime || mode != 0 || mtime != 0 {
@@ -329,6 +356,18 @@ See 'dag export' and 'dag import' for more information.
 			opts = append(opts, options.Unixfs.RawLeaves(rawblks))
 		}
 
+		if maxFileLinksSet {
+			opts = append(opts, options.Unixfs.MaxFileLinks(maxFileLinks))
+		}
+
+		if maxDirectoryLinksSet {
+			opts = append(opts, options.Unixfs.MaxDirectoryLinks(maxDirectoryLinks))
+		}
+
+		if maxHAMTFanoutSet {
+			opts = append(opts, options.Unixfs.MaxHAMTFanout(maxHAMTFanout))
+		}
+
 		if trickle {
 			opts = append(opts, options.Unixfs.Layout(options.TrickleLayout))
 		}

core/coreapi/unixfs.go

@@ -50,6 +50,12 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 		attribute.Int("inlinelimit", settings.InlineLimit),
 		attribute.Bool("rawleaves", settings.RawLeaves),
 		attribute.Bool("rawleavesset", settings.RawLeavesSet),
+		attribute.Int("maxfilelinks", settings.MaxFileLinks),
+		attribute.Bool("maxfilelinksset", settings.MaxFileLinksSet),
+		attribute.Int("maxdirectorylinks", settings.MaxDirectoryLinks),
+		attribute.Bool("maxdirectorylinksset", settings.MaxDirectoryLinksSet),
+		attribute.Int("maxhamtfanout", settings.MaxHAMTFanout),
+		attribute.Bool("maxhamtfanoutset", settings.MaxHAMTFanoutSet),
 		attribute.Int("layout", int(settings.Layout)),
 		attribute.Bool("pin", settings.Pin),
 		attribute.Bool("onlyhash", settings.OnlyHash),
@@ -132,6 +138,16 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 	fileAdder.Pin = settings.Pin && !settings.OnlyHash
 	fileAdder.Silent = settings.Silent
 	fileAdder.RawLeaves = settings.RawLeaves
+	if settings.MaxFileLinksSet {
+		fileAdder.MaxLinks = settings.MaxFileLinks
+	}
+	if settings.MaxDirectoryLinksSet {
+		fileAdder.MaxDirectoryLinks = settings.MaxDirectoryLinks
+	}
+
+	if settings.MaxHAMTFanoutSet {
+		fileAdder.MaxHAMTFanout = settings.MaxHAMTFanout
+	}
 	fileAdder.NoCopy = settings.NoCopy
 	fileAdder.CidBuilder = prefix
 	fileAdder.PreserveMode = settings.PreserveMode

core/coreiface/options/unixfs.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	dag "github.com/ipfs/boxo/ipld/merkledag"
+	"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
+	"github.com/ipfs/boxo/ipld/unixfs/io"
 	cid "github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
 )
@@ -22,10 +24,16 @@ type UnixfsAddSettings struct {
 	CidVersion int
 	MhType     uint64
 
-	Inline       bool
-	InlineLimit  int
-	RawLeaves    bool
-	RawLeavesSet bool
+	Inline               bool
+	InlineLimit          int
+	RawLeaves            bool
+	RawLeavesSet         bool
+	MaxFileLinks         int
+	MaxFileLinksSet      bool
+	MaxDirectoryLinks    int
+	MaxDirectoryLinksSet bool
+	MaxHAMTFanout        int
+	MaxHAMTFanoutSet     bool
 
 	Chunker string
 	Layout  Layout
@@ -60,10 +68,16 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix,
 		CidVersion: -1,
 		MhType:     mh.SHA2_256,
 
-		Inline:       false,
-		InlineLimit:  32,
-		RawLeaves:    false,
-		RawLeavesSet: false,
+		Inline:               false,
+		InlineLimit:          32,
+		RawLeaves:            false,
+		RawLeavesSet:         false,
+		MaxFileLinks:         helpers.DefaultLinksPerBlock,
+		MaxFileLinksSet:      false,
+		MaxDirectoryLinks:    0,
+		MaxDirectoryLinksSet: false,
+		MaxHAMTFanout:        io.DefaultShardWidth,
+		MaxHAMTFanoutSet:     false,
 
 		Chunker: "size-262144",
 		Layout:  BalancedLayout,
@@ -190,6 +204,35 @@ func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption {
 	}
 }
 
+// MaxFileLinks specifies the maximum number of children for UnixFS file
+// nodes.
+func (unixfsOpts) MaxFileLinks(n int) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.MaxFileLinks = n
+		settings.MaxFileLinksSet = true
+		return nil
+	}
+}
+
+// MaxDirectoryLinks specifies the maximum number of children for UnixFS basic
+// directory nodes.
+func (unixfsOpts) MaxDirectoryLinks(n int) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.MaxDirectoryLinks = n
+		settings.MaxDirectoryLinksSet = true
+		return nil
+	}
+}
+
+// MaxHAMTFanout specifies the maximum width of the HAMT directory shards.
+func (unixfsOpts) MaxHAMTFanout(n int) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.MaxHAMTFanout = n
+		settings.MaxHAMTFanoutSet = true
+		return nil
+	}
+}
+
 // Inline tells the adder to inline small blocks into CIDs
 func (unixfsOpts) Inline(enable bool) UnixfsAddOption {
 	return func(settings *UnixfsAddSettings) error {