feat(api): add directory manifest support for folder uploads

Implements directory manifests for organizing multiple files into a tree:

- New DirectoryCodec (0xCD04) for directory manifest CIDs
- DirectoryManifest type with entries, totalSize, filesCount
- Two-phase upload: files uploaded individually, then /directory endpoint
  finalizes them into a tree structure with POST JSON body
- HTML directory browsing with Archivist branding at /data/{cid}
- JSON directory listing when Accept header doesn't include text/html
- Path resolution within directories via /data/{cid}/path?p=subdir/file
- Auto-promotion of single-child root directories

API changes:
- POST /api/archivist/v1/directory - finalize directory from uploaded files
- GET /api/archivist/v1/data/{cid} - now serves HTML for directories
- Added MIME types: audio/mpeg, audio/flac, video/webm, etc.
- Relaxed filename validation to allow paths like "Album/track.mp3"

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: Zorlin

archivist/archivisttypes.nim

@@ -42,6 +42,7 @@ const
   Pos2Bn128MrklCodec* = multiCodec("poseidon2-alt_bn_128-merkle-2kb")
 
   ManifestCodec* = multiCodec("codex-manifest")
+  DirectoryCodec* = MultiCodec(0xCD04) # codex-directory (not yet registered in libp2p)
   DatasetRootCodec* = multiCodec("codex-root")
   BlockCodec* = multiCodec("codex-block")
   SlotRootCodec* = multiCodec("codex-slot-root")
@@ -51,8 +52,8 @@ const
   HashesCodecs* = [Sha256HashCodec, Pos2Bn128SpngCodec, Pos2Bn128MrklCodec]
 
   PrimitivesCodecs* = [
-    ManifestCodec, DatasetRootCodec, BlockCodec, SlotRootCodec, SlotProvingRootCodec,
-    SlotCellCodec,
+    ManifestCodec, DirectoryCodec, DatasetRootCodec, BlockCodec, SlotRootCodec,
+    SlotProvingRootCodec, SlotCellCodec,
   ]
 
 proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =

archivist/conf.nim

@@ -608,6 +608,16 @@ proc readValue*(
 ) {.raises: [SerializationError, IOError].} =
   val = EthAddress.init(r.readValue(string)).get()
 
+proc readValue*(
+    r: var TomlReader, val: var Cid
+) {.raises: [SerializationError, IOError].} =
+  let cidStr = r.readValue(string)
+  let cidResult = Cid.init(cidStr)
+  if cidResult.isOk:
+    val = cidResult.get()
+  else:
+    raise newException(SerializationError, "Invalid CID: " & cidStr)
+
 proc readValue*(r: var TomlReader, val: var SignedPeerRecord) =
   without uri =? r.readValue(string).catch, err:
     error "invalid SignedPeerRecord configuration value", error = err.msg

archivist/directorynode.nim

@@ -0,0 +1,168 @@
+## Copyright (c) 2025 Archivist Authors
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+## Directory operations for ArchivistNode
+##
+## This module is kept separate from node.nim to avoid importing minprotobuf
+## into the main compilation path, which triggers serialization conflicts
+## with TOML config loading.
+
+{.push raises: [].}
+
+import pkg/chronos
+import pkg/questionable
+import pkg/questionable/results
+import pkg/libp2p/cid
+
+# Import minprotobuf explicitly to avoid leaking symbols that conflict with serialization
+from pkg/libp2p/protobuf/minprotobuf import
+  ProtoBuffer, initProtoBuffer, getField, getRequiredRepeatedField,
+  write, finish, ProtoResult
+
+import ./manifest/directory
+import ./blocktype as bt
+import ./stores
+import ./archivisttypes
+import ./units
+import ./errors
+import ./logutils
+
+logScope:
+  topics = "archivist directorynode"
+
+proc decodeDirectoryManifest*(blk: bt.Block): ?!DirectoryManifest =
+  ## Decode a directory manifest from a block
+  ##
+  if not ?blk.cid.isDirectory:
+    return failure "Cid not a directory codec"
+
+  var
+    pbNode = initProtoBuffer(blk.data)
+    pbEntries: seq[seq[byte]]
+    totalSize: uint64
+    name: string
+    entries: seq[DirectoryEntry]
+
+  if pbNode.getRequiredRepeatedField(1, pbEntries).isErr:
+    return failure("Unable to decode `entries` from directory manifest!")
+
+  if pbNode.getField(2, totalSize).isErr:
+    return failure("Unable to decode `totalSize` from directory manifest!")
+
+  if pbNode.getField(3, name).isErr:
+    return failure("Unable to decode `name` from directory manifest!")
+
+  for pbEntryData in pbEntries:
+    var
+      pbEntry = initProtoBuffer(pbEntryData)
+      entryName: string
+      cidBuf: seq[byte]
+      size: uint64
+      isDir: uint32
+      mimetype: string
+
+    if pbEntry.getField(1, entryName).isErr:
+      return failure("Unable to decode entry `name` from directory manifest!")
+
+    if pbEntry.getField(2, cidBuf).isErr:
+      return failure("Unable to decode entry `cid` from directory manifest!")
+
+    if pbEntry.getField(3, size).isErr:
+      return failure("Unable to decode entry `size` from directory manifest!")
+
+    if pbEntry.getField(4, isDir).isErr:
+      return failure("Unable to decode entry `isDirectory` from directory manifest!")
+
+    if pbEntry.getField(5, mimetype).isErr:
+      return failure("Unable to decode entry `mimetype` from directory manifest!")
+
+    let entryCid = ?Cid.init(cidBuf).mapFailure
+
+    entries.add(DirectoryEntry(
+      name: entryName,
+      cid: entryCid,
+      size: size.NBytes,
+      isDirectory: isDir != 0,
+      mimetype: mimetype,
+    ))
+
+  success DirectoryManifest(
+    entries: entries,
+    totalSize: totalSize.NBytes,
+    name: name,
+  )
+
+proc encodeDirectoryManifest*(directory: DirectoryManifest): seq[byte] =
+  ## Encode a directory manifest to protobuf bytes
+  ##
+  var pbNode = initProtoBuffer()
+
+  for entry in directory.entries:
+    var pbEntry = initProtoBuffer()
+    pbEntry.write(1, entry.name)
+    pbEntry.write(2, entry.cid.data.buffer)
+    pbEntry.write(3, entry.size.uint64)
+    pbEntry.write(4, entry.isDirectory.uint32)
+    if entry.mimetype.len > 0:
+      pbEntry.write(5, entry.mimetype)
+    pbEntry.finish()
+    pbNode.write(1, pbEntry)
+
+  pbNode.write(2, directory.totalSize.uint64)
+
+  if directory.name.len > 0:
+    pbNode.write(3, directory.name)
+
+  pbNode.finish()
+  pbNode.buffer
+
+proc storeDirectoryManifest*(
+    networkStore: NetworkStore, directory: DirectoryManifest
+): Future[?!bt.Block] {.async: (raises: [CancelledError]).} =
+  ## Store a directory manifest and return its block
+  ##
+  let encoded = encodeDirectoryManifest(directory)
+
+  without blk =? bt.Block.new(data = encoded, codec = DirectoryCodec), error:
+    trace "Unable to create block from directory manifest"
+    return failure(error)
+
+  if err =? (await networkStore.putBlock(blk)).errorOption:
+    trace "Unable to store directory manifest block", cid = blk.cid, err = err.msg
+    return failure(err)
+
+  info "Stored directory manifest",
+    cid = blk.cid,
+    entries = directory.entries.len,
+    totalSize = directory.totalSize
+
+  success blk
+
+proc fetchDirectoryManifest*(
+    networkStore: NetworkStore, cid: Cid
+): Future[?!DirectoryManifest] {.async: (raises: [CancelledError]).} =
+  ## Fetch and decode a directory manifest block
+  ##
+  if err =? cid.isDirectory.errorOption:
+    return failure "CID has invalid content type for directory manifest {$cid}"
+
+  trace "Retrieving directory manifest for cid", cid
+
+  without blk =? await networkStore.getBlock(BlockAddress.init(cid)), err:
+    trace "Error retrieving directory manifest block", cid, err = err.msg
+    return failure err
+
+  trace "Decoding directory manifest for cid", cid
+
+  without directory =? decodeDirectoryManifest(blk), err:
+    trace "Unable to decode as directory manifest", err = err.msg
+    return failure("Unable to decode as directory manifest")
+
+  trace "Decoded directory manifest", cid, entries = directory.entries.len
+
+  return directory.success

archivist/manifest/directory.nim

@@ -0,0 +1,186 @@
+## Copyright (c) 2025 Archivist Authors
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+# This module defines DirectoryManifest for folder/directory support
+
+{.push raises: [].}
+
+# Note: minprotobuf import is NOT included here - it triggers a serialization
+# error when imported into a module that gets pulled into TOML config loading.
+# The encode/decode procs that need protobuf are in coders.nim instead.
+
+import pkg/libp2p/[cid, multihash, multicodec]
+import pkg/questionable/results
+
+import ../errors
+import ../units
+import ../archivisttypes
+
+type
+  DirectoryEntry* = object
+    name*: string
+    cid*: Cid
+    size*: NBytes
+    isDirectory*: bool
+    mimetype*: string  # Empty string = not set
+
+  DirectoryManifest* = object
+    entries*: seq[DirectoryEntry]
+    totalSize*: NBytes
+    name*: string  # Empty string = not set
+
+############################################################
+# Accessors
+############################################################
+
+func entries*(self: DirectoryManifest): seq[DirectoryEntry] =
+  self.entries
+
+func totalSize*(self: DirectoryManifest): NBytes =
+  self.totalSize
+
+func name*(self: DirectoryManifest): string =
+  self.name
+
+func filesCount*(self: DirectoryManifest): int =
+  var count = 0
+  for entry in self.entries:
+    if not entry.isDirectory:
+      inc count
+  count
+
+func dirsCount*(self: DirectoryManifest): int =
+  var count = 0
+  for entry in self.entries:
+    if entry.isDirectory:
+      inc count
+  count
+
+############################################################
+# Predicates
+############################################################
+
+func isDirectory*(cid: Cid): ?!bool =
+  success (DirectoryCodec == ?cid.contentType().mapFailure(ArchivistError))
+
+func isDirectory*(mc: MultiCodec): ?!bool =
+  success mc == DirectoryCodec
+
+## Note: encode/decode procs are in coders.nim to avoid minprotobuf import
+## which triggers a serialization error when imported here.
+
+############################################################
+# Constructors
+############################################################
+
+func new*(
+    T: type DirectoryManifest,
+    entries: seq[DirectoryEntry] = @[],
+    name: string = "",
+): DirectoryManifest =
+  var total: NBytes = 0.NBytes
+  for entry in entries:
+    total = NBytes(total.int + entry.size.int)
+
+  T(
+    entries: entries,
+    totalSize: total,
+    name: name,
+  )
+
+func new*(
+    T: type DirectoryEntry,
+    name: string,
+    cid: Cid,
+    size: NBytes,
+    isDirectory: bool = false,
+    mimetype: string = "",
+): DirectoryEntry =
+  T(
+    name: name,
+    cid: cid,
+    size: size,
+    isDirectory: isDirectory,
+    mimetype: mimetype,
+  )
+
+############################################################
+# String representation
+############################################################
+
+func `$`*(entry: DirectoryEntry): string =
+  result = "DirectoryEntry(name: " & entry.name
+  result &= ", cid: " & $entry.cid
+  result &= ", size: " & $entry.size
+  result &= ", isDirectory: " & $entry.isDirectory
+  if entry.mimetype.len > 0:
+    result &= ", mimetype: " & entry.mimetype
+  result &= ")"
+
+func `$`*(self: DirectoryManifest): string =
+  result = "DirectoryManifest("
+  if self.name.len > 0:
+    result &= "name: " & self.name & ", "
+  result &= "totalSize: " & $self.totalSize
+  result &= ", entries: " & $self.entries.len & " items)"
+
+############################################################
+# Equality
+############################################################
+
+func `==`*(a, b: DirectoryEntry): bool =
+  a.name == b.name and
+  a.cid == b.cid and
+  a.size == b.size and
+  a.isDirectory == b.isDirectory and
+  a.mimetype == b.mimetype
+
+func `==`*(a, b: DirectoryManifest): bool =
+  a.entries == b.entries and
+  a.totalSize == b.totalSize and
+  a.name == b.name
+
+############################################################
+# Helpers
+############################################################
+
+proc findEntry*(self: DirectoryManifest, name: string, foundEntry: var DirectoryEntry): bool =
+  ## Find an entry by name in the directory
+  ## Sets foundEntry and returns true if found, returns false otherwise
+  for entry in self.entries:
+    if entry.name == name:
+      foundEntry = entry
+      return true
+  return false
+
+proc sortByName(entries: var seq[DirectoryEntry]) =
+  ## Simple insertion sort to avoid importing std/algorithm which triggers
+  ## a serialization error during TOML config loading
+  for i in 1 ..< entries.len:
+    let key = entries[i]
+    var j = i - 1
+    while j >= 0 and entries[j].name > key.name:
+      entries[j + 1] = entries[j]
+      dec j
+    entries[j + 1] = key
+
+proc sortedEntries*(self: DirectoryManifest): seq[DirectoryEntry] =
+  ## Return entries sorted: directories first, then files, alphabetically
+  var dirs: seq[DirectoryEntry]
+  var files: seq[DirectoryEntry]
+
+  for entry in self.entries:
+    if entry.isDirectory:
+      dirs.add(entry)
+    else:
+      files.add(entry)
+
+  sortByName(dirs)
+  sortByName(files)
+
+  result = dirs & files