Batch require cache deletion to avoid quadratic scanning

deleteFromRequireCache() scans the entire require.cache for every
single module deletion to clean up parent-child references. When
called in a loop for N files, this is O(N * C) where C is the cache
size.

Rewrite deleteFromRequireCache to accept string[], collecting modules
into a Set and doing one scan with O(1) Set.has() lookups. Add
deleteCacheBatch() for callers that delete multiple modules. Update
all three batch call sites: turbopack hot-reloader (serverPaths loop),
manifest-loader (~15 manifest files per writeManifests), and webpack
plugin (afterEmit runtime + page entries).

Author: lukesandberg

packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts

@@ -22,16 +22,14 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance {
     compiler.hooks.assetEmitted.tap(PLUGIN_NAME, (_file, { targetPath }) => {
       // Clear module context in this process
       clearModuleContext(targetPath)
-      deleteCache(targetPath)
+      deleteCache([targetPath])
     })
 
     compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async (compilation) => {
+      const allPaths: string[] = []
+
       for (const name of RUNTIME_NAMES) {
-        const runtimeChunkPath = path.join(
-          compilation.outputOptions.path!,
-          `${name}.js`
-        )
-        deleteCache(runtimeChunkPath)
+        allPaths.push(path.join(compilation.outputOptions.path!, `${name}.js`))
       }
 
       // we need to make sure to clear all server entries from cache
@@ -43,12 +41,10 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance {
       })
 
       for (const page of entries) {
-        const outputPath = path.join(
-          compilation.outputOptions.path!,
-          page + '.js'
-        )
-        deleteCache(outputPath)
+        allPaths.push(path.join(compilation.outputOptions.path!, page + '.js'))
       }
+
+      deleteCache(allPaths)
     })
   }
 }

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -592,6 +592,7 @@ export async function createHotReloaderTurbopack(
     // use the edge runtime, and App Router edge routes all don't support server HMR.
     const usesServerHmr = entryType === 'app' && writtenEndpoint.type !== 'edge'
 
+    const filesToDelete: string[] = []
     for (const file of serverPaths) {
       const relativePath = relative(distDir, file)
 
@@ -603,11 +604,12 @@ export async function createHotReloaderTurbopack(
       }
 
       clearModuleContext(file)
-      // For Pages Router, edge routes, middleware, and manifest files
-      // (e.g., *_client-reference-manifest.js): clear the sharedCache in
-      // evalManifest(), Node.js require.cache, and edge runtime module contexts.
-      deleteCache(file)
+      filesToDelete.push(file)
     }
+    // For Pages Router, edge routes, middleware, and manifest files
+    // (e.g., *_client-reference-manifest.js): clear the sharedCache in
+    // evalManifest(), Node.js require.cache, and edge runtime module contexts.
+    deleteCache(filesToDelete)
 
     // Clear Turbopack's chunk-loading cache so chunks are re-required from disk on
     // the next request.

packages/next/src/server/dev/require-cache.ts

@@ -2,34 +2,59 @@ import isError from '../../lib/is-error'
 import { realpathSync } from '../../lib/realpath'
 import { clearManifestCache } from '../load-manifest.external'
 
-function deleteFromRequireCache(filePath: string) {
-  try {
-    filePath = realpathSync(filePath)
-  } catch (e) {
-    if (isError(e) && e.code !== 'ENOENT') throw e
+/**
+ * Batch delete modules from require.cache with a single scan.
+ *
+ * When deleting N modules, this performs ONE scan of require.cache
+ * instead of N scans, reducing complexity from O(N * C) to O(C + N)
+ * where C = size of require.cache.
+ */
+function deleteFromRequireCache(filePaths: string[]): void {
+  // Phase 1: Resolve all paths and collect modules to delete
+  const resolvedPaths: string[] = []
+  const modsToDelete = new Set<NodeModule>()
+
+  for (let filePath of filePaths) {
+    try {
+      filePath = realpathSync(filePath)
+    } catch (e) {
+      if (isError(e) && e.code !== 'ENOENT') throw e
+    }
+    const mod = require.cache[filePath]
+    if (mod) {
+      resolvedPaths.push(filePath)
+      modsToDelete.add(mod)
+    }
   }
-  const mod = require.cache[filePath]
-  if (mod) {
-    // remove the child reference from all parent modules
-    for (const parent of Object.values(require.cache)) {
-      if (parent?.children) {
-        const idx = parent.children.indexOf(mod)
-        if (idx >= 0) parent.children.splice(idx, 1)
+
+  if (modsToDelete.size === 0) return
+
+  // Phase 2: Single scan of require.cache to remove child references
+  for (const parent of Object.values(require.cache)) {
+    if (parent?.children) {
+      for (let i = parent.children.length - 1; i >= 0; i--) {
+        if (modsToDelete.has(parent.children[i])) {
+          parent.children.splice(i, 1)
+        }
       }
     }
-    // remove parent references from external modules
+  }
+
+  // Phase 3: Clear parent references from children and delete cache entries
+  for (const mod of modsToDelete) {
     for (const child of mod.children) {
       child.parent = null
     }
+  }
+
+  for (const filePath of resolvedPaths) {
     delete require.cache[filePath]
-    return true
   }
-  return false
 }
 
-export function deleteCache(filePath: string) {
-  // try to clear it from the fs cache
-  clearManifestCache(filePath)
-
-  deleteFromRequireCache(filePath)
+export function deleteCache(filePaths: string[]) {
+  for (const filePath of filePaths) {
+    clearManifestCache(filePath)
+  }
+  deleteFromRequireCache(filePaths)
 }

packages/next/src/shared/lib/turbopack/manifest-loader.ts

@@ -209,6 +209,7 @@ export class TurbopackManifestLoader {
   /// interceptionRewrites that have been written to disk
   /// This is used to avoid unnecessary writes if the rewrites haven't changed
   private cachedInterceptionRewrites: string | undefined = undefined
+  private pendingCacheDeletes: string[] = []
 
   private readonly distDir: string
   private readonly buildId: string
@@ -318,8 +319,8 @@ export class TurbopackManifestLoader {
       `${SERVER_REFERENCE_MANIFEST}.js`
     )
     const json = JSON.stringify(actionManifest, null, 2)
-    deleteCache(actionManifestJsonPath)
-    deleteCache(actionManifestJsPath)
+    this.pendingCacheDeletes.push(actionManifestJsonPath)
+    this.pendingCacheDeletes.push(actionManifestJsPath)
     writeFileAtomic(actionManifestJsonPath, json)
     writeFileAtomic(
       actionManifestJsPath,
@@ -351,7 +352,7 @@ export class TurbopackManifestLoader {
       'server',
       APP_PATHS_MANIFEST
     )
-    deleteCache(appPathsManifestPath)
+    this.pendingCacheDeletes.push(appPathsManifestPath)
     writeFileAtomic(
       appPathsManifestPath,
       JSON.stringify(appPathsManifest, null, 2)
@@ -364,7 +365,7 @@ export class TurbopackManifestLoader {
     }
     const webpackStats = this.mergeWebpackStats(this.webpackStats.values())
     const path = join(this.distDir, 'server', WEBPACK_STATS)
-    deleteCache(path)
+    this.pendingCacheDeletes.push(path)
     writeFileAtomic(path, JSON.stringify(webpackStats, null, 2))
   }
 
@@ -383,8 +384,8 @@ export class TurbopackManifestLoader {
       'server',
       `${SUBRESOURCE_INTEGRITY_MANIFEST}.js`
     )
-    deleteCache(pathJson)
-    deleteCache(pathJs)
+    this.pendingCacheDeletes.push(pathJson)
+    this.pendingCacheDeletes.push(pathJs)
     writeFileAtomic(pathJson, JSON.stringify(sriManifest, null, 2))
     writeFileAtomic(
       pathJs,
@@ -560,7 +561,7 @@ export class TurbopackManifestLoader {
       'server',
       `${INTERCEPTION_ROUTE_REWRITE_MANIFEST}.js`
     )
-    deleteCache(interceptionRewriteManifestPath)
+    this.pendingCacheDeletes.push(interceptionRewriteManifestPath)
 
     writeFileAtomic(
       interceptionRewriteManifestPath,
@@ -586,8 +587,8 @@ export class TurbopackManifestLoader {
       `${MIDDLEWARE_BUILD_MANIFEST}.js`
     )
 
-    deleteCache(buildManifestPath)
-    deleteCache(middlewareBuildManifestPath)
+    this.pendingCacheDeletes.push(buildManifestPath)
+    this.pendingCacheDeletes.push(middlewareBuildManifestPath)
     writeFileAtomic(buildManifestPath, JSON.stringify(buildManifest, null, 2))
     writeFileAtomic(
       middlewareBuildManifestPath,
@@ -606,7 +607,7 @@ export class TurbopackManifestLoader {
       this.distDir,
       `fallback-${BUILD_MANIFEST}`
     )
-    deleteCache(fallbackBuildManifestPath)
+    this.pendingCacheDeletes.push(fallbackBuildManifestPath)
     writeFileAtomic(
       fallbackBuildManifestPath,
       JSON.stringify(fallbackBuildManifest, null, 2)
@@ -727,8 +728,8 @@ export class TurbopackManifestLoader {
       'server',
       `${NEXT_FONT_MANIFEST}.js`
     )
-    deleteCache(fontManifestJsonPath)
-    deleteCache(fontManifestJsPath)
+    this.pendingCacheDeletes.push(fontManifestJsonPath)
+    this.pendingCacheDeletes.push(fontManifestJsPath)
     writeFileAtomic(fontManifestJsonPath, json)
     writeFileAtomic(
       fontManifestJsPath,
@@ -872,7 +873,7 @@ export class TurbopackManifestLoader {
       'server',
       MIDDLEWARE_MANIFEST
     )
-    deleteCache(middlewareManifestPath)
+    this.pendingCacheDeletes.push(middlewareManifestPath)
     writeFileAtomic(
       middlewareManifestPath,
       JSON.stringify(middlewareManifest, null, 2)
@@ -888,7 +889,7 @@ export class TurbopackManifestLoader {
       2
     )};self.__MIDDLEWARE_MATCHERS_CB && self.__MIDDLEWARE_MATCHERS_CB()`
 
-    deleteCache(clientMiddlewareManifestPath)
+    this.pendingCacheDeletes.push(clientMiddlewareManifestPath)
     writeFileAtomic(
       join(this.distDir, clientMiddlewareManifestPath),
       clientMiddlewareManifestJs
@@ -928,7 +929,7 @@ export class TurbopackManifestLoader {
     }
     const pagesManifest = this.mergePagesManifests(this.pagesManifests.values())
     const pagesManifestPath = join(this.distDir, 'server', PAGES_MANIFEST)
-    deleteCache(pagesManifestPath)
+    this.pendingCacheDeletes.push(pagesManifestPath)
     writeFileAtomic(pagesManifestPath, JSON.stringify(pagesManifest, null, 2))
   }
 
@@ -959,6 +960,12 @@ export class TurbopackManifestLoader {
     if (process.env.TURBOPACK_STATS != null) {
       this.writeWebpackStats()
     }
+
+    // Flush all queued cache deletions in a single require.cache scan
+    if (this.pendingCacheDeletes.length > 0) {
+      deleteCache(this.pendingCacheDeletes)
+      this.pendingCacheDeletes = []
+    }
   }
 }