perf: create only one fetcher per project (#8762)

Author: sheremet-va

packages/vitest/src/node/core.ts

@@ -6,18 +6,19 @@ import type { ModuleRunner } from 'vite/module-runner'
 import type { SerializedCoverageConfig } from '../runtime/config'
 import type { ArgumentsType, ProvidedContext, UserConsoleLog } from '../types/general'
 import type { CliOptions } from './cli/cli-api'
+import type { VitestFetchFunction } from './environments/fetchModule'
 import type { ProcessPool } from './pool'
 import type { TestModule } from './reporters/reported-tasks'
 import type { TestSpecification } from './spec'
 import type { ResolvedConfig, TestProjectConfiguration, UserConfig, VitestRunMode } from './types/config'
 import type { CoverageProvider, ResolvedCoverageOptions } from './types/coverage'
 import type { Reporter } from './types/reporter'
 import type { TestRunResult } from './types/tests'
-import os from 'node:os'
+import os, { tmpdir } from 'node:os'
 import { getTasks, hasFailed, limitConcurrency } from '@vitest/runner/utils'
 import { SnapshotManager } from '@vitest/snapshot/manager'
-import { deepClone, deepMerge, noop, toArray } from '@vitest/utils/helpers'
-import { normalize, relative } from 'pathe'
+import { deepClone, deepMerge, nanoid, noop, toArray } from '@vitest/utils/helpers'
+import { join, normalize, relative } from 'pathe'
 import { version } from '../../package.json' with { type: 'json' }
 import { WebSocketReporter } from '../api/setup'
 import { distDir } from '../paths'
@@ -28,6 +29,7 @@ import { BrowserSessions } from './browser/sessions'
 import { VitestCache } from './cache'
 import { resolveConfig } from './config/resolveConfig'
 import { getCoverageProvider } from './coverage'
+import { createFetchModuleFunction } from './environments/fetchModule'
 import { ServerModuleRunner } from './environments/serverRunner'
 import { FilesNotFoundError } from './errors'
 import { Logger } from './logger'
@@ -105,6 +107,8 @@ export class Vitest {
   /** @internal */ runner!: ModuleRunner
   /** @internal */ _testRun: TestRun = undefined!
   /** @internal */ _resolver!: VitestResolver
+  /** @internal */ _fetcher!: VitestFetchFunction
+  /** @internal */ _tmpDir = join(tmpdir(), nanoid())
 
   private isFirstRun = true
   private restartsCount = 0
@@ -212,10 +216,18 @@ export class Vitest {
     }
 
     this._resolver = new VitestResolver(server.config.cacheDir, resolved)
+    this._fetcher = createFetchModuleFunction(
+      this._resolver,
+      this._tmpDir,
+      {
+        dumpFolder: this.config.dumpDir,
+        readFromDump: this.config.server.debug?.load ?? process.env.VITEST_DEBUG_LOAD_DUMP != null,
+      },
+    )
     const environment = server.environments.__vitest__
     this.runner = new ServerModuleRunner(
       environment,
-      this._resolver,
+      this._fetcher,
       resolved,
     )
 

packages/vitest/src/node/environments/fetchModule.ts

@@ -18,22 +18,26 @@ interface DumpOptions {
   readFromDump?: boolean
 }
 
+export interface VitestFetchFunction {
+  (
+    url: string,
+    importer: string | undefined,
+    environment: DevEnvironment,
+    cacheFs: boolean,
+    options?: FetchFunctionOptions
+  ): Promise<FetchResult | FetchCachedFileSystemResult>
+}
+
 export function createFetchModuleFunction(
   resolver: VitestResolver,
-  cacheFs: boolean = false,
   tmpDir: string = join(tmpdir(), nanoid()),
   dump?: DumpOptions,
-): (
-  url: string,
-  importer: string | undefined,
-  environment: DevEnvironment,
-  options?: FetchFunctionOptions
-) => Promise<FetchResult | FetchCachedFileSystemResult> {
-  const cachedFsResults = new Map<string, string>()
+): VitestFetchFunction {
   return async (
     url,
     importer,
     environment,
+    cacheFs,
     options,
   ) => {
     // We are copy pasting Vite's externalization logic from `fetchModule` because
@@ -117,10 +121,14 @@ export function createFetchModuleFunction(
     }
 
     const code = result.code
+    const transformResult = result.transformResult!
+    if (!transformResult) {
+      throw new Error(`"transformResult" in not defined. This is a bug in Vitest.`)
+    }
     // to avoid serialising large chunks of code,
     // we store them in a tmp file and read in the test thread
-    if (cachedFsResults.has(result.id)) {
-      return getCachedResult(result, cachedFsResults)
+    if ('_vitestTmp' in transformResult) {
+      return getCachedResult(result, Reflect.get(transformResult as any, '_vitestTmp'))
     }
     const dir = join(tmpDir, environment.name)
     const name = hash('sha1', result.id, 'hex')
@@ -131,8 +139,8 @@ export function createFetchModuleFunction(
     }
     if (promises.has(tmp)) {
       await promises.get(tmp)
-      cachedFsResults.set(result.id, tmp)
-      return getCachedResult(result, cachedFsResults)
+      Reflect.set(transformResult, '_vitestTmp', tmp)
+      return getCachedResult(result, tmp)
     }
     promises.set(
       tmp,
@@ -143,8 +151,7 @@ export function createFetchModuleFunction(
         .finally(() => promises.delete(tmp)),
     )
     await promises.get(tmp)
-    cachedFsResults.set(result.id, tmp)
-    return getCachedResult(result, cachedFsResults)
+    return getCachedResult(result, tmp)
   }
 }
 
@@ -153,7 +160,9 @@ SOURCEMAPPING_URL += 'ppingURL'
 
 const MODULE_RUNNER_SOURCEMAPPING_SOURCE = '//# sourceMappingSource=vite-generated'
 
-function processResultSource(environment: DevEnvironment, result: FetchResult): FetchResult {
+function processResultSource(environment: DevEnvironment, result: FetchResult): FetchResult & {
+  transformResult?: TransformResult | null
+} {
   if (!('code' in result)) {
     return result
   }
@@ -168,6 +177,7 @@ function processResultSource(environment: DevEnvironment, result: FetchResult):
   return {
     ...result,
     code: node?.transformResult?.code || result.code,
+    transformResult: node?.transformResult,
   }
 }
 
@@ -220,11 +230,7 @@ function genSourceMapUrl(map: Rollup.SourceMap | string): string {
   return `data:application/json;base64,${Buffer.from(map).toString('base64')}`
 }
 
-function getCachedResult(result: Extract<FetchResult, { code: string }>, cachedFsResults: Map<string, string>): FetchCachedFileSystemResult {
-  const tmp = cachedFsResults.get(result.id)
-  if (!tmp) {
-    throw new Error(`The cached result was returned too early for ${result.id}.`)
-  }
+function getCachedResult(result: Extract<FetchResult, { code: string }>, tmp: string): FetchCachedFileSystemResult {
   return {
     cached: true as const,
     file: result.file,

packages/vitest/src/node/environments/serverRunner.ts

@@ -1,21 +1,16 @@
 import type { DevEnvironment } from 'vite'
-import type { VitestResolver } from '../resolver'
 import type { ResolvedConfig } from '../types/config'
+import type { VitestFetchFunction } from './fetchModule'
 import { VitestModuleEvaluator } from '#module-evaluator'
 import { ModuleRunner } from 'vite/module-runner'
-import { createFetchModuleFunction } from './fetchModule'
 import { normalizeResolvedIdToUrl } from './normalizeUrl'
 
 export class ServerModuleRunner extends ModuleRunner {
   constructor(
     private environment: DevEnvironment,
-    resolver: VitestResolver,
+    fetcher: VitestFetchFunction,
     private config: ResolvedConfig,
   ) {
-    const fetchModule = createFetchModuleFunction(
-      resolver,
-      false,
-    )
     super(
       {
         hmr: false,
@@ -26,7 +21,7 @@ export class ServerModuleRunner extends ModuleRunner {
             }
             const { data } = event.data
             try {
-              const result = await fetchModule(data[0], data[1], environment, data[2])
+              const result = await fetcher(data[0], data[1], environment, false, data[2])
               return { result }
             }
             catch (error) {

packages/vitest/src/node/pools/rpc.ts

@@ -4,7 +4,7 @@ import type { ResolveSnapshotPathHandlerContext } from '../types/config'
 import { existsSync, mkdirSync } from 'node:fs'
 import { fileURLToPath } from 'node:url'
 import { cleanUrl } from '@vitest/utils/helpers'
-import { createFetchModuleFunction, handleRollupError } from '../environments/fetchModule'
+import { handleRollupError } from '../environments/fetchModule'
 import { normalizeResolvedIdToUrl } from '../environments/normalizeUrl'
 
 interface MethodsOptions {
@@ -25,15 +25,6 @@ export function createMethodsRPC(project: TestProject, options: MethodsOptions =
     mkdirSync(project.config.dumpDir, { recursive: true })
   }
   project.vitest.state.metadata[project.name].dumpDir = project.config.dumpDir
-  const fetch = createFetchModuleFunction(
-    project._resolver,
-    cacheFs,
-    project.tmpDir,
-    {
-      dumpFolder: project.config.dumpDir,
-      readFromDump: project.config.server.debug?.load ?? process.env.VITEST_DEBUG_LOAD_DUMP != null,
-    },
-  )
   return {
     async fetch(
       url,
@@ -48,7 +39,7 @@ export function createMethodsRPC(project: TestProject, options: MethodsOptions =
 
       const start = performance.now()
 
-      return await fetch(url, importer, environment, options).then((result) => {
+      return await project._fetcher(url, importer, environment, cacheFs, options).then((result) => {
         const duration = performance.now() - start
         project.vitest.state.transformTime += duration
         const metadata = project.vitest.state.metadata[project.name]

packages/vitest/src/node/project.ts

@@ -4,6 +4,7 @@ import type { ModuleRunner } from 'vite/module-runner'
 import type { Typechecker } from '../typecheck/typechecker'
 import type { ProvidedContext } from '../types/general'
 import type { OnTestsRerunHandler, Vitest } from './core'
+import type { VitestFetchFunction } from './environments/fetchModule'
 import type { GlobalSetupFile } from './globalSetup'
 import type { ParentProjectBrowser, ProjectBrowser } from './types/browser'
 import type {
@@ -25,6 +26,7 @@ import { setup } from '../api/setup'
 import { createDefinesScript } from '../utils/config-helpers'
 import { isBrowserEnabled, resolveConfig } from './config/resolveConfig'
 import { serializeConfig } from './config/serializeConfig'
+import { createFetchModuleFunction } from './environments/fetchModule'
 import { ServerModuleRunner } from './environments/serverRunner'
 import { loadGlobalSetupFiles } from './globalSetup'
 import { CoverageTransform } from './plugins/coverageTransform'
@@ -55,13 +57,14 @@ export class TestProject {
   /**
    * Temporary directory for the project. This is unique for each project. Vitest stores transformed content here.
    */
-  public readonly tmpDir: string = join(tmpdir(), nanoid())
+  public readonly tmpDir: string
 
   /** @internal */ typechecker?: Typechecker
   /** @internal */ _config?: ResolvedConfig
   /** @internal */ _vite?: ViteDevServer
   /** @internal */ _hash?: string
   /** @internal */ _resolver!: VitestResolver
+  /** @internal */ _fetcher!: VitestFetchFunction
   /** @internal */ _serializedDefines?: string
   /** @inetrnal */ testFilesList: string[] | null = null
 
@@ -77,9 +80,11 @@ export class TestProject {
   constructor(
     vitest: Vitest,
     public options?: InitializeProjectOptions | undefined,
+    tmpDir?: string,
   ) {
     this.vitest = vitest
     this.globalConfig = vitest.config
+    this.tmpDir = tmpDir || join(tmpdir(), nanoid())
   }
 
   /**
@@ -545,11 +550,19 @@ export class TestProject {
     this._resolver = new VitestResolver(server.config.cacheDir, this._config)
     this._vite = server
     this._serializedDefines = createDefinesScript(server.config.define)
+    this._fetcher = createFetchModuleFunction(
+      this._resolver,
+      this.tmpDir,
+      {
+        dumpFolder: this.config.dumpDir,
+        readFromDump: this.config.server.debug?.load ?? process.env.VITEST_DEBUG_LOAD_DUMP != null,
+      },
+    )
 
     const environment = server.environments.__vitest__
     this.runner = new ServerModuleRunner(
       environment,
-      this._resolver,
+      this._fetcher,
       this._config,
     )
   }
@@ -600,11 +613,14 @@ export class TestProject {
   static _createBasicProject(vitest: Vitest): TestProject {
     const project = new TestProject(
       vitest,
+      undefined,
+      vitest._tmpDir,
     )
     project.runner = vitest.runner
     project._vite = vitest.vite
     project._config = vitest.config
     project._resolver = vitest._resolver
+    project._fetcher = vitest._fetcher
     project._serializedDefines = createDefinesScript(vitest.vite.config.define)
     project._setHash()
     project._provideObject(vitest.config.provide)
@@ -613,10 +629,11 @@ export class TestProject {
 
   /** @internal */
   static _cloneBrowserProject(parent: TestProject, config: ResolvedConfig): TestProject {
-    const clone = new TestProject(parent.vitest)
+    const clone = new TestProject(parent.vitest, undefined, parent.tmpDir)
     clone.runner = parent.runner
     clone._vite = parent._vite
     clone._resolver = parent._resolver
+    clone._fetcher = parent._fetcher
     clone._config = config
     clone._setHash()
     clone._parent = parent