fix(vm): support network imports (#5610)

Author: sheremet-va

packages/vitest/src/runtime/external-executor.ts

@@ -27,7 +27,7 @@ export interface ExternalModulesExecutorOptions {
 }
 
 interface ModuleInformation {
-  type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs'
+  type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs' | 'network'
   url: string
   path: string
 }
@@ -41,6 +41,8 @@ export class ExternalModulesExecutor {
   private fs: FileMap
   private resolvers: ((id: string, parent: string) => string | undefined)[] = []
 
+  #networkSupported: boolean | null = null
+
   constructor(private options: ExternalModulesExecutorOptions) {
     this.context = options.context
 
@@ -62,6 +64,20 @@ export class ExternalModulesExecutor {
     this.resolvers = [this.vite.resolve]
   }
 
+  async import(identifier: string) {
+    const module = await this.createModule(identifier)
+    await this.esm.evaluateModule(module)
+    return module.namespace
+  }
+
+  require(identifier: string) {
+    return this.cjs.require(identifier)
+  }
+
+  createRequire(identifier: string) {
+    return this.cjs.createRequire(identifier)
+  }
+
   // dynamic import can be used in both ESM and CJS, so we have it in the executor
   public importModuleDynamically = async (specifier: string, referencer: VMModule) => {
     const module = await this.resolveModule(specifier, referencer.identifier)
@@ -161,6 +177,9 @@ export class ExternalModulesExecutor {
     if (extension === '.node' || isNodeBuiltin(identifier))
       return { type: 'builtin', url: identifier, path: identifier }
 
+    if (this.isNetworkSupported && (identifier.startsWith('http:') || identifier.startsWith('https:')))
+      return { type: 'network', url: identifier, path: identifier }
+
     const isFileUrl = identifier.startsWith('file://')
     const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier
     const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString()
@@ -209,31 +228,32 @@ export class ExternalModulesExecutor {
       case 'vite':
         return await this.vite.createViteModule(url)
       case 'wasm':
-        return await this.esm.createWebAssemblyModule(url, this.fs.readBuffer(path))
+        return await this.esm.createWebAssemblyModule(url, () => this.fs.readBuffer(path))
       case 'module':
-        return await this.esm.createEsModule(url, this.fs.readFile(path))
+        return await this.esm.createEsModule(url, () => this.fs.readFile(path))
       case 'commonjs': {
         const exports = this.require(path)
         return this.wrapCommonJsSynteticModule(identifier, exports)
       }
+      case 'network': {
+        return this.esm.createNetworkModule(url)
+      }
       default: {
         const _deadend: never = type
         return _deadend
       }
     }
   }
 
-  async import(identifier: string) {
-    const module = await this.createModule(identifier)
-    await this.esm.evaluateModule(module)
-    return module.namespace
-  }
-
-  require(identifier: string) {
-    return this.cjs.require(identifier)
-  }
-
-  createRequire(identifier: string) {
-    return this.cjs.createRequire(identifier)
+  private get isNetworkSupported() {
+    if (this.#networkSupported == null) {
+      if (process.execArgv.includes('--experimental-network-imports'))
+        this.#networkSupported = true
+      else if (process.env.NODE_OPTIONS?.includes('--experimental-network-imports'))
+        this.#networkSupported = true
+      else
+        this.#networkSupported = false
+    }
+    return this.#networkSupported
   }
 }

packages/vitest/src/runtime/vm/esm-executor.ts

@@ -18,6 +18,8 @@ export class EsmExecutor {
   private esmLinkMap = new WeakMap<VMModule, Promise<void>>()
   private context: vm.Context
 
+  #httpIp = IPnumber('127.0.0.0')
+
   constructor(private executor: ExternalModulesExecutor, options: EsmExecutorOptions) {
     this.context = options.context
   }
@@ -38,10 +40,11 @@ export class EsmExecutor {
     return m
   }
 
-  public async createEsModule(fileUrl: string, code: string) {
+  public async createEsModule(fileUrl: string, getCode: () => Promise<string> | string) {
     const cached = this.moduleCache.get(fileUrl)
     if (cached)
       return cached
+    const code = await getCode()
     // TODO: should not be allowed in strict mode, implement in #2854
     if (fileUrl.endsWith('.json')) {
       const m = new SyntheticModule(
@@ -77,15 +80,35 @@ export class EsmExecutor {
     return m
   }
 
-  public async createWebAssemblyModule(fileUrl: string, code: Buffer) {
+  public async createWebAssemblyModule(fileUrl: string, getCode: () => Buffer) {
     const cached = this.moduleCache.get(fileUrl)
     if (cached)
       return cached
-    const m = this.loadWebAssemblyModule(code, fileUrl)
+    const m = this.loadWebAssemblyModule(getCode(), fileUrl)
     this.moduleCache.set(fileUrl, m)
     return m
   }
 
+  public async createNetworkModule(fileUrl: string) {
+    // https://nodejs.org/api/esm.html#https-and-http-imports
+    if (fileUrl.startsWith('http:')) {
+      const url = new URL(fileUrl)
+      if (
+        url.hostname !== 'localhost'
+        && url.hostname !== '::1'
+        && (IPnumber(url.hostname) & IPmask(8)) !== this.#httpIp
+      ) {
+        throw new Error(
+          // we don't know the importer, so it's undefined (the same happens in --pool=threads)
+          `import of '${fileUrl}' by undefined is not supported: `
+          + 'http can only be used to load local resources (use https instead).',
+        )
+      }
+    }
+
+    return this.createEsModule(fileUrl, () => fetch(fileUrl).then(r => r.text()))
+  }
+
   public async loadWebAssemblyModule(source: Buffer, identifier: string) {
     const cached = this.moduleCache.get(identifier)
     if (cached)
@@ -187,6 +210,18 @@ export class EsmExecutor {
       return module
     }
 
-    return this.createEsModule(identifier, code)
+    return this.createEsModule(identifier, () => code)
   }
 }
+
+function IPnumber(address: string) {
+  const ip = address.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/)
+  if (ip)
+    return (+ip[1] << 24) + (+ip[2] << 16) + (+ip[3] << 8) + (+ip[4])
+
+  throw new Error(`Expected IP address, received ${address}`)
+}
+
+function IPmask(maskSize: number) {
+  return -1 << (32 - maskSize)
+}

packages/vitest/src/runtime/vm/vite-executor.ts

@@ -56,10 +56,12 @@ export class ViteExecutor {
     const cached = this.esm.resolveCachedModule(fileUrl)
     if (cached)
       return cached
-    const result = await this.options.transform(fileUrl, 'web')
-    if (!result.code)
-      throw new Error(`[vitest] Failed to transform ${fileUrl}. Does the file exist?`)
-    return this.esm.createEsModule(fileUrl, result.code)
+    return this.esm.createEsModule(fileUrl, async () => {
+      const result = await this.options.transform(fileUrl, 'web')
+      if (!result.code)
+        throw new Error(`[vitest] Failed to transform ${fileUrl}. Does the file exist?`)
+      return result.code
+    })
   }
 
   private createViteClientModule() {

test/cli/fixtures/network-imports/basic.test.ts

@@ -9,3 +9,10 @@ import slash from 'http://localhost:9602/[email protected]'
 test('network imports', () => {
   expect(slash('foo\\bar')).toBe('foo/bar')
 })
+
+test('doesn\'t work for http outside localhost', async () => {
+  // @ts-expect-error network imports
+  await expect(() => import('http://100.0.0.0/')).rejects.toThrowError(
+    'import of \'http://100.0.0.0/\' by undefined is not supported: http can only be used to load local resources (use https instead).',
+  )
+})

test/cli/test/network-imports.test.ts

@@ -9,11 +9,6 @@ const config = {
     forks: {
       execArgv: ['--experimental-network-imports'],
     },
-    // not supported?
-    //   FAIL  test/basic.test.ts [ test/basic.test.ts ]
-    //   Error: ENOENT: no such file or directory, open 'http://localhost:9602/[email protected]'
-    //    ❯ Object.openSync node:fs:596:3
-    //    ❯ readFileSync node:fs:464:35
     vmThreads: {
       execArgv: ['--experimental-network-imports'],
     },
@@ -25,7 +20,7 @@ const config = {
 it.each([
   'threads',
   'forks',
-  // 'vmThreads',
+  'vmThreads',
 ])('importing from network in %s', async (pool) => {
   const { stderr, exitCode } = await runVitest({
     ...config,