fix(runner): async assertion auto await should timeout (#6391)

Author: hi-ogawa

packages/expect/src/jest-expect.ts

@@ -960,7 +960,7 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => {
             return result instanceof chai.Assertion ? proxy : result
           }
 
-          return async (...args: any[]) => {
+          return (...args: any[]) => {
             const promise = obj.then(
               (value: any) => {
                 utils.flag(this, 'object', value)
@@ -1022,7 +1022,7 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => {
             return result instanceof chai.Assertion ? proxy : result
           }
 
-          return async (...args: any[]) => {
+          return (...args: any[]) => {
             const promise = wrapper.then(
               (value: any) => {
                 const _error = new AssertionError(

packages/runner/src/context.ts

@@ -41,7 +41,8 @@ export function withTimeout<T extends (...args: any[]) => any>(
 
   const { setTimeout, clearTimeout } = getSafeTimers()
 
-  return ((...args: T extends (...args: infer A) => any ? A : never) => {
+  // this function name is used to filter error in test/cli/test/fails.test.ts
+  return (function runWithTimeout(...args: T extends (...args: infer A) => any ? A : never) {
     return Promise.race([
       fn(...args),
       new Promise((resolve, reject) => {

packages/runner/src/run.ts

@@ -224,16 +224,6 @@ export async function runTest(test: Test | Custom, runner: VitestRunner): Promis
           }
           await fn()
         }
-        // some async expect will be added to this array, in case user forget to await theme
-        if (test.promises) {
-          const result = await Promise.allSettled(test.promises)
-          const errors = result
-            .map(r => (r.status === 'rejected' ? r.reason : undefined))
-            .filter(Boolean)
-          if (errors.length) {
-            throw errors
-          }
-        }
 
         await runner.onAfterTryTask?.(test, {
           retry: retryCount,

packages/runner/src/suite.ts

@@ -20,6 +20,7 @@ import type {
   SuiteHooks,
   Task,
   TaskCustomOptions,
+  TaskPopulated,
   Test,
   TestAPI,
   TestFunction,
@@ -346,7 +347,7 @@ function createSuiteCollector(
       setFn(
         task,
         withTimeout(
-          withFixtures(handler, context),
+          withAwaitAsyncAssetions(withFixtures(handler, context), task),
           options?.timeout ?? runner.config.testTimeout,
         ),
       )
@@ -481,6 +482,22 @@ function createSuiteCollector(
   return collector
 }
 
+function withAwaitAsyncAssetions<T extends (...args: any[]) => any>(fn: T, task: TaskPopulated): T {
+  return (async (...args: any[]) => {
+    await fn(...args)
+    // some async expect will be added to this array, in case user forget to await them
+    if (task.promises) {
+      const result = await Promise.allSettled(task.promises)
+      const errors = result
+        .map(r => (r.status === 'rejected' ? r.reason : undefined))
+        .filter(Boolean)
+      if (errors.length) {
+        throw errors
+      }
+    }
+  }) as T
+}
+
 function createSuite() {
   // eslint-disable-next-line unicorn/consistent-function-scoping
   function suiteFn(

test/cli/fixtures/fails/async-assertion.test.ts

@@ -0,0 +1,6 @@
+import { test, expect } from "vitest"
+
+test('multiple errors', () => {
+  expect(new Promise((r) => r("xx"))).resolves.toBe("yy");
+  expect(new Promise((r) => setTimeout(() => r("xx"), 10))).resolves.toBe("zz");
+})

test/cli/fixtures/fails/test-timeout.test.ts

@@ -1,4 +1,4 @@
-import { suite, test } from 'vitest'
+import { expect, suite, test } from 'vitest'
 
 test('hi', async () => {
   await new Promise(resolve => setTimeout(resolve, 1000))
@@ -17,3 +17,7 @@ suite('suite timeout simple input', () => {
     await new Promise(resolve => setTimeout(resolve, 500))
   })
 }, 200)
+
+test('auto await async assertion', { timeout: 20 }, () => {
+  expect(new Promise(() => {})).resolves.toBe(0)
+})

test/cli/test/__snapshots__/fails.test.ts.snap

@@ -2,6 +2,11 @@
 
 exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
 
+exports[`should fail async-assertion.test.ts > async-assertion.test.ts 1`] = `
+"AssertionError: expected 'xx' to be 'zz' // Object.is equality
+AssertionError: expected 'xx' to be 'yy' // Object.is equality"
+`;
+
 exports[`should fail concurrent-suite-deadlock.test.ts > concurrent-suite-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`;
 
 exports[`should fail concurrent-test-deadlock.test.ts > concurrent-test-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`;
@@ -85,7 +90,8 @@ exports[`should fail test-extend/test-rest-props.test.ts > test-extend/test-rest
 exports[`should fail test-extend/test-without-destructuring.test.ts > test-extend/test-without-destructuring.test.ts 1`] = `"Error: The first argument inside a fixture must use object destructuring pattern, e.g. ({ test } => {}). Instead, received "context"."`;
 
 exports[`should fail test-timeout.test.ts > test-timeout.test.ts 1`] = `
-"Error: Test timed out in 200ms.
+"Error: Test timed out in 20ms.
+Error: Test timed out in 200ms.
 Error: Test timed out in 100ms.
 Error: Test timed out in 10ms."
 `;

test/cli/test/fails.test.ts

@@ -17,7 +17,7 @@ it.each(files)('should fail %s', async (file) => {
   const msg = String(stderr)
     .split(/\n/g)
     .reverse()
-    .filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest'))
+    .filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest') && !i.includes('at runWithTimeout'))
     .map(i => i.trim().replace(root, '<rootDir>'),
     ).join('\n')
   expect(msg).toMatchSnapshot(file)

test/core/test/jest-expect.test.ts

@@ -789,7 +789,7 @@ describe('async expect', () => {
 
   describe('promise auto queuing', () => {
     it.fails('fails', () => {
-      expect(() => new Promise((resolve, reject) => setTimeout(reject, 500)))
+      expect(new Promise((resolve, reject) => setTimeout(reject, 500)))
         .resolves
         .toBe('true')
     })