feat(runner): enhance retry options (#9370)

Co-authored-by: Matan Shavit <[email protected]>
Co-authored-by: Ari Perkkiö <[email protected]>
Co-authored-by: Vladimir <[email protected]>

Author: 4 people

docs/config/retry.md

@@ -5,8 +5,141 @@ outline: deep
 
 # retry
 
-- **Type:** `number`
+Retry the test specific number of times if it fails.
+
+- **Type:** `number | { count?: number, delay?: number, condition?: RegExp }`
 - **Default:** `0`
-- **CLI:** `--retry=<value>`
+- **CLI:** `--retry <times>`, `--retry.count <times>`, `--retry.delay <ms>`, `--retry.condition <pattern>`
 
-Retry the test specific number of times if it fails.
+## Basic Usage
+
+Specify a number to retry failed tests:
+
+```ts
+export default defineConfig({
+  test: {
+    retry: 3,
+  },
+})
+```
+
+## CLI Usage
+
+You can also configure retry options from the command line:
+
+```bash
+# Simple retry count
+vitest --retry 3
+
+# Advanced options using dot notation
+vitest --retry.count 3 --retry.delay 500 --retry.condition 'ECONNREFUSED|timeout'
+```
+
+## Advanced Options <Version>4.1.0</Version> {#advanced-options}
+
+Use an object to configure retry behavior:
+
+```ts
+export default defineConfig({
+  test: {
+    retry: {
+      count: 3, // Number of times to retry
+      delay: 1000, // Delay in milliseconds between retries
+      condition: /ECONNREFUSED|timeout/i, // RegExp to match errors that should trigger retry
+    },
+  },
+})
+```
+
+### count
+
+Number of times to retry a test if it fails. Default is `0`.
+
+```ts
+export default defineConfig({
+  test: {
+    retry: {
+      count: 2,
+    },
+  },
+})
+```
+
+### delay
+
+Delay in milliseconds between retry attempts. Useful for tests that interact with rate-limited APIs or need time to recover. Default is `0`.
+
+```ts
+export default defineConfig({
+  test: {
+    retry: {
+      count: 3,
+      delay: 500, // Wait 500ms between retries
+    },
+  },
+})
+```
+
+### condition
+
+A RegExp pattern or a function to determine if a test should be retried based on the error.
+
+- When a **RegExp**, it's tested against the error message
+- When a **function**, it receives the error and returns a boolean
+
+::: warning
+When defining `condition` as a function, it must be done in a test file directly, not in a configuration file (configurations are serialized for worker threads).
+:::
+
+#### RegExp condition (in config file):
+
+```ts
+export default defineConfig({
+  test: {
+    retry: {
+      count: 2,
+      condition: /ECONNREFUSED|ETIMEDOUT/i, // Retry on connection/timeout errors
+    },
+  },
+})
+```
+
+#### Function condition (in test file):
+
+```ts
+import { describe, test } from 'vitest'
+
+describe('tests with advanced retry condition', () => {
+  test('with function condition', { retry: { count: 2, condition: error => error.message.includes('Network') } }, () => {
+    // test code
+  })
+})
+```
+
+## Test File Override
+
+You can also define retry options per test or suite in test files:
+
+```ts
+import { describe, test } from 'vitest'
+
+describe('flaky tests', {
+  retry: {
+    count: 2,
+    delay: 100,
+  },
+}, () => {
+  test('network request', () => {
+    // test code
+  })
+})
+
+test('another test', {
+  retry: {
+    count: 3,
+    condition: error => error.message.includes('timeout'),
+  },
+}, () => {
+  // test code
+})
+```

packages/runner/src/run.ts

@@ -1,4 +1,4 @@
-import type { Awaitable } from '@vitest/utils'
+import type { Awaitable, TestError } from '@vitest/utils'
 import type { DiffOptions } from '@vitest/utils/diff'
 import type { FileSpecification, VitestRunner } from './types/runner'
 import type {
@@ -34,6 +34,42 @@ const now = globalThis.performance ? globalThis.performance.now.bind(globalThis.
 const unixNow = Date.now
 const { clearTimeout, setTimeout } = getSafeTimers()
 
+/**
+ * Normalizes retry configuration to extract individual values.
+ * Handles both number and object forms.
+ */
+function getRetryCount(retry: number | { count?: number } | undefined): number {
+  if (retry === undefined) {
+    return 0
+  }
+  if (typeof retry === 'number') {
+    return retry
+  }
+  return retry.count ?? 0
+}
+
+function getRetryDelay(retry: number | { delay?: number } | undefined): number {
+  if (retry === undefined) {
+    return 0
+  }
+  if (typeof retry === 'number') {
+    return 0
+  }
+  return retry.delay ?? 0
+}
+
+function getRetryCondition(
+  retry: number | { condition?: RegExp | ((error: TestError) => boolean) } | undefined,
+): RegExp | ((error: TestError) => boolean) | undefined {
+  if (retry === undefined) {
+    return undefined
+  }
+  if (typeof retry === 'number') {
+    return undefined
+  }
+  return retry.condition
+}
+
 function updateSuiteHookState(
   task: Task,
   name: keyof SuiteHooks,
@@ -266,6 +302,32 @@ async function callCleanupHooks(runner: VitestRunner, cleanups: unknown[]) {
   }
 }
 
+/**
+ * Determines if a test should be retried based on its retryCondition configuration
+ */
+function passesRetryCondition(test: Test, errors: TestError[] | undefined): boolean {
+  const condition = getRetryCondition(test.retry)
+
+  if (!errors || errors.length === 0) {
+    return false
+  }
+
+  if (!condition) {
+    return true
+  }
+
+  const error = errors[errors.length - 1]
+
+  if (condition instanceof RegExp) {
+    return condition.test(error.message || '')
+  }
+  else if (typeof condition === 'function') {
+    return condition(error)
+  }
+
+  return false
+}
+
 export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
   await runner.onBeforeRunTask?.(test)
 
@@ -300,7 +362,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
 
   const repeats = test.repeats ?? 0
   for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) {
-    const retry = test.retry ?? 0
+    const retry = getRetryCount(test.retry)
     for (let retryCount = 0; retryCount <= retry; retryCount++) {
       let beforeEachCleanups: unknown[] = []
       try {
@@ -412,9 +474,19 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
       }
 
       if (retryCount < retry) {
-        // reset state when retry test
+        const shouldRetry = passesRetryCondition(test, test.result.errors)
+
+        if (!shouldRetry) {
+          break
+        }
+
         test.result.state = 'run'
         test.result.retryCount = (test.result.retryCount ?? 0) + 1
+
+        const delay = getRetryDelay(test.retry)
+        if (delay > 0) {
+          await new Promise(resolve => setTimeout(resolve, delay))
+        }
       }
 
       // update retry info

packages/runner/src/types.ts

@@ -20,10 +20,12 @@ export type {
   InferFixturesTypes,
   OnTestFailedHandler,
   OnTestFinishedHandler,
+  Retry,
   RunMode,
   RuntimeContext,
   SequenceHooks,
   SequenceSetupFiles,
+  SerializableRetry,
   Suite,
   SuiteAPI,
   SuiteCollector,

packages/runner/src/types/runner.ts

@@ -4,6 +4,7 @@ import type {
   ImportDuration,
   SequenceHooks,
   SequenceSetupFiles,
+  SerializableRetry,
   Suite,
   TaskEventPack,
   TaskResultPack,
@@ -36,7 +37,7 @@ export interface VitestRunnerConfig {
   maxConcurrency: number
   testTimeout: number
   hookTimeout: number
-  retry: number
+  retry: SerializableRetry
   includeTaskLocation?: boolean
   diffOptions?: DiffOptions
 }

packages/runner/src/types/tasks.ts

@@ -87,10 +87,12 @@ export interface TaskBase {
    */
   result?: TaskResult
   /**
-   * The amount of times the task should be retried if it fails.
+   * Retry configuration for the task.
+   * - If a number, specifies how many times to retry
+   * - If an object, allows fine-grained retry control
    * @default 0
    */
-  retry?: number
+  retry?: Retry
   /**
    * The amount of times the task should be repeated after the successful run.
    * If the task fails, it will not be retried unless `retry` is specified.
@@ -461,18 +463,70 @@ type ChainableTestAPI<ExtraContext = object> = ChainableFunction<
 
 type TestCollectorOptions = Omit<TestOptions, 'shuffle'>
 
+/**
+ * Retry configuration for tests.
+ * Can be a number for simple retry count, or an object for advanced retry control.
+ */
+export type Retry = number | {
+  /**
+   * The number of times to retry the test if it fails.
+   * @default 0
+   */
+  count?: number
+  /**
+   * Delay in milliseconds between retry attempts.
+   * @default 0
+   */
+  delay?: number
+  /**
+   * Condition to determine if a test should be retried based on the error.
+   * - If a RegExp, it is tested against the error message
+   * - If a function, called with the TestError object; return true to retry
+   *
+   * NOTE: Functions can only be used in test files, not in vitest.config.ts,
+   * because the configuration is serialized when passed to worker threads.
+   *
+   * @default undefined (retry on all errors)
+   */
+  condition?: RegExp | ((error: TestError) => boolean)
+}
+
+/**
+ * Serializable retry configuration (used in config files).
+ * Functions cannot be serialized, so only string conditions are allowed.
+ */
+export type SerializableRetry = number | {
+  /**
+   * The number of times to retry the test if it fails.
+   * @default 0
+   */
+  count?: number
+  /**
+   * Delay in milliseconds between retry attempts.
+   * @default 0
+   */
+  delay?: number
+  /**
+   * Condition to determine if a test should be retried based on the error.
+   * Must be a RegExp tested against the error message.
+   *
+   * @default undefined (retry on all errors)
+   */
+  condition?: RegExp
+}
+
 export interface TestOptions {
   /**
    * Test timeout.
    */
   timeout?: number
   /**
-   * Times to retry the test if fails. Useful for making flaky tests more stable.
-   * When retries is up, the last test error will be thrown.
-   *
+   * Retry configuration for the test.
+   * - If a number, specifies how many times to retry
+   * - If an object, allows fine-grained retry control
    * @default 0
    */
-  retry?: number
+  retry?: Retry
   /**
    * How many times the test will run again.
    * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default.