diff --git a/docs/guide/browser/config.md b/docs/guide/browser/config.md index c1f8c7242c9b..7dd724a73ce9 100644 --- a/docs/guide/browser/config.md +++ b/docs/guide/browser/config.md @@ -492,3 +492,116 @@ For example, to store diffs in a subdirectory of attachments: resolveDiffPath: ({ arg, attachmentsDir, browserName, ext, root, testFileName }) => `${root}/${attachmentsDir}/screenshot-diffs/${testFileName}/${arg}-${browserName}${ext}` ``` + +#### browser.expect.toMatchScreenshot.comparators + +- **Type:** `Record` + +Register custom screenshot comparison algorithms, like [SSIM](https://en.wikipedia.org/wiki/Structural_similarity_index_measure) or other perceptual similarity metrics. + +To create a custom comparator, you need to register it in your config. If using TypeScript, declare its options in the `ScreenshotComparatorRegistry` interface. + +```ts +import { defineConfig } from 'vitest/config' + +// 1. Declare the comparator's options type +declare module 'vitest/browser' { + interface ScreenshotComparatorRegistry { + myCustomComparator: { + sensitivity?: number + ignoreColors?: boolean + } + } +} + +// 2. Implement the comparator +export default defineConfig({ + test: { + browser: { + expect: { + toMatchScreenshot: { + comparators: { + myCustomComparator: async ( + reference, + actual, + { + createDiff, // always provided by Vitest + sensitivity = 0.01, + ignoreColors = false, + } + ) => { + // ...algorithm implementation + return { pass, diff, message } + }, + }, + }, + }, + }, + }, +}) +``` + +Then use it in your tests: + +```ts +await expect(locator).toMatchScreenshot({ + comparatorName: 'myCustomComparator', + comparatorOptions: { + sensitivity: 0.08, + ignoreColors: true, + }, +}) +``` + +**Comparator Function Signature:** + +```ts +type Comparator = ( + reference: { + metadata: { height: number; width: number } + data: TypedArray + }, + actual: { + metadata: { height: number; width: number } + data: TypedArray + }, + options: { + createDiff: boolean + } & Options +) => Promise<{ + pass: boolean + diff: TypedArray | null + message: string | null +}> | { + pass: boolean + diff: TypedArray | null + message: string | null +} +``` + +The `reference` and `actual` images are decoded using the appropriate codec (currently only PNG). The `data` property is a flat `TypedArray` (`Buffer`, `Uint8Array`, or `Uint8ClampedArray`) containing pixel data in RGBA format: + +- **4 bytes per pixel**: red, green, blue, alpha (from `0` to `255` each) +- **Row-major order**: pixels are stored left-to-right, top-to-bottom +- **Total length**: `width × height × 4` bytes +- **Alpha channel**: always present. Images without transparency have alpha values set to `255` (fully opaque) + +::: tip Performance Considerations +The `createDiff` option indicates whether a diff image is needed. During [stable screenshot detection](/guide/browser/visual-regression-testing#how-visual-tests-work), Vitest calls comparators with `createDiff: false` to avoid unnecessary work. + +**Respect this flag to keep your tests fast**. +::: + +::: warning Handle Missing Options +The `options` parameter in `toMatchScreenshot()` is optional, so users might not provide all your comparator options. Always make them optional with default values: + +```ts +myCustomComparator: ( + reference, + actual, + { createDiff, threshold = 0.1, maxDiff = 100 }, +) => { + // ...comparison logic +} +``` +::: diff --git a/docs/guide/browser/visual-regression-testing.md b/docs/guide/browser/visual-regression-testing.md index 5049c2728b25..101253f8d134 100644 --- a/docs/guide/browser/visual-regression-testing.md +++ b/docs/guide/browser/visual-regression-testing.md @@ -121,6 +121,24 @@ $ vitest --update Review updated screenshots before committing to make sure changes are intentional. +## How Visual Tests Work + +Visual regression tests need stable screenshots to compare against. But pages aren't instantly stable as images load, animations finish, fonts render, and layouts settle. + +Vitest handles this automatically through "Stable Screenshot Detection": + +1. Vitest takes a first screenshot (or uses the reference screenshot if available) as baseline +1. It takes another screenshot and compares it with the baseline + - If the screenshots match, the page is stable and testing continues + - If they differ, Vitest uses the newest screenshot as the baseline and repeats +1. This continues until stability is achieved or the timeout is reached + +This ensures that transient visual changes (like loading spinners or animations) don't cause false failures. If something never stops animating though, you'll hit the timeout, so consider [disabling animations during testing](#disable-animations). + +If a stable screenshot is captured after retries (one or more) and a reference screenshot exists, Vitest performs a final comparison with the reference using `createDiff: true`. This will generate a diff image if they don't match. + +During stability detection, Vitest calls comparators with `createDiff: false` since it only needs to know if screenshots match. This keeps the detection process fast. + ## Configuring Visual Tests ### Global Configuration diff --git a/packages/browser-playwright/src/playwright.ts b/packages/browser-playwright/src/playwright.ts index 278ab3f39540..02239b938e64 100644 --- a/packages/browser-playwright/src/playwright.ts +++ b/packages/browser-playwright/src/playwright.ts @@ -1,5 +1,6 @@ /* eslint-disable ts/method-signature-style */ +import type { CustomComparatorsRegistry } from '@vitest/browser' import type { MockedModule } from '@vitest/mocker' import type { Browser, @@ -556,7 +557,7 @@ declare module 'vitest/node' { extends Omit< ScreenshotMatcherOptions, 'comparatorName' | 'comparatorOptions' - > {} + >, CustomComparatorsRegistry {} export interface ToMatchScreenshotComparators extends ScreenshotComparatorRegistry {} diff --git a/packages/browser-webdriverio/src/webdriverio.ts b/packages/browser-webdriverio/src/webdriverio.ts index 57040dd3fa46..7996fb265ace 100644 --- a/packages/browser-webdriverio/src/webdriverio.ts +++ b/packages/browser-webdriverio/src/webdriverio.ts @@ -1,3 +1,4 @@ +import type { CustomComparatorsRegistry } from '@vitest/browser' import type { Capabilities } from '@wdio/types' import type { ScreenshotComparatorRegistry, @@ -306,7 +307,7 @@ declare module 'vitest/node' { extends Omit< ScreenshotMatcherOptions, 'comparatorName' | 'comparatorOptions' - > {} + >, CustomComparatorsRegistry {} export interface ToMatchScreenshotComparators extends ScreenshotComparatorRegistry {} diff --git a/packages/browser/context.d.ts b/packages/browser/context.d.ts index d338909d5ccd..dec09a89659e 100644 --- a/packages/browser/context.d.ts +++ b/packages/browser/context.d.ts @@ -40,7 +40,7 @@ export interface ScreenshotOptions { save?: boolean } -export interface ScreenshotComparatorRegistry { +interface StandardScreenshotComparators { pixelmatch: { /** * The maximum number of pixels that are allowed to differ between the captured @@ -136,6 +136,13 @@ export interface ScreenshotComparatorRegistry { } } +export interface ScreenshotComparatorRegistry extends StandardScreenshotComparators {} + +export type NonStandardScreenshotComparators = Omit< + ScreenshotComparatorRegistry, + keyof StandardScreenshotComparators +> + export interface ScreenshotMatcherOptions< ComparatorName extends keyof ScreenshotComparatorRegistry = keyof ScreenshotComparatorRegistry > { diff --git a/packages/browser/src/node/commands/screenshotMatcher/comparators/index.ts b/packages/browser/src/node/commands/screenshotMatcher/comparators/index.ts index 696bae0beca1..8ef9307a206d 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/comparators/index.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/comparators/index.ts @@ -1,20 +1,34 @@ +import type { BrowserCommandContext } from 'vitest/node' import type { ScreenshotComparatorRegistry } from '../../../../../context' import type { Comparator } from '../types' import { pixelmatch } from './pixelmatch' -const comparators = new Map(Object.entries({ - pixelmatch, -} satisfies { +const comparators: { [ComparatorName in keyof ScreenshotComparatorRegistry]: Comparator< ScreenshotComparatorRegistry[ComparatorName] > -})) +} = { + pixelmatch, +} export function getComparator( comparator: ComparatorName, + context: BrowserCommandContext, ): Comparator { - if (comparators.has(comparator)) { - return comparators.get(comparator)! + if (comparator in comparators) { + return comparators[comparator] + } + + const customComparators = context + .project + .config + .browser + .expect + ?.toMatchScreenshot + ?.comparators + + if (customComparators && comparator in customComparators) { + return customComparators[comparator] } throw new Error(`Unrecognized comparator ${comparator}`) diff --git a/packages/browser/src/node/commands/screenshotMatcher/types.ts b/packages/browser/src/node/commands/screenshotMatcher/types.ts index b1c0cd12702b..b7bd081e3834 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/types.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/types.ts @@ -1,4 +1,5 @@ import type { + NonStandardScreenshotComparators, ScreenshotComparatorRegistry, ScreenshotMatcherOptions, } from '@vitest/browser/context' @@ -47,12 +48,21 @@ export type Comparator> = ( } & Options ) => Promisable<{ pass: boolean; diff: TypedArray | null; message: string | null }> +type CustomComparatorsToRegister = { + [Key in keyof NonStandardScreenshotComparators]: Comparator +} + +export type CustomComparatorsRegistry + = keyof CustomComparatorsToRegister extends never + ? { comparators?: Record>> } + : { comparators: CustomComparatorsToRegister } + declare module 'vitest/node' { export interface ToMatchScreenshotOptions extends Omit< ScreenshotMatcherOptions, 'comparatorName' | 'comparatorOptions' - > {} + >, CustomComparatorsRegistry {} export interface ToMatchScreenshotComparators extends ScreenshotComparatorRegistry {} diff --git a/packages/browser/src/node/commands/screenshotMatcher/utils.ts b/packages/browser/src/node/commands/screenshotMatcher/utils.ts index 96b95e3e3966..5c0dddf6c0c9 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/utils.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/utils.ts @@ -8,12 +8,13 @@ import { basename, dirname, extname, join, relative, resolve } from 'pathe' import { getCodec } from './codecs' import { getComparator } from './comparators' -type GlobalOptions = Required< +type GlobalOptions = Required['toMatchScreenshot'] & NonNullable> - > -> + >, + 'comparators' +>> const defaultOptions = { comparatorName: 'pixelmatch', @@ -140,7 +141,7 @@ export function resolveOptions( return { codec: getCodec(extension), - comparator: getComparator(resolvedOptions.comparatorName), + comparator: getComparator(resolvedOptions.comparatorName, context), resolvedOptions, paths: { reference: resolvedOptions.resolveScreenshotPath(resolvePathData), diff --git a/packages/browser/src/node/index.ts b/packages/browser/src/node/index.ts index d2c7f16bcaff..0b490100ffaf 100644 --- a/packages/browser/src/node/index.ts +++ b/packages/browser/src/node/index.ts @@ -9,6 +9,8 @@ import BrowserPlugin from './plugin' import { ParentBrowserProject } from './projectParent' import { setupBrowserRpc } from './rpc' +export type { CustomComparatorsRegistry } from './commands/screenshotMatcher/types' + export function defineBrowserCommand( fn: BrowserCommand, ): BrowserCommand { diff --git a/test/browser/fixtures/expect-dom/toMatchScreenshot.test.ts b/test/browser/fixtures/expect-dom/toMatchScreenshot.test.ts index 7ac7ced6ae73..22679fe4ffe0 100644 --- a/test/browser/fixtures/expect-dom/toMatchScreenshot.test.ts +++ b/test/browser/fixtures/expect-dom/toMatchScreenshot.test.ts @@ -16,6 +16,12 @@ const renderTestCase = (colors: readonly [string, string, string]) => `) +declare module 'vitest/browser' { + interface ScreenshotComparatorRegistry { + failing: Record + } +} + /** * ## Screenshot Testing Strategy * @@ -370,4 +376,49 @@ describe('.toMatchScreenshot', () => { ) }, ) + + test('can use custom comparators', async ({ onTestFinished }) => { + const filename = globalThis.crypto.randomUUID() + const path = join( + '__screenshots__', + 'toMatchScreenshot.test.ts', + `${filename}-${server.browser}-${server.platform}.png`, + ) + + onTestFinished(async () => { + await server.commands.removeFile(path) + }) + + renderTestCase([ + 'oklch(39.6% 0.141 25.723)', + 'oklch(40.5% 0.101 131.063)', + 'oklch(37.9% 0.146 265.522)', + ]) + + const locator = page.getByTestId(dataTestId) + + // Create a reference screenshot by explicitly saving one + await locator.screenshot({ + save: true, + path, + }) + + // Test that `toMatchScreenshot()` correctly uses a custom comparator; since + // the element hasn't changed, it should match, but this custom comparator + // will always fail + await expect(locator).toMatchScreenshot(filename) + + let errorMessage: string + + try { + await expect(locator).toMatchScreenshot(filename, { + comparatorName: 'failing', + timeout: 100, + }) + } catch (error) { + errorMessage = error.message + } + + expect(errorMessage).matches(/^Could not capture a stable screenshot within 100ms\.$/m) + }) }) diff --git a/test/browser/fixtures/expect-dom/vitest.config.js b/test/browser/fixtures/expect-dom/vitest.config.js index ccbaf85a5770..f2f2fcaf40e3 100644 --- a/test/browser/fixtures/expect-dom/vitest.config.js +++ b/test/browser/fixtures/expect-dom/vitest.config.js @@ -10,7 +10,14 @@ export default defineConfig({ provider, instances, isolate: false, + expect: { + toMatchScreenshot: { + comparators: { + failing: () => ({ pass: false, diff: null, message: null }), + }, + }, + }, }, setupFiles: './setup.ts', }, -}) \ No newline at end of file +})