feat: fixed visual regression testing, by using VisualWorld base class

Author: dnotes

.changeset/afraid-glasses-move.md

@@ -0,0 +1,5 @@
+---
+"@quickpickle/vitest-browser": minor
+---
+
+Switched to VisualWorld class for visual regression testing

package.json

@@ -30,7 +30,7 @@
   "devDependencies": {
     "playwright": "^1.52.0",
     "pnpm": "^9.15.9",
-    "vitest": "^3.1.3"
+    "vitest": "^3.1.4"
   },
   "dependencies": {
     "@changesets/cli": "^2.29.3"

packages/browser/README.md

@@ -57,8 +57,9 @@ The following is a pretty basic setup for testing components in Svelte, Vue, or
 ## Known Issues:
 
 * Reactivity is currently broken for Svelte and Vue tests.
-* Screenshot comparisons don't work yet.
 * Selecting elements by css selector doesn't work yet.
+* Performing screenshot comparisons may result in an extra screenshot
+  being created for @vitest/browser versions below 3.2.0
 
 ## Suspected Issues:
 
@@ -70,5 +71,5 @@ The following is a pretty basic setup for testing components in Svelte, Vue, or
 
 [x] basic actions and outcomes in English, to match @quickpickle/playwright
 [x] basic tests for rendering Svelte, Vue, and React components
-[ ] full tests for all actions and outcomes, matching @quickpickle/playwright
+[x] full tests for all actions and outcomes, matching @quickpickle/playwright
 [ ] some sort of Storybook-esque presentation using Vitest UI

packages/browser/package.json

@@ -97,14 +97,15 @@
     "@types/react": "^19.1.4",
     "@vitejs/plugin-react": "^4.4.1",
     "@vitejs/plugin-vue": "^5.2.4",
-    "@vitest/browser": "^3.1.3",
+    "@vitest/browser": "^3.1.4",
     "react": "^19.1.0",
     "rollup": "^4",
     "svelte": "^5",
     "vite": "^6",
-    "vitest": "^3.1.3"
+    "vitest": "^3.1.4"
   },
   "dependencies": {
+    "buffer": "^6.0.3",
     "quickpickle": "workspace:^",
     "vitest-browser-react": "^0.1.1",
     "vitest-browser-svelte": "^0.1.0",

packages/browser/rollup.config.js

@@ -43,6 +43,7 @@ export default {
     }),
   ],
   external: [
+    /^buffer/,
     'quickpickle',
     'vite',
     'vitest',

packages/browser/src/VitestBrowserWorld.ts

@@ -2,28 +2,33 @@ import { Before, QuickPickleWorld, QuickPickleWorldInterface } from 'quickpickle
 import type { BrowserPage, Locator, UserEvent, ScreenshotOptions } from '@vitest/browser/context'
 import { defaultsDeep } from 'lodash-es'
 import type { TestContext } from 'vitest';
-import type { InfoConstructor } from 'quickpickle/dist/world';
+import { ScreenshotComparisonOptions, VisualConfigSetting, VisualWorld, VisualWorldInterface, type InfoConstructor } from 'quickpickle';
+import { commands } from '@vitest/browser/context';
+import { Buffer } from 'buffer'
+
 
 /// <reference types="@vitest/browser/providers/playwright" />
 
-export type VitestWorldConfig = {
+export interface VitestWorldConfigSetting extends VisualConfigSetting {
   componentDir?:        string;
-  screenshotDir?:       string;
-  screenshotOptions?:   Partial<ScreenshotOptions>;
 }
 
-export const defaultVitestWorldConfig:VitestWorldConfig = {
+export const defaultVitestWorldConfig:VitestWorldConfigSetting = {
   componentDir: '',               // directory in which components are kept, relative to project root
   screenshotDir: 'screenshots',   // directory in which to save screenshots, relative to project root (default: "screenshots")
-  screenshotOptions: {},          // options for the default screenshot comparisons
+  screenshotOpts: {               // options for the default screenshot comparisons
+    threshold: 0.1,
+    alpha: 0.6,
+    maxDiffPercentage: .01,
+  },
 }
 
 export type ActionsInterface = {
   clicks: any[];
   doubleClicks: any[];
 }
 
-export type VitestBrowserWorldInterface = QuickPickleWorldInterface & {
+export interface VitestBrowserWorldInterface extends VisualWorldInterface {
   /**
    * The `render` function must be provided by the World Constructor
    * and must be tailored for the framework being used. It should render
@@ -49,7 +54,7 @@ export type VitestBrowserWorldInterface = QuickPickleWorldInterface & {
   userEvent: UserEvent;
 }
 
-export class VitestBrowserWorld extends QuickPickleWorld implements VitestBrowserWorldInterface {
+export class VitestBrowserWorld extends VisualWorld implements VitestBrowserWorldInterface {
 
   actions:ActionsInterface = {
     clicks: [],
@@ -59,7 +64,7 @@ export class VitestBrowserWorld extends QuickPickleWorld implements VitestBrowse
   userEvent!: UserEvent;
   async render(name:string|any,props?:any,renderOptions?:any){};
   async cleanup(){};
-  private _page:Locator|null = null;
+  _page!:Locator|null;
 
   constructor(context:TestContext, info:InfoConstructor) {
     info = defaultsDeep(info || {}, { config: { worldConfig: defaultVitestWorldConfig } } )
@@ -75,22 +80,6 @@ export class VitestBrowserWorld extends QuickPickleWorld implements VitestBrowse
     this.userEvent = browserContext.userEvent;
   }
 
-  get screenshotDir() {
-    return this.sanitizePath(this.worldConfig.screenshotDir)
-  }
-
-  get screenshotFilename() {
-    return `${this.toString().replace(/^.+?Feature: /, '').replace(' ' + this.info.step, '')}.png`
-  }
-
-  async screenshot(options:{name?:string, locator?:Locator} = {}) {
-    let locator = options.locator || this.page
-    let path = options.name
-      ? this.fullPath(`${this.screenshotDir}/${options.name}${(this.info.explodedIdx ? ` (${this.info.tags.join(',')})` : '')}.png`.replace(/\.png\.png$/i, '.png'))
-      : this.fullPath(`${this.screenshotDir}/${this.screenshotFilename}`)
-    await locator.screenshot({ path })
-  }
-
   get page():Locator {
     if (!this._page) throw new Error('You must render a component before running tests.')
     return this._page
@@ -220,9 +209,75 @@ export class VitestBrowserWorld extends QuickPickleWorld implements VitestBrowse
     if (toBePresent === (matchingElements.length === 0)) throw new Error(`The${toBeVisible ? '' : ' hidden'} element "${locator}" was unexpectedly ${toBePresent ? 'not present' : 'present'}.`)
   }
 
+  async screenshot(opts?:{
+    bufferOnly?:boolean
+    name?:string
+    locator?:Locator
+  }):Promise<any> {
+    let path
+    if (!opts?.bufferOnly) path = this.getScreenshotPath(opts?.name)
+    let locator = opts?.locator ?? this.page
+    return locator.screenshot({ path })
+  }
+
+  async expectScreenshotMatch(locator:Locator, filename?:string, opts:ScreenshotComparisonOptions={}) {
+
+    const filepath = this.getScreenshotPath(filename)
+    let expectedImg:string
+
+    /**
+     * Load existing screenshot, or save it if it doesn't yet exist
+     */
+    try {
+      expectedImg = await commands.readFile(this.getScreenshotPath(filename), 'base64')
+    }
+    catch(e) {
+      // If the screenshot doesn't exist, save it and pass the test
+      await locator.screenshot()
+      console.warn(`new visual regression test: ${this.screenshotDir}/${this.screenshotFilename}`)
+      return
+    }
+
+    let expected = Buffer.from(expectedImg, 'base64')
+
+    /**
+     * Get the screenshot
+     */
+    // type does not include the "save" option in the docs: see https://vitest.dev/guide/browser/locators#screenshot
+    let screenshotOptions = { save:false, base64:true } as ScreenshotOptions
+    let actualImg = await locator.screenshot(screenshotOptions) as string|{ base64:string }
+    let actual = Buffer.from(typeof actualImg === 'string' ? actualImg : actualImg.base64, 'base64')
+
+    /**
+     * Compare the two screenshots
+     */
+    let screenshotOpts = defaultsDeep(opts, this.worldConfig.screenshotOpts)
+    let matchResult = null
+    try {
+      matchResult = await this.screenshotDiff(actual, expected, screenshotOpts)
+    }
+    catch(e) {}
+    console.log({
+      pass: matchResult?.pass,
+      diffPercentage: matchResult?.diffPercentage,
+      filename,
+      locator,
+    }.toString())
+
+    if (!matchResult?.pass) {
+      await commands.writeFile(`${filepath}.actual.png`, actual.toString('base64'), 'base64');
+      if (matchResult?.diff) await commands.writeFile(`${filepath}.diff.png`, matchResult.diff.toString('base64'), 'base64');
+      throw new Error(`Screenshot does not match the snapshot.
+  Diff percentage: ${matchResult?.diffPercentage?.toFixed(2) ?? '100'}%
+  Max allowed: ${screenshotOpts.maxDiffPercentage}%
+  Diffs at: ${filepath}.(diff|actual).png`)
+    }
+  }
+
   /**
    * Waits for a certain amount of time
    * @param ms number
+   * @deprecated use `wait` method instead
    */
   async waitForTimeout(ms:number) {
     await new Promise(r => setTimeout(r, ms))

packages/browser/src/actions.steps.ts

@@ -148,10 +148,10 @@ When('I uncheck (the ){string}( checkbox)( box)', async function (world:VitestBr
 // })
 
 When('I wait (for ){int}ms', async function (world:VitestBrowserWorld, num) {
-  await world.waitForTimeout(num)
+  await world.wait(num)
 })
 When('I wait (for ){float} second(s)', async function (world:VitestBrowserWorld, num) {
-  await world.waitForTimeout(num * 1000)
+  await world.wait(num * 1000)
 })
 
 // ================

packages/browser/src/outcomes.steps.ts

@@ -232,34 +232,16 @@ Then('a/an/the (value of )(the ){string} {word} should not/NOT be/equal {int}',
 
 // Visual regression testing
 Then('(the )screenshot/snapshot should match', async function (world:VitestBrowserWorld) {
-  await expect(world.page).toMatchImageSnapshot({
-    ...world.worldConfig.screenshotOptions,
-    customSnapshotsDir: world.screenshotDir,
-    customSnapshotIdentifier: world.screenshotFilename,
-  })
+  await world.expectScreenshotMatch(world.page)
 })
 Then('(the )screenshot/snapshot {string} should match', async function (world:VitestBrowserWorld, name:string) {
-  let explodedTags = world.info.explodedIdx ? `_(${world.info.tags.join(',')})` : ''
-  await expect(world.page).toMatchImageSnapshot({
-    ...world.worldConfig.screenshotOptions,
-    customSnapshotsDir: world.screenshotDir,
-    customSnapshotIdentifier: `${name}${explodedTags}`,
-  })
+  await world.expectScreenshotMatch(world.page, name)
 })
 Then('(the )screenshot/snapshot of the {string} {word} should match', async function (world:VitestBrowserWorld, identifier, role) {
   let locator = await world.getLocator(world.page, identifier, role)
-  await expect(locator).toMatchImageSnapshot({
-    ...world.worldConfig.screenshotOptions,
-    customSnapshotsDir: world.screenshotDir,
-    customSnapshotIdentifier: world.screenshotFilename,
-  })
+  await world.expectScreenshotMatch(locator)
 })
 Then('(the )screenshot/snapshot {string} of the {string} {word} should match', async function (world:VitestBrowserWorld, name, identifier, role) {
   let locator = await world.getLocator(world.page, identifier, role)
-  let explodedTags = world.info.explodedIdx ? `_(${world.info.tags.join(',')})` : ''
-  await expect(locator).toMatchImageSnapshot({
-    ...world.worldConfig.screenshotOptions,
-    customSnapshotsDir: world.screenshotDir,
-    customSnapshotIdentifier: `${name}${explodedTags}`,
-  })
+  await world.expectScreenshotMatch(locator, name)
 })

packages/browser/tests/.gitignore

@@ -0,0 +1 @@
+__screenshots__/

packages/browser/tests/generic/browser-actions.feature

@@ -168,7 +168,7 @@ Feature: Actions step definitions in Vitest Browser
 
     Scenario: Taking a default screenshot
       When I take a screenshot
-      Then the screenshot "Actions step definitions in Vitest Browser_Taking a default screenshot_01.png" should exist--delete it
+      Then the screenshot "Feature Actions step definitions in Vitest Browser_Taking a default screenshot_01.png" should exist--delete it
 
     Scenario: Taking a named screenshot
       When I take a screenshot named "pickles"
@@ -184,10 +184,10 @@ Feature: Actions step definitions in Vitest Browser
 
     @sequential
     Scenario: Cleaning up the screenshots with exploded tags
-      Then the screenshot "Actions step definitions in Vitest Browser_Taking a default screenshot with exploded tags (sequential,tag1)_01.png" should exist--delete it
-      And the screenshot "Actions step definitions in Vitest Browser_Taking a default screenshot with exploded tags (sequential,tag2)_01.png" should exist--delete it
+      Then the screenshot "Feature Actions step definitions in Vitest Browser_Taking a default screenshot with exploded tags (sequential,tag1)_01.png" should exist--delete it
+      And the screenshot "Feature Actions step definitions in Vitest Browser_Taking a default screenshot with exploded tags (sequential,tag2)_01.png" should exist--delete it
 
     @sequential @skip-ci
     Scenario: Cleaning up the screenshots with exploded tags
-      And the screenshot "temp (sequential,tag1).png" should exist--delete it
-      And the screenshot "temp (sequential,tag2).png" should exist--delete it
+      And the screenshot "temp_(sequential,tag1).png" should exist--delete it
+      And the screenshot "temp_(sequential,tag2).png" should exist--delete it