ci: fix flaky browser tests (#7887)

Author: sheremet-va

packages/browser/src/node/commands/upload.ts

@@ -1,3 +1,4 @@
+import type { UserEventUploadOptions } from '@vitest/browser/context'
 import type { UserEventCommand } from './utils'
 import { dirname, resolve } from 'pathe'
 import { PlaywrightBrowserProvider } from '../providers/playwright'
@@ -7,10 +8,11 @@ export const upload: UserEventCommand<(element: string, files: Array<string | {
   name: string
   mimeType: string
   base64: string
-}>) => void> = async (
+}>, options: UserEventUploadOptions) => void> = async (
   context,
   selector,
   files,
+  options,
 ) => {
   const testPath = context.testPath
   if (!testPath) {
@@ -30,7 +32,7 @@ export const upload: UserEventCommand<(element: string, files: Array<string | {
         buffer: Buffer.from(file.base64, 'base64'),
       }
     })
-    await iframe.locator(selector).setInputFiles(playwrightFiles as string[])
+    await iframe.locator(selector).setInputFiles(playwrightFiles as string[], options)
   }
   else if (context.provider instanceof WebdriverBrowserProvider) {
     for (const file of files) {

packages/browser/src/node/index.ts

@@ -45,6 +45,7 @@ export async function createBrowserServer(
   const vite = await createViteServer({
     ...project.options, // spread project config inlined in root workspace config
     base: '/',
+    root: project.config.root,
     logLevel,
     customLogger: {
       ...logger,

packages/browser/src/node/pool.ts

@@ -82,6 +82,8 @@ export function createBrowserPool(vitest: Vitest): ProcessPool {
           return
         }
 
+        debug?.('provider is ready for %s project', project.name)
+
         const pool = ensurePool(project)
         vitest.state.clearFiles(project, files)
         providers.add(project.browser!.provider)
@@ -112,12 +114,14 @@ export function createBrowserPool(vitest: Vitest): ProcessPool {
     name: 'browser',
     async close() {
       await Promise.all([...providers].map(provider => provider.close()))
+      vitest._browserSessions.sessionIds.clear()
       providers.clear()
       vitest.projects.forEach((project) => {
         project.browser?.state.orchestrators.forEach((orchestrator) => {
           orchestrator.$close()
         })
       })
+      debug?.('browser pool closed all providers')
     },
     runTests: files => runWorkspaceTests('run', files),
     collectTests: files => runWorkspaceTests('collect', files),
@@ -161,6 +165,7 @@ class BrowserPool {
     this._promise ??= createDefer<void>()
 
     if (!files.length) {
+      debug?.('no tests found, finishing test run immediately')
       this._promise.resolve()
       return this._promise
     }
@@ -177,6 +182,7 @@ class BrowserPool {
     })
 
     if (this.orchestrators.size >= this.options.maxWorkers) {
+      debug?.('all orchestrators are ready, not creating more')
       return this._promise
     }
 
@@ -190,13 +196,17 @@ class BrowserPool {
     const promises: Promise<void>[] = []
     for (let i = 0; i < workerCount; i++) {
       const sessionId = crypto.randomUUID()
+      this.project.vitest._browserSessions.sessionIds.add(sessionId)
+      const project = this.project.name
+      debug?.('[%s] creating session for %s', sessionId, project)
       const page = this.openPage(sessionId).then(() => {
         // start running tests on the page when it's ready
         this.runNextTest(method, sessionId)
       })
       promises.push(page)
     }
     await Promise.all(promises)
+    debug?.('all sessions are created')
     return this._promise
   }
 
@@ -230,7 +240,14 @@ class BrowserPool {
     if (this.readySessions.size === this.orchestrators.size) {
       this._promise?.resolve()
       this._promise = undefined
-      debug?.('all tests finished running')
+      debug?.('[%s] all tests finished running', sessionId)
+    }
+    else {
+      debug?.(
+        `did not finish sessions for ${sessionId}: |ready - %s| |overall - %s|`,
+        [...this.readySessions].join(', '),
+        [...this.orchestrators.keys()].join(', '),
+      )
     }
   }
 
@@ -288,6 +305,7 @@ class BrowserPool {
             this.cancel()
             this._promise?.resolve()
             this._promise = undefined
+            debug?.('[%s] browser connection was closed', sessionId)
             return
           }
           debug?.('[%s] error during %s test run: %s', sessionId, file, error)

packages/browser/src/node/projectParent.ts

@@ -79,7 +79,7 @@ export class ParentBrowserProject {
         if (mod) {
           return id
         }
-        const resolvedPath = resolve(project.config.root, id.slice(1))
+        const resolvedPath = resolve(this.vite.config.root, id.slice(1))
         const modUrl = this.vite.moduleGraph.getModuleById(resolvedPath)
         if (modUrl) {
           return resolvedPath

packages/browser/src/node/providers/playwright.ts

@@ -18,6 +18,9 @@ import type {
   TestProject,
 } from 'vitest/node'
 import { createManualModuleSource } from '@vitest/mocker/node'
+import { createDebugger } from 'vitest/node'
+
+const debug = createDebugger('vitest:browser:playwright')
 
 export const playwrightBrowsers = ['firefox', 'webkit', 'chromium'] as const
 export type PlaywrightBrowser = (typeof playwrightBrowsers)[number]
@@ -48,6 +51,8 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
 
   public mocker: BrowserModuleMocker | undefined
 
+  private closing = false
+
   getSupportedBrowsers(): readonly string[] {
     return playwrightBrowsers
   }
@@ -56,18 +61,23 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
     project: TestProject,
     { browser, options }: PlaywrightProviderOptions,
   ): void {
+    this.closing = false
     this.project = project
     this.browserName = browser
     this.options = options as any
     this.mocker = this.createMocker()
   }
 
   private async openBrowser() {
+    await this._throwIfClosing()
+
     if (this.browserPromise) {
+      debug?.('[%s] the browser is resolving, reusing the promise', this.browserName)
       return this.browserPromise
     }
 
     if (this.browser) {
+      debug?.('[%s] the browser is resolved, reusing it', this.browserName)
       return this.browser
     }
 
@@ -103,8 +113,8 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
         }
       }
 
-      const browser = await playwright[this.browserName].launch(launchOptions)
-      this.browser = browser
+      debug?.('[%s] initializing the browser with launch options: %O', this.browserName, launchOptions)
+      this.browser = await playwright[this.browserName].launch(launchOptions)
       this.browserPromise = null
       return this.browser
     })()
@@ -243,11 +253,15 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
   }
 
   private async createContext(sessionId: string) {
+    await this._throwIfClosing()
+
     if (this.contexts.has(sessionId)) {
+      debug?.('[%s][%s] the context already exists, reusing it', sessionId, this.browserName)
       return this.contexts.get(sessionId)!
     }
 
     const browser = await this.openBrowser()
+    await this._throwIfClosing(browser)
     const { actionTimeout, ...contextOptions } = this.options?.context ?? {}
     const options = {
       ...contextOptions,
@@ -257,9 +271,11 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
       options.viewport = null
     }
     const context = await browser.newContext(options)
+    await this._throwIfClosing(context)
     if (actionTimeout) {
       context.setDefaultTimeout(actionTimeout)
     }
+    debug?.('[%s][%s] the context is ready', sessionId, this.browserName)
     this.contexts.set(sessionId, context)
     return context
   }
@@ -306,14 +322,19 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
   }
 
   private async openBrowserPage(sessionId: string) {
+    await this._throwIfClosing()
+
     if (this.pages.has(sessionId)) {
+      debug?.('[%s][%s] the page already exists, closing the old one', sessionId, this.browserName)
       const page = this.pages.get(sessionId)!
       await page.close()
       this.pages.delete(sessionId)
     }
 
     const context = await this.createContext(sessionId)
     const page = await context.newPage()
+    debug?.('[%s][%s] the page is ready', sessionId, this.browserName)
+    await this._throwIfClosing(page)
     this.pages.set(sessionId, page)
 
     if (process.env.VITEST_PW_DEBUG) {
@@ -333,9 +354,24 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
   }
 
   async openPage(sessionId: string, url: string, beforeNavigate?: () => Promise<void>): Promise<void> {
+    debug?.('[%s][%s] creating the browser page for %s', sessionId, this.browserName, url)
     const browserPage = await this.openBrowserPage(sessionId)
     await beforeNavigate?.()
+    debug?.('[%s][%s] browser page is created, opening %s', sessionId, this.browserName, url)
     await browserPage.goto(url, { timeout: 0 })
+    await this._throwIfClosing(browserPage)
+  }
+
+  private async _throwIfClosing(disposable?: { close: () => Promise<void> }) {
+    if (this.closing) {
+      debug?.('[%s] provider was closed, cannot perform the action on %s', this.browserName, String(disposable))
+      await disposable?.close()
+      this.pages.clear()
+      this.contexts.clear()
+      this.browser = null
+      this.browserPromise = null
+      throw new Error(`[vitest] The provider was closed.`)
+    }
   }
 
   async getCDPSession(sessionid: string): Promise<CDPSession> {
@@ -359,13 +395,20 @@ export class PlaywrightBrowserProvider implements BrowserProvider {
   }
 
   async close(): Promise<void> {
+    debug?.('[%s] closing provider', this.browserName)
+    this.closing = true
     const browser = this.browser
     this.browser = null
+    if (this.browserPromise) {
+      await this.browserPromise
+      this.browserPromise = null
+    }
     await Promise.all([...this.pages.values()].map(p => p.close()))
     this.pages.clear()
     await Promise.all([...this.contexts.values()].map(c => c.close()))
     this.contexts.clear()
     await browser?.close()
+    debug?.('[%s] provider is closed', this.browserName)
   }
 }
 

packages/browser/src/node/providers/webdriver.ts

@@ -4,6 +4,9 @@ import type {
   BrowserProviderInitializationOptions,
   TestProject,
 } from 'vitest/node'
+import { createDebugger } from 'vitest/node'
+
+const debug = createDebugger('vitest:browser:wdio')
 
 const webdriverBrowsers = ['firefox', 'chrome', 'edge', 'safari'] as const
 type WebdriverBrowser = (typeof webdriverBrowsers)[number]
@@ -24,6 +27,8 @@ export class WebdriverBrowserProvider implements BrowserProvider {
 
   private options?: Capabilities.WebdriverIOConfig
 
+  private closing = false
+
   getSupportedBrowsers(): readonly string[] {
     return webdriverBrowsers
   }
@@ -32,6 +37,7 @@ export class WebdriverBrowserProvider implements BrowserProvider {
     ctx: TestProject,
     { browser, options }: WebdriverProviderOptions,
   ): Promise<void> {
+    this.closing = false
     this.project = ctx
     this.browserName = browser
     this.options = options as Capabilities.WebdriverIOConfig
@@ -71,7 +77,10 @@ export class WebdriverBrowserProvider implements BrowserProvider {
   }
 
   async openBrowser(): Promise<WebdriverIO.Browser> {
+    await this._throwIfClosing('opening the browser')
+
     if (this.browser) {
+      debug?.('[%s] the browser is already opened, reusing it', this.browserName)
       return this.browser
     }
 
@@ -87,12 +96,16 @@ export class WebdriverBrowserProvider implements BrowserProvider {
 
     const { remote } = await import('webdriverio')
 
-    // TODO: close everything, if browser is closed from the outside
-    this.browser = await remote({
+    const remoteOptions: Capabilities.WebdriverIOConfig = {
       ...this.options,
       logLevel: 'error',
       capabilities: this.buildCapabilities(),
-    })
+    }
+
+    debug?.('[%s] opening the browser with options: %O', this.browserName, remoteOptions)
+    // TODO: close everything, if browser is closed from the outside
+    this.browser = await remote(remoteOptions)
+    await this._throwIfClosing()
 
     return this.browser
   }
@@ -134,12 +147,26 @@ export class WebdriverBrowserProvider implements BrowserProvider {
     return capabilities
   }
 
-  async openPage(_sessionId: string, url: string): Promise<void> {
+  async openPage(sessionId: string, url: string): Promise<void> {
+    await this._throwIfClosing('creating the browser')
+    debug?.('[%s][%s] creating the browser page for %s', sessionId, this.browserName, url)
     const browserInstance = await this.openBrowser()
+    debug?.('[%s][%s] browser page is created, opening %s', sessionId, this.browserName, url)
     await browserInstance.url(url)
+    await this._throwIfClosing('opening the url')
+  }
+
+  private async _throwIfClosing(action?: string) {
+    if (this.closing) {
+      debug?.(`[%s] provider was closed, cannot perform the action${action ? ` ${action}` : ''}`, this.browserName)
+      await (this.browser?.sessionId ? this.browser?.deleteSession?.() : null)
+      throw new Error(`[vitest] The provider was closed.`)
+    }
   }
 
   async close(): Promise<void> {
+    debug?.('[%s] closing provider', this.browserName)
+    this.closing = true
     await Promise.all([
       this.browser?.sessionId ? this.browser?.deleteSession?.() : null,
     ])

packages/browser/src/node/rpc.ts

@@ -58,6 +58,13 @@ export function setupBrowserRpc(globalServer: ParentBrowserProject, defaultMocke
       )
     }
 
+    if (!vitest._browserSessions.sessionIds.has(sessionId)) {
+      const ids = [...vitest._browserSessions.sessionIds].join(', ')
+      return error(
+        new Error(`[vitest] Unknown session id "${sessionId}". Expected one of ${ids}.`),
+      )
+    }
+
     if (type === 'orchestrator') {
       const session = vitest._browserSessions.getSession(sessionId)
       // it's possible the session was already resolved by the preview provider

packages/browser/src/node/serverOrchestrator.ts

@@ -26,6 +26,11 @@ export async function resolveOrchestrator(
     return
   }
 
+  // ignore uknown pages
+  if (sessionId && sessionId !== 'none' && !globalServer.vitest._browserSessions.sessionIds.has(sessionId)) {
+    return
+  }
+
   const injectorJs = typeof globalServer.injectorJs === 'string'
     ? globalServer.injectorJs
     : await globalServer.injectorJs

packages/vitest/src/node/browser/sessions.ts

@@ -5,6 +5,8 @@ import { createDefer } from '@vitest/utils'
 export class BrowserSessions {
   private sessions = new Map<string, BrowserServerStateSession>()
 
+  public sessionIds: Set<string> = new Set()
+
   getSession(sessionId: string): BrowserServerStateSession | undefined {
     return this.sessions.get(sessionId)
   }