refactor!: Rename PlaywrightCrawler kwargs: browser_options, page_options (#831)

### Description

Motivation is for the names to match their actual usage. Rename them
also in internal components.
Change `PlaywrightBrowserController.close` method to rely on context to
close pages, as per PW documentation recommendations.

### Breaking changes
- Renamed `PlaywrightCrawler.__init__` keyword arguments:
`browser_options` to `browser_launch_options`, `page_options` to
`browser_new_context_options`.
- Same keyword arguments renaming happened in `PlaywrightBrowserPlugin`,
`BaseBrowserPlugin`, `PlaywrightBrowserController`,
`BaseBrowserController` in all methods where previously named arguments
were present.

---------

Co-authored-by: Vlada Dusek <[email protected]>

Author: Pijukatel

docs/examples/code/playwright_crawler_with_camoufox.py

@@ -18,7 +18,7 @@ async def new_browser(self) -> PlaywrightBrowserController:
             raise RuntimeError('Playwright browser plugin is not initialized.')
 
         return PlaywrightBrowserController(
-            browser=await AsyncNewBrowser(self._playwright, headless=True, **self._browser_options),
+            browser=await AsyncNewBrowser(self._playwright, headless=True, **self._browser_launch_options),
             max_open_pages_per_browser=1,  # Increase, if camoufox can handle it in your use case.
             header_generator=None,  # This turns off the crawlee header_generation. Camoufox has its own.
         )

docs/upgrading/upgrading_to_v0x.md

@@ -40,6 +40,10 @@ This section summarizes the breaking changes between v0.4.x and v0.5.0.
 ### PlaywrightCrawler
 
 - The `PlaywrightPreNavigationContext` was renamed to `PlaywrightPreNavCrawlingContext`.
+- The input arguments in `PlaywrightCrawler.__init__` have been renamed:
+    - `browser_options` is now `browser_launch_options`,
+    - `page_options` is now `browser_new_context_options`.
+- These argument renaming changes have also been applied to `BrowserPool`, `PlaywrightBrowserPlugin`, and `PlaywrightBrowserController`.
 
 ## Upgrading to v0.4
 

src/crawlee/browsers/_base_browser_controller.py

@@ -59,13 +59,15 @@ def browser_type(self) -> BrowserType:
     @abstractmethod
     async def new_page(
         self,
-        page_options: Mapping[str, Any] | None = None,
+        browser_new_context_options: Mapping[str, Any] | None = None,
         proxy_info: ProxyInfo | None = None,
     ) -> Page:
         """Create a new page with the given context options.
 
         Args:
-            page_options: Options to configure the new page.
+            browser_new_context_options: Keyword arguments to pass to the browser new context method. These options
+                are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
+                Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             proxy_info: The proxy configuration to use for the new page.
 
         Returns:

src/crawlee/browsers/_base_browser_plugin.py

@@ -35,13 +35,23 @@ def browser_type(self) -> BrowserType:
 
     @property
     @abstractmethod
-    def browser_options(self) -> Mapping[str, Any]:
-        """Return the options for a new browser."""
+    def browser_launch_options(self) -> Mapping[str, Any]:
+        """Return the options for the `browser.launch` method.
+
+        Keyword arguments to pass to the browser launch method. These options are provided directly to Playwright's
+        `browser_type.launch` method. For more details, refer to the Playwright documentation:
+         https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
+        """
 
     @property
     @abstractmethod
-    def page_options(self) -> Mapping[str, Any]:
-        """Return the options for a new page."""
+    def browser_new_context_options(self) -> Mapping[str, Any]:
+        """Return the options for the `browser.new_context` method.
+
+        Keyword arguments to pass to the browser new context method. These options are provided directly to Playwright's
+        `browser.new_context` method. For more details, refer to the Playwright documentation:
+        https://playwright.dev/python/docs/api/class-browser#browser-new-context.
+        """
 
     @property
     @abstractmethod

src/crawlee/browsers/_browser_pool.py

@@ -100,30 +100,30 @@ def with_default_plugin(
         cls,
         *,
         browser_type: BrowserType | None = None,
-        browser_options: Mapping[str, Any] | None = None,
-        page_options: Mapping[str, Any] | None = None,
+        browser_launch_options: Mapping[str, Any] | None = None,
+        browser_new_context_options: Mapping[str, Any] | None = None,
         headless: bool | None = None,
         **kwargs: Any,
     ) -> BrowserPool:
         """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options.
 
         Args:
             browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
-            browser_options: Keyword arguments to pass to the browser launch method. These options are provided
+            browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided
                 directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
                 documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
-            page_options: Keyword arguments to pass to the page object is set at the playwright context level.
-                These options are provided directly to Playwright's `browser.new_context` method. For more details,
-                refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
+            browser_new_context_options: Keyword arguments to pass to the browser new context method. These options
+                are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
+                Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             headless: Whether to run the browser in headless mode.
             kwargs: Additional arguments for default constructor.
         """
         plugin_options: dict = defaultdict(dict)
-        plugin_options['browser_options'] = browser_options or {}
-        plugin_options['page_options'] = page_options or {}
+        plugin_options['browser_launch_options'] = browser_launch_options or {}
+        plugin_options['browser_new_context_options'] = browser_new_context_options or {}
 
         if headless is not None:
-            plugin_options['browser_options']['headless'] = headless
+            plugin_options['browser_launch_options']['headless'] = headless
 
         if browser_type:
             plugin_options['browser_type'] = browser_type
@@ -262,13 +262,16 @@ async def _get_new_page(
     ) -> CrawleePage:
         """Internal method to initialize a new page in a browser using the specified plugin."""
         timeout = self._operation_timeout.total_seconds()
-        browser = self._pick_browser_with_free_capacity(plugin)
+        browser_controller = self._pick_browser_with_free_capacity(plugin)
 
         try:
-            if not browser:
-                browser = await asyncio.wait_for(self._launch_new_browser(plugin), timeout)
+            if not browser_controller:
+                browser_controller = await asyncio.wait_for(self._launch_new_browser(plugin), timeout)
             page = await asyncio.wait_for(
-                browser.new_page(page_options=plugin.page_options, proxy_info=proxy_info),
+                browser_controller.new_page(
+                    browser_new_context_options=plugin.browser_new_context_options,
+                    proxy_info=proxy_info,
+                ),
                 timeout,
             )
         except asyncio.TimeoutError as exc:

src/crawlee/browsers/_playwright_browser_controller.py

@@ -20,6 +20,10 @@
 
     from crawlee.proxy_configuration import ProxyInfo
 
+from logging import getLogger
+
+logger = getLogger(__name__)
+
 
 @docs_group('Classes')
 class PlaywrightBrowserController(BaseBrowserController):
@@ -94,11 +98,25 @@ def browser_type(self) -> BrowserType:
     @override
     async def new_page(
         self,
-        page_options: Mapping[str, Any] | None = None,
+        browser_new_context_options: Mapping[str, Any] | None = None,
         proxy_info: ProxyInfo | None = None,
     ) -> Page:
+        """Create a new page with the given context options.
+
+        Args:
+            browser_new_context_options: Keyword arguments to pass to the browser new context method. These options
+                are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
+                Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
+            proxy_info: The proxy configuration to use for the new page.
+
+        Returns:
+            Page: The newly created page.
+
+        Raises:
+            ValueError: If the browser has reached the maximum number of open pages.
+        """
         if not self._browser_context:
-            self._browser_context = await self._create_browser_context(page_options, proxy_info)
+            self._browser_context = await self._create_browser_context(browser_new_context_options, proxy_info)
 
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
@@ -116,21 +134,27 @@ async def new_page(
 
     @override
     async def close(self, *, force: bool = False) -> None:
-        if force:
-            for page in self._pages:
-                await page.close()
+        """Close the browser.
 
-        if self.pages_count > 0:
+        Args:
+            force: Whether to force close all open pages before closing the browser.
+
+        Raises:
+            ValueError: If there are still open pages when trying to close the browser.
+        """
+        if self.pages_count > 0 and not force:
             raise ValueError('Cannot close the browser while there are open pages.')
 
+        if self._browser_context:
+            await self._browser_context.close()
         await self._browser.close()
 
     def _on_page_close(self, page: Page) -> None:
         """Handle actions after a page is closed."""
         self._pages.remove(page)
 
     async def _create_browser_context(
-        self, page_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None
+        self, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None
     ) -> BrowserContext:
         """Create a new browser context with the specified proxy settings."""
         if self._header_generator:
@@ -141,17 +165,19 @@ async def _create_browser_context(
         else:
             extra_http_headers = None
 
-        page_options = dict(page_options) if page_options else {}
-        page_options['extra_http_headers'] = page_options.get('extra_http_headers', extra_http_headers)
+        browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {}
+        browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
+            'extra_http_headers', extra_http_headers
+        )
+
+        if proxy_info:
+            if browser_new_context_options['proxy']:
+                logger.warning("browser_new_context_options['proxy'] overriden by explicit `proxy_info` argument.")
 
-        proxy = (
-            ProxySettings(
+            browser_new_context_options['proxy'] = ProxySettings(
                 server=f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}',
                 username=proxy_info.username,
                 password=proxy_info.password,
             )
-            if proxy_info
-            else None
-        )
 
-        return await self._browser.new_context(proxy=proxy, **page_options)
+        return await self._browser.new_context(**browser_new_context_options)

src/crawlee/browsers/_playwright_browser_plugin.py

@@ -35,26 +35,26 @@ def __init__(
         self,
         *,
         browser_type: BrowserType = 'chromium',
-        browser_options: Mapping[str, Any] | None = None,
-        page_options: Mapping[str, Any] | None = None,
+        browser_launch_options: Mapping[str, Any] | None = None,
+        browser_new_context_options: Mapping[str, Any] | None = None,
         max_open_pages_per_browser: int = 20,
     ) -> None:
         """A default constructor.
 
         Args:
             browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
-            browser_options: Keyword arguments to pass to the browser launch method. These options are provided
+            browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided
                 directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
                 documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
-            page_options: Keyword arguments to pass to the page object is set at the playwright context level.
-                These options are provided directly to Playwright's `browser.new_context` method. For more details,
-                refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
+            browser_new_context_options: Keyword arguments to pass to the browser new context method. These options
+                are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
+                Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance.
                 Once reached, a new browser instance will be launched to handle the excess.
         """
         self._browser_type = browser_type
-        self._browser_options = browser_options or {}
-        self._page_options = page_options or {}
+        self._browser_launch_options = browser_launch_options or {}
+        self._browser_new_context_options = browser_new_context_options or {}
         self._max_open_pages_per_browser = max_open_pages_per_browser
 
         self._playwright_context_manager = async_playwright()
@@ -75,13 +75,25 @@ def browser_type(self) -> BrowserType:
 
     @property
     @override
-    def browser_options(self) -> Mapping[str, Any]:
-        return self._browser_options
+    def browser_launch_options(self) -> Mapping[str, Any]:
+        """Return the options for the `browser.launch` method.
+
+        Keyword arguments to pass to the browser launch method. These options are provided directly to Playwright's
+        `browser_type.launch` method. For more details, refer to the Playwright documentation:
+         https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
+        """
+        return self._browser_launch_options
 
     @property
     @override
-    def page_options(self) -> Mapping[str, Any]:
-        return self._page_options
+    def browser_new_context_options(self) -> Mapping[str, Any]:
+        """Return the options for the `browser.new_context` method.
+
+        Keyword arguments to pass to the browser new context method. These options are provided directly to Playwright's
+        `browser.new_context` method. For more details, refer to the Playwright documentation:
+        https://playwright.dev/python/docs/api/class-browser#browser-new-context.
+        """
+        return self._browser_new_context_options
 
     @property
     @override
@@ -117,11 +129,11 @@ async def new_browser(self) -> PlaywrightBrowserController:
             raise RuntimeError('Playwright browser plugin is not initialized.')
 
         if self._browser_type == 'chromium':
-            browser = await self._playwright.chromium.launch(**self._browser_options)
+            browser = await self._playwright.chromium.launch(**self._browser_launch_options)
         elif self._browser_type == 'firefox':
-            browser = await self._playwright.firefox.launch(**self._browser_options)
+            browser = await self._playwright.firefox.launch(**self._browser_launch_options)
         elif self._browser_type == 'webkit':
-            browser = await self._playwright.webkit.launch(**self._browser_options)
+            browser = await self._playwright.webkit.launch(**self._browser_launch_options)
         else:
             raise ValueError(f'Invalid browser type: {self._browser_type}')
 

src/crawlee/playwright_crawler/_playwright_crawler.py

@@ -71,8 +71,8 @@ def __init__(
         self,
         browser_pool: BrowserPool | None = None,
         browser_type: BrowserType | None = None,
-        browser_options: Mapping[str, Any] | None = None,
-        page_options: Mapping[str, Any] | None = None,
+        browser_launch_options: Mapping[str, Any] | None = None,
+        browser_new_context_options: Mapping[str, Any] | None = None,
         headless: bool | None = None,
         **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]],
     ) -> None:
@@ -82,33 +82,36 @@ def __init__(
             browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages.
             browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
                 This option should not be used if `browser_pool` is provided.
-            browser_options: Keyword arguments to pass to the browser launch method. These options are provided
+            browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided
                 directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
                 documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
                 This option should not be used if `browser_pool` is provided.
-            page_options: Keyword arguments to pass to the new page method. These options are provided directly to
-                Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation:
-                https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
+            browser_new_context_options: Keyword arguments to pass to the browser new context method. These options
+                are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
+                Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
                 This option should not be used if `browser_pool` is provided.
             headless: Whether to run the browser in headless mode.
                 This option should not be used if `browser_pool` is provided.
             kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`.
         """
         if browser_pool:
             # Raise an exception if browser_pool is provided together with other browser-related arguments.
-            if any(param is not None for param in (headless, browser_type, browser_options, page_options)):
+            if any(
+                param is not None
+                for param in (headless, browser_type, browser_launch_options, browser_new_context_options)
+            ):
                 raise ValueError(
-                    'You cannot provide `headless`, `browser_type`, `browser_options` or `page_options` '
-                    'arguments when `browser_pool` is provided.'
+                    'You cannot provide `headless`, `browser_type`, `browser_launch_options` or '
+                    '`browser_new_context_options` arguments when `browser_pool` is provided.'
                 )
 
         # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments.
         else:
             browser_pool = BrowserPool.with_default_plugin(
                 headless=headless,
                 browser_type=browser_type,
-                browser_options=browser_options,
-                page_options=page_options,
+                browser_launch_options=browser_launch_options,
+                browser_new_context_options=browser_new_context_options,
             )
 
         self._browser_pool = browser_pool

tests/unit/browsers/test_browser_pool.py

@@ -154,7 +154,7 @@ async def test_methods_raise_error_when_not_active() -> None:
 
 
 async def test_with_plugin_contains_page_options(httpbin: URL) -> None:
-    plugin = PlaywrightBrowserPlugin(page_options={'user_agent': 'My Best User-Agent'})
+    plugin = PlaywrightBrowserPlugin(browser_new_context_options={'user_agent': 'My Best User-Agent'})
     async with BrowserPool(plugins=[plugin]) as browser_pool:
         test_page = await browser_pool.new_page()
         await test_page.page.goto(str(httpbin / 'user-agent'))