Update CDP Mode

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -445,7 +445,8 @@ sb.cdp.get_text(selector)
 sb.cdp.get_title()
 sb.cdp.get_current_url()
 sb.cdp.get_origin()
-sb.cdp.get_page_source()
+sb.cdp.get_html(include_shadow_dom=True)
+sb.cdp.get_page_source(include_shadow_dom=True)
 sb.cdp.get_user_agent()
 sb.cdp.get_cookie_string()
 sb.cdp.get_locale_code()

help_docs/method_summary.md

@@ -40,6 +40,7 @@ self.refresh()
 # Duplicates: self.refresh_page(), self.reload_page(), self.reload()
 self.get_current_url()
 self.get_origin()
+self.get_html()
 self.get_page_source()
 self.get_title()
 # Duplicates: self.get_page_title()

seleniumbase/core/browser_launcher.py

@@ -838,6 +838,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     cdp.get_element_position = CDPM.get_element_position
     cdp.get_gui_element_rect = CDPM.get_gui_element_rect
     cdp.get_gui_element_center = CDPM.get_gui_element_center
+    cdp.get_html = CDPM.get_html
     cdp.get_page_source = CDPM.get_page_source
     cdp.get_user_agent = CDPM.get_user_agent
     cdp.get_cookie_string = CDPM.get_cookie_string
@@ -934,6 +935,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     cdp.core = core_items
     cdp.loop = cdp.get_event_loop()
     driver.cdp = cdp
+    driver.solve_captcha = CDPM.solve_captcha
     driver._is_using_cdp = True
 
 

seleniumbase/core/sb_cdp.py

@@ -1185,7 +1185,14 @@ def get_origin(self):
             self.page.evaluate("window.location.origin")
         )
 
-    def get_page_source(self):
+    def get_html(self, include_shadow_dom=True):
+        return self.get_page_source(
+            include_shadow_dom=include_shadow_dom
+        )
+
+    def get_page_source(self, include_shadow_dom=True):
+        if include_shadow_dom:
+            return self.find_element("html").get_html()
         try:
             source = self.loop.run_until_complete(
                 self.page.evaluate("document.documentElement.outerHTML")

seleniumbase/fixtures/base_case.py

@@ -1293,8 +1293,11 @@ def get_origin(self):
         self.__check_scope()
         return self.execute_script("return window.location.origin;")
 
-    def get_page_source(self):
-        if self.__is_cdp_swap_needed():
+    def get_html(self, *args, **kwargs):
+        return self.get_page_source(*args, **kwargs)
+
+    def get_page_source(self, *args, **kwargs):
+        if self.__is_cdp_swap_needed(*args, **kwargs):
             return self.cdp.get_page_source()
         self.wait_for_ready_state_complete()
         if self.__needs_minimum_wait:

seleniumbase/undetected/cdp_driver/config.py

@@ -130,6 +130,7 @@ def __init__(
             "--no-default-browser-check",
             "--homepage=about:blank",
             "--no-pings",
+            "--enable-unsafe-extension-debugging",
             "--wm-window-animations-disabled",
             "--animation-duration-scale=0",
             "--enable-privacy-sandbox-ads-apis",

seleniumbase/undetected/cdp_driver/element.py

@@ -806,7 +806,10 @@ async def set_text_async(self, value):
 
     async def get_html_async(self):
         return await self._tab.send(
-            cdp.dom.get_outer_html(backend_node_id=self.backend_node_id)
+            cdp.dom.get_outer_html(
+                backend_node_id=self.backend_node_id,
+                include_shadow_dom=True,
+            )
         )
 
     @property

seleniumbase/undetected/cdp_driver/tab.py

@@ -899,7 +899,10 @@ async def get_content(self):
         """Gets the current page source content (html)"""
         doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True))
         return await self.send(
-            cdp.dom.get_outer_html(backend_node_id=doc.backend_node_id)
+            cdp.dom.get_outer_html(
+                backend_node_id=doc.backend_node_id,
+                include_shadow_dom=True,
+            )
         )
 
     async def maximize(self):