From e39e6a75cc4934b7dcb73894cd7d2d0ea42bbac8 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 18 Mar 2026 12:36:51 -0700 Subject: [PATCH 01/10] feat: add ref system for aria snapshots --- tools/src/gcu/browser/refs.py | 191 +++++++++++++++++++ tools/src/gcu/browser/session.py | 4 + tools/src/gcu/browser/tools/advanced.py | 20 ++ tools/src/gcu/browser/tools/inspection.py | 7 + tools/src/gcu/browser/tools/interactions.py | 72 +++++++- tools/tests/test_refs.py | 192 ++++++++++++++++++++ 6 files changed, 478 insertions(+), 8 deletions(-) create mode 100644 tools/src/gcu/browser/refs.py create mode 100644 tools/tests/test_refs.py diff --git a/tools/src/gcu/browser/refs.py b/tools/src/gcu/browser/refs.py new file mode 100644 index 0000000000..6e9ece722d --- /dev/null +++ b/tools/src/gcu/browser/refs.py @@ -0,0 +1,191 @@ +"""Ref system for aria snapshots. + +Assigns short `[ref=eN]` markers to interactive elements in Playwright's +aria_snapshot() output so the LLM can reference elements by ref instead of +constructing fragile CSS selectors. + +Usage: + annotated, ref_map = annotate_snapshot(raw_snapshot) + # ... later, when the LLM says selector="e5" ... + playwright_selector = resolve_ref("e5", ref_map) +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .session import BrowserSession + +# --------------------------------------------------------------------------- +# Role sets (matching Playwright's aria roles that matter for interaction) +# --------------------------------------------------------------------------- + +INTERACTIVE_ROLES: frozenset[str] = frozenset({ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "scrollbar", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +}) + +NAMED_CONTENT_ROLES: frozenset[str] = frozenset({ + "cell", + "heading", + "img", +}) + +# Regex: captures indent, role, optional quoted name, and trailing text. +# Example line: " - button \"Submit\" [disabled]" +# group(1)=indent " ", group(2)=role "button", +# group(3)=name "Submit" (or None), group(4)=rest " [disabled]" +_LINE_RE = re.compile(r"^(\s*-\s+)(\w+)(?:\s+\"([^\"]*)\")?(.*?)$") + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RefEntry: + """A single ref entry mapping to a Playwright role selector.""" + + role: str + name: str | None + nth: int + + +# ref_id (e.g. "e0") -> RefEntry +RefMap = dict[str, RefEntry] + +# --------------------------------------------------------------------------- +# annotate_snapshot +# --------------------------------------------------------------------------- + + +def annotate_snapshot(snapshot: str) -> tuple[str, RefMap]: + """Inject ``[ref=eN]`` markers into an aria snapshot. + + Returns: + (annotated_text, ref_map) where ref_map maps ref ids to RefEntry. + """ + lines = snapshot.split("\n") + + # First pass: identify which lines get refs and count (role, name) pairs + # for nth disambiguation. + candidates: list[tuple[int, str, str | None]] = [] # (line_idx, role, name) + pair_counts: dict[tuple[str, str | None], int] = {} + + for i, line in enumerate(lines): + m = _LINE_RE.match(line) + if not m: + continue + role = m.group(2) + name = m.group(3) # None if no quoted name + + if role in INTERACTIVE_ROLES or (role in NAMED_CONTENT_ROLES and name): + candidates.append((i, role, name)) + key = (role, name) + pair_counts[key] = pair_counts.get(key, 0) + 1 + + # Second pass: assign refs with nth indices. + ref_map: RefMap = {} + pair_seen: dict[tuple[str, str | None], int] = {} + ref_counter = 0 + + for line_idx, role, name in candidates: + key = (role, name) + nth = pair_seen.get(key, 0) + pair_seen[key] = nth + 1 + + ref_id = f"e{ref_counter}" + ref_counter += 1 + + ref_map[ref_id] = RefEntry(role=role, name=name, nth=nth) + + # Inject [ref=eN] at end of line (before any trailing whitespace) + lines[line_idx] = lines[line_idx].rstrip() + f" [ref={ref_id}]" + + return "\n".join(lines), ref_map + + +# --------------------------------------------------------------------------- +# resolve_ref +# --------------------------------------------------------------------------- + +_REF_PATTERN = re.compile(r"^e\d+$") + + +def resolve_ref(selector: str, ref_map: RefMap | None) -> str: + """Resolve a ref id (e.g. ``"e5"``) to a Playwright role selector. + + If *selector* doesn't look like a ref (``e\\d+``), it's returned as-is + so that plain CSS selectors keep working. + + Raises: + ValueError: If the ref is not found or no snapshot has been taken. + """ + if not _REF_PATTERN.match(selector): + return selector # Pass through CSS / XPath / role selectors + + if ref_map is None: + raise ValueError( + f"Ref '{selector}' used but no snapshot has been taken yet. " + "Call browser_snapshot first." + ) + + entry = ref_map.get(selector) + if entry is None: + valid = ", ".join(sorted(ref_map.keys(), key=lambda k: int(k[1:]))) + raise ValueError( + f"Ref '{selector}' not found. Valid refs: {valid}. " + "The page may have changed — take a new snapshot." + ) + + # Build Playwright role selector + if entry.name is not None: + escaped_name = entry.name.replace("\\", "\\\\").replace('"', '\\"') + sel = f'role={entry.role}[name="{escaped_name}"]' + else: + sel = f"role={entry.role}" + + # Always include nth to disambiguate + sel += f" >> nth={entry.nth}" + return sel + + +# --------------------------------------------------------------------------- +# Convenience wrapper +# --------------------------------------------------------------------------- + + +def resolve_selector( + selector: str, + session: BrowserSession, + target_id: str | None, +) -> str: + """Resolve a selector that might be a ref, using the session's ref maps. + + Args: + selector: A CSS selector or ref id (e.g. ``"e5"``). + session: The current BrowserSession. + target_id: The target page id (falls back to session.active_page_id). + """ + tid = target_id or session.active_page_id + ref_map = session.ref_maps.get(tid) if tid else None + return resolve_ref(selector, ref_map) diff --git a/tools/src/gcu/browser/session.py b/tools/src/gcu/browser/session.py index 89ab73e62f..626db76294 100644 --- a/tools/src/gcu/browser/session.py +++ b/tools/src/gcu/browser/session.py @@ -353,6 +353,7 @@ class BrowserSession: active_page_id: str | None = None console_messages: dict[str, list[dict]] = field(default_factory=dict) page_meta: dict[str, TabMeta] = field(default_factory=dict) + ref_maps: dict[str, dict] = field(default_factory=dict) # target_id → RefMap _playwright: Any = None _lock: asyncio.Lock = field(default_factory=asyncio.Lock) @@ -447,6 +448,7 @@ async def _cleanup_after_failed_start(self) -> None: self.active_page_id = None self.console_messages.clear() self.page_meta.clear() + self.ref_maps.clear() async def start(self, headless: bool = True, persistent: bool = True) -> dict: """ @@ -623,6 +625,7 @@ async def stop(self) -> dict: self.active_page_id = None self.console_messages.clear() self.page_meta.clear() + self.ref_maps.clear() self.user_data_dir = None self.persistent = False @@ -801,6 +804,7 @@ def _handle_page_close(self, target_id: str) -> None: self.pages.pop(target_id, None) self.console_messages.pop(target_id, None) self.page_meta.pop(target_id, None) + self.ref_maps.pop(target_id, None) if self.active_page_id == target_id: self.active_page_id = next(iter(self.pages), None) diff --git a/tools/src/gcu/browser/tools/advanced.py b/tools/src/gcu/browser/tools/advanced.py index 67384e0eb2..7294a55fb6 100644 --- a/tools/src/gcu/browser/tools/advanced.py +++ b/tools/src/gcu/browser/tools/advanced.py @@ -16,6 +16,7 @@ ) from ..highlight import highlight_element +from ..refs import resolve_selector from ..session import DEFAULT_TIMEOUT_MS, get_session @@ -52,6 +53,10 @@ async def browser_wait( return {"ok": False, "error": "No active tab"} if selector: + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} await page.wait_for_selector(selector, timeout=timeout_ms) return {"ok": True, "action": "wait", "condition": "selector", "selector": selector} elif text: @@ -122,6 +127,11 @@ async def browser_get_text( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + element = await page.wait_for_selector(selector, timeout=timeout_ms) if not element: return {"ok": False, "error": f"Element not found: {selector}"} @@ -160,6 +170,11 @@ async def browser_get_attribute( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + element = await page.wait_for_selector(selector, timeout=timeout_ms) if not element: return {"ok": False, "error": f"Element not found: {selector}"} @@ -238,6 +253,11 @@ async def browser_upload( if not Path(path).exists(): return {"ok": False, "error": f"File not found: {path}"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + await highlight_element(page, selector) element = await page.wait_for_selector(selector, timeout=timeout_ms) diff --git a/tools/src/gcu/browser/tools/inspection.py b/tools/src/gcu/browser/tools/inspection.py index 68c452d456..1564a160c3 100644 --- a/tools/src/gcu/browser/tools/inspection.py +++ b/tools/src/gcu/browser/tools/inspection.py @@ -196,6 +196,13 @@ async def browser_snapshot( await cdp.detach() else: snapshot = await page.locator(":root").aria_snapshot() + # Annotate with [ref=eN] markers for interactive elements + from ..refs import annotate_snapshot + + snapshot, ref_map = annotate_snapshot(snapshot) + tid = target_id or session.active_page_id + if tid: + session.ref_maps[tid] = ref_map return { "ok": True, diff --git a/tools/src/gcu/browser/tools/interactions.py b/tools/src/gcu/browser/tools/interactions.py index 26d7e78c5d..0437427a45 100644 --- a/tools/src/gcu/browser/tools/interactions.py +++ b/tools/src/gcu/browser/tools/interactions.py @@ -17,7 +17,8 @@ ) from ..highlight import highlight_coordinate, highlight_element -from ..session import DEFAULT_TIMEOUT_MS, get_session +from ..refs import annotate_snapshot, resolve_selector +from ..session import DEFAULT_TIMEOUT_MS, BrowserSession, get_session logger = logging.getLogger(__name__) @@ -27,6 +28,8 @@ async def _auto_snapshot( page: Page, *, + session: BrowserSession | None = None, + target_id: str | None = None, wait_for_nav: bool = False, max_chars: int = _AUTO_SNAPSHOT_MAX_CHARS, ) -> str | None: @@ -34,6 +37,8 @@ async def _auto_snapshot( Args: page: Playwright Page instance. + session: BrowserSession to store ref maps in. + target_id: Target page id for ref map storage. wait_for_nav: If True, briefly wait for any in-flight navigation to settle before snapshotting. Used after click actions that may trigger page navigation. @@ -48,6 +53,14 @@ async def _auto_snapshot( except Exception: pass # No navigation happened — that's fine snapshot = await page.locator(":root").aria_snapshot() + + # Annotate with refs before truncation so the full RefMap is captured + if snapshot and session: + snapshot, ref_map = annotate_snapshot(snapshot) + tid = target_id or session.active_page_id + if tid: + session.ref_maps[tid] = ref_map + if snapshot and max_chars > 0 and len(snapshot) > max_chars: snapshot = ( snapshot[:max_chars] @@ -96,6 +109,11 @@ async def browser_click( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + await highlight_element(page, selector) if double_click: @@ -105,7 +123,9 @@ async def browser_click( result: dict = {"ok": True, "action": "click", "selector": selector} if auto_snapshot: - snapshot = await _auto_snapshot(page, wait_for_nav=True) + snapshot = await _auto_snapshot( + page, session=session, target_id=target_id, wait_for_nav=True, + ) if snapshot: result["snapshot"] = snapshot result["url"] = page.url @@ -151,7 +171,9 @@ async def browser_click_coordinate( await page.mouse.click(x, y, button=button) result: dict = {"ok": True, "action": "click_coordinate", "x": x, "y": y} if auto_snapshot: - snapshot = await _auto_snapshot(page, wait_for_nav=True) + snapshot = await _auto_snapshot( + page, session=session, target_id=target_id, wait_for_nav=True, + ) if snapshot: result["snapshot"] = snapshot result["url"] = page.url @@ -194,6 +216,11 @@ async def browser_type( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + await highlight_element(page, selector) if clear_first: @@ -202,7 +229,7 @@ async def browser_type( await page.type(selector, text, delay=delay_ms, timeout=timeout_ms) result: dict = {"ok": True, "action": "type", "selector": selector, "length": len(text)} if auto_snapshot: - snapshot = await _auto_snapshot(page) + snapshot = await _auto_snapshot(page, session=session, target_id=target_id) if snapshot: result["snapshot"] = snapshot result["url"] = page.url @@ -244,12 +271,17 @@ async def browser_fill( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + await highlight_element(page, selector) await page.fill(selector, value, timeout=timeout_ms) result: dict = {"ok": True, "action": "fill", "selector": selector} if auto_snapshot: - snapshot = await _auto_snapshot(page) + snapshot = await _auto_snapshot(page, session=session, target_id=target_id) if snapshot: result["snapshot"] = snapshot result["url"] = page.url @@ -287,6 +319,10 @@ async def browser_press( return {"ok": False, "error": "No active tab"} if selector: + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} await page.press(selector, key, timeout=timeout_ms) else: await page.keyboard.press(key) @@ -322,6 +358,11 @@ async def browser_hover( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + await page.hover(selector, timeout=timeout_ms) return {"ok": True, "action": "hover", "selector": selector} except PlaywrightTimeout: @@ -360,6 +401,11 @@ async def browser_select( if not page: return {"ok": False, "error": "No active tab"} + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + selected = await page.select_option(selector, values, timeout=timeout_ms) result: dict = { "ok": True, @@ -368,7 +414,7 @@ async def browser_select( "selected": selected, } if auto_snapshot: - snapshot = await _auto_snapshot(page) + snapshot = await _auto_snapshot(page, session=session, target_id=target_id) if snapshot: result["snapshot"] = snapshot result["url"] = page.url @@ -422,6 +468,10 @@ async def browser_scroll( delta_x = -amount if selector: + try: + selector = resolve_selector(selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} element = await page.query_selector(selector) if element: await element.evaluate(f"e => e.scrollBy({delta_x}, {delta_y})") @@ -435,7 +485,7 @@ async def browser_scroll( "amount": amount, } if auto_snapshot: - snapshot = await _auto_snapshot(page) + snapshot = await _auto_snapshot(page, session=session, target_id=target_id) if snapshot: result["snapshot"] = snapshot result["url"] = page.url @@ -474,6 +524,12 @@ async def browser_drag( if not page: return {"ok": False, "error": "No active tab"} + try: + start_selector = resolve_selector(start_selector, session, target_id) + end_selector = resolve_selector(end_selector, session, target_id) + except ValueError as e: + return {"ok": False, "error": str(e)} + await page.drag_and_drop( start_selector, end_selector, @@ -486,7 +542,7 @@ async def browser_drag( "to": end_selector, } if auto_snapshot: - snapshot = await _auto_snapshot(page) + snapshot = await _auto_snapshot(page, session=session, target_id=target_id) if snapshot: result["snapshot"] = snapshot result["url"] = page.url diff --git a/tools/tests/test_refs.py b/tools/tests/test_refs.py new file mode 100644 index 0000000000..165240c4e0 --- /dev/null +++ b/tools/tests/test_refs.py @@ -0,0 +1,192 @@ +"""Tests for the browser ref system (annotate_snapshot / resolve_ref).""" + +from __future__ import annotations + +import pytest + +from gcu.browser.refs import ( + RefEntry, + annotate_snapshot, + resolve_ref, +) + + +# --------------------------------------------------------------------------- +# annotate_snapshot +# --------------------------------------------------------------------------- + +SAMPLE_SNAPSHOT = """\ +- navigation "Main": + - link "Home" + - link "About" +- main: + - heading "Welcome" + - textbox "Search" + - button "Submit" + - paragraph: some text here + - img "Logo" + - list: + - listitem: + - link "Item 1" + - listitem: + - link "Item 2\"""" + + +class TestAnnotateSnapshot: + def test_assigns_refs_to_interactive_roles(self): + annotated, ref_map = annotate_snapshot(SAMPLE_SNAPSHOT) + # link, textbox, button should all get refs + assert "[ref=e" in annotated + # Check that specific interactive elements got refs + roles_in_map = {entry.role for entry in ref_map.values()} + assert "link" in roles_in_map + assert "textbox" in roles_in_map + assert "button" in roles_in_map + + def test_skips_structural_roles(self): + annotated, ref_map = annotate_snapshot(SAMPLE_SNAPSHOT) + roles_in_map = {entry.role for entry in ref_map.values()} + # navigation, main, list, listitem, paragraph are structural — no refs + assert "navigation" not in roles_in_map + assert "main" not in roles_in_map + assert "list" not in roles_in_map + assert "listitem" not in roles_in_map + assert "paragraph" not in roles_in_map + + def test_named_content_roles_get_refs(self): + annotated, ref_map = annotate_snapshot(SAMPLE_SNAPSHOT) + roles_in_map = {entry.role for entry in ref_map.values()} + # heading and img have names, so they should get refs + assert "heading" in roles_in_map + assert "img" in roles_in_map + + def test_unnamed_content_roles_skip(self): + snapshot = "- heading\n- img" + _, ref_map = annotate_snapshot(snapshot) + # No names → no refs for content roles + assert len(ref_map) == 0 + + def test_preserves_non_matching_lines(self): + snapshot = "some random text\n- button \"OK\"\nanother line" + annotated, _ = annotate_snapshot(snapshot) + lines = annotated.split("\n") + assert lines[0] == "some random text" + assert lines[2] == "another line" + + def test_nth_disambiguation(self): + snapshot = '- button "Save"\n- button "Save"\n- button "Cancel"' + annotated, ref_map = annotate_snapshot(snapshot) + + # Two "Save" buttons should have nth=0 and nth=1 + save_entries = [ + (rid, e) for rid, e in ref_map.items() + if e.role == "button" and e.name == "Save" + ] + assert len(save_entries) == 2 + nths = sorted(e.nth for _, e in save_entries) + assert nths == [0, 1] + + # "Cancel" should have nth=0 + cancel_entries = [ + e for e in ref_map.values() + if e.role == "button" and e.name == "Cancel" + ] + assert len(cancel_entries) == 1 + assert cancel_entries[0].nth == 0 + + def test_sequential_ref_ids(self): + snapshot = '- link "A"\n- link "B"\n- link "C"' + _, ref_map = annotate_snapshot(snapshot) + assert set(ref_map.keys()) == {"e0", "e1", "e2"} + + def test_empty_snapshot(self): + annotated, ref_map = annotate_snapshot("") + assert annotated == "" + assert ref_map == {} + + +# --------------------------------------------------------------------------- +# resolve_ref +# --------------------------------------------------------------------------- + + +class TestResolveRef: + def test_resolves_valid_ref(self): + ref_map = { + "e0": RefEntry(role="button", name="Submit", nth=0), + } + result = resolve_ref("e0", ref_map) + assert result == 'role=button[name="Submit"] >> nth=0' + + def test_passes_through_css_selectors(self): + ref_map = {"e0": RefEntry(role="button", name="OK", nth=0)} + assert resolve_ref("#my-button", ref_map) == "#my-button" + assert resolve_ref(".btn-primary", ref_map) == ".btn-primary" + assert resolve_ref("div > button", ref_map) == "div > button" + + def test_passes_through_role_selectors(self): + ref_map = {"e0": RefEntry(role="button", name="OK", nth=0)} + sel = 'role=button[name="OK"]' + assert resolve_ref(sel, ref_map) == sel + + def test_raises_on_unknown_ref(self): + ref_map = {"e0": RefEntry(role="button", name="OK", nth=0)} + with pytest.raises(ValueError, match="not found"): + resolve_ref("e99", ref_map) + + def test_raises_when_no_ref_map(self): + with pytest.raises(ValueError, match="no snapshot"): + resolve_ref("e0", None) + + def test_escapes_quotes_in_name(self): + ref_map = { + "e0": RefEntry(role="button", name='Say "Hello"', nth=0), + } + result = resolve_ref("e0", ref_map) + assert result == 'role=button[name="Say \\"Hello\\""] >> nth=0' + + def test_no_name_produces_role_only_selector(self): + ref_map = { + "e0": RefEntry(role="textbox", name=None, nth=0), + } + result = resolve_ref("e0", ref_map) + assert result == "role=textbox >> nth=0" + + def test_empty_name(self): + ref_map = { + "e0": RefEntry(role="button", name="", nth=0), + } + result = resolve_ref("e0", ref_map) + assert result == 'role=button[name=""] >> nth=0' + + def test_nth_in_selector(self): + ref_map = { + "e0": RefEntry(role="link", name="Next", nth=2), + } + result = resolve_ref("e0", ref_map) + assert result == 'role=link[name="Next"] >> nth=2' + + +# --------------------------------------------------------------------------- +# Round-trip: annotate → resolve +# --------------------------------------------------------------------------- + + +class TestRoundTrip: + def test_annotate_then_resolve(self): + snapshot = '- button "Submit"\n- textbox "Email"\n- link "Home"' + _, ref_map = annotate_snapshot(snapshot) + + # Each ref should resolve to a valid Playwright role selector + for ref_id, entry in ref_map.items(): + resolved = resolve_ref(ref_id, ref_map) + assert resolved.startswith(f"role={entry.role}") + if entry.name is not None: + assert f'name="{entry.name}"' in resolved + assert f"nth={entry.nth}" in resolved + + def test_css_selectors_still_work_after_annotate(self): + snapshot = '- button "OK"' + _, ref_map = annotate_snapshot(snapshot) + # CSS selectors pass through even when a ref_map exists + assert resolve_ref("#submit-btn", ref_map) == "#submit-btn" From 9dadb5264d87002d57b5e9aec64e2762f358d81b Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 18 Mar 2026 12:40:18 -0700 Subject: [PATCH 02/10] feat: add screenshot image passthrough to LLM --- .../agents/queen/reference/gcu_guide.md | 2 +- core/framework/graph/conversation.py | 16 ++ core/framework/graph/event_loop_node.py | 4 + core/framework/graph/gcu.py | 7 +- core/framework/llm/provider.py | 1 + core/framework/runner/mcp_client.py | 33 ++-- core/framework/runner/tool_registry.py | 11 +- tools/pyproject.toml | 3 + tools/src/gcu/browser/tools/inspection.py | 154 ++++++++++++++++-- tools/tests/test_screenshot_normalization.py | 151 +++++++++++++++++ uv.lock | 6 +- 11 files changed, 360 insertions(+), 28 deletions(-) create mode 100644 tools/tests/test_screenshot_normalization.py diff --git a/core/framework/agents/queen/reference/gcu_guide.md b/core/framework/agents/queen/reference/gcu_guide.md index f9a552b677..c27db24d8e 100644 --- a/core/framework/agents/queen/reference/gcu_guide.md +++ b/core/framework/agents/queen/reference/gcu_guide.md @@ -150,7 +150,7 @@ Call all three subagents in a single response to run them in parallel: ## GCU Anti-Patterns -- Using `browser_screenshot` to read text (use `browser_snapshot`) +- Using `browser_screenshot` to read text (use `browser_snapshot` instead; screenshots are for visual context only) - Re-navigating after scrolling (resets scroll position) - Attempting login on auth walls - Forgetting `target_id` in multi-tab scenarios diff --git a/core/framework/graph/conversation.py b/core/framework/graph/conversation.py index 99f541528b..97bd56cda5 100644 --- a/core/framework/graph/conversation.py +++ b/core/framework/graph/conversation.py @@ -33,6 +33,8 @@ class Message: is_transition_marker: bool = False # True when this message is real human input (from /chat), not a system prompt is_client_input: bool = False + # Optional image content blocks (e.g. from browser_screenshot) + image_content: list[dict[str, Any]] | None = None def to_llm_dict(self) -> dict[str, Any]: """Convert to OpenAI-format message dict.""" @@ -47,6 +49,15 @@ def to_llm_dict(self) -> dict[str, Any]: # role == "tool" content = f"ERROR: {self.content}" if self.is_error else self.content + if self.image_content: + # Multimodal tool result: text + image content blocks + blocks: list[dict[str, Any]] = [{"type": "text", "text": content}] + blocks.extend(self.image_content) + return { + "role": "tool", + "tool_call_id": self.tool_use_id, + "content": blocks, + } return { "role": "tool", "tool_call_id": self.tool_use_id, @@ -72,6 +83,8 @@ def to_storage_dict(self) -> dict[str, Any]: d["is_transition_marker"] = self.is_transition_marker if self.is_client_input: d["is_client_input"] = self.is_client_input + if self.image_content is not None: + d["image_content"] = self.image_content return d @classmethod @@ -87,6 +100,7 @@ def from_storage_dict(cls, data: dict[str, Any]) -> Message: phase_id=data.get("phase_id"), is_transition_marker=data.get("is_transition_marker", False), is_client_input=data.get("is_client_input", False), + image_content=data.get("image_content"), ) @@ -409,6 +423,7 @@ async def add_tool_result( tool_use_id: str, content: str, is_error: bool = False, + image_content: list[dict[str, Any]] | None = None, ) -> Message: msg = Message( seq=self._next_seq, @@ -417,6 +432,7 @@ async def add_tool_result( tool_use_id=tool_use_id, is_error=is_error, phase_id=self._current_phase, + image_content=image_content, ) self._messages.append(msg) self._next_seq += 1 diff --git a/core/framework/graph/event_loop_node.py b/core/framework/graph/event_loop_node.py index cd65d6b839..a5bcfdf1c4 100644 --- a/core/framework/graph/event_loop_node.py +++ b/core/framework/graph/event_loop_node.py @@ -2707,6 +2707,7 @@ async def _timed_subagent( tool_use_id=tc.tool_use_id, content=result.content, is_error=result.is_error, + image_content=result.image_content, ) if tc.tool_name in ("ask_user", "ask_user_multiple"): # Defer tool_call_completed until after user responds @@ -3813,6 +3814,7 @@ def _truncate_tool_result( tool_use_id=result.tool_use_id, content=truncated, is_error=False, + image_content=result.image_content, ) spill_dir = self._config.spillover_dir @@ -3885,6 +3887,7 @@ def _truncate_tool_result( tool_use_id=result.tool_use_id, content=content, is_error=False, + image_content=result.image_content, ) # No spillover_dir — truncate in-place if needed @@ -3927,6 +3930,7 @@ def _truncate_tool_result( tool_use_id=result.tool_use_id, content=truncated, is_error=False, + image_content=result.image_content, ) return result diff --git a/core/framework/graph/gcu.py b/core/framework/graph/gcu.py index d3a35c814a..c8ace4f39d 100644 --- a/core/framework/graph/gcu.py +++ b/core/framework/graph/gcu.py @@ -43,8 +43,11 @@ `browser_snapshot` separately after every action. Only call `browser_snapshot` when you need a fresh view without performing an action, or after setting `auto_snapshot=false`. -- Do NOT use `browser_screenshot` for reading text content - — it produces huge base64 images with no searchable text. +- Do NOT use `browser_screenshot` to read text — use + `browser_snapshot` for that (compact, searchable, fast). +- DO use `browser_screenshot` when you need visual context: + charts, images, canvas elements, layout verification, or when + the snapshot doesn't capture what you need. - Only fall back to `browser_get_text` for extracting specific small elements by CSS selector. diff --git a/core/framework/llm/provider.py b/core/framework/llm/provider.py index d505619222..9b21d15600 100644 --- a/core/framework/llm/provider.py +++ b/core/framework/llm/provider.py @@ -45,6 +45,7 @@ class ToolResult: tool_use_id: str content: str is_error: bool = False + image_content: list[dict[str, Any]] | None = None class LLMProvider(ABC): diff --git a/core/framework/runner/mcp_client.py b/core/framework/runner/mcp_client.py index 8149249ac5..cd731fed64 100644 --- a/core/framework/runner/mcp_client.py +++ b/core/framework/runner/mcp_client.py @@ -391,17 +391,30 @@ async def _call_tool_stdio_async(self, tool_name: str, arguments: dict[str, Any] error_text = content_item.text raise RuntimeError(f"MCP tool '{tool_name}' failed: {error_text}") - # Extract content + # Extract content — preserve image blocks alongside text if result.content: - # MCP returns content as a list of content items - if len(result.content) > 0: - content_item = result.content[0] - # Check if it's a text content item - if hasattr(content_item, "text"): - return content_item.text - elif hasattr(content_item, "data"): - return content_item.data - return result.content + text_parts: list[str] = [] + image_parts: list[dict[str, Any]] = [] + for item in result.content: + if hasattr(item, "text"): + text_parts.append(item.text) + elif hasattr(item, "data") and hasattr(item, "mimeType"): + # MCP ImageContent — preserve as structured image block + image_parts.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{item.mimeType};base64,{item.data}", + }, + } + ) + elif hasattr(item, "data"): + text_parts.append(str(item.data)) + + text = "\n".join(text_parts) if text_parts else "" + if image_parts: + return {"_text": text, "_images": image_parts} + return text if text else None return None diff --git a/core/framework/runner/tool_registry.py b/core/framework/runner/tool_registry.py index cc7fa2921c..590171671f 100644 --- a/core/framework/runner/tool_registry.py +++ b/core/framework/runner/tool_registry.py @@ -243,6 +243,13 @@ def get_executor(self) -> Callable[[ToolUse], ToolResult]: def _wrap_result(tool_use_id: str, result: Any) -> ToolResult: if isinstance(result, ToolResult): return result + # MCP client returns dict with _images when image content is present + if isinstance(result, dict) and "_images" in result: + return ToolResult( + tool_use_id=tool_use_id, + content=result.get("_text", ""), + image_content=result["_images"], + ) return ToolResult( tool_use_id=tool_use_id, content=json.dumps(result) if not isinstance(result, str) else result, @@ -560,7 +567,9 @@ def executor(inputs: dict) -> Any: } merged_inputs = {**clean_inputs, **filtered_context} result = client_ref.call_tool(tool_name, merged_inputs) - # MCP tools return content array, extract the result + # MCP client already extracts content (returns str + # or {"_text": ..., "_images": ...} for image results). + # Handle legacy list format from HTTP transport. if isinstance(result, list) and len(result) > 0: if isinstance(result[0], dict) and "text" in result[0]: return result[0]["text"] diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 19a7d9c952..643237ba86 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -48,6 +48,9 @@ dev = [ sandbox = [ "RestrictedPython>=7.0", ] +browser = [ + "pillow>=10.0.0", +] ocr = [ "pytesseract>=0.3.10", "pillow>=10.0.0", diff --git a/tools/src/gcu/browser/tools/inspection.py b/tools/src/gcu/browser/tools/inspection.py index 1564a160c3..78b617b87f 100644 --- a/tools/src/gcu/browser/tools/inspection.py +++ b/tools/src/gcu/browser/tools/inspection.py @@ -7,14 +7,113 @@ from __future__ import annotations import base64 +import io +import json +import logging from pathlib import Path from typing import Any, Literal from fastmcp import FastMCP +from mcp.types import ImageContent, TextContent from playwright.async_api import Error as PlaywrightError from ..session import get_session +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Screenshot normalization +# --------------------------------------------------------------------------- + +_QUALITY_STEPS = (85, 70, 50) +_MIN_DIMENSION = 400 +_DIMENSION_STEP = 200 + + +def _normalize_screenshot( + raw_bytes: bytes, + image_type: str, + *, + max_dimension: int = 2000, + max_bytes: int = 5_000_000, +) -> tuple[bytes, str]: + """Normalize a screenshot to fit within size and dimension limits. + + Progressively resizes and compresses to JPEG until the image fits + under *max_bytes* and *max_dimension*. If Pillow is not installed + the original bytes are returned unchanged. + + Args: + raw_bytes: Raw PNG or JPEG image bytes from Playwright. + image_type: Original format (``"png"`` or ``"jpeg"``). + max_dimension: Maximum width or height in pixels. + max_bytes: Maximum file size in bytes. + + Returns: + ``(normalized_bytes, image_type)`` where *image_type* may change + to ``"jpeg"`` if compression was applied. + """ + try: + from PIL import Image + except ImportError: + logger.debug("Pillow not installed — skipping screenshot normalization") + return raw_bytes, image_type + + try: + img = Image.open(io.BytesIO(raw_bytes)) + width, height = img.size + max_dim = max(width, height) + + # Already within limits — return as-is + if len(raw_bytes) <= max_bytes and max_dim <= max_dimension: + return raw_bytes, image_type + + # Build candidate dimensions (descending), skip anything >= original + candidates = [ + d for d in range(max_dimension, _MIN_DIMENSION - 1, -_DIMENSION_STEP) if d < max_dim + ] + # If the original is already <= max_dimension but over max_bytes, + # still try compressing at original size first. + if max_dim <= max_dimension: + candidates = [max_dim] + candidates + + smallest: tuple[bytes, int] | None = None + + for side in candidates: + # Re-open from source each iteration (thumbnail is destructive) + img = Image.open(io.BytesIO(raw_bytes)) + img.thumbnail((side, side), Image.LANCZOS) + + # JPEG doesn't support alpha + if img.mode in ("RGBA", "LA", "P"): + img = img.convert("RGB") + + for quality in _QUALITY_STEPS: + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=quality, optimize=True) + out_bytes = buf.getvalue() + + if smallest is None or len(out_bytes) < smallest[1]: + smallest = (out_bytes, len(out_bytes)) + + if len(out_bytes) <= max_bytes: + return out_bytes, "jpeg" + + # Nothing fit — return the smallest we produced + if smallest is not None: + logger.warning( + "Screenshot normalization: could not fit under %d bytes (best: %d bytes)", + max_bytes, + smallest[1], + ) + return smallest[0], "jpeg" + + return raw_bytes, image_type + + except Exception: + logger.warning("Screenshot normalization failed — returning original", exc_info=True) + return raw_bytes, image_type + def _format_ax_tree(nodes: list[dict[str, Any]]) -> str: """Format a CDP Accessibility.getFullAXTree result into an indented text tree. @@ -102,10 +201,13 @@ async def browser_screenshot( full_page: bool = False, selector: str | None = None, image_type: Literal["png", "jpeg"] = "png", - ) -> dict: + ) -> list: """ Take a screenshot of the current page. + Returns the screenshot as an image the LLM can see, alongside + text metadata (URL, size, etc.). + Args: target_id: Tab ID (default: active tab) profile: Browser profile name (default: "default") @@ -114,18 +216,29 @@ async def browser_screenshot( image_type: Image format - png or jpeg (default: png) Returns: - Dict with screenshot data (base64 encoded) and metadata + List of content blocks: text metadata + image """ try: session = get_session(profile) page = session.get_page(target_id) if not page: - return {"ok": False, "error": "No active tab"} + return [ + TextContent( + type="text", text=json.dumps({"ok": False, "error": "No active tab"}) + ) + ] if selector: element = await page.query_selector(selector) if not element: - return {"ok": False, "error": f"Element not found: {selector}"} + return [ + TextContent( + type="text", + text=json.dumps( + {"ok": False, "error": f"Element not found: {selector}"} + ), + ) + ] screenshot_bytes = await element.screenshot(type=image_type) else: screenshot_bytes = await page.screenshot( @@ -133,16 +246,31 @@ async def browser_screenshot( type=image_type, ) - return { - "ok": True, - "targetId": target_id or session.active_page_id, - "url": page.url, - "imageType": image_type, - "imageBase64": base64.b64encode(screenshot_bytes).decode(), - "size": len(screenshot_bytes), - } + normalized_bytes, normalized_type = _normalize_screenshot(screenshot_bytes, image_type) + meta = json.dumps( + { + "ok": True, + "targetId": target_id or session.active_page_id, + "url": page.url, + "imageType": normalized_type, + "size": len(normalized_bytes), + "originalSize": len(screenshot_bytes), + } + ) + return [ + TextContent(type="text", text=meta), + ImageContent( + type="image", + data=base64.b64encode(normalized_bytes).decode(), + mimeType=f"image/{normalized_type}", + ), + ] except PlaywrightError as e: - return {"ok": False, "error": f"Browser error: {e!s}"} + return [ + TextContent( + type="text", text=json.dumps({"ok": False, "error": f"Browser error: {e!s}"}) + ) + ] @mcp.tool() async def browser_snapshot( diff --git a/tools/tests/test_screenshot_normalization.py b/tools/tests/test_screenshot_normalization.py new file mode 100644 index 0000000000..cc64c54f78 --- /dev/null +++ b/tools/tests/test_screenshot_normalization.py @@ -0,0 +1,151 @@ +"""Tests for screenshot normalization.""" + +from __future__ import annotations + +import io +from unittest.mock import patch + +from PIL import Image + +from gcu.browser.tools.inspection import _normalize_screenshot + + +def _make_png(width: int, height: int, *, mode: str = "RGB") -> bytes: + """Create a solid-color PNG image of the given size.""" + img = Image.new( + mode, (width, height), color=(100, 150, 200) if mode == "RGB" else (100, 150, 200, 128) + ) + buf = io.BytesIO() + img.save(buf, format="PNG") + return buf.getvalue() + + +def _make_large_png(width: int, height: int, min_bytes: int) -> bytes: + """Create a PNG that's at least *min_bytes* by using random-ish pixel data.""" + # Gradient with noise produces poorly-compressible PNGs + img = Image.new("RGB", (width, height)) + pixels = img.load() + for y in range(height): + for x in range(width): + pixels[x, y] = ((x * 7 + y * 13) % 256, (x * 11 + y * 3) % 256, (x * 5 + y * 17) % 256) + buf = io.BytesIO() + img.save(buf, format="PNG") + raw = buf.getvalue() + # If still under target, that's fine for most tests — the important + # thing is we have a large-dimension image. + return raw + + +class TestPassthrough: + """Images already within limits should pass through unchanged.""" + + def test_small_image_unchanged(self): + raw = _make_png(100, 100) + result_bytes, result_type = _normalize_screenshot(raw, "png") + assert result_bytes is raw + assert result_type == "png" + + def test_within_dimension_and_size_unchanged(self): + raw = _make_png(1920, 1080) + result_bytes, result_type = _normalize_screenshot(raw, "png") + assert result_bytes is raw + assert result_type == "png" + + +class TestDimensionResize: + """Images exceeding max_dimension should be resized.""" + + def test_large_dimension_gets_resized(self): + raw = _make_png(4000, 3000) + result_bytes, result_type = _normalize_screenshot(raw, "png") + + # Should be JPEG after normalization + assert result_type == "jpeg" + + # Verify dimensions are within limit + img = Image.open(io.BytesIO(result_bytes)) + assert max(img.size) <= 2000 + + def test_custom_max_dimension(self): + raw = _make_png(2000, 1500) + result_bytes, result_type = _normalize_screenshot(raw, "png", max_dimension=800) + assert result_type == "jpeg" + + img = Image.open(io.BytesIO(result_bytes)) + assert max(img.size) <= 800 + + def test_aspect_ratio_preserved(self): + raw = _make_png(4000, 2000) # 2:1 ratio + result_bytes, _ = _normalize_screenshot(raw, "png") + + img = Image.open(io.BytesIO(result_bytes)) + w, h = img.size + ratio = w / h + assert abs(ratio - 2.0) < 0.1 # Allow small rounding error + + +class TestSizeCompression: + """Images exceeding max_bytes should be compressed.""" + + def test_custom_max_bytes(self): + raw = _make_large_png(1500, 1500, min_bytes=100_000) + result_bytes, result_type = _normalize_screenshot(raw, "png", max_bytes=50_000) + assert result_type == "jpeg" + assert len(result_bytes) <= 50_000 + + def test_over_size_within_dimension_compresses(self): + """Image within dimension limit but over byte limit gets JPEG-compressed.""" + raw = _make_large_png(1800, 1800, min_bytes=100_000) + result_bytes, result_type = _normalize_screenshot(raw, "png", max_bytes=50_000) + assert result_type == "jpeg" + assert len(result_bytes) <= 50_000 + + +class TestAlphaChannel: + """RGBA images should be converted to RGB for JPEG output.""" + + def test_rgba_to_rgb(self): + raw = _make_png(4000, 3000, mode="RGBA") + result_bytes, result_type = _normalize_screenshot(raw, "png") + + assert result_type == "jpeg" + img = Image.open(io.BytesIO(result_bytes)) + assert img.mode == "RGB" + + +class TestGracefulDegradation: + """Normalization should never break screenshots.""" + + def test_pillow_not_available(self): + raw = _make_png(4000, 3000) + with patch.dict("sys.modules", {"PIL": None, "PIL.Image": None}): + # Need to force reimport failure — patch builtins.__import__ + original_import = ( + __builtins__.__import__ if hasattr(__builtins__, "__import__") else __import__ + ) + + def mock_import(name, *args, **kwargs): + if name == "PIL" or name.startswith("PIL."): + raise ImportError("No module named 'PIL'") + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=mock_import): + result_bytes, result_type = _normalize_screenshot(raw, "png") + + # Should return original unchanged + assert result_bytes is raw + assert result_type == "png" + + def test_corrupt_bytes_returns_original(self): + raw = b"not an image at all" + result_bytes, result_type = _normalize_screenshot(raw, "png") + + assert result_bytes is raw + assert result_type == "png" + + def test_empty_bytes_returns_original(self): + raw = b"" + result_bytes, result_type = _normalize_screenshot(raw, "png") + + assert result_bytes is raw + assert result_type == "png" diff --git a/uv.lock b/uv.lock index bbac251f0b..1cdb69a3d4 100644 --- a/uv.lock +++ b/uv.lock @@ -3523,6 +3523,9 @@ all = [ bigquery = [ { name = "google-cloud-bigquery" }, ] +browser = [ + { name = "pillow" }, +] databricks = [ { name = "databricks-mcp" }, { name = "databricks-sdk" }, @@ -3577,6 +3580,7 @@ requires-dist = [ { name = "openpyxl", marker = "extra == 'excel'", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=2.0.0" }, { name = "pillow", marker = "extra == 'all'", specifier = ">=10.0.0" }, + { name = "pillow", marker = "extra == 'browser'", specifier = ">=10.0.0" }, { name = "pillow", marker = "extra == 'ocr'", specifier = ">=10.0.0" }, { name = "playwright", specifier = ">=1.40.0" }, { name = "playwright-stealth", specifier = ">=1.0.5" }, @@ -3594,7 +3598,7 @@ requires-dist = [ { name = "restrictedpython", marker = "extra == 'sandbox'", specifier = ">=7.0" }, { name = "stripe", specifier = ">=14.3.0" }, ] -provides-extras = ["dev", "sandbox", "ocr", "excel", "sql", "bigquery", "databricks", "all"] +provides-extras = ["dev", "sandbox", "browser", "ocr", "excel", "sql", "bigquery", "databricks", "all"] [package.metadata.requires-dev] dev = [ From c0da3bec0288a6238f4f83416f56ddcdf545db18 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 18 Mar 2026 12:40:30 -0700 Subject: [PATCH 03/10] feat: strip image content for non-vision models --- core/framework/graph/event_loop_node.py | 13 +++++- core/framework/llm/capabilities.py | 34 +++++++++++++++ core/tests/test_capabilities.py | 58 +++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 core/framework/llm/capabilities.py create mode 100644 core/tests/test_capabilities.py diff --git a/core/framework/graph/event_loop_node.py b/core/framework/graph/event_loop_node.py index a5bcfdf1c4..02534fef2d 100644 --- a/core/framework/graph/event_loop_node.py +++ b/core/framework/graph/event_loop_node.py @@ -24,6 +24,7 @@ from framework.graph.conversation import ConversationStore, NodeConversation from framework.graph.node import NodeContext, NodeProtocol, NodeResult +from framework.llm.capabilities import supports_image_tool_results from framework.llm.provider import Tool, ToolResult, ToolUse from framework.llm.stream_events import ( FinishEvent, @@ -2703,11 +2704,21 @@ async def _timed_subagent( real_tool_results.append(tool_entry) logged_tool_calls.append(tool_entry) + # Strip image content for models that can't handle it + image_content = result.image_content + if image_content and ctx.llm and not supports_image_tool_results(ctx.llm.model): + logger.info( + "Stripping image_content from tool result — model '%s' " + "does not support images in tool results", + ctx.llm.model, + ) + image_content = None + await conversation.add_tool_result( tool_use_id=tc.tool_use_id, content=result.content, is_error=result.is_error, - image_content=result.image_content, + image_content=image_content, ) if tc.tool_name in ("ask_user", "ask_user_multiple"): # Defer tool_call_completed until after user responds diff --git a/core/framework/llm/capabilities.py b/core/framework/llm/capabilities.py new file mode 100644 index 0000000000..19f6b6443b --- /dev/null +++ b/core/framework/llm/capabilities.py @@ -0,0 +1,34 @@ +"""Model capability checks for LLM providers.""" + +from __future__ import annotations + +# Prefixes of models/providers known to NOT support image content blocks +# inside tool result messages. We use a deny-list (rather than an allow-list) +# because most OpenAI-compatible providers pass content lists through to the +# API unchanged — only a few are known to silently strip or break on images. +_IMAGE_TOOL_RESULT_DENY_PREFIXES: tuple[str, ...] = ( + # DeepSeek: LiteLLM explicitly flattens all content lists to strings, + # silently dropping image blocks. + "deepseek/", + "deepseek-", + # Local model providers: most models lack vision support, and those that + # do typically handle images in user messages only, not tool results. + "ollama/", + "ollama_chat/", + "lm_studio/", + "vllm/", + "llamacpp/", + # Cerebras: no known vision/multimodal support. + "cerebras/", +) + + +def supports_image_tool_results(model: str) -> bool: + """Return whether *model* can receive image content in tool result messages. + + Models on the deny-list are known to either silently strip images or lack + vision support entirely. Everything else is assumed to work (OpenAI, + Anthropic, Gemini, Mistral, Groq, etc. all handle it correctly via LiteLLM). + """ + model_lower = model.lower() + return not any(model_lower.startswith(prefix) for prefix in _IMAGE_TOOL_RESULT_DENY_PREFIXES) diff --git a/core/tests/test_capabilities.py b/core/tests/test_capabilities.py new file mode 100644 index 0000000000..a6b84e3d05 --- /dev/null +++ b/core/tests/test_capabilities.py @@ -0,0 +1,58 @@ +"""Tests for LLM model capability checks.""" + +from __future__ import annotations + +import pytest + +from framework.llm.capabilities import supports_image_tool_results + + +class TestSupportsImageToolResults: + """Verify the deny-list correctly identifies models that can't handle images.""" + + @pytest.mark.parametrize( + "model", + [ + "gpt-4o", + "gpt-4o-mini", + "gpt-4-turbo", + "openai/gpt-4o", + "anthropic/claude-sonnet-4-20250514", + "claude-haiku-4-5-20251001", + "gemini/gemini-1.5-pro", + "google/gemini-1.5-flash", + "mistral/mistral-large", + "groq/llama3-70b", + "together/meta-llama/Llama-3-70b", + "fireworks_ai/llama-v3-70b", + "azure/gpt-4o", + "kimi/claude-sonnet-4-20250514", + "hive/claude-sonnet-4-20250514", + ], + ) + def test_supported_models(self, model: str): + assert supports_image_tool_results(model) is True + + @pytest.mark.parametrize( + "model", + [ + "deepseek/deepseek-chat", + "deepseek/deepseek-coder", + "deepseek-chat", + "deepseek-reasoner", + "ollama/llama3", + "ollama/mistral", + "ollama_chat/llama3", + "lm_studio/my-model", + "vllm/meta-llama/Llama-3-70b", + "llamacpp/model", + "cerebras/llama3-70b", + ], + ) + def test_unsupported_models(self, model: str): + assert supports_image_tool_results(model) is False + + def test_case_insensitive(self): + assert supports_image_tool_results("DeepSeek/deepseek-chat") is False + assert supports_image_tool_results("OLLAMA/llama3") is False + assert supports_image_tool_results("GPT-4o") is True From 0f1f0090b0d89afa8907b98ebb4950495b3d265e Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 18 Mar 2026 12:41:01 -0700 Subject: [PATCH 04/10] chore: linter update --- tools/src/gcu/browser/refs.py | 58 +++++++++++---------- tools/src/gcu/browser/tools/interactions.py | 10 +++- tools/tests/test_refs.py | 11 ++-- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/tools/src/gcu/browser/refs.py b/tools/src/gcu/browser/refs.py index 6e9ece722d..af10ad88c9 100644 --- a/tools/src/gcu/browser/refs.py +++ b/tools/src/gcu/browser/refs.py @@ -13,7 +13,7 @@ from __future__ import annotations import re -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -23,32 +23,36 @@ # Role sets (matching Playwright's aria roles that matter for interaction) # --------------------------------------------------------------------------- -INTERACTIVE_ROLES: frozenset[str] = frozenset({ - "button", - "checkbox", - "combobox", - "link", - "listbox", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "option", - "radio", - "scrollbar", - "searchbox", - "slider", - "spinbutton", - "switch", - "tab", - "textbox", - "treeitem", -}) - -NAMED_CONTENT_ROLES: frozenset[str] = frozenset({ - "cell", - "heading", - "img", -}) +INTERACTIVE_ROLES: frozenset[str] = frozenset( + { + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "scrollbar", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", + } +) + +NAMED_CONTENT_ROLES: frozenset[str] = frozenset( + { + "cell", + "heading", + "img", + } +) # Regex: captures indent, role, optional quoted name, and trailing text. # Example line: " - button \"Submit\" [disabled]" diff --git a/tools/src/gcu/browser/tools/interactions.py b/tools/src/gcu/browser/tools/interactions.py index 0437427a45..625f9a5811 100644 --- a/tools/src/gcu/browser/tools/interactions.py +++ b/tools/src/gcu/browser/tools/interactions.py @@ -124,7 +124,10 @@ async def browser_click( result: dict = {"ok": True, "action": "click", "selector": selector} if auto_snapshot: snapshot = await _auto_snapshot( - page, session=session, target_id=target_id, wait_for_nav=True, + page, + session=session, + target_id=target_id, + wait_for_nav=True, ) if snapshot: result["snapshot"] = snapshot @@ -172,7 +175,10 @@ async def browser_click_coordinate( result: dict = {"ok": True, "action": "click_coordinate", "x": x, "y": y} if auto_snapshot: snapshot = await _auto_snapshot( - page, session=session, target_id=target_id, wait_for_nav=True, + page, + session=session, + target_id=target_id, + wait_for_nav=True, ) if snapshot: result["snapshot"] = snapshot diff --git a/tools/tests/test_refs.py b/tools/tests/test_refs.py index 165240c4e0..109fb47c17 100644 --- a/tools/tests/test_refs.py +++ b/tools/tests/test_refs.py @@ -10,7 +10,6 @@ resolve_ref, ) - # --------------------------------------------------------------------------- # annotate_snapshot # --------------------------------------------------------------------------- @@ -67,7 +66,7 @@ def test_unnamed_content_roles_skip(self): assert len(ref_map) == 0 def test_preserves_non_matching_lines(self): - snapshot = "some random text\n- button \"OK\"\nanother line" + snapshot = 'some random text\n- button "OK"\nanother line' annotated, _ = annotate_snapshot(snapshot) lines = annotated.split("\n") assert lines[0] == "some random text" @@ -79,18 +78,14 @@ def test_nth_disambiguation(self): # Two "Save" buttons should have nth=0 and nth=1 save_entries = [ - (rid, e) for rid, e in ref_map.items() - if e.role == "button" and e.name == "Save" + (rid, e) for rid, e in ref_map.items() if e.role == "button" and e.name == "Save" ] assert len(save_entries) == 2 nths = sorted(e.nth for _, e in save_entries) assert nths == [0, 1] # "Cancel" should have nth=0 - cancel_entries = [ - e for e in ref_map.values() - if e.role == "button" and e.name == "Cancel" - ] + cancel_entries = [e for e in ref_map.values() if e.role == "button" and e.name == "Cancel"] assert len(cancel_entries) == 1 assert cancel_entries[0].nth == 0 From d84c3364d06fb26fdfe2f6068dd1e871beca0d71 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 18 Mar 2026 13:20:56 -0700 Subject: [PATCH 05/10] chore: update to pass make test --- tools/src/gcu/browser/refs.py | 3 --- tools/src/gcu/browser/tools/inspection.py | 3 +++ tools/tests/test_screenshot_normalization.py | 14 +++++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/src/gcu/browser/refs.py b/tools/src/gcu/browser/refs.py index af10ad88c9..a954341849 100644 --- a/tools/src/gcu/browser/refs.py +++ b/tools/src/gcu/browser/refs.py @@ -93,7 +93,6 @@ def annotate_snapshot(snapshot: str) -> tuple[str, RefMap]: # First pass: identify which lines get refs and count (role, name) pairs # for nth disambiguation. candidates: list[tuple[int, str, str | None]] = [] # (line_idx, role, name) - pair_counts: dict[tuple[str, str | None], int] = {} for i, line in enumerate(lines): m = _LINE_RE.match(line) @@ -104,8 +103,6 @@ def annotate_snapshot(snapshot: str) -> tuple[str, RefMap]: if role in INTERACTIVE_ROLES or (role in NAMED_CONTENT_ROLES and name): candidates.append((i, role, name)) - key = (role, name) - pair_counts[key] = pair_counts.get(key, 0) + 1 # Second pass: assign refs with nth indices. ref_map: RefMap = {} diff --git a/tools/src/gcu/browser/tools/inspection.py b/tools/src/gcu/browser/tools/inspection.py index 78b617b87f..5fbad6a6b7 100644 --- a/tools/src/gcu/browser/tools/inspection.py +++ b/tools/src/gcu/browser/tools/inspection.py @@ -229,6 +229,9 @@ async def browser_screenshot( ] if selector: + from ..refs import resolve_selector + + selector = resolve_selector(selector, session, target_id) element = await page.query_selector(selector) if not element: return [ diff --git a/tools/tests/test_screenshot_normalization.py b/tools/tests/test_screenshot_normalization.py index cc64c54f78..b7b3a2337a 100644 --- a/tools/tests/test_screenshot_normalization.py +++ b/tools/tests/test_screenshot_normalization.py @@ -1,13 +1,21 @@ -"""Tests for screenshot normalization.""" +"""Tests for screenshot normalization. + +Requires the ``browser`` extra (Pillow). Skipped automatically when +Pillow is not installed. +""" from __future__ import annotations import io from unittest.mock import patch -from PIL import Image +import pytest + +Image = pytest.importorskip( + "PIL.Image", reason="Pillow not installed (install with: pip install pillow)" +) -from gcu.browser.tools.inspection import _normalize_screenshot +from gcu.browser.tools.inspection import _normalize_screenshot # noqa: E402 def _make_png(width: int, height: int, *, mode: str = "RGB") -> bytes: From 648bad26eda2a3aef775a31a31a14d96d7c263f9 Mon Sep 17 00:00:00 2001 From: Timothy Date: Fri, 20 Mar 2026 18:40:28 -0700 Subject: [PATCH 06/10] feat: user input image content --- core/framework/graph/conversation.py | 8 ++ core/framework/graph/event_loop_node.py | 40 ++++++++-- core/framework/llm/capabilities.py | 46 +++++++----- core/framework/runtime/agent_runtime.py | 11 ++- core/framework/runtime/execution_stream.py | 5 +- core/framework/server/routes_execution.py | 12 ++- core/frontend/src/api/execution.ts | 4 +- core/frontend/src/components/ChatPanel.tsx | 87 ++++++++++++++++++++-- core/frontend/src/pages/workspace.tsx | 5 +- 9 files changed, 176 insertions(+), 42 deletions(-) diff --git a/core/framework/graph/conversation.py b/core/framework/graph/conversation.py index 9da6f1c09b..b1bbba400b 100644 --- a/core/framework/graph/conversation.py +++ b/core/framework/graph/conversation.py @@ -41,6 +41,12 @@ class Message: def to_llm_dict(self) -> dict[str, Any]: """Convert to OpenAI-format message dict.""" if self.role == "user": + if self.image_content: + blocks: list[dict[str, Any]] = [] + if self.content: + blocks.append({"type": "text", "text": self.content}) + blocks.extend(self.image_content) + return {"role": "user", "content": blocks} return {"role": "user", "content": self.content} if self.role == "assistant": @@ -389,6 +395,7 @@ async def add_user_message( *, is_transition_marker: bool = False, is_client_input: bool = False, + image_content: list[dict[str, Any]] | None = None, ) -> Message: msg = Message( seq=self._next_seq, @@ -397,6 +404,7 @@ async def add_user_message( phase_id=self._current_phase, is_transition_marker=is_transition_marker, is_client_input=is_client_input, + image_content=image_content, ) self._messages.append(msg) self._next_seq += 1 diff --git a/core/framework/graph/event_loop_node.py b/core/framework/graph/event_loop_node.py index 33cc52d8c2..da572a1a60 100644 --- a/core/framework/graph/event_loop_node.py +++ b/core/framework/graph/event_loop_node.py @@ -427,7 +427,9 @@ def __init__( self._config = config or LoopConfig() self._tool_executor = tool_executor self._conversation_store = conversation_store - self._injection_queue: asyncio.Queue[tuple[str, bool]] = asyncio.Queue() + self._injection_queue: asyncio.Queue[ + tuple[str, bool, list[dict[str, Any]] | None] + ] = asyncio.Queue() self._trigger_queue: asyncio.Queue[TriggerEvent] = asyncio.Queue() # Client-facing input blocking state self._input_ready = asyncio.Event() @@ -767,7 +769,7 @@ async def execute(self, ctx: NodeContext) -> NodeResult: ) # 6b. Drain injection queue - await self._drain_injection_queue(conversation) + await self._drain_injection_queue(conversation, ctx) # 6b1. Drain trigger queue (framework-level signals) await self._drain_trigger_queue(conversation) @@ -1893,7 +1895,13 @@ async def execute(self, ctx: NodeContext) -> NodeResult: conversation=conversation if _is_continuous else None, ) - async def inject_event(self, content: str, *, is_client_input: bool = False) -> None: + async def inject_event( + self, + content: str, + *, + is_client_input: bool = False, + image_content: list[dict[str, Any]] | None = None, + ) -> None: """Inject an external event or user input into the running loop. The content becomes a user message prepended to the next iteration. @@ -1909,8 +1917,10 @@ async def inject_event(self, content: str, *, is_client_input: bool = False) -> human user (e.g. /chat endpoint), False for external events (e.g. worker question forwarded by the frontend). Controls message formatting in _drain_injection_queue, not wake behavior. + image_content: Optional list of image content blocks (OpenAI + image_url format) to include alongside the text. """ - await self._injection_queue.put((content, is_client_input)) + await self._injection_queue.put((content, is_client_input, image_content)) self._input_ready.set() async def inject_trigger(self, trigger: TriggerEvent) -> None: @@ -4674,20 +4684,34 @@ async def _write_cursor( ] await self._conversation_store.write_cursor(cursor) - async def _drain_injection_queue(self, conversation: NodeConversation) -> int: + async def _drain_injection_queue( + self, conversation: NodeConversation, ctx: NodeContext + ) -> int: """Drain all pending injected events as user messages. Returns count.""" count = 0 while not self._injection_queue.empty(): try: - content, is_client_input = self._injection_queue.get_nowait() + content, is_client_input, image_content = self._injection_queue.get_nowait() logger.info( - "[drain] injected message (client_input=%s): %s", + "[drain] injected message (client_input=%s, images=%d): %s", is_client_input, + len(image_content) if image_content else 0, content[:200] if content else "(empty)", ) + # Strip images for models that don't support them in user messages + if image_content and ctx.llm: + if not supports_image_tool_results(ctx.llm.model): + logger.info( + "Stripping image_content from user message — model '%s' " + "does not support image input", + ctx.llm.model, + ) + image_content = None # Real user input is stored as-is; external events get a prefix if is_client_input: - await conversation.add_user_message(content, is_client_input=True) + await conversation.add_user_message( + content, is_client_input=True, image_content=image_content + ) else: await conversation.add_user_message(f"[External event]: {content}") count += 1 diff --git a/core/framework/llm/capabilities.py b/core/framework/llm/capabilities.py index 19f6b6443b..96b313df75 100644 --- a/core/framework/llm/capabilities.py +++ b/core/framework/llm/capabilities.py @@ -2,33 +2,45 @@ from __future__ import annotations -# Prefixes of models/providers known to NOT support image content blocks -# inside tool result messages. We use a deny-list (rather than an allow-list) -# because most OpenAI-compatible providers pass content lists through to the -# API unchanged — only a few are known to silently strip or break on images. -_IMAGE_TOOL_RESULT_DENY_PREFIXES: tuple[str, ...] = ( +# Provider prefixes that are known to NOT support image content. +# These are checked against the full model string (e.g. "deepseek/deepseek-chat"). +_IMAGE_DENY_PROVIDER_PREFIXES: tuple[str, ...] = ( # DeepSeek: LiteLLM explicitly flattens all content lists to strings, # silently dropping image blocks. "deepseek/", "deepseek-", - # Local model providers: most models lack vision support, and those that - # do typically handle images in user messages only, not tool results. - "ollama/", - "ollama_chat/", - "lm_studio/", - "vllm/", - "llamacpp/", # Cerebras: no known vision/multimodal support. "cerebras/", ) +# Model name fragments (checked after stripping any "provider/" prefix) that +# indicate a text-only model regardless of which provider routes the request. +_IMAGE_DENY_MODEL_FRAGMENTS: tuple[str, ...] = ( + # ZAI / GLM family: GLM-5 and ZAI-GLM-4.x are text-only. + # Accessed as "openai/glm-5", "cerebras/zai-glm-4.7", "glm-5", etc. + "glm-", + "zai-glm", + # MiniMax M2.5: text-only, no vision. + "minimax-text", + "abab", +) + + +def _model_name(model: str) -> str: + """Return the bare model name, stripping any 'provider/' prefix.""" + if "/" in model: + return model.split("/", 1)[1] + return model + def supports_image_tool_results(model: str) -> bool: - """Return whether *model* can receive image content in tool result messages. + """Return whether *model* can receive image content in messages. - Models on the deny-list are known to either silently strip images or lack - vision support entirely. Everything else is assumed to work (OpenAI, - Anthropic, Gemini, Mistral, Groq, etc. all handle it correctly via LiteLLM). + Used to gate both tool-result image blocks and user-message images. + Deny-list approach: everything is assumed capable unless explicitly listed. """ model_lower = model.lower() - return not any(model_lower.startswith(prefix) for prefix in _IMAGE_TOOL_RESULT_DENY_PREFIXES) + if any(model_lower.startswith(p) for p in _IMAGE_DENY_PROVIDER_PREFIXES): + return False + model_name_lower = _model_name(model_lower) + return not any(model_name_lower.startswith(f) for f in _IMAGE_DENY_MODEL_FRAGMENTS) diff --git a/core/framework/runtime/agent_runtime.py b/core/framework/runtime/agent_runtime.py index b9a71bd659..9bdfffafbf 100644 --- a/core/framework/runtime/agent_runtime.py +++ b/core/framework/runtime/agent_runtime.py @@ -1474,6 +1474,7 @@ async def inject_input( graph_id: str | None = None, *, is_client_input: bool = False, + image_content: list[dict[str, Any]] | None = None, ) -> bool: """Inject user input into a running client-facing node. @@ -1486,6 +1487,8 @@ async def inject_input( graph_id: Optional graph to search first (defaults to active graph) is_client_input: True when the message originates from a real human user (e.g. /chat endpoint), False for external events. + image_content: Optional list of image content blocks (OpenAI + image_url format) to include alongside the text. Returns: True if input was delivered, False if no matching node found @@ -1497,7 +1500,9 @@ async def inject_input( target = graph_id or self._active_graph_id if target in self._graphs: for stream in self._graphs[target].streams.values(): - if await stream.inject_input(node_id, content, is_client_input=is_client_input): + if await stream.inject_input( + node_id, content, is_client_input=is_client_input, image_content=image_content + ): return True # Then search all other graphs @@ -1505,7 +1510,9 @@ async def inject_input( if gid == target: continue for stream in reg.streams.values(): - if await stream.inject_input(node_id, content, is_client_input=is_client_input): + if await stream.inject_input( + node_id, content, is_client_input=is_client_input, image_content=image_content + ): return True return False diff --git a/core/framework/runtime/execution_stream.py b/core/framework/runtime/execution_stream.py index f5f98e8f57..cb6852df37 100644 --- a/core/framework/runtime/execution_stream.py +++ b/core/framework/runtime/execution_stream.py @@ -433,6 +433,7 @@ async def inject_input( content: str, *, is_client_input: bool = False, + image_content: list[dict[str, Any]] | None = None, ) -> bool: """Inject user input into a running client-facing EventLoopNode. @@ -444,7 +445,9 @@ async def inject_input( for executor in self._active_executors.values(): node = executor.node_registry.get(node_id) if node is not None and hasattr(node, "inject_event"): - await node.inject_event(content, is_client_input=is_client_input) + await node.inject_event( + content, is_client_input=is_client_input, image_content=image_content + ) return True return False diff --git a/core/framework/server/routes_execution.py b/core/framework/server/routes_execution.py index 2d4c9bba43..fb6307ec4f 100644 --- a/core/framework/server/routes_execution.py +++ b/core/framework/server/routes_execution.py @@ -108,7 +108,10 @@ async def handle_chat(request: web.Request) -> web.Response: The input box is permanently connected to the queen agent. Worker input is handled separately via /worker-input. - Body: {"message": "hello"} + Body: {"message": "hello", "images": [{"type": "image_url", "image_url": {"url": "data:..."}}]} + + The optional ``images`` field accepts a list of OpenAI-format image_url + content blocks. The frontend encodes images as base64 data URIs. """ session, err = resolve_session(request) if err: @@ -116,15 +119,16 @@ async def handle_chat(request: web.Request) -> web.Response: body = await request.json() message = body.get("message", "") + image_content = body.get("images") or None # list[dict] | None - if not message: + if not message and not image_content: return web.json_response({"error": "message is required"}, status=400) queen_executor = session.queen_executor if queen_executor is not None: node = queen_executor.node_registry.get("queen") if node is not None and hasattr(node, "inject_event"): - await node.inject_event(message, is_client_input=True) + await node.inject_event(message, is_client_input=True, image_content=image_content) # Publish to EventBus so the session event log captures user messages from framework.runtime.event_bus import AgentEvent, EventType @@ -134,7 +138,7 @@ async def handle_chat(request: web.Request) -> web.Response: stream_id="queen", node_id="queen", execution_id=session.id, - data={"content": message}, + data={"content": message, "image_count": len(image_content) if image_content else 0}, ) ) return web.json_response( diff --git a/core/frontend/src/api/execution.ts b/core/frontend/src/api/execution.ts index edfe6bba3c..fa5411edd0 100644 --- a/core/frontend/src/api/execution.ts +++ b/core/frontend/src/api/execution.ts @@ -34,8 +34,8 @@ export const executionApi = { graph_id: graphId, }), - chat: (sessionId: string, message: string) => - api.post(`/sessions/${sessionId}/chat`, { message }), + chat: (sessionId: string, message: string, images?: { type: string; image_url: { url: string } }[]) => + api.post(`/sessions/${sessionId}/chat`, { message, ...(images?.length ? { images } : {}) }), /** Queue context for the queen without triggering an LLM response. */ queenContext: (sessionId: string, message: string) => diff --git a/core/frontend/src/components/ChatPanel.tsx b/core/frontend/src/components/ChatPanel.tsx index 0e02082de3..4be9ded390 100644 --- a/core/frontend/src/components/ChatPanel.tsx +++ b/core/frontend/src/components/ChatPanel.tsx @@ -1,5 +1,10 @@ import { memo, useState, useRef, useEffect } from "react"; -import { Send, Square, Crown, Cpu, Check, Loader2 } from "lucide-react"; +import { Send, Square, Crown, Cpu, Check, Loader2, Paperclip, X } from "lucide-react"; + +export interface ImageContent { + type: "image_url"; + image_url: { url: string }; +} export interface ContextUsageEntry { usagePct: number; @@ -25,11 +30,13 @@ export interface ChatMessage { createdAt?: number; /** Queen phase active when this message was created */ phase?: "planning" | "building" | "staging" | "running"; + /** Images attached to a user message */ + images?: ImageContent[]; } interface ChatPanelProps { messages: ChatMessage[]; - onSend: (message: string, thread: string) => void; + onSend: (message: string, thread: string, images?: ImageContent[]) => void; isWaiting?: boolean; /** When true a worker is thinking (not yet streaming) */ isWorkerWaiting?: boolean; @@ -195,7 +202,19 @@ const MessageBubble = memo(function MessageBubble({ msg, queenPhase }: { msg: Ch return (
-

{msg.content}

+ {msg.images && msg.images.length > 0 && ( +
+ {msg.images.map((img, i) => ( + {`attachment + ))} +
+ )} + {msg.content &&

{msg.content}

}
); @@ -252,11 +271,13 @@ const MessageBubble = memo(function MessageBubble({ msg, queenPhase }: { msg: Ch export default function ChatPanel({ messages, onSend, isWaiting, isWorkerWaiting, isBusy, activeThread, disabled, onCancel, pendingQuestion, pendingOptions, pendingQuestions, onQuestionSubmit, onMultiQuestionSubmit, onQuestionDismiss, queenPhase, contextUsage }: ChatPanelProps) { const [input, setInput] = useState(""); + const [pendingImages, setPendingImages] = useState([]); const [readMap, setReadMap] = useState>({}); const bottomRef = useRef(null); const scrollRef = useRef(null); const stickToBottom = useRef(true); const textareaRef = useRef(null); + const fileInputRef = useRef(null); const threadMessages = messages.filter((m) => { if (m.type === "system" && !m.thread) return false; @@ -299,12 +320,28 @@ export default function ChatPanel({ messages, onSend, isWaiting, isWorkerWaiting const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); - if (!input.trim()) return; - onSend(input.trim(), activeThread); + if (!input.trim() && pendingImages.length === 0) return; + onSend(input.trim(), activeThread, pendingImages.length > 0 ? pendingImages : undefined); setInput(""); + setPendingImages([]); if (textareaRef.current) textareaRef.current.style.height = "auto"; }; + const handleFileChange = (e: React.ChangeEvent) => { + const files = Array.from(e.target.files ?? []); + if (files.length === 0) return; + files.forEach((file) => { + const reader = new FileReader(); + reader.onload = (ev) => { + const url = ev.target?.result as string; + setPendingImages((prev) => [...prev, { type: "image_url", image_url: { url } }]); + }; + reader.readAsDataURL(file); + }); + // Reset so the same file can be re-selected + e.target.value = ""; + }; + return (
{/* Compact sub-header */} @@ -432,7 +469,45 @@ export default function ChatPanel({ messages, onSend, isWaiting, isWorkerWaiting /> ) : (
+ {/* Image preview strip */} + {pendingImages.length > 0 && ( +
+ {pendingImages.map((img, i) => ( +
+ {`preview + +
+ ))} +
+ )}
+ +