feat: image capabilities — upload, screenshot passthrough, vision detection & fallback, aria refs

core/framework/agents/queen/nodes/__init__.py

@@ -702,6 +702,15 @@ class definition, `validate()` method, `default_agent` export, and \
 _queen_behavior_always = """
 # Behavior
 
+## Images attached by the user
+
+Users can attach images directly to their chat messages. When you see an \
+image in the conversation, analyze it using your native vision capability — \
+do NOT say you cannot see images or that you lack access to files. The image \
+is embedded in the message; no tool call is needed to view it. Describe what \
+you see, answer questions about it, and use the visual content to inform your \
+response just as you would text.
+
 ## CRITICAL RULE — ask_user / ask_user_multiple
 
 Every response that ends with a question, a prompt, or expects user \

core/framework/agents/queen/reference/gcu_guide.md

@@ -150,7 +150,7 @@ Call all three subagents in a single response to run them in parallel:
 
 ## GCU Anti-Patterns
 
-- Using `browser_screenshot` to read text (use `browser_snapshot`)
+- Using `browser_screenshot` to read text (use `browser_snapshot` instead; screenshots are for visual context only)
 - Re-navigating after scrolling (resets scroll position)
 - Attempting login on auth walls
 - Forgetting `target_id` in multi-tab scenarios

core/framework/graph/conversation.py

@@ -33,12 +33,20 @@ class Message:
     is_transition_marker: bool = False
     # True when this message is real human input (from /chat), not a system prompt
     is_client_input: bool = False
+    # Optional image content blocks (e.g. from browser_screenshot)
+    image_content: list[dict[str, Any]] | None = None
     # True when message contains an activated skill body (AS-10: never prune)
     is_skill_content: bool = False
 
     def to_llm_dict(self) -> dict[str, Any]:
         """Convert to OpenAI-format message dict."""
         if self.role == "user":
+            if self.image_content:
+                blocks: list[dict[str, Any]] = []
+                if self.content:
+                    blocks.append({"type": "text", "text": self.content})
+                blocks.extend(self.image_content)
+                return {"role": "user", "content": blocks}
             return {"role": "user", "content": self.content}
 
         if self.role == "assistant":
@@ -49,6 +57,15 @@ def to_llm_dict(self) -> dict[str, Any]:
 
         # role == "tool"
         content = f"ERROR: {self.content}" if self.is_error else self.content
+        if self.image_content:
+            # Multimodal tool result: text + image content blocks
+            blocks: list[dict[str, Any]] = [{"type": "text", "text": content}]
+            blocks.extend(self.image_content)
+            return {
+                "role": "tool",
+                "tool_call_id": self.tool_use_id,
+                "content": blocks,
+            }
         return {
             "role": "tool",
             "tool_call_id": self.tool_use_id,
@@ -74,6 +91,8 @@ def to_storage_dict(self) -> dict[str, Any]:
             d["is_transition_marker"] = self.is_transition_marker
         if self.is_client_input:
             d["is_client_input"] = self.is_client_input
+        if self.image_content is not None:
+            d["image_content"] = self.image_content
         return d
 
     @classmethod
@@ -89,6 +108,7 @@ def from_storage_dict(cls, data: dict[str, Any]) -> Message:
             phase_id=data.get("phase_id"),
             is_transition_marker=data.get("is_transition_marker", False),
             is_client_input=data.get("is_client_input", False),
+            image_content=data.get("image_content"),
         )
 
 
@@ -375,6 +395,7 @@ async def add_user_message(
         *,
         is_transition_marker: bool = False,
         is_client_input: bool = False,
+        image_content: list[dict[str, Any]] | None = None,
     ) -> Message:
         msg = Message(
             seq=self._next_seq,
@@ -383,6 +404,7 @@ async def add_user_message(
             phase_id=self._current_phase,
             is_transition_marker=is_transition_marker,
             is_client_input=is_client_input,
+            image_content=image_content,
         )
         self._messages.append(msg)
         self._next_seq += 1
@@ -411,6 +433,7 @@ async def add_tool_result(
         tool_use_id: str,
         content: str,
         is_error: bool = False,
+        image_content: list[dict[str, Any]] | None = None,
         is_skill_content: bool = False,
     ) -> Message:
         msg = Message(
@@ -420,6 +443,7 @@ async def add_tool_result(
             tool_use_id=tool_use_id,
             is_error=is_error,
             phase_id=self._current_phase,
+            image_content=image_content,
             is_skill_content=is_skill_content,
         )
         self._messages.append(msg)

core/framework/graph/event_loop_node.py

@@ -14,6 +14,7 @@
 import asyncio
 import json
 import logging
+import os
 import re
 import time
 from collections.abc import Awaitable, Callable
@@ -24,6 +25,7 @@
 
 from framework.graph.conversation import ConversationStore, NodeConversation
 from framework.graph.node import NodeContext, NodeProtocol, NodeResult
+from framework.llm.capabilities import supports_image_tool_results
 from framework.llm.provider import Tool, ToolResult, ToolUse
 from framework.llm.stream_events import (
     FinishEvent,
@@ -37,6 +39,56 @@
 logger = logging.getLogger(__name__)
 
 
+async def _describe_images_as_text(image_content: list[dict[str, Any]]) -> str | None:
+    """Describe images using the best available vision model.
+
+    Called when the queen's model lacks vision support.  Tries vision-capable
+    models in priority order based on available API keys and returns a bracketed
+    description to inject into the message text, or None if no vision model is
+    reachable.
+    """
+    import litellm
+
+    # Build content blocks: prompt + all images
+    blocks: list[dict[str, Any]] = [
+        {
+            "type": "text",
+            "text": (
+                "Describe the following image(s) concisely but with enough detail "
+                "that a text-only AI assistant can understand the content and context."
+            ),
+        }
+    ]
+    blocks.extend(image_content)
+
+    # Ordered candidates based on available env vars
+    candidates: list[str] = []
+    if os.environ.get("OPENAI_API_KEY"):
+        candidates.append("gpt-4o-mini")
+    if os.environ.get("ANTHROPIC_API_KEY"):
+        candidates.append("claude-3-haiku-20240307")
+    if os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY"):
+        candidates.append("gemini/gemini-1.5-flash")
+
+    for model in candidates:
+        try:
+            response = await litellm.acompletion(
+                model=model,
+                messages=[{"role": "user", "content": blocks}],
+                max_tokens=512,
+            )
+            description = (response.choices[0].message.content or "").strip()
+            if description:
+                count = len(image_content)
+                label = "image" if count == 1 else f"{count} images"
+                return f"[{label} attached — description: {description}]"
+        except Exception as exc:
+            logger.debug("Vision fallback model '%s' failed: %s", model, exc)
+            continue
+
+    return None
+
+
 @dataclass
 class TriggerEvent:
     """A framework-level trigger signal (timer tick or webhook hit).
@@ -90,7 +142,13 @@ def __init__(self) -> None:
         self._response: str | None = None
         self._awaiting_input = True  # So inject_worker_message() can prefer us
 
-    async def inject_event(self, content: str, *, is_client_input: bool = False) -> None:
+    async def inject_event(
+        self,
+        content: str,
+        *,
+        is_client_input: bool = False,
+        image_content: list[dict] | None = None,
+    ) -> None:
         """Called by ExecutionStream.inject_input() when the user responds."""
         self._response = content
         self._event.set()
@@ -426,7 +484,9 @@ def __init__(
         self._config = config or LoopConfig()
         self._tool_executor = tool_executor
         self._conversation_store = conversation_store
-        self._injection_queue: asyncio.Queue[tuple[str, bool]] = asyncio.Queue()
+        self._injection_queue: asyncio.Queue[tuple[str, bool, list[dict[str, Any]] | None]] = (
+            asyncio.Queue()
+        )
         self._trigger_queue: asyncio.Queue[TriggerEvent] = asyncio.Queue()
         # Client-facing input blocking state
         self._input_ready = asyncio.Event()
@@ -784,7 +844,7 @@ async def execute(self, ctx: NodeContext) -> NodeResult:
                 )
 
             # 6b. Drain injection queue
-            await self._drain_injection_queue(conversation)
+            await self._drain_injection_queue(conversation, ctx)
             # 6b1. Drain trigger queue (framework-level signals)
             await self._drain_trigger_queue(conversation)
 
@@ -1910,7 +1970,13 @@ async def execute(self, ctx: NodeContext) -> NodeResult:
             conversation=conversation if _is_continuous else None,
         )
 
-    async def inject_event(self, content: str, *, is_client_input: bool = False) -> None:
+    async def inject_event(
+        self,
+        content: str,
+        *,
+        is_client_input: bool = False,
+        image_content: list[dict[str, Any]] | None = None,
+    ) -> None:
         """Inject an external event or user input into the running loop.
 
         The content becomes a user message prepended to the next iteration.
@@ -1926,8 +1992,10 @@ async def inject_event(self, content: str, *, is_client_input: bool = False) ->
                 human user (e.g. /chat endpoint), False for external events
                 (e.g. worker question forwarded by the frontend).  Controls
                 message formatting in _drain_injection_queue, not wake behavior.
+            image_content: Optional list of image content blocks (OpenAI
+                image_url format) to include alongside the text.
         """
-        await self._injection_queue.put((content, is_client_input))
+        await self._injection_queue.put((content, is_client_input, image_content))
         self._input_ready.set()
 
     async def inject_trigger(self, trigger: TriggerEvent) -> None:
@@ -2101,6 +2169,24 @@ async def _run_single_turn(
 
             messages = conversation.to_llm_messages()
 
+            # Debug: log whether the last user message contains image blocks
+            for _m in reversed(messages):
+                if _m.get("role") == "user":
+                    _content = _m.get("content")
+                    if isinstance(_content, list):
+                        _img_count = sum(
+                            1
+                            for _b in _content
+                            if isinstance(_b, dict) and _b.get("type") == "image_url"
+                        )
+                        if _img_count:
+                            logger.info(
+                                "[%s] LLM call: last user message has %d image block(s)",
+                                node_id,
+                                _img_count,
+                            )
+                    break
+
             # Defensive guard: ensure messages don't end with an assistant
             # message.  The Anthropic API rejects "assistant message prefill"
             # (conversations must end with a user or tool message).  This can
@@ -2770,10 +2856,21 @@ async def _timed_subagent(
                     real_tool_results.append(tool_entry)
                     logged_tool_calls.append(tool_entry)
 
+                # Strip image content for models that can't handle it
+                image_content = result.image_content
+                if image_content and ctx.llm and not supports_image_tool_results(ctx.llm.model):
+                    logger.info(
+                        "Stripping image_content from tool result — model '%s' "
+                        "does not support images in tool results",
+                        ctx.llm.model,
+                    )
+                    image_content = None
+
                 await conversation.add_tool_result(
                     tool_use_id=tc.tool_use_id,
                     content=result.content,
                     is_error=result.is_error,
+                    image_content=image_content,
                     is_skill_content=result.is_skill_content,
                 )
                 if (
@@ -3914,6 +4011,7 @@ def _truncate_tool_result(
                 tool_use_id=result.tool_use_id,
                 content=truncated,
                 is_error=False,
+                image_content=result.image_content,
             )
 
         spill_dir = self._config.spillover_dir
@@ -3986,6 +4084,7 @@ def _truncate_tool_result(
                 tool_use_id=result.tool_use_id,
                 content=content,
                 is_error=False,
+                image_content=result.image_content,
             )
 
         # No spillover_dir — truncate in-place if needed
@@ -4028,6 +4127,7 @@ def _truncate_tool_result(
                 tool_use_id=result.tool_use_id,
                 content=truncated,
                 is_error=False,
+                image_content=result.image_content,
             )
 
         return result
@@ -4698,20 +4798,37 @@ async def _write_cursor(
                 ]
             await self._conversation_store.write_cursor(cursor)
 
-    async def _drain_injection_queue(self, conversation: NodeConversation) -> int:
+    async def _drain_injection_queue(self, conversation: NodeConversation, ctx: NodeContext) -> int:
         """Drain all pending injected events as user messages. Returns count."""
         count = 0
         while not self._injection_queue.empty():
             try:
-                content, is_client_input = self._injection_queue.get_nowait()
+                content, is_client_input, image_content = self._injection_queue.get_nowait()
                 logger.info(
-                    "[drain] injected message (client_input=%s): %s",
+                    "[drain] injected message (client_input=%s, images=%d): %s",
                     is_client_input,
+                    len(image_content) if image_content else 0,
                     content[:200] if content else "(empty)",
                 )
+                # For models that don't support images, fall back to a text description
+                if image_content and ctx.llm:
+                    if not supports_image_tool_results(ctx.llm.model):
+                        logger.info(
+                            "Model '%s' does not support images — attempting vision fallback",
+                            ctx.llm.model,
+                        )
+                        description = await _describe_images_as_text(image_content)
+                        if description:
+                            content = f"{content}\n\n{description}" if content else description
+                            logger.info("[drain] image described as text via vision fallback")
+                        else:
+                            logger.info("[drain] no vision fallback available — images dropped")
+                        image_content = None
                 # Real user input is stored as-is; external events get a prefix
                 if is_client_input:
-                    await conversation.add_user_message(content, is_client_input=True)
+                    await conversation.add_user_message(
+                        content, is_client_input=True, image_content=image_content
+                    )
                 else:
                     await conversation.add_user_message(f"[External event]: {content}")
                 count += 1

core/framework/graph/gcu.py

@@ -43,8 +43,11 @@
   `browser_snapshot` separately after every action.
   Only call `browser_snapshot` when you need a fresh view without
   performing an action, or after setting `auto_snapshot=false`.
-- Do NOT use `browser_screenshot` for reading text content
-  — it produces huge base64 images with no searchable text.
+- Do NOT use `browser_screenshot` to read text — use
+  `browser_snapshot` for that (compact, searchable, fast).
+- DO use `browser_screenshot` when you need visual context:
+  charts, images, canvas elements, layout verification, or when
+  the snapshot doesn't capture what you need.
 - Only fall back to `browser_get_text` for extracting specific
   small elements by CSS selector.