refactor(tarko): some enhancement for gui agent (#1198)

Author: ulivz

multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/ToolCalls.tsx

@@ -195,6 +195,8 @@ export const ToolCalls: React.FC<ToolCallsProps> = ({
     switch (toolName) {
       case 'browser_navigate':
         return 'Navigate';
+      case 'browser_vision_control':
+        return 'Browser';
       case 'browser_get_markdown':
         return 'Extract Content';
       case 'browser_click':

multimodal/tarko/agent-web-ui/src/standalone/workspace/components/WorkspaceHeader.tsx

@@ -82,7 +82,7 @@ export const WorkspaceHeader: React.FC<WorkspaceHeaderProps> = ({
         <div className="min-w-0 flex-1">
           <div className="flex items-baseline gap-2">
             <h2 className="font-medium text-gray-900 dark:text-gray-100 text-base leading-tight truncate">
-              {panelContent.title}
+              {panelContent.title === 'browser_vision_control' ? 'Browser' : panelContent.title}
             </h2>
             <div className="text-xs text-gray-400 dark:text-gray-500 whitespace-nowrap flex-shrink-0 font-mono">
               {formatTimestamp(panelContent.timestamp, true)}

multimodal/tarko/agent-web-ui/src/standalone/workspace/renderers/BrowserControlRenderer.tsx

@@ -73,24 +73,28 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
 
   // Find the most recent environment input (screenshot) before this operation
   useEffect(() => {
-    if (!activeSessionId) return;
-
-    const sessionMessages = messages[activeSessionId] || [];
+    // Initialize: clear current screenshot if no direct environment image provided
+    if (!environmentImage) {
+      setRelatedImage(null);
+    }
 
-    if (!toolCallId) return;
+    if (!activeSessionId || !toolCallId) return;
 
-    // Get the index of current tool call in messages
+    const sessionMessages = messages[activeSessionId] || [];
     const currentToolCallIndex = sessionMessages.findIndex((msg) =>
       msg.toolCalls?.some((tc) => tc.id === toolCallId),
     );
 
-    if (currentToolCallIndex === -1) return;
+    if (currentToolCallIndex === -1) {
+      console.warn(`[BrowserControlRenderer] Tool call ${toolCallId} not found in messages`);
+      if (!environmentImage) setRelatedImage(null);
+      return;
+    }
 
-    // Find the environment input closest to the current tool call
     let foundImage = false;
 
-    // Search forward for environment input, find the most recent screenshot
-    for (let i = currentToolCallIndex; i >= 0; i--) {
+    // Only search for screenshots BEFORE the current tool call
+    for (let i = currentToolCallIndex - 1; i >= 0; i--) {
       const msg = sessionMessages[i];
       if (msg.role === 'environment' && Array.isArray(msg.content)) {
         const imgContent = msg.content.find(
@@ -105,29 +109,14 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
       }
     }
 
-    // If no image is found before the current tool call, search all environment messages as fallback
-    if (!foundImage) {
+    // If no valid screenshot found before the tool call, clear the display
+    if (!foundImage && !environmentImage) {
       console.warn(
-        `[BrowserControlRenderer] Could not find preceding screenshot for toolCallId: ${toolCallId}. Falling back to search all environment messages.`,
+        `[BrowserControlRenderer] No valid screenshot found before toolCallId: ${toolCallId}. Clearing screenshot display.`,
       );
-      const envMessages = sessionMessages.filter(
-        (msg) => msg.role === 'environment' && Array.isArray(msg.content),
-      );
-
-      // Search backwards to find the most recent screenshot
-      for (let i = envMessages.length - 1; i >= 0; i--) {
-        const msg = envMessages[i];
-        const imgContent = msg.content.find(
-          (c) => typeof c === 'object' && 'type' in c && c.type === 'image_url',
-        );
-
-        if (imgContent && 'image_url' in imgContent && imgContent.image_url.url) {
-          setRelatedImage(imgContent.image_url.url);
-          break; // Stop when the latest one is found
-        }
-      }
+      setRelatedImage(null);
     }
-  }, [activeSessionId, messages, toolCallId]);
+  }, [activeSessionId, messages, toolCallId, environmentImage]);
 
   // Handler to get image dimensions when loaded
   const handleImageLoad = () => {
@@ -142,7 +131,7 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
   return (
     <div className="space-y-4">
       {/* Screenshot section - moved to the top */}
-      {relatedImage && (
+      {relatedImage ? (
         <div>
           <BrowserShell className="mb-4">
             <div className="relative">
@@ -275,6 +264,23 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
             </div>
           </BrowserShell>
         </div>
+      ) : (
+        // No screenshot available - show placeholder
+        <div>
+          <BrowserShell className="mb-4">
+            <div className="flex items-center justify-center h-64 bg-gray-50 dark:bg-gray-800">
+              <div className="text-center">
+                <FiImage className="mx-auto text-gray-400 dark:text-gray-500 mb-2" size={48} />
+                <p className="text-gray-500 dark:text-gray-400 font-medium">
+                  No Screenshot Available
+                </p>
+                <p className="text-sm text-gray-400 dark:text-gray-500 mt-1">
+                  Unable to find environment screenshot for this operation
+                </p>
+              </div>
+            </div>
+          </BrowserShell>
+        </div>
       )}
 
       {/* Visual operation details card */}