feat: support multimodal tool results and improve tool message handling

web-app/src/lib/completion.ts

@@ -32,7 +32,7 @@
 }
 import { ulid } from 'ulidx'
 import { MCPTool } from '@/types/completion'
-import { CompletionMessagesBuilder } from './messages'
+import { CompletionMessagesBuilder, ToolResult } from './messages'
 import { ChatCompletionMessageToolCall } from 'openai/resources'
 import { ExtensionManager } from './extension'
 import { useAppState } from '@/hooks/useAppState'
@@ -66,11 +66,11 @@
   const docMetadata = documents
     .filter((doc) => doc.id) // Only include processed documents
     .map((doc) => ({
       id: doc.id!,
       name: doc.name,
       type: doc.fileType,
       size: typeof doc.size === 'number' ? doc.size : undefined,
       chunkCount: typeof doc.chunkCount === 'number' ? doc.chunkCount : undefined,
     }))
 
   const textWithFiles =
@@ -88,15 +88,15 @@
 
   // Add image attachments to content array
   images.forEach((img) => {
     if (img.base64 && img.mimeType) {
       contentParts.push({
         type: ContentType.Image,
         image_url: {
           url: `data:${img.mimeType};base64,${img.base64}`,
           detail: 'auto',
         },
       } as any)
     }
   })
 
   return {
@@ -168,45 +168,45 @@
  * @returns
  */
 export const sendCompletion = async (
   thread: Thread,
   provider: ModelProvider,
   messages: ChatCompletionMessageParam[],
   abortController: AbortController,
   tools: MCPTool[] = [],
   stream: boolean = true,
   params: Record<string, object> = {}
 ): Promise<ChatCompletionResponse | undefined> => {
   if (!thread?.model?.id || !provider) return undefined
 
   let providerName = provider.provider as unknown as keyof typeof models
 
   if (!Object.keys(models).some((key) => key === providerName))
     providerName = 'openai-compatible'
 
   const tokenJS = new TokenJS({
     apiKey:
       provider.api_key ?? (await getServiceHub().core().getAppToken()) ?? '',
     // TODO: Retrieve from extension settings
     baseURL: provider.base_url,
     // Use Tauri's fetch to avoid CORS issues only for openai-compatible provider
     fetch: IS_DEV ? fetch : getServiceHub().providers().fetch(),
     // OpenRouter identification headers for Jan
     // ref: https://openrouter.ai/docs/api-reference/overview#headers
     ...(provider.provider === 'openrouter' && {
       defaultHeaders: {
         'HTTP-Referer': 'https://jan.ai',
         'X-Title': 'Jan',
       },
     }),
     // Add Origin header for local providers to avoid CORS issues
     ...((provider.base_url?.includes('localhost:') ||
       provider.base_url?.includes('127.0.0.1:')) && {
       fetch: getServiceHub().providers().fetch(),
       defaultHeaders: {
         Origin: 'tauri://localhost',
       },
     }),
   } as ExtendedConfigOptions)
 
   if (
     thread.model.id &&
@@ -543,7 +543,7 @@
           },
         ],
       }
-      builder.addToolMessage(result.content[0]?.text ?? '', toolCall.id)
+      builder.addToolMessage(result as ToolResult, toolCall.id)
       // update message metadata
     }
     return message

web-app/src/lib/messages.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { ChatCompletionMessageParam } from 'token.js'
 import { ChatCompletionMessageToolCall } from 'openai/resources'
 import { ThreadMessage, ContentType } from '@janhq/core'
@@ -6,6 +7,48 @@ import { removeReasoningContent } from '@/utils/reasoning'
 
 type ThreadContent = NonNullable<ThreadMessage['content']>[number]
 
+// Define a temporary type for the expected tool result shape (ToolResult as before)
+export type ToolResult = {
+  content: Array<{
+    type?: string
+    text?: string
+    data?: string
+    image_url?: { url: string; detail?: string }
+  }>
+  error?: string
+}
+
+// Helper function to convert the tool's output part into an API content part
+const convertToolPartToApiContentPart = (part: ToolResult['content'][0]) => {
+  if (part.text) {
+    return { type: 'text', text: part.text }
+  }
+
+  // Handle base64 image data
+  if (part.data) {
+    // Assume default image type, though a proper tool should return the mime type
+    const mimeType =
+      part.type === 'image' ? 'image/png' : part.type || 'image/png'
+    const dataUrl = `data:${mimeType};base64,${part.data}`
+
+    return {
+      type: 'image_url',
+      image_url: {
+        url: dataUrl,
+        detail: 'auto',
+      },
+    }
+  }
+
+  // Handle pre-formatted image URL
+  if (part.image_url) {
+    return { type: 'image_url', image_url: part.image_url }
+  }
+
+  // Fallback to text stringification for structured but unhandled data
+  return { type: 'text', text: JSON.stringify(part) }
+}
+
 /**
  * @fileoverview Helper functions for creating chat completion request.
  * These functions are used to create chat completion request objects
@@ -26,7 +69,11 @@ export class CompletionMessagesBuilder {
         .map<ChatCompletionMessageParam>((msg) => {
           const param = this.toCompletionParamFromThread(msg)
           // In constructor context, normalize empty user text to a placeholder
-          if (param.role === 'user' && typeof param.content === 'string' && param.content === '') {
+          if (
+            param.role === 'user' &&
+            typeof param.content === 'string' &&
+            param.content === ''
+          ) {
             return { ...param, content: '.' }
           }
           return param
@@ -35,7 +82,9 @@ export class CompletionMessagesBuilder {
   }
 
   // Normalize a ThreadMessage into a ChatCompletionMessageParam for Token.js
-  private toCompletionParamFromThread(msg: ThreadMessage): ChatCompletionMessageParam {
+  private toCompletionParamFromThread(
+    msg: ThreadMessage
+  ): ChatCompletionMessageParam {
     if (msg.role === 'assistant') {
       return {
         role: 'assistant',
@@ -60,7 +109,10 @@ export class CompletionMessagesBuilder {
         if (part.type === ContentType.Image) {
           return {
             type: 'image_url' as const,
-            image_url: { url: part.image_url?.url || '', detail: part.image_url?.detail || 'auto' },
+            image_url: {
+              url: part.image_url?.url || '',
+              detail: part.image_url?.detail || 'auto',
+            },
           }
         }
         // Fallback for unknown content types
@@ -110,13 +162,43 @@ export class CompletionMessagesBuilder {
 
   /**
    * Add a tool message to the messages array.
-   * @param content - The content of the tool message.
+   * @param content - The content of the tool message (string or ToolResult object).
    * @param toolCallId - The ID of the tool call associated with the message.
    */
-  addToolMessage(content: string, toolCallId: string) {
+  addToolMessage(result: string | ToolResult, toolCallId: string) {
+    let content: string | any[] = ''
+
+    // Handle simple string case
+    if (typeof result === 'string') {
+      content = result
+    } else {
+      // Check for multimodal content (more than just a simple text string)
+      const hasMultimodalContent = result.content?.some(
+        (p) => p.data || p.image_url
+      )
+
+      if (hasMultimodalContent) {
+        // Build the structured content array
+        content = result.content.map(convertToolPartToApiContentPart)
+      } else if (result.content?.[0]?.text) {
+        // Standard text case
+        content = result.content[0].text
+      } else if (result.error) {
+        // Error case
+        content = `Tool execution failed: ${result.error}`
+      } else {
+        // Fallback: serialize the whole result structure if content is unexpected
+        try {
+          content = JSON.stringify(result)
+        } catch {
+          content = 'Tool call completed, unexpected output format.'
+        }
+      }
+    }
     this.messages.push({
       role: 'tool',
-      content: content,
+      // for role 'tool',  need to use 'as ChatCompletionMessageParam'
+      content: content as any,
       tool_call_id: toolCallId,
     })
   }