fix(openai/v1): better responses multimodal handling (#9169)

Author: hntrl

dependency_range_tests/scripts/with_standard_tests/openai/test-with-latest-deps.sh

@@ -37,5 +37,5 @@ cd "$monorepo_openai_dir"
 
 # Clean and reinstall to avoid dependency conflicts
 pnpm install --no-frozen-lockfile
-pnpm add @langchain/core
+pnpm add @langchain/core@alpha
 pnpm test

libs/providers/langchain-openai/package.json

@@ -35,7 +35,7 @@
     "zod": "^3.25.76 || ^4"
   },
   "peerDependencies": {
-    "@langchain/core": "^1.0.0-alpha.1"
+    "@langchain/core": "^1.0.0-alpha.6"
   },
   "devDependencies": {
     "@azure/identity": "^4.2.1",

libs/providers/langchain-openai/src/chat_models.ts

@@ -1626,10 +1626,10 @@ export class ChatOpenAIResponses<
   protected _convertMessagesToResponsesParams(messages: BaseMessage[]) {
     return messages.flatMap(
       (lcMsg): ResponsesInputItem | ResponsesInputItem[] => {
-        if (
-          isAIMessage(lcMsg) &&
-          lcMsg.response_metadata?.output_version === "v1"
-        ) {
+        const responseMetadata = lcMsg.response_metadata as
+          | Record<string, unknown>
+          | undefined;
+        if (responseMetadata?.output_version === "v1") {
           return _convertToResponsesMessageFromV1(lcMsg);
         }
 
@@ -1697,7 +1697,7 @@ export class ChatOpenAIResponses<
           }
 
           // Handle custom tool output
-          if (toolMessage.metadata?.customTool) {
+          if (toolMessage.additional_kwargs?.customTool) {
             return {
               type: "custom_tool_call_output",
               call_id: toolMessage.tool_call_id,
@@ -1720,12 +1720,12 @@ export class ChatOpenAIResponses<
           // if we have the original response items, just reuse them
           if (
             !this.zdrEnabled &&
-            lcMsg.response_metadata.output != null &&
-            Array.isArray(lcMsg.response_metadata.output) &&
-            lcMsg.response_metadata.output.length > 0 &&
-            lcMsg.response_metadata.output.every((item) => "type" in item)
+            responseMetadata?.output != null &&
+            Array.isArray(responseMetadata?.output) &&
+            responseMetadata?.output.length > 0 &&
+            responseMetadata?.output.every((item) => "type" in item)
           ) {
-            return lcMsg.response_metadata.output;
+            return responseMetadata?.output;
           }
 
           // otherwise, try to reconstruct the response from what we have
@@ -1827,9 +1827,9 @@ export class ChatOpenAIResponses<
           }
 
           const toolOutputs = (
-            lcMsg.response_metadata.output as Array<ResponsesInputItem>
+            responseMetadata?.output as Array<ResponsesInputItem>
           )?.length
-            ? lcMsg.response_metadata.output
+            ? responseMetadata?.output
             : additional_kwargs.tool_outputs;
 
           const fallthroughCallTypes: ResponsesInputItem["type"][] = [

libs/providers/langchain-openai/src/utils/message_inputs.ts

@@ -19,8 +19,6 @@ import type {
 import { _convertToCompletionsMessageFromV1 } from "./standard.js";
 import { isReasoningModel, messageToOpenAIRole } from "./misc.js";
 
-export type ResponsesInputItem = OpenAIClient.Responses.ResponseInputItem;
-
 export const completionsApiContentBlockConverter: StandardContentBlockConverter<{
   text: ChatCompletionContentPartText;
   image: ChatCompletionContentPartImage;
@@ -198,7 +196,7 @@ export function _convertMessagesToOpenAIParams(
 ): OpenAIClient.Chat.Completions.ChatCompletionMessageParam[] {
   return messages.flatMap((message) => {
     if (
-      isAIMessage(message) &&
+      "output_version" in message.response_metadata &&
       message.response_metadata?.output_version === "v1"
     ) {
       return _convertToCompletionsMessageFromV1(message);

libs/providers/langchain-openai/src/utils/misc.ts

@@ -1,5 +1,5 @@
 import type { OpenAI as OpenAIClient } from "openai";
-import type { BaseMessage, ChatMessage } from "@langchain/core/messages";
+import { BaseMessage, ChatMessage } from "@langchain/core/messages";
 
 export const iife = <T>(fn: () => T) => fn();