Skip to content

Support multimodal custom tool outputs #12948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fjord-oai merged 1 commit into from
Feb 27, 2026
Merged

Support multimodal custom tool outputs #12948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/json/ClientRequest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1340,7 +1340,7 @@
"type": "string"
},
"output": {
"type": "string"
"$ref": "#/definitions/FunctionCallOutputPayload"
},
"type": {
"enum": [
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/json/EventMsg.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4822,7 +4822,7 @@
"type": "string"
},
"output": {
"type": "string"
"$ref": "#/definitions/FunctionCallOutputPayload"
},
"type": {
"enum": [
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11421,7 +11421,7 @@
"type": "string"
},
"output": {
"type": "string"
"$ref": "#/definitions/v2/FunctionCallOutputPayload"
},
"type": {
"enum": [
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,7 @@
"type": "string"
},
"output": {
"type": "string"
"$ref": "#/definitions/FunctionCallOutputPayload"
},
"type": {
"enum": [
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -615,7 +615,7 @@
"type": "string"
},
"output": {
"type": "string"
"$ref": "#/definitions/FunctionCallOutputPayload"
},
"type": {
"enum": [
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ export type ResponseItem = { "type": "message", role: string, content: Array<Con
/**
* Set when using the Responses API.
*/
call_id: string | null, status: LocalShellStatus, action: LocalShellAction, } | { "type": "function_call", name: string, arguments: string, call_id: string, } | { "type": "function_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "custom_tool_call", status?: string, call_id: string, name: string, input: string, } | { "type": "custom_tool_call_output", call_id: string, output: string, } | { "type": "web_search_call", status?: string, action?: WebSearchAction, } | { "type": "ghost_snapshot", ghost_commit: GhostCommit, } | { "type": "compaction", encrypted_content: string, } | { "type": "other" };
call_id: string | null, status: LocalShellStatus, action: LocalShellAction, } | { "type": "function_call", name: string, arguments: string, call_id: string, } | { "type": "function_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "custom_tool_call", status?: string, call_id: string, name: string, input: string, } | { "type": "custom_tool_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "web_search_call", status?: string, action?: WebSearchAction, } | { "type": "ghost_snapshot", ghost_commit: GhostCommit, } | { "type": "compaction", encrypted_content: string, } | { "type": "other" };
69 changes: 61 additions & 8 deletions codex-rs/core/src/client_common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,19 +84,13 @@ fn reserialize_shell_outputs(items: &mut [ResponseItem]) {
shell_call_ids.insert(call_id.clone());
}
}
ResponseItem::CustomToolCallOutput { call_id, output } => {
if shell_call_ids.remove(call_id)
&& let Some(structured) = parse_structured_shell_output(output)
{
*output = structured
}
}
ResponseItem::FunctionCall { name, call_id, .. }
if is_shell_tool_name(name) || name == "apply_patch" =>
{
shell_call_ids.insert(call_id.clone());
}
ResponseItem::FunctionCallOutput { call_id, output } => {
ResponseItem::FunctionCallOutput { call_id, output }
| ResponseItem::CustomToolCallOutput { call_id, output } => {
if shell_call_ids.remove(call_id)
&& let Some(structured) = output
.text_content()
Expand Down Expand Up @@ -240,6 +234,7 @@ mod tests {
use codex_api::common::OpenAiVerbosity;
use codex_api::common::TextControls;
use codex_api::create_text_param_for_request;
use codex_protocol::models::FunctionCallOutputPayload;
use pretty_assertions::assert_eq;

use super::*;
Expand Down Expand Up @@ -343,4 +338,62 @@ mod tests {
let v = serde_json::to_value(&req).expect("json");
assert!(v.get("text").is_none());
}

#[test]
fn reserializes_shell_outputs_for_function_and_custom_tool_calls() {
let raw_output = r#"{"output":"hello","metadata":{"exit_code":0,"duration_seconds":0.5}}"#;
let expected_output = "Exit code: 0\nWall time: 0.5 seconds\nOutput:\nhello";
let mut items = vec![
ResponseItem::FunctionCall {
id: None,
name: "shell".to_string(),
arguments: "{}".to_string(),
call_id: "call-1".to_string(),
},
ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload::from_text(raw_output.to_string()),
},
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "call-2".to_string(),
name: "apply_patch".to_string(),
input: "*** Begin Patch".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "call-2".to_string(),
output: FunctionCallOutputPayload::from_text(raw_output.to_string()),
},
];

reserialize_shell_outputs(&mut items);

assert_eq!(
items,
vec![
ResponseItem::FunctionCall {
id: None,
name: "shell".to_string(),
arguments: "{}".to_string(),
call_id: "call-1".to_string(),
},
ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload::from_text(expected_output.to_string()),
},
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "call-2".to_string(),
name: "apply_patch".to_string(),
input: "*** Begin Patch".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "call-2".to_string(),
output: FunctionCallOutputPayload::from_text(expected_output.to_string()),
},
]
);
}
}
49 changes: 29 additions & 20 deletions codex-rs/core/src/context_manager/history.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,32 +344,21 @@ impl ContextManager {
let policy_with_serialization_budget = policy * 1.2;
match item {
ResponseItem::FunctionCallOutput { call_id, output } => {
let body = match &output.body {
FunctionCallOutputBody::Text(content) => FunctionCallOutputBody::Text(
truncate_text(content, policy_with_serialization_budget),
),
FunctionCallOutputBody::ContentItems(items) => {
FunctionCallOutputBody::ContentItems(
truncate_function_output_items_with_policy(
items,
policy_with_serialization_budget,
),
)
}
};
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
body,
success: output.success,
},
output: truncate_function_output_payload(
output,
policy_with_serialization_budget,
),
}
}
ResponseItem::CustomToolCallOutput { call_id, output } => {
let truncated = truncate_text(output, policy_with_serialization_budget);
ResponseItem::CustomToolCallOutput {
call_id: call_id.clone(),
output: truncated,
output: truncate_function_output_payload(
output,
policy_with_serialization_budget,
),
}
}
ResponseItem::Message { .. }
Expand All @@ -385,6 +374,25 @@ impl ContextManager {
}
}

fn truncate_function_output_payload(
output: &FunctionCallOutputPayload,
policy: TruncationPolicy,
) -> FunctionCallOutputPayload {
let body = match &output.body {
FunctionCallOutputBody::Text(content) => {
FunctionCallOutputBody::Text(truncate_text(content, policy))
}
FunctionCallOutputBody::ContentItems(items) => FunctionCallOutputBody::ContentItems(
truncate_function_output_items_with_policy(items, policy),
),
};

FunctionCallOutputPayload {
body,
success: output.success,
}
}

/// API messages include every non-system item (user/assistant messages, reasoning,
/// tool calls, tool outputs, shell calls, and web-search calls).
fn is_api_message(message: &ResponseItem) -> bool {
Expand Down Expand Up @@ -508,7 +516,8 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
}
}
}
ResponseItem::FunctionCallOutput { output, .. } => {
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for content_item in items {
if let FunctionCallOutputContentItem::InputImage { image_url } = content_item {
Expand Down
76 changes: 68 additions & 8 deletions codex-rs/core/src/context_manager/history_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ fn user_input_text_msg(text: &str) -> ResponseItem {
fn custom_tool_call_output(call_id: &str, output: &str) -> ResponseItem {
ResponseItem::CustomToolCallOutput {
call_id: call_id.to_string(),
output: output.to_string(),
output: FunctionCallOutputPayload::from_text(output.to_string()),
}
}

Expand Down Expand Up @@ -279,6 +279,24 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
},
]),
},
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "tool-1".to_string(),
name: "js_repl".to_string(),
input: "view_image".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "tool-1".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputText {
text: "js repl result".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/js-repl-result.png".to_string(),
},
]),
},
];
let history = create_history_with_items(items);
let text_only_modalities = vec![InputModality::Text];
Expand Down Expand Up @@ -321,6 +339,25 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
},
]),
},
ResponseItem::CustomToolCall {
id: None,
status: None,
call_id: "tool-1".to_string(),
name: "js_repl".to_string(),
input: "view_image".to_string(),
},
ResponseItem::CustomToolCallOutput {
call_id: "tool-1".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputText {
text: "js repl result".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: "image content omitted because you do not support image input"
.to_string(),
},
]),
},
];
assert_eq!(stripped, expected);

Expand Down Expand Up @@ -671,7 +708,7 @@ fn remove_first_item_handles_custom_tool_pair() {
},
ResponseItem::CustomToolCallOutput {
call_id: "tool-1".to_string(),
output: "ok".to_string(),
output: FunctionCallOutputPayload::from_text("ok".to_string()),
},
];
let mut h = create_history_with_items(items);
Expand Down Expand Up @@ -750,15 +787,16 @@ fn record_items_truncates_custom_tool_call_output_content() {
let long_output = line.repeat(2_500);
let item = ResponseItem::CustomToolCallOutput {
call_id: "tool-200".to_string(),
output: long_output.clone(),
output: FunctionCallOutputPayload::from_text(long_output.clone()),
};

history.record_items([&item], policy);

assert_eq!(history.items.len(), 1);
match &history.items[0] {
ResponseItem::CustomToolCallOutput { output, .. } => {
assert_ne!(output, &long_output);
let output = output.text_content().unwrap_or_default();
assert_ne!(output, long_output);
assert!(
output.contains("tokens truncated"),
"expected token-based truncation marker, got {output}"
Expand Down Expand Up @@ -949,7 +987,7 @@ fn normalize_adds_missing_output_for_custom_tool_call() {
},
ResponseItem::CustomToolCallOutput {
call_id: "tool-x".to_string(),
output: "aborted".to_string(),
output: FunctionCallOutputPayload::from_text("aborted".to_string()),
},
]
);
Expand Down Expand Up @@ -1016,7 +1054,7 @@ fn normalize_removes_orphan_function_call_output() {
fn normalize_removes_orphan_custom_tool_call_output() {
let items = vec![ResponseItem::CustomToolCallOutput {
call_id: "orphan-2".to_string(),
output: "ok".to_string(),
output: FunctionCallOutputPayload::from_text("ok".to_string()),
}];
let mut h = create_history_with_items(items);

Expand Down Expand Up @@ -1089,7 +1127,7 @@ fn normalize_mixed_inserts_and_removals() {
},
ResponseItem::CustomToolCallOutput {
call_id: "t1".to_string(),
output: "aborted".to_string(),
output: FunctionCallOutputPayload::from_text("aborted".to_string()),
},
ResponseItem::LocalShellCall {
id: None,
Expand Down Expand Up @@ -1191,7 +1229,7 @@ fn normalize_removes_orphan_function_call_output_panics_in_debug() {
fn normalize_removes_orphan_custom_tool_call_output_panics_in_debug() {
let items = vec![ResponseItem::CustomToolCallOutput {
call_id: "orphan-2".to_string(),
output: "ok".to_string(),
output: FunctionCallOutputPayload::from_text("ok".to_string()),
}];
let mut h = create_history_with_items(items);
h.normalize_history(&default_input_modalities());
Expand Down Expand Up @@ -1294,6 +1332,28 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
assert!(estimated < raw_len);
}

#[test]
fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
let payload = "C".repeat(50_000);
let image_url = format!("data:image/png;base64,{payload}");
let item = ResponseItem::CustomToolCallOutput {
call_id: "call-js-repl".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputText {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage { image_url },
]),
};

let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;

assert_eq!(estimated, expected);
assert!(estimated < raw_len);
}

#[test]
fn non_base64_image_urls_are_unchanged() {
let message_item = ResponseItem::Message {
Expand Down
Loading
Loading