From 2ab4bc05dfd6aec4402e4252070609f07c6efd5d Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Fri, 10 Apr 2026 18:31:23 -0700 Subject: [PATCH 1/4] [codex-analytics] feature plumbing and emittance --- .../analytics/src/analytics_client_tests.rs | 497 +++++++++++++++++- codex-rs/analytics/src/client.rs | 22 + codex-rs/analytics/src/events.rs | 56 +- codex-rs/analytics/src/facts.rs | 55 ++ codex-rs/analytics/src/lib.rs | 3 + codex-rs/analytics/src/reducer.rs | 392 +++++++++++++- .../app-server/src/bespoke_event_handling.rs | 33 ++ .../app-server/src/codex_message_processor.rs | 12 + codex-rs/app-server/src/main.rs | 23 +- codex-rs/app-server/src/message_processor.rs | 9 + codex-rs/app-server/tests/common/config.rs | 28 + codex-rs/app-server/tests/common/lib.rs | 1 + .../app-server/tests/common/mcp_process.rs | 5 + .../app-server/tests/suite/v2/analytics.rs | 53 ++ .../app-server/tests/suite/v2/thread_fork.rs | 4 +- .../tests/suite/v2/thread_resume.rs | 4 +- .../app-server/tests/suite/v2/thread_start.rs | 20 +- .../tests/suite/v2/turn_interrupt.rs | 21 +- .../app-server/tests/suite/v2/turn_start.rs | 147 ++++++ .../app-server/tests/suite/v2/turn_steer.rs | 6 +- codex-rs/core/src/codex.rs | 55 ++ codex-rs/core/src/codex_delegate.rs | 2 +- codex-rs/core/src/codex_tests_guardian.rs | 2 +- codex-rs/core/src/state/session.rs | 12 + .../core/src/thread_rollout_truncation.rs | 12 + codex-rs/protocol/src/protocol.rs | 8 + 26 files changed, 1416 insertions(+), 66 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index d86159560b3..0720b47b887 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -7,7 +7,7 @@ use crate::events::CodexCompactionEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; -use crate::events::ThreadInitializationMode; +use crate::events::CodexTurnEventRequest; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; use crate::events::TrackEventRequest; @@ -34,29 +34,46 @@ use crate::facts::PluginUsedInput; use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnStatus; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer; use codex_app_server_protocol::AskForApproval as AppServerAskForApproval; use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy; +use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::SessionSource as AppServerSessionSource; use codex_app_server_protocol::Thread; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus; +use codex_app_server_protocol::Turn; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnError as AppServerTurnError; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartedNotification; +use codex_app_server_protocol::TurnStatus as AppServerTurnStatus; +use codex_app_server_protocol::UserInput; use codex_login::default_client::DEFAULT_ORIGINATOR; use codex_login::default_client::originator; use codex_plugin::AppConnectorId; use codex_plugin::PluginCapabilitySummary; use codex_plugin::PluginId; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use pretty_assertions::assert_eq; use serde_json::json; @@ -163,6 +180,181 @@ fn sample_thread_resume_response_with_source( } } +fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest { + ClientRequest::TurnStart { + request_id: RequestId::Integer(request_id), + params: TurnStartParams { + thread_id: thread_id.to_string(), + input: vec![ + UserInput::Text { + text: "hello".to_string(), + text_elements: vec![], + }, + UserInput::Image { + url: "https://example.com/a.png".to_string(), + }, + ], + ..Default::default() + }, + } +} + +fn sample_turn_start_response(turn_id: &str, request_id: i64) -> ClientResponse { + ClientResponse::TurnStart { + request_id: RequestId::Integer(request_id), + response: codex_app_server_protocol::TurnStartResponse { + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status: AppServerTurnStatus::InProgress, + error: None, + started_at: None, + completed_at: None, + duration_ms: None, + }, + }, + } +} + +fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNotification { + ServerNotification::TurnStarted(TurnStartedNotification { + thread_id: thread_id.to_string(), + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status: AppServerTurnStatus::InProgress, + error: None, + started_at: Some(455), + completed_at: None, + duration_ms: None, + }, + }) +} + +fn sample_turn_completed_notification( + thread_id: &str, + turn_id: &str, + status: AppServerTurnStatus, + codex_error_info: Option, +) -> ServerNotification { + ServerNotification::TurnCompleted(TurnCompletedNotification { + thread_id: thread_id.to_string(), + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status, + error: codex_error_info.map(|codex_error_info| AppServerTurnError { + message: "turn failed".to_string(), + codex_error_info: Some(codex_error_info), + additional_details: None, + }), + started_at: None, + completed_at: Some(456), + duration_ms: Some(1234), + }, + }) +} + +fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { + TurnResolvedConfigFact { + turn_id: turn_id.to_string(), + thread_id: "thread-2".to_string(), + num_input_images: 1, + submission_type: None, + ephemeral: false, + session_source: SessionSource::Exec, + model: "gpt-5".to_string(), + model_provider: "openai".to_string(), + sandbox_policy: SandboxPolicy::new_read_only_policy(), + reasoning_effort: None, + reasoning_summary: None, + service_tier: None, + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: ApprovalsReviewer::GuardianSubagent, + sandbox_network_access: true, + collaboration_mode: ModeKind::Plan, + personality: None, + is_first_turn: true, + } +} + +async fn ingest_turn_prerequisites( + reducer: &mut AnalyticsReducer, + out: &mut Vec, + include_initialize: bool, + include_resolved_config: bool, + include_started: bool, +) { + if include_initialize { + reducer + .ingest( + AnalyticsFact::Initialize { + connection_id: 7, + params: InitializeParams { + client_info: ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "1.0.0".to_string(), + }, + capabilities: None, + }, + product_client_id: "codex-tui".to_string(), + runtime: CodexRuntimeMetadata { + codex_rs_version: "0.1.0".to_string(), + runtime_os: "macos".to_string(), + runtime_os_version: "15.3.1".to_string(), + runtime_arch: "aarch64".to_string(), + }, + rpc_transport: AppServerRpcTransport::Stdio, + }, + out, + ) + .await; + } + + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)), + }, + out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)), + }, + out, + ) + .await; + + if include_resolved_config { + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + sample_turn_resolved_config("turn-2"), + ))), + out, + ) + .await; + } + + if include_started { + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_started_notification( + "thread-2", "turn-2", + ))), + out, + ) + .await; + } +} + fn expected_absolute_path(path: &PathBuf) -> String { std::fs::canonicalize(path) .unwrap_or_else(|_| path.to_path_buf()) @@ -563,10 +755,6 @@ async fn initialize_caches_client_and_thread_lifecycle_publishes_once_initialize payload[0]["event_params"]["runtime"]["runtime_arch"], "x86_64" ); - assert_eq!(payload[0]["event_params"]["initialization_mode"], "resumed"); - assert_eq!(payload[0]["event_params"]["thread_source"], "user"); - assert_eq!(payload[0]["event_params"]["subagent_source"], json!(null)); - assert_eq!(payload[0]["event_params"]["parent_thread_id"], json!(null)); } #[tokio::test] @@ -1087,6 +1275,305 @@ async fn reducer_ingests_plugin_state_changed_fact() { ); } +#[test] +fn turn_event_serializes_expected_shape() { + let event = TrackEventRequest::TurnEvent(Box::new(CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: crate::events::CodexTurnEventParams { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + app_server_client: sample_app_server_client_metadata(), + runtime: sample_runtime_metadata(), + submission_type: None, + ephemeral: false, + model: Some("gpt-5".to_string()), + model_provider: "openai".to_string(), + sandbox_policy: Some("read_only"), + reasoning_effort: Some("high".to_string()), + reasoning_summary: Some("detailed".to_string()), + service_tier: "flex".to_string(), + approval_policy: "on-request".to_string(), + approvals_reviewer: "guardian_subagent".to_string(), + sandbox_network_access: true, + collaboration_mode: Some("plan"), + personality: Some("pragmatic".to_string()), + num_input_images: 2, + is_first_turn: true, + status: Some(TurnStatus::Completed), + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + duration_ms: Some(1234), + started_at: Some(455), + completed_at: Some(456), + }, + })); + + let payload = serde_json::to_value(&event).expect("serialize turn event"); + let expected = serde_json::from_str::( + r#"{ + "event_type": "codex_turn_event", + "event_params": { + "thread_id": "thread-2", + "turn_id": "turn-2", + "submission_type": null, + "app_server_client": { + "product_client_id": "codex_cli_rs", + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "stdio", + "experimental_api_enabled": true + }, + "runtime": { + "codex_rs_version": "0.1.0", + "runtime_os": "macos", + "runtime_os_version": "15.3.1", + "runtime_arch": "aarch64" + }, + "ephemeral": false, + "model": "gpt-5", + "model_provider": "openai", + "sandbox_policy": "read_only", + "reasoning_effort": "high", + "reasoning_summary": "detailed", + "service_tier": "flex", + "approval_policy": "on-request", + "approvals_reviewer": "guardian_subagent", + "sandbox_network_access": true, + "collaboration_mode": "plan", + "personality": "pragmatic", + "num_input_images": 2, + "is_first_turn": true, + "status": "completed", + "turn_error": null, + "steer_count": null, + "total_tool_call_count": null, + "shell_command_count": null, + "file_change_count": null, + "mcp_tool_call_count": null, + "dynamic_tool_call_count": null, + "subagent_tool_call_count": null, + "web_search_count": null, + "image_generation_count": null, + "duration_ms": 1234, + "started_at": 455, + "completed_at": 456 + } + }"#, + ) + .expect("parse expected turn event"); + + assert_eq!(payload, expected); +} + +#[tokio::test] +async fn turn_lifecycle_emits_turn_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_type"], json!("codex_turn_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["turn_id"], json!("turn-2")); + assert_eq!( + payload["event_params"]["app_server_client"], + json!({ + "product_client_id": "codex-tui", + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "stdio", + "experimental_api_enabled": null, + }) + ); + assert_eq!( + payload["event_params"]["runtime"], + json!({ + "codex_rs_version": "0.1.0", + "runtime_os": "macos", + "runtime_os_version": "15.3.1", + "runtime_arch": "aarch64", + }) + ); + assert!(payload["event_params"].get("product_client_id").is_none()); + assert_eq!(payload["event_params"]["ephemeral"], json!(false)); + assert_eq!(payload["event_params"]["num_input_images"], json!(1)); + assert_eq!(payload["event_params"]["status"], json!("completed")); + assert_eq!(payload["event_params"]["started_at"], json!(455)); + assert_eq!(payload["event_params"]["completed_at"], json!(456)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); +} + +#[tokio::test] +async fn turn_does_not_emit_without_required_prerequisites() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ false, + /*include_resolved_config*/ true, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + assert!(out.is_empty()); + + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ false, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + assert!(out.is_empty()); +} + +#[tokio::test] +async fn turn_lifecycle_emits_failed_turn_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Failed, + Some(codex_app_server_protocol::CodexErrorInfo::BadRequest), + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["status"], json!("failed")); + assert_eq!(payload["event_params"]["turn_error"], json!("badRequest")); +} + +#[tokio::test] +async fn turn_lifecycle_emits_interrupted_turn_event_without_error() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Interrupted, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["status"], json!("interrupted")); + assert_eq!(payload["event_params"]["turn_error"], json!(null)); +} + +#[tokio::test] +async fn turn_completed_without_started_notification_emits_null_started_at() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["started_at"], json!(null)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); +} + fn sample_plugin_metadata() -> PluginTelemetryMetadata { PluginTelemetryMetadata { plugin_id: PluginId::parse("sample@test").expect("valid plugin id"), diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 0d96ee60617..bd4cfdd4739 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -14,9 +14,13 @@ use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; use crate::reducer::AnalyticsReducer; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; use codex_login::AuthManager; use codex_login::default_client::create_client; use codex_plugin::PluginTelemetryMetadata; @@ -167,6 +171,14 @@ impl AnalyticsEventsClient { ))); } + pub fn track_request(&self, connection_id: u64, request_id: RequestId, request: ClientRequest) { + self.record_fact(AnalyticsFact::Request { + connection_id, + request_id, + request: Box::new(request), + }); + } + pub fn track_app_used(&self, tracking: TrackEventsContext, app: AppInvocation) { if !self.queue.should_enqueue_app_used(&tracking, &app) { return; @@ -191,6 +203,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)), + )); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { @@ -240,6 +258,10 @@ impl AnalyticsEventsClient { response: Box::new(response), }); } + + pub fn track_notification(&self, notification: ServerNotification) { + self.record_fact(AnalyticsFact::Notification(Box::new(notification))); + } } async fn send_track_events( diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 618dd8ffebe..4e073c123ea 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -3,7 +3,11 @@ use crate::facts::CodexCompactionEvent; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; +use crate::facts::TurnStatus; +use crate::facts::TurnSubmissionType; +use codex_app_server_protocol::CodexErrorInfo; use codex_login::default_client::originator; use codex_plugin::PluginTelemetryMetadata; use codex_protocol::approvals::NetworkApprovalProtocol; @@ -21,14 +25,6 @@ pub enum AppServerRpcTransport { InProcess, } -#[derive(Clone, Copy, Debug, Serialize)] -#[serde(rename_all = "snake_case")] -pub(crate) enum ThreadInitializationMode { - New, - Forked, - Resumed, -} - #[derive(Serialize)] pub(crate) struct TrackEventsRequest { pub(crate) events: Vec, @@ -43,6 +39,7 @@ pub(crate) enum TrackEventRequest { AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), Compaction(Box), + TurnEvent(Box), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -330,6 +327,49 @@ pub(crate) struct CodexCompactionEventRequest { pub(crate) event_params: CodexCompactionEventParams, } +#[derive(Serialize)] +pub(crate) struct CodexTurnEventParams { + pub(crate) thread_id: String, + pub(crate) turn_id: String, + pub(crate) submission_type: Option, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) ephemeral: bool, + pub(crate) model: Option, + pub(crate) model_provider: String, + pub(crate) sandbox_policy: Option<&'static str>, + pub(crate) reasoning_effort: Option, + pub(crate) reasoning_summary: Option, + pub(crate) service_tier: String, + pub(crate) approval_policy: String, + pub(crate) approvals_reviewer: String, + pub(crate) sandbox_network_access: bool, + pub(crate) collaboration_mode: Option<&'static str>, + pub(crate) personality: Option, + pub(crate) num_input_images: usize, + pub(crate) is_first_turn: bool, + pub(crate) status: Option, + pub(crate) turn_error: Option, + pub(crate) steer_count: Option, + pub(crate) total_tool_call_count: Option, + pub(crate) shell_command_count: Option, + pub(crate) file_change_count: Option, + pub(crate) mcp_tool_call_count: Option, + pub(crate) dynamic_tool_call_count: Option, + pub(crate) subagent_tool_call_count: Option, + pub(crate) web_search_count: Option, + pub(crate) image_generation_count: Option, + pub(crate) duration_ms: Option, + pub(crate) started_at: Option, + pub(crate) completed_at: Option, +} + +#[derive(Serialize)] +pub(crate) struct CodexTurnEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexTurnEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexPluginMetadata { pub(crate) plugin_id: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 931ae010138..a36ba125a57 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -7,6 +7,15 @@ use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; use serde::Serialize; @@ -31,6 +40,51 @@ pub fn build_track_events_context( } } +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSubmissionType { + Default, + Queued, +} + +#[derive(Clone)] +pub struct TurnResolvedConfigFact { + pub turn_id: String, + pub thread_id: String, + pub num_input_images: usize, + pub submission_type: Option, + pub ephemeral: bool, + pub session_source: SessionSource, + pub model: String, + pub model_provider: String, + pub sandbox_policy: SandboxPolicy, + pub reasoning_effort: Option, + pub reasoning_summary: Option, + pub service_tier: Option, + pub approval_policy: AskForApproval, + pub approvals_reviewer: ApprovalsReviewer, + pub sandbox_network_access: bool, + pub collaboration_mode: ModeKind, + pub personality: Option, + pub is_first_turn: bool, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ThreadInitializationMode { + New, + Forked, + Resumed, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnStatus { + Completed, + Failed, + Interrupted, +} + #[derive(Clone, Debug)] pub struct SkillInvocation { pub skill_name: String, @@ -156,6 +210,7 @@ pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), GuardianReview(Box), + TurnResolvedConfig(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 9b4cc1e9bcd..acc88acf5eb 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -27,7 +27,10 @@ pub use facts::CompactionTrigger; pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; +pub use facts::ThreadInitializationMode; pub use facts::TrackEventsContext; +pub use facts::TurnResolvedConfigFact; +pub use facts::TurnStatus; pub use facts::build_track_events_context; #[cfg(test)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 0ed28997626..e52b7e7d3ba 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -6,12 +6,13 @@ use crate::events::CodexCompactionEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::CodexTurnEventParams; +use crate::events::CodexTurnEventRequest; use crate::events::GuardianReviewEventParams; use crate::events::GuardianReviewEventPayload; use crate::events::GuardianReviewEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; -use crate::events::ThreadInitializationMode; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; use crate::events::TrackEventRequest; @@ -34,11 +35,23 @@ use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::ThreadInitializationMode; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnStatus; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; +use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::UserInput; use codex_git_utils::collect_git_info; use codex_git_utils::get_git_repo_root; use codex_login::default_client::originator; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use sha1::Digest; @@ -47,6 +60,8 @@ use std::path::Path; #[derive(Default)] pub(crate) struct AnalyticsReducer { + requests: HashMap<(u64, RequestId), RequestState>, + turns: HashMap, connections: HashMap, thread_connections: HashMap, thread_metadata: HashMap, @@ -86,6 +101,32 @@ impl ThreadMetadataState { } } +enum RequestState { + TurnStart(PendingTurnStartState), +} + +struct PendingTurnStartState { + thread_id: String, + num_input_images: usize, +} + +#[derive(Clone)] +struct CompletedTurnState { + status: Option, + turn_error: Option, + completed_at: u64, + duration_ms: Option, +} + +struct TurnState { + connection_id: Option, + thread_id: Option, + num_input_images: Option, + resolved_config: Option, + started_at: Option, + completed: Option, +} + impl AnalyticsReducer { pub(crate) async fn ingest(&mut self, input: AnalyticsFact, out: &mut Vec) { match input { @@ -105,17 +146,21 @@ impl AnalyticsReducer { ); } AnalyticsFact::Request { - connection_id: _connection_id, - request_id: _request_id, - request: _request, - } => {} + connection_id, + request_id, + request, + } => { + self.ingest_request(connection_id, request_id, *request); + } AnalyticsFact::Response { connection_id, response, } => { self.ingest_response(connection_id, *response, out); } - AnalyticsFact::Notification(_notification) => {} + AnalyticsFact::Notification(notification) => { + self.ingest_notification(*notification, out); + } AnalyticsFact::Custom(input) => match input { CustomAnalyticsFact::SubAgentThreadStarted(input) => { self.ingest_subagent_thread_started(input, out); @@ -126,6 +171,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::GuardianReview(input) => { self.ingest_guardian_review(*input, out); } + CustomAnalyticsFact::TurnResolvedConfig(input) => { + self.ingest_turn_resolved_config(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -216,6 +264,52 @@ impl AnalyticsReducer { ))); } + fn ingest_request( + &mut self, + connection_id: u64, + request_id: RequestId, + request: ClientRequest, + ) { + let ClientRequest::TurnStart { params, .. } = request else { + return; + }; + self.requests.insert( + (connection_id, request_id), + RequestState::TurnStart(PendingTurnStartState { + thread_id: params.thread_id, + num_input_images: params + .input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + }), + ); + } + + fn ingest_turn_resolved_config( + &mut self, + input: TurnResolvedConfigFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let thread_id = input.thread_id.clone(); + let num_input_images = input.num_input_images; + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.thread_id = Some(thread_id); + turn_state.num_input_images = Some(num_input_images); + turn_state.resolved_config = Some(input); + self.maybe_emit_turn_event(&turn_id, out); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -316,24 +410,124 @@ impl AnalyticsReducer { response: ClientResponse, out: &mut Vec, ) { - let (thread, model, initialization_mode) = match response { - ClientResponse::ThreadStart { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::New, - ), - ClientResponse::ThreadResume { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::Resumed, - ), - ClientResponse::ThreadFork { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::Forked, - ), - _ => return, - }; + match response { + ClientResponse::ThreadStart { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::New, + out, + ); + } + ClientResponse::ThreadResume { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::Resumed, + out, + ); + } + ClientResponse::ThreadFork { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::Forked, + out, + ); + } + ClientResponse::TurnStart { + request_id, + response, + } => { + let turn_id = response.turn.id; + let Some(RequestState::TurnStart(pending_request)) = + self.requests.remove(&(connection_id, request_id)) + else { + return; + }; + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.connection_id = Some(connection_id); + turn_state.thread_id = Some(pending_request.thread_id); + turn_state.num_input_images = Some(pending_request.num_input_images); + self.maybe_emit_turn_event(&turn_id, out); + } + _ => {} + } + } + + fn ingest_notification( + &mut self, + notification: ServerNotification, + out: &mut Vec, + ) { + match notification { + ServerNotification::TurnStarted(notification) => { + let turn_state = self.turns.entry(notification.turn.id).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.started_at = notification + .turn + .started_at + .and_then(|started_at| u64::try_from(started_at).ok()); + } + ServerNotification::TurnCompleted(notification) => { + let turn_state = + self.turns + .entry(notification.turn.id.clone()) + .or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.completed = Some(CompletedTurnState { + status: analytics_turn_status(notification.turn.status), + turn_error: notification + .turn + .error + .and_then(|error| error.codex_error_info), + completed_at: notification + .turn + .completed_at + .and_then(|completed_at| u64::try_from(completed_at).ok()) + .unwrap_or_default(), + duration_ms: notification + .turn + .duration_ms + .and_then(|duration_ms| u64::try_from(duration_ms).ok()), + }); + let turn_id = notification.turn.id; + self.maybe_emit_turn_event(&turn_id, out); + } + _ => {} + } + } + + fn emit_thread_initialized( + &mut self, + connection_id: u64, + thread: codex_app_server_protocol::Thread, + model: String, + initialization_mode: ThreadInitializationMode, + out: &mut Vec, + ) { let thread_source: SessionSource = thread.source.into(); let thread_id = thread.id; let Some(connection_state) = self.connections.get(&connection_id) else { @@ -403,6 +597,156 @@ impl AnalyticsReducer { }, ))); } + + fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec) { + let Some(turn_state) = self.turns.get(turn_id) else { + return; + }; + if turn_state.thread_id.is_none() + || turn_state.num_input_images.is_none() + || turn_state.resolved_config.is_none() + || turn_state.completed.is_none() + { + return; + } + let connection_metadata = turn_state + .connection_id + .and_then(|connection_id| self.connections.get(&connection_id)) + .map(|connection_state| { + ( + connection_state.app_server_client.clone(), + connection_state.runtime.clone(), + ) + }); + let Some((app_server_client, runtime)) = connection_metadata else { + return; + }; + out.push(TrackEventRequest::TurnEvent(Box::new( + CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: codex_turn_event_params( + app_server_client, + runtime, + turn_id.to_string(), + turn_state, + ), + }, + ))); + self.turns.remove(turn_id); + } +} + +fn codex_turn_event_params( + app_server_client: CodexAppServerClientMetadata, + runtime: CodexRuntimeMetadata, + turn_id: String, + turn_state: &TurnState, +) -> CodexTurnEventParams { + let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = ( + turn_state.thread_id.clone(), + turn_state.num_input_images, + turn_state.resolved_config.clone(), + turn_state.completed.clone(), + ) else { + unreachable!("turn event params require a fully populated turn state"); + }; + let started_at = turn_state.started_at; + let TurnResolvedConfigFact { + turn_id: _resolved_turn_id, + thread_id: _resolved_thread_id, + num_input_images: _resolved_num_input_images, + submission_type, + ephemeral, + session_source: _session_source, + model, + model_provider, + sandbox_policy, + reasoning_effort, + reasoning_summary, + service_tier, + approval_policy, + approvals_reviewer, + sandbox_network_access, + collaboration_mode, + personality, + is_first_turn, + } = resolved_config; + CodexTurnEventParams { + thread_id, + turn_id, + app_server_client, + runtime, + submission_type, + ephemeral, + model: Some(model), + model_provider, + sandbox_policy: Some(sandbox_policy_mode(&sandbox_policy)), + reasoning_effort: reasoning_effort.map(|value| value.to_string()), + reasoning_summary: reasoning_summary_mode(reasoning_summary), + service_tier: service_tier + .map(|value| value.to_string()) + .unwrap_or_else(|| "default".to_string()), + approval_policy: approval_policy.to_string(), + approvals_reviewer: approvals_reviewer.to_string(), + sandbox_network_access, + collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)), + personality: personality_mode(personality), + num_input_images, + is_first_turn, + status: completed.status, + turn_error: completed.turn_error, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + duration_ms: completed.duration_ms, + started_at, + completed_at: Some(completed.completed_at), + } +} + +fn sandbox_policy_mode(sandbox_policy: &SandboxPolicy) -> &'static str { + match sandbox_policy { + SandboxPolicy::DangerFullAccess => "full_access", + SandboxPolicy::ReadOnly { .. } => "read_only", + SandboxPolicy::WorkspaceWrite { .. } => "workspace_write", + SandboxPolicy::ExternalSandbox { .. } => "external_sandbox", + } +} + +fn collaboration_mode_mode(mode: ModeKind) -> &'static str { + match mode { + ModeKind::Plan => "plan", + ModeKind::Default | ModeKind::PairProgramming | ModeKind::Execute => "default", + } +} + +fn reasoning_summary_mode(summary: Option) -> Option { + match summary { + Some(ReasoningSummary::None) | None => None, + Some(summary) => Some(summary.to_string()), + } +} + +fn personality_mode(personality: Option) -> Option { + match personality { + Some(Personality::None) | None => None, + Some(personality) => Some(personality.to_string()), + } +} + +fn analytics_turn_status(status: codex_app_server_protocol::TurnStatus) -> Option { + match status { + codex_app_server_protocol::TurnStatus::Completed => Some(TurnStatus::Completed), + codex_app_server_protocol::TurnStatus::Failed => Some(TurnStatus::Failed), + codex_app_server_protocol::TurnStatus::Interrupted => Some(TurnStatus::Interrupted), + codex_app_server_protocol::TurnStatus::InProgress => None, + } } pub(crate) fn skill_id_for_local_skill( diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index d8990483b79..53361bc3c33 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -12,6 +12,7 @@ use crate::thread_state::TurnSummary; use crate::thread_state::resolve_server_request_on_thread_listener; use crate::thread_status::ThreadWatchActiveGuard; use crate::thread_status::ThreadWatchManager; +use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::AccountRateLimitsUpdatedNotification; use codex_app_server_protocol::AdditionalPermissionProfile as V2AdditionalPermissionProfile; use codex_app_server_protocol::AgentMessageDeltaNotification; @@ -167,6 +168,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id: ThreadId, conversation: Arc, thread_manager: Arc, + analytics_events_client: Option, outgoing: ThreadScopedOutgoingMessageSender, thread_state: Arc>, thread_watch_manager: ThreadWatchManager, @@ -202,6 +204,10 @@ pub(crate) async fn apply_bespoke_event_handling( thread_id: conversation_id.to_string(), turn, }; + if let Some(analytics_events_client) = analytics_events_client.as_ref() { + analytics_events_client + .track_notification(ServerNotification::TurnStarted(notification.clone())); + } outgoing .send_server_notification(ServerNotification::TurnStarted(notification)) .await; @@ -218,6 +224,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id, event_turn_id, turn_complete_event, + analytics_events_client.as_ref(), &outgoing, &thread_state, ) @@ -1738,6 +1745,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id, event_turn_id, turn_aborted_event, + analytics_events_client.as_ref(), &outgoing, &thread_state, ) @@ -1915,6 +1923,7 @@ async fn emit_turn_completed_with_status( conversation_id: ThreadId, event_turn_id: String, turn_completion_metadata: TurnCompletionMetadata, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, ) { let notification = TurnCompletedNotification { @@ -1929,6 +1938,10 @@ async fn emit_turn_completed_with_status( duration_ms: turn_completion_metadata.duration_ms, }, }; + if let Some(analytics_events_client) = analytics_events_client { + analytics_events_client + .track_notification(ServerNotification::TurnCompleted(notification.clone())); + } outgoing .send_server_notification(ServerNotification::TurnCompleted(notification)) .await; @@ -2121,6 +2134,7 @@ async fn handle_turn_complete( conversation_id: ThreadId, event_turn_id: String, turn_complete_event: TurnCompleteEvent, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, thread_state: &Arc>, ) { @@ -2141,6 +2155,7 @@ async fn handle_turn_complete( completed_at: turn_complete_event.completed_at, duration_ms: turn_complete_event.duration_ms, }, + analytics_events_client, outgoing, ) .await; @@ -2150,6 +2165,7 @@ async fn handle_turn_interrupted( conversation_id: ThreadId, event_turn_id: String, turn_aborted_event: TurnAbortedEvent, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, thread_state: &Arc>, ) { @@ -2165,6 +2181,7 @@ async fn handle_turn_interrupted( completed_at: turn_aborted_event.completed_at, duration_ms: turn_aborted_event.duration_ms, }, + analytics_events_client, outgoing, ) .await; @@ -2905,6 +2922,7 @@ mod tests { use codex_app_server_protocol::GuardianApprovalReviewStatus; use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::TurnPlanStepStatus; + use codex_login::AuthManager; use codex_login::CodexAuth; use codex_protocol::items::HookPromptFragment; use codex_protocol::items::build_hook_prompt_message; @@ -3032,6 +3050,7 @@ mod tests { outgoing: ThreadScopedOutgoingMessageSender, thread_state: Arc>, thread_watch_manager: ThreadWatchManager, + analytics_events_client: AnalyticsEventsClient, codex_home: PathBuf, } @@ -3046,6 +3065,7 @@ mod tests { self.conversation_id, self.conversation.clone(), self.thread_manager.clone(), + Some(self.analytics_events_client.clone()), self.outgoing.clone(), self.thread_state.clone(), self.thread_watch_manager.clone(), @@ -3372,6 +3392,13 @@ mod tests { outgoing: outgoing.clone(), thread_state: thread_state.clone(), thread_watch_manager: thread_watch_manager.clone(), + analytics_events_client: AnalyticsEventsClient::new( + AuthManager::from_auth_for_testing( + CodexAuth::create_dummy_chatgpt_auth_for_testing(), + ), + "http://localhost".to_string(), + Some(false), + ), codex_home: codex_home.path().to_path_buf(), }; @@ -3822,6 +3849,7 @@ mod tests { conversation_id, event_turn_id.clone(), turn_complete_event(&event_turn_id), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -3870,6 +3898,7 @@ mod tests { conversation_id, event_turn_id.clone(), turn_aborted_event(&event_turn_id), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -3917,6 +3946,7 @@ mod tests { conversation_id, event_turn_id.clone(), turn_complete_event(&event_turn_id), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -4183,6 +4213,7 @@ mod tests { conversation_a, a_turn1.clone(), turn_complete_event(&a_turn1), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -4204,6 +4235,7 @@ mod tests { conversation_b, b_turn1.clone(), turn_complete_event(&b_turn1), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -4215,6 +4247,7 @@ mod tests { conversation_a, a_turn2.clone(), turn_complete_event(&a_turn2), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index e0509907b93..ebc6a1aab9c 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -6669,6 +6669,15 @@ impl CodexMessageProcessor { }; let response = TurnStartResponse { turn }; + if self.config.features.enabled(Feature::GeneralAnalytics) { + self.analytics_events_client.track_response( + request_id.connection_id.0, + ClientResponse::TurnStart { + request_id: request_id.request_id.clone(), + response: response.clone(), + }, + ); + } self.outgoing.send_response(request_id, response).await; } Err(err) => { @@ -7480,6 +7489,9 @@ impl CodexMessageProcessor { conversation_id, conversation.clone(), thread_manager.clone(), + listener_task_context + .general_analytics_enabled + .then(|| listener_task_context.analytics_events_client.clone()), thread_outgoing, thread_state.clone(), thread_watch_manager.clone(), diff --git a/codex-rs/app-server/src/main.rs b/codex-rs/app-server/src/main.rs index 9a23680fb9c..d896f2f8ecc 100644 --- a/codex-rs/app-server/src/main.rs +++ b/codex-rs/app-server/src/main.rs @@ -12,6 +12,7 @@ use std::path::PathBuf; // Debug-only test hook: lets integration tests point the server at a temporary // managed config file without writing to /etc. const MANAGED_CONFIG_PATH_ENV_VAR: &str = "CODEX_APP_SERVER_MANAGED_CONFIG_PATH"; +const DISABLE_MANAGED_CONFIG_ENV_VAR: &str = "CODEX_APP_SERVER_DISABLE_MANAGED_CONFIG"; #[derive(Debug, Parser)] struct AppServerArgs { @@ -40,10 +41,13 @@ struct AppServerArgs { fn main() -> anyhow::Result<()> { arg0_dispatch_or_else(|arg0_paths: Arg0DispatchPaths| async move { let args = AppServerArgs::parse(); - let managed_config_path = managed_config_path_from_debug_env(); - let loader_overrides = LoaderOverrides { - managed_config_path, - ..Default::default() + let loader_overrides = if disable_managed_config_from_debug_env() { + LoaderOverrides::without_managed_config_for_tests() + } else { + LoaderOverrides { + managed_config_path: managed_config_path_from_debug_env(), + ..Default::default() + } }; let transport = args.listen; let session_source = args.session_source; @@ -63,6 +67,17 @@ fn main() -> anyhow::Result<()> { }) } +fn disable_managed_config_from_debug_env() -> bool { + #[cfg(debug_assertions)] + { + if let Ok(value) = std::env::var(DISABLE_MANAGED_CONFIG_ENV_VAR) { + return matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"); + } + } + + false +} + fn managed_config_path_from_debug_env() -> Option { #[cfg(debug_assertions)] { diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 8c2bba00a32..4c47ec15aa6 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -677,6 +677,15 @@ impl MessageProcessor { self.outgoing.send_error(connection_request_id, error).await; return; } + if self.config.features.enabled(Feature::GeneralAnalytics) + && let ClientRequest::TurnStart { request_id, .. } = &codex_request + { + self.analytics_events_client.track_request( + connection_id.0, + request_id.clone(), + codex_request.clone(), + ); + } match codex_request { ClientRequest::ConfigRead { request_id, params } => { diff --git a/codex-rs/app-server/tests/common/config.rs b/codex-rs/app-server/tests/common/config.rs index deb16c63221..1ac2572fa25 100644 --- a/codex-rs/app-server/tests/common/config.rs +++ b/codex-rs/app-server/tests/common/config.rs @@ -78,3 +78,31 @@ model_provider = "{model_provider_id}" ), ) } + +pub fn write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home: &Path, + server_uri: &str, + chatgpt_base_url: &str, +) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +chatgpt_base_url = "{chatgpt_base_url}" + +model_provider = "mock_provider" + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{server_uri}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 3f89765851e..90553760d97 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -14,6 +14,7 @@ pub use auth_fixtures::encode_id_token; pub use auth_fixtures::write_chatgpt_auth; use codex_app_server_protocol::JSONRPCResponse; pub use config::write_mock_responses_config_toml; +pub use config::write_mock_responses_config_toml_with_chatgpt_base_url; pub use core_test_support::format_with_current_shell; pub use core_test_support::format_with_current_shell_display; pub use core_test_support::format_with_current_shell_display_non_login; diff --git a/codex-rs/app-server/tests/common/mcp_process.rs b/codex-rs/app-server/tests/common/mcp_process.rs index 03c2284b841..69d209c0206 100644 --- a/codex-rs/app-server/tests/common/mcp_process.rs +++ b/codex-rs/app-server/tests/common/mcp_process.rs @@ -98,12 +98,17 @@ pub struct McpProcess { } pub const DEFAULT_CLIENT_NAME: &str = "codex-app-server-tests"; +const DISABLE_MANAGED_CONFIG_ENV_VAR: &str = "CODEX_APP_SERVER_DISABLE_MANAGED_CONFIG"; impl McpProcess { pub async fn new(codex_home: &Path) -> anyhow::Result { Self::new_with_env_and_args(codex_home, &[], &[]).await } + pub async fn new_without_managed_config(codex_home: &Path) -> anyhow::Result { + Self::new_with_env(codex_home, &[(DISABLE_MANAGED_CONFIG_ENV_VAR, Some("1"))]).await + } + pub async fn new_with_args(codex_home: &Path, args: &[&str]) -> anyhow::Result { Self::new_with_env_and_args(codex_home, &[], args).await } diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index a4d7a7f349d..a3ecdbc1f43 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -80,6 +80,24 @@ async fn app_server_default_analytics_enabled_with_flag() -> Result<()> { } pub(crate) async fn enable_analytics_capture(server: &MockServer, codex_home: &Path) -> Result<()> { + let config_path = codex_home.join("config.toml"); + let config_toml = std::fs::read_to_string(&config_path)?; + if !config_toml.contains("[features]") { + std::fs::write( + &config_path, + format!("{config_toml}\n[features]\ngeneral_analytics = true\n"), + )?; + } else if !config_toml.contains("general_analytics") { + std::fs::write( + &config_path, + config_toml.replace("[features]\n", "[features]\ngeneral_analytics = true\n"), + )?; + } + + mount_analytics_capture(server, codex_home).await +} + +pub(crate) async fn mount_analytics_capture(server: &MockServer, codex_home: &Path) -> Result<()> { Mock::given(method("POST")) .and(path("/codex/analytics-events/events")) .respond_with(ResponseTemplate::new(200)) @@ -120,6 +138,41 @@ pub(crate) async fn wait_for_analytics_payload( serde_json::from_slice(&body).map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}")) } +pub(crate) async fn wait_for_analytics_event( + server: &MockServer, + read_timeout: Duration, + event_type: &str, +) -> Result { + timeout(read_timeout, async { + loop { + let Some(requests) = server.received_requests().await else { + tokio::time::sleep(Duration::from_millis(25)).await; + continue; + }; + for request in &requests { + if request.method != "POST" + || request.url.path() != "/codex/analytics-events/events" + { + continue; + } + let payload: Value = serde_json::from_slice(&request.body) + .map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?; + let Some(events) = payload["events"].as_array() else { + continue; + }; + if let Some(event) = events + .iter() + .find(|event| event["event_type"] == event_type) + { + return Ok::(event.clone()); + } + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await? +} + pub(crate) fn thread_initialized_event(payload: &Value) -> Result<&Value> { let events = payload["events"] .as_array() diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs index 9907fc4b1de..19bf00f64a4 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs @@ -205,7 +205,7 @@ async fn thread_fork_tracks_thread_initialized_analytics() -> Result<()> { /*git_info*/ None, )?; - let mut mcp = McpProcess::new(codex_home.path()).await?; + let mut mcp = McpProcess::new_without_managed_config(codex_home.path()).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let fork_id = mcp @@ -565,7 +565,7 @@ fn create_config_toml_with_chatgpt_base_url( let general_analytics_toml = if general_analytics_enabled { "\ngeneral_analytics = true".to_string() } else { - String::new() + "\ngeneral_analytics = false".to_string() }; let config_toml = codex_home.join("config.toml"); std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 80ae7568881..0a302c23806 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -178,7 +178,7 @@ async fn thread_resume_tracks_thread_initialized_analytics() -> Result<()> { /*git_info*/ None, )?; - let mut mcp = McpProcess::new(codex_home.path()).await?; + let mut mcp = McpProcess::new_without_managed_config(codex_home.path()).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let resume_id = mcp @@ -1901,7 +1901,7 @@ fn create_config_toml_with_chatgpt_base_url( let general_analytics_toml = if general_analytics_enabled { "\ngeneral_analytics = true".to_string() } else { - String::new() + "\ngeneral_analytics = false".to_string() }; let config_toml = codex_home.join("config.toml"); std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/thread_start.rs b/codex-rs/app-server/tests/suite/v2/thread_start.rs index 50c373ac93c..4fcba242721 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_start.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_start.rs @@ -39,8 +39,9 @@ use wiremock::matchers::method; use wiremock::matchers::path; use super::analytics::assert_basic_thread_initialized_event; -use super::analytics::enable_analytics_capture; +use super::analytics::mount_analytics_capture; use super::analytics::thread_initialized_event; +use super::analytics::wait_for_analytics_event; use super::analytics::wait_for_analytics_payload; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); @@ -173,9 +174,9 @@ async fn thread_start_tracks_thread_initialized_analytics() -> Result<()> { &server.uri(), /*general_analytics_enabled*/ true, )?; - enable_analytics_capture(&server, codex_home.path()).await?; + mount_analytics_capture(&server, codex_home.path()).await?; - let mut mcp = McpProcess::new(codex_home.path()).await?; + let mut mcp = McpProcess::new_without_managed_config(codex_home.path()).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let req_id = mcp @@ -206,9 +207,9 @@ async fn thread_start_does_not_track_thread_initialized_analytics_without_featur &server.uri(), /*general_analytics_enabled*/ false, )?; - enable_analytics_capture(&server, codex_home.path()).await?; + mount_analytics_capture(&server, codex_home.path()).await?; - let mut mcp = McpProcess::new(codex_home.path()).await?; + let mut mcp = McpProcess::new_without_managed_config(codex_home.path()).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let req_id = mcp @@ -221,7 +222,12 @@ async fn thread_start_does_not_track_thread_initialized_analytics_without_featur .await??; let _ = to_response::(resp)?; - let payload = wait_for_analytics_payload(&server, Duration::from_millis(250)).await; + let payload = wait_for_analytics_event( + &server, + Duration::from_millis(250), + "codex_thread_initialized", + ) + .await; assert!( payload.is_err(), "thread analytics should be gated off when general_analytics is disabled" @@ -829,7 +835,7 @@ fn create_config_toml_with_chatgpt_base_url( let general_analytics_toml = if general_analytics_enabled { "\ngeneral_analytics = true".to_string() } else { - String::new() + "\ngeneral_analytics = false".to_string() }; let config_toml = codex_home.join("config.toml"); std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index 2850c7b74f2..b5531377526 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -3,6 +3,7 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_mock_responses_server_sequence; +use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; use codex_app_server_protocol::JSONRPCNotification; @@ -43,14 +44,15 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { std::fs::create_dir(&working_directory)?; // Mock server: long-running shell command then (after abort) nothing else needed. - let server = create_mock_responses_server_sequence(vec![create_shell_command_sse_response( - shell_command.clone(), - Some(&working_directory), - Some(10_000), - "call_sleep", - )?]) - .await; - create_config_toml(&codex_home, &server.uri(), "never", "danger-full-access")?; + let server = + create_mock_responses_server_sequence_unchecked(vec![create_shell_command_sse_response( + shell_command.clone(), + Some(&working_directory), + Some(10_000), + "call_sleep", + )?]) + .await; + create_config_toml(&codex_home, &server.uri(), "never", "workspace-write")?; let mut mcp = McpProcess::new(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -87,6 +89,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { ) .await??; let TurnStartResponse { turn } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); // Give the command a brief moment to start. tokio::time::sleep(std::time::Duration::from_secs(1)).await; @@ -96,7 +99,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { let interrupt_id = mcp .send_turn_interrupt_request(TurnInterruptParams { thread_id: thread_id.clone(), - turn_id: turn.id, + turn_id: turn_id.clone(), }) .await?; let interrupt_resp: JSONRPCResponse = timeout( diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index d81a5d27a05..ef1b2c0400e 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use app_test_support::DEFAULT_CLIENT_NAME; use app_test_support::McpProcess; use app_test_support::create_apply_patch_sse_response; use app_test_support::create_exec_command_sse_response; @@ -9,6 +10,7 @@ use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::format_with_current_shell_display; use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::ByteRange; @@ -64,6 +66,10 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +use super::analytics::enable_analytics_capture; +use super::analytics::mount_analytics_capture; +use super::analytics::wait_for_analytics_event; + #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); #[cfg(not(windows))] @@ -328,6 +334,147 @@ async fn thread_start_omits_empty_instruction_overrides_from_model_request() -> Ok(()) } +#[tokio::test] +async fn turn_start_tracks_turn_event_analytics() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, codex_home.path()).await?; + + let mut mcp = McpProcess::new_without_managed_config(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Image { + url: "https://example.com/a.png".to_string(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["turn_id"], turn.id); + assert_eq!( + event["event_params"]["app_server_client"]["product_client_id"], + DEFAULT_CLIENT_NAME + ); + assert_eq!(event["event_params"]["model"], "mock-model"); + assert_eq!(event["event_params"]["model_provider"], "mock_provider"); + assert_eq!(event["event_params"]["sandbox_policy"], "read_only"); + assert_eq!(event["event_params"]["num_input_images"], 1); + assert_eq!(event["event_params"]["status"], "completed"); + assert!(event["event_params"]["started_at"].as_u64().is_some()); + assert!(event["event_params"]["completed_at"].as_u64().is_some()); + assert!(event["event_params"]["duration_ms"].as_u64().is_some()); + + Ok(()) +} + +#[tokio::test] +async fn turn_start_does_not_track_turn_event_analytics_without_feature() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + &server.uri(), + &server.uri(), + )?; + let config_path = codex_home.path().join("config.toml"); + let config_toml = std::fs::read_to_string(&config_path)?; + std::fs::write( + &config_path, + format!("{config_toml}\n[features]\ngeneral_analytics = false\n"), + )?; + mount_analytics_capture(&server, codex_home.path()).await?; + + let mut mcp = McpProcess::new_without_managed_config(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "hello".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _ = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let turn_event = wait_for_analytics_event( + &server, + std::time::Duration::from_millis(250), + "codex_turn_event", + ) + .await; + assert!( + turn_event.is_err(), + "turn analytics should be gated off when general_analytics is disabled" + ); + Ok(()) +} + #[tokio::test] async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> { let responses = vec![create_final_assistant_message_sse_response("Done")?]; diff --git a/codex-rs/app-server/tests/suite/v2/turn_steer.rs b/codex-rs/app-server/tests/suite/v2/turn_steer.rs index a93bf6c6ab7..e66dfc9e0b0 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_steer.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_steer.rs @@ -34,7 +34,7 @@ async fn turn_steer_requires_active_turn() -> Result<()> { let server = create_mock_responses_server_sequence(vec![]).await; create_config_toml(&codex_home, &server.uri())?; - let mut mcp = McpProcess::new(&codex_home).await?; + let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let thread_req = mcp @@ -98,7 +98,7 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> { .await; create_config_toml(&codex_home, &server.uri())?; - let mut mcp = McpProcess::new(&codex_home).await?; + let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let thread_req = mcp @@ -202,7 +202,7 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { .await; create_config_toml(&codex_home, &server.uri())?; - let mut mcp = McpProcess::new(&codex_home).await?; + let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let thread_req = mcp diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cd6f1e84fdd..49588490ae8 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -55,6 +55,7 @@ use codex_analytics::CompactionPhase; use codex_analytics::CompactionReason; use codex_analytics::InvocationType; use codex_analytics::SubAgentThreadStartedInput; +use codex_analytics::TurnResolvedConfigFact; use codex_analytics::build_track_events_context; use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::McpServerElicitationRequest; @@ -188,6 +189,7 @@ use crate::config::resolve_web_search_mode_for_turn; use crate::context_manager::ContextManager; use crate::context_manager::TotalTokenUsageBreakdown; use crate::environment_context::EnvironmentContext; +use crate::thread_rollout_truncation::initial_history_has_prior_user_turns; use codex_config::CONFIG_TOML_FILE; use codex_config::types::McpServerConfig; use codex_config::types::ShellEnvironmentPolicy; @@ -2339,6 +2341,11 @@ impl Session { SessionSource::SubAgent(_) ) }; + let has_prior_user_turns = initial_history_has_prior_user_turns(&conversation_history); + { + let mut state = self.state.lock().await; + state.set_next_turn_is_first(!has_prior_user_turns); + } match conversation_history { InitialHistory::New | InitialHistory::Cleared => { // Defer initial context insertion until the first real turn starts so @@ -6187,6 +6194,8 @@ pub(crate) async fn run_turn( .await; } + track_turn_resolved_config_analytics(&sess, &turn_context, &input).await; + let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) .await; @@ -6490,6 +6499,52 @@ pub(crate) async fn run_turn( last_agent_message } +async fn track_turn_resolved_config_analytics( + sess: &Session, + turn_context: &TurnContext, + input: &[UserInput], +) { + if !sess.enabled(Feature::GeneralAnalytics) { + return; + } + + let thread_config = { + let state = sess.state.lock().await; + state.session_configuration.thread_config_snapshot() + }; + let is_first_turn = { + let mut state = sess.state.lock().await; + state.take_next_turn_is_first() + }; + sess.services + .analytics_events_client + .track_turn_resolved_config(TurnResolvedConfigFact { + turn_id: turn_context.sub_id.clone(), + thread_id: sess.conversation_id.to_string(), + num_input_images: input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + submission_type: None, + ephemeral: thread_config.ephemeral, + session_source: thread_config.session_source, + model: turn_context.model_info.slug.clone(), + model_provider: turn_context.config.model_provider_id.clone(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + reasoning_effort: turn_context.reasoning_effort, + reasoning_summary: Some(turn_context.reasoning_summary), + service_tier: turn_context.config.service_tier, + approval_policy: turn_context.approval_policy.value(), + approvals_reviewer: turn_context.config.approvals_reviewer, + sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), + collaboration_mode: turn_context.collaboration_mode.mode, + personality: turn_context.personality, + is_first_turn, + }); +} + async fn run_pre_sampling_compact( sess: &Arc, turn_context: &Arc, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 4deb7d3cfdb..82573690483 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -78,7 +78,6 @@ pub(crate) async fn run_codex_thread_interactive( let CodexSpawnOk { codex, .. } = Codex::spawn(CodexSpawnArgs { config, auth_manager, - analytics_events_client: Some(parent_session.services.analytics_events_client.clone()), models_manager, environment_manager: Arc::new(EnvironmentManager::from_environment( parent_ctx.environment.as_deref(), @@ -97,6 +96,7 @@ pub(crate) async fn run_codex_thread_interactive( user_shell_override: None, inherited_exec_policy: Some(Arc::clone(&parent_session.services.exec_policy)), parent_trace: None, + analytics_events_client: Some(parent_session.services.analytics_events_client.clone()), }) .await?; if parent_session.enabled(codex_features::Feature::GeneralAnalytics) { diff --git a/codex-rs/core/src/codex_tests_guardian.rs b/codex-rs/core/src/codex_tests_guardian.rs index 70a70a367c2..4762d5d8c65 100644 --- a/codex-rs/core/src/codex_tests_guardian.rs +++ b/codex-rs/core/src/codex_tests_guardian.rs @@ -436,7 +436,6 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { let CodexSpawnOk { codex, .. } = Codex::spawn(CodexSpawnArgs { config, auth_manager, - analytics_events_client: None, models_manager, environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), skills_manager, @@ -455,6 +454,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { inherited_exec_policy: Some(Arc::new(parent_exec_policy)), user_shell_override: None, parent_trace: None, + analytics_events_client: None, }) .await .expect("spawn guardian subagent"); diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 206f75060c7..4360b16de4c 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -33,6 +33,7 @@ pub(crate) struct SessionState { pub(crate) active_connector_selection: HashSet, pub(crate) pending_session_start_source: Option, granted_permissions: Option, + next_turn_is_first: bool, } impl SessionState { @@ -51,6 +52,7 @@ impl SessionState { active_connector_selection: HashSet::new(), pending_session_start_source: None, granted_permissions: None, + next_turn_is_first: true, } } @@ -73,6 +75,16 @@ impl SessionState { self.previous_turn_settings = previous_turn_settings; } + pub(crate) fn set_next_turn_is_first(&mut self, value: bool) { + self.next_turn_is_first = value; + } + + pub(crate) fn take_next_turn_is_first(&mut self) -> bool { + let is_first_turn = self.next_turn_is_first; + self.next_turn_is_first = false; + is_first_turn + } + pub(crate) fn clone_history(&self) -> ContextManager { self.history.clone() } diff --git a/codex-rs/core/src/thread_rollout_truncation.rs b/codex-rs/core/src/thread_rollout_truncation.rs index 97370ce41e3..e20ee53d47d 100644 --- a/codex-rs/core/src/thread_rollout_truncation.rs +++ b/codex-rs/core/src/thread_rollout_truncation.rs @@ -8,9 +8,21 @@ use crate::event_mapping; use codex_protocol::items::TurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::RolloutItem; +pub(crate) fn initial_history_has_prior_user_turns(conversation_history: &InitialHistory) -> bool { + conversation_history.scan_rollout_items(rollout_item_is_user_turn_boundary) +} + +fn rollout_item_is_user_turn_boundary(item: &RolloutItem) -> bool { + match item { + RolloutItem::ResponseItem(item) => is_user_turn_boundary(item), + _ => false, + } +} + /// Return the indices of user message boundaries in a rollout. /// /// A user message boundary is a `RolloutItem::ResponseItem(ResponseItem::Message { .. })` diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 2c2c9cabeca..fc20a527cb0 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2430,6 +2430,14 @@ pub enum InitialHistory { } impl InitialHistory { + pub fn scan_rollout_items(&self, mut predicate: impl FnMut(&RolloutItem) -> bool) -> bool { + match self { + InitialHistory::New | InitialHistory::Cleared => false, + InitialHistory::Resumed(resumed) => resumed.history.iter().any(&mut predicate), + InitialHistory::Forked(items) => items.iter().any(predicate), + } + } + pub fn forked_from_id(&self) -> Option { match self { InitialHistory::New | InitialHistory::Cleared => None, From de7d0810ddfa1833d171453fdc2d99d641a5fede Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Fri, 10 Apr 2026 18:34:52 -0700 Subject: [PATCH 2/4] [codex-analytics] add token usage metadata --- .../analytics/src/analytics_client_tests.rs | 60 +++++++++++++++++++ codex-rs/analytics/src/client.rs | 7 +++ codex-rs/analytics/src/events.rs | 5 ++ codex-rs/analytics/src/facts.rs | 9 +++ codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/reducer.rs | 46 ++++++++++++++ .../app-server/tests/suite/v2/turn_start.rs | 5 ++ codex-rs/core/src/tasks/mod.rs | 8 +++ 8 files changed, 141 insertions(+) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 0720b47b887..68874ff1f3a 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -38,6 +38,7 @@ use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; @@ -75,6 +76,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::TokenUsage; use pretty_assertions::assert_eq; use serde_json::json; use std::collections::HashSet; @@ -231,6 +233,20 @@ fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNot }) } +fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsageFact { + TurnTokenUsageFact { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_string(), + token_usage: TokenUsage { + total_tokens: 321, + input_tokens: 123, + cached_input_tokens: 45, + output_tokens: 140, + reasoning_output_tokens: 13, + }, + } +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -284,6 +300,7 @@ async fn ingest_turn_prerequisites( include_initialize: bool, include_resolved_config: bool, include_started: bool, + include_token_usage: bool, ) { if include_initialize { reducer @@ -353,6 +370,17 @@ async fn ingest_turn_prerequisites( ) .await; } + + if include_token_usage { + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage(Box::new( + sample_turn_token_usage_fact("thread-2", "turn-2"), + ))), + out, + ) + .await; + } } fn expected_absolute_path(path: &PathBuf) -> String { @@ -1310,6 +1338,11 @@ fn turn_event_serializes_expected_shape() { subagent_tool_call_count: None, web_search_count: None, image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, duration_ms: Some(1234), started_at: Some(455), completed_at: Some(456), @@ -1362,6 +1395,11 @@ fn turn_event_serializes_expected_shape() { "subagent_tool_call_count": null, "web_search_count": null, "image_generation_count": null, + "input_tokens": null, + "cached_input_tokens": null, + "output_tokens": null, + "reasoning_output_tokens": null, + "total_tokens": null, "duration_ms": 1234, "started_at": 455, "completed_at": 456 @@ -1384,6 +1422,7 @@ async fn turn_lifecycle_emits_turn_event() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ true, + /*include_token_usage*/ true, ) .await; reducer @@ -1429,6 +1468,14 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["started_at"], json!(455)); assert_eq!(payload["event_params"]["completed_at"], json!(456)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["input_tokens"], json!(123)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(45)); + assert_eq!(payload["event_params"]["output_tokens"], json!(140)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(13) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } #[tokio::test] @@ -1442,6 +1489,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { /*include_initialize*/ false, /*include_resolved_config*/ true, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1466,6 +1514,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { /*include_initialize*/ true, /*include_resolved_config*/ false, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1493,6 +1542,7 @@ async fn turn_lifecycle_emits_failed_turn_event() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ true, + /*include_token_usage*/ false, ) .await; reducer @@ -1524,6 +1574,7 @@ async fn turn_lifecycle_emits_interrupted_turn_event_without_error() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ true, + /*include_token_usage*/ false, ) .await; reducer @@ -1555,6 +1606,7 @@ async fn turn_completed_without_started_notification_emits_null_started_at() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1572,6 +1624,14 @@ async fn turn_completed_without_started_notification_emits_null_started_at() { let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); assert_eq!(payload["event_params"]["started_at"], json!(null)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["output_tokens"], json!(null)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(null) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(null)); } fn sample_plugin_metadata() -> PluginTelemetryMetadata { diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index bd4cfdd4739..8524ddc1e28 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -15,6 +15,7 @@ use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; @@ -209,6 +210,12 @@ impl AnalyticsEventsClient { )); } + pub fn track_turn_token_usage(&self, fact: TurnTokenUsageFact) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage( + Box::new(fact), + ))); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 4e073c123ea..ce2745cd674 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -359,6 +359,11 @@ pub(crate) struct CodexTurnEventParams { pub(crate) subagent_tool_call_count: Option, pub(crate) web_search_count: Option, pub(crate) image_generation_count: Option, + pub(crate) input_tokens: Option, + pub(crate) cached_input_tokens: Option, + pub(crate) output_tokens: Option, + pub(crate) reasoning_output_tokens: Option, + pub(crate) total_tokens: Option, pub(crate) duration_ms: Option, pub(crate) started_at: Option, pub(crate) completed_at: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index a36ba125a57..2ed0fba7ff3 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -18,6 +18,7 @@ use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::TokenUsage; use serde::Serialize; use std::path::PathBuf; @@ -77,6 +78,13 @@ pub enum ThreadInitializationMode { Resumed, } +#[derive(Clone)] +pub struct TurnTokenUsageFact { + pub turn_id: String, + pub thread_id: String, + pub token_usage: TokenUsage, +} + #[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "snake_case")] pub enum TurnStatus { @@ -211,6 +219,7 @@ pub(crate) enum CustomAnalyticsFact { Compaction(Box), GuardianReview(Box), TurnResolvedConfig(Box), + TurnTokenUsage(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index acc88acf5eb..a93f1f80511 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -31,6 +31,7 @@ pub use facts::ThreadInitializationMode; pub use facts::TrackEventsContext; pub use facts::TurnResolvedConfigFact; pub use facts::TurnStatus; +pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; #[cfg(test)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index e52b7e7d3ba..d49e94a653f 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -38,6 +38,7 @@ use crate::facts::SubAgentThreadStartedInput; use crate::facts::ThreadInitializationMode; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnTokenUsageFact; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::CodexErrorInfo; @@ -54,6 +55,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; +use codex_protocol::protocol::TokenUsage; use sha1::Digest; use std::collections::HashMap; use std::path::Path; @@ -124,6 +126,7 @@ struct TurnState { num_input_images: Option, resolved_config: Option, started_at: Option, + token_usage: Option, completed: Option, } @@ -174,6 +177,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnResolvedConfig(input) => { self.ingest_turn_resolved_config(*input, out); } + CustomAnalyticsFact::TurnTokenUsage(input) => { + self.ingest_turn_token_usage(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -302,6 +308,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.thread_id = Some(thread_id); @@ -310,6 +317,26 @@ impl AnalyticsReducer { self.maybe_emit_turn_event(&turn_id, out); } + fn ingest_turn_token_usage( + &mut self, + input: TurnTokenUsageFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + token_usage: None, + completed: None, + }); + turn_state.thread_id = Some(input.thread_id); + turn_state.token_usage = Some(input.token_usage); + self.maybe_emit_turn_event(&turn_id, out); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -454,6 +481,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.connection_id = Some(connection_id); @@ -478,6 +506,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.started_at = notification @@ -495,6 +524,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.completed = Some(CompletedTurnState { @@ -671,6 +701,7 @@ fn codex_turn_event_params( personality, is_first_turn, } = resolved_config; + let token_usage = turn_state.token_usage.clone(); CodexTurnEventParams { thread_id, turn_id, @@ -704,6 +735,21 @@ fn codex_turn_event_params( subagent_tool_call_count: None, web_search_count: None, image_generation_count: None, + input_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.input_tokens), + cached_input_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.cached_input_tokens), + output_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.output_tokens), + reasoning_output_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.reasoning_output_tokens), + total_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.total_tokens), duration_ms: completed.duration_ms, started_at, completed_at: Some(completed.completed_at), diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index ef1b2c0400e..ea3ba02b468 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -400,6 +400,11 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert!(event["event_params"]["started_at"].as_u64().is_some()); assert!(event["event_params"]["completed_at"].as_u64().is_some()); assert!(event["event_params"]["duration_ms"].as_u64().is_some()); + assert_eq!(event["event_params"]["input_tokens"], 0); + assert_eq!(event["event_params"]["cached_input_tokens"], 0); + assert_eq!(event["event_params"]["output_tokens"], 0); + assert_eq!(event["event_params"]["reasoning_output_tokens"], 0); + assert_eq!(event["event_params"]["total_tokens"], 0); Ok(()) } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index b46db8e98f2..f17017316e5 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -30,6 +30,7 @@ use crate::hook_runtime::record_pending_input; use crate::state::ActiveTurn; use crate::state::RunningTask; use crate::state::TaskKind; +use codex_analytics::TurnTokenUsageFact; use codex_login::AuthManager; use codex_models_manager::manager::ModelsManager; use codex_otel::SessionTelemetry; @@ -497,6 +498,13 @@ impl Session { - token_usage_at_turn_start.total_tokens) .max(0), }; + self.services + .analytics_events_client + .track_turn_token_usage(TurnTokenUsageFact { + turn_id: turn_context.sub_id.clone(), + thread_id: self.conversation_id.to_string(), + token_usage: turn_token_usage.clone(), + }); self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.total_tokens, From 48a801a22cb85fd7c6e97ef8e52e2ed8fbdb92af Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Fri, 10 Apr 2026 18:34:52 -0700 Subject: [PATCH 3/4] [codex-analytics] add steering metadata --- .../analytics/src/analytics_client_tests.rs | 504 +++++++++++++++++- codex-rs/analytics/src/client.rs | 17 + codex-rs/analytics/src/events.rs | 42 ++ codex-rs/analytics/src/facts.rs | 75 +++ codex-rs/analytics/src/lib.rs | 16 + codex-rs/analytics/src/reducer.rs | 208 +++++++- .../app-server/src/codex_message_processor.rs | 85 ++- codex-rs/app-server/src/message_processor.rs | 3 +- .../app-server/tests/suite/v2/turn_steer.rs | 77 ++- codex-rs/core/src/compact.rs | 10 +- 10 files changed, 942 insertions(+), 95 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 68874ff1f3a..e57c1ed375c 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -16,6 +16,7 @@ use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; use crate::events::subagent_thread_started_event_request; use crate::facts::AnalyticsFact; +use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; @@ -27,6 +28,7 @@ use crate::facts::CompactionStatus; use crate::facts::CompactionStrategy; use crate::facts::CompactionTrigger; use crate::facts::CustomAnalyticsFact; +use crate::facts::InputError; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; @@ -38,6 +40,7 @@ use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnSteerRequestError; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; @@ -47,8 +50,11 @@ use codex_app_server_protocol::AskForApproval as AppServerAskForApproval; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; +use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::NonSteerableTurnKind; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy; use codex_app_server_protocol::ServerNotification; @@ -63,6 +69,8 @@ use codex_app_server_protocol::TurnError as AppServerTurnError; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartedNotification; use codex_app_server_protocol::TurnStatus as AppServerTurnStatus; +use codex_app_server_protocol::TurnSteerParams; +use codex_app_server_protocol::TurnSteerResponse; use codex_app_server_protocol::UserInput; use codex_login::default_client::DEFAULT_ORIGINATOR; use codex_login::default_client::originator; @@ -294,6 +302,149 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { } } +fn sample_turn_steer_request( + thread_id: &str, + expected_turn_id: &str, + request_id: i64, +) -> ClientRequest { + ClientRequest::TurnSteer { + request_id: RequestId::Integer(request_id), + params: TurnSteerParams { + thread_id: thread_id.to_string(), + expected_turn_id: expected_turn_id.to_string(), + input: vec![ + UserInput::Text { + text: "more".to_string(), + text_elements: vec![], + }, + UserInput::LocalImage { + path: "/tmp/a.png".into(), + }, + ], + responsesapi_client_metadata: None, + }, + } +} + +fn sample_turn_steer_response(turn_id: &str, request_id: i64) -> ClientResponse { + ClientResponse::TurnSteer { + request_id: RequestId::Integer(request_id), + response: TurnSteerResponse { + turn_id: turn_id.to_string(), + }, + } +} + +fn no_active_turn_steer_error() -> JSONRPCErrorError { + JSONRPCErrorError { + code: -32600, + message: "no active turn to steer".to_string(), + data: None, + } +} + +fn no_active_turn_steer_error_type() -> AnalyticsJsonRpcError { + AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::NoActiveTurn) +} + +fn non_steerable_review_error() -> JSONRPCErrorError { + JSONRPCErrorError { + code: -32600, + message: "cannot steer a review turn".to_string(), + data: Some( + serde_json::to_value(AppServerTurnError { + message: "cannot steer a review turn".to_string(), + codex_error_info: Some(CodexErrorInfo::ActiveTurnNotSteerable { + turn_kind: NonSteerableTurnKind::Review, + }), + additional_details: None, + }) + .expect("serialize turn error"), + ), + } +} + +fn non_steerable_review_error_type() -> AnalyticsJsonRpcError { + AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::NonSteerableReview) +} + +fn input_too_large_steer_error() -> JSONRPCErrorError { + JSONRPCErrorError { + code: -32602, + message: "Input exceeds the maximum length of 1048576 characters.".to_string(), + data: Some(json!({ + "input_error_code": "input_too_large", + "actual_chars": 1048577, + "max_chars": 1048576, + })), + } +} + +fn input_too_large_error_type() -> AnalyticsJsonRpcError { + AnalyticsJsonRpcError::Input(InputError::TooLarge) +} + +async fn ingest_rejected_turn_steer( + reducer: &mut AnalyticsReducer, + out: &mut Vec, + error: JSONRPCErrorError, + error_type: Option, +) -> serde_json::Value { + ingest_turn_prerequisites( + reducer, out, /*include_initialize*/ true, /*include_resolved_config*/ false, + /*include_started*/ false, /*include_token_usage*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(4), + request: Box::new(sample_turn_steer_request( + "thread-2", "turn-2", /*request_id*/ 4, + )), + }, + out, + ) + .await; + reducer + .ingest( + AnalyticsFact::ErrorResponse { + connection_id: 7, + request_id: RequestId::Integer(4), + error, + error_type, + }, + out, + ) + .await; + + assert_eq!(out.len(), 1); + serde_json::to_value(&out[0]).expect("serialize turn steer event") +} + +async fn ingest_initialize(reducer: &mut AnalyticsReducer, out: &mut Vec) { + reducer + .ingest( + AnalyticsFact::Initialize { + connection_id: 7, + params: InitializeParams { + client_info: ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "1.0.0".to_string(), + }, + capabilities: None, + }, + product_client_id: "codex-tui".to_string(), + runtime: sample_runtime_metadata(), + rpc_transport: AppServerRpcTransport::Stdio, + }, + out, + ) + .await; +} + async fn ingest_turn_prerequisites( reducer: &mut AnalyticsReducer, out: &mut Vec, @@ -303,30 +454,7 @@ async fn ingest_turn_prerequisites( include_token_usage: bool, ) { if include_initialize { - reducer - .ingest( - AnalyticsFact::Initialize { - connection_id: 7, - params: InitializeParams { - client_info: ClientInfo { - name: "codex-tui".to_string(), - title: None, - version: "1.0.0".to_string(), - }, - capabilities: None, - }, - product_client_id: "codex-tui".to_string(), - runtime: CodexRuntimeMetadata { - codex_rs_version: "0.1.0".to_string(), - runtime_os: "macos".to_string(), - runtime_os_version: "15.3.1".to_string(), - runtime_arch: "aarch64".to_string(), - }, - rpc_transport: AppServerRpcTransport::Stdio, - }, - out, - ) - .await; + ingest_initialize(reducer, out).await; } reducer @@ -1329,7 +1457,7 @@ fn turn_event_serializes_expected_shape() { is_first_turn: true, status: Some(TurnStatus::Completed), turn_error: None, - steer_count: None, + steer_count: Some(0), total_tool_call_count: None, shell_command_count: None, file_change_count: None, @@ -1386,7 +1514,7 @@ fn turn_event_serializes_expected_shape() { "is_first_turn": true, "status": "completed", "turn_error": null, - "steer_count": null, + "steer_count": 0, "total_tool_call_count": null, "shell_command_count": null, "file_change_count": null, @@ -1411,6 +1539,226 @@ fn turn_event_serializes_expected_shape() { assert_eq!(payload, expected); } +#[tokio::test] +async fn accepted_turn_steer_emits_expected_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ false, + /*include_started*/ false, + /*include_token_usage*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(4), + request: Box::new(sample_turn_steer_request( + "thread-2", "turn-2", /*request_id*/ 4, + )), + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 4)), + }, + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event"); + assert_eq!(payload["event_type"], json!("codex_turn_steer_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2")); + assert_eq!(payload["event_params"]["accepted_turn_id"], json!("turn-2")); + assert_eq!(payload["event_params"]["num_input_images"], json!(1)); + assert_eq!(payload["event_params"]["result"], json!("accepted")); + assert_eq!(payload["event_params"]["rejection_reason"], json!(null)); + assert!( + payload["event_params"]["created_at"] + .as_u64() + .expect("created_at") + > 0 + ); + assert_eq!( + payload["event_params"]["app_server_client"]["product_client_id"], + json!("codex-tui") + ); + assert_eq!( + payload["event_params"]["runtime"]["codex_rs_version"], + json!("0.1.0") + ); + assert!(payload["event_params"].get("product_client_id").is_none()); +} + +#[tokio::test] +async fn rejected_turn_steer_uses_request_connection_metadata() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + let payload = ingest_rejected_turn_steer( + &mut reducer, + &mut out, + no_active_turn_steer_error(), + Some(no_active_turn_steer_error_type()), + ) + .await; + + assert_eq!(payload["event_type"], json!("codex_turn_steer_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2")); + assert_eq!(payload["event_params"]["accepted_turn_id"], json!(null)); + assert_eq!(payload["event_params"]["num_input_images"], json!(1)); + assert_eq!( + payload["event_params"]["app_server_client"]["product_client_id"], + json!("codex-tui") + ); + assert_eq!( + payload["event_params"]["runtime"]["codex_rs_version"], + json!("0.1.0") + ); + assert_eq!(payload["event_params"]["result"], json!("rejected")); + assert_eq!( + payload["event_params"]["rejection_reason"], + json!("no_active_turn") + ); + assert!( + payload["event_params"]["created_at"] + .as_u64() + .expect("created_at") + > 0 + ); +} + +#[tokio::test] +async fn rejected_turn_steer_maps_active_turn_not_steerable_error_type() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + let payload = ingest_rejected_turn_steer( + &mut reducer, + &mut out, + non_steerable_review_error(), + Some(non_steerable_review_error_type()), + ) + .await; + + assert_eq!( + payload["event_params"]["rejection_reason"], + json!("non_steerable_review") + ); +} + +#[tokio::test] +async fn rejected_turn_steer_maps_input_too_large_error_type() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + let payload = ingest_rejected_turn_steer( + &mut reducer, + &mut out, + input_too_large_steer_error(), + Some(input_too_large_error_type()), + ) + .await; + + assert_eq!( + payload["event_params"]["rejection_reason"], + json!("input_too_large") + ); +} + +#[tokio::test] +async fn turn_steer_does_not_emit_without_pending_request() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + reducer + .ingest( + AnalyticsFact::ErrorResponse { + connection_id: 7, + request_id: RequestId::Integer(4), + error: no_active_turn_steer_error(), + error_type: Some(no_active_turn_steer_error_type()), + }, + &mut out, + ) + .await; + + assert!(out.is_empty()); +} + +#[tokio::test] +async fn turn_start_error_response_discards_pending_start_request() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_initialize(&mut reducer, &mut out).await; + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)), + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::ErrorResponse { + connection_id: 7, + request_id: RequestId::Integer(3), + error: no_active_turn_steer_error(), + error_type: None, + }, + &mut out, + ) + .await; + + // A late/synthetic response for the same request id must not resurrect the + // failed turn/start request and attach request-scoped connection metadata. + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)), + }, + &mut out, + ) + .await; + assert!(out.is_empty()); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + sample_turn_resolved_config("turn-2"), + ))), + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert!(out.is_empty()); +} + #[tokio::test] async fn turn_lifecycle_emits_turn_event() { let mut reducer = AnalyticsReducer::default(); @@ -1465,6 +1813,7 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["ephemeral"], json!(false)); assert_eq!(payload["event_params"]["num_input_images"], json!(1)); assert_eq!(payload["event_params"]["status"], json!("completed")); + assert_eq!(payload["event_params"]["steer_count"], json!(0)); assert_eq!(payload["event_params"]["started_at"], json!(455)); assert_eq!(payload["event_params"]["completed_at"], json!(456)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); @@ -1478,6 +1827,109 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } +#[tokio::test] +async fn accepted_steers_increment_turn_steer_count() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(4), + request: Box::new(sample_turn_steer_request( + "thread-2", "turn-2", /*request_id*/ 4, + )), + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 4)), + }, + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(5), + request: Box::new(sample_turn_steer_request( + "thread-2", "turn-2", /*request_id*/ 5, + )), + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::ErrorResponse { + connection_id: 7, + request_id: RequestId::Integer(5), + error: no_active_turn_steer_error(), + error_type: Some(no_active_turn_steer_error_type()), + }, + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(6), + request: Box::new(sample_turn_steer_request( + "thread-2", "turn-2", /*request_id*/ 6, + )), + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 6)), + }, + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + let turn_event = out + .iter() + .find(|event| matches!(event, TrackEventRequest::TurnEvent(_))) + .expect("turn event should be emitted"); + let payload = serde_json::to_value(turn_event).expect("serialize turn event"); + assert_eq!(payload["event_params"]["steer_count"], json!(2)); +} + #[tokio::test] async fn turn_does_not_emit_without_required_prerequisites() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 8524ddc1e28..5ba60e3ef44 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -4,6 +4,7 @@ use crate::events::TrackEventRequest; use crate::events::TrackEventsRequest; use crate::events::current_runtime_metadata; use crate::facts::AnalyticsFact; +use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; @@ -20,6 +21,7 @@ use crate::reducer::AnalyticsReducer; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_login::AuthManager; @@ -266,6 +268,21 @@ impl AnalyticsEventsClient { }); } + pub fn track_error_response( + &self, + connection_id: u64, + request_id: RequestId, + error: JSONRPCErrorError, + error_type: Option, + ) { + self.record_fact(AnalyticsFact::ErrorResponse { + connection_id, + request_id, + error, + error_type, + }); + } + pub fn track_notification(&self, notification: ServerNotification) { self.record_fact(AnalyticsFact::Notification(Box::new(notification))); } diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index ce2745cd674..3f149b493ae 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1,11 +1,14 @@ use crate::facts::AppInvocation; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexTurnSteerEvent; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; use crate::facts::TurnStatus; +use crate::facts::TurnSteerRejectionReason; +use crate::facts::TurnSteerResult; use crate::facts::TurnSubmissionType; use codex_app_server_protocol::CodexErrorInfo; use codex_login::default_client::originator; @@ -40,6 +43,7 @@ pub(crate) enum TrackEventRequest { AppUsed(CodexAppUsedEventRequest), Compaction(Box), TurnEvent(Box), + TurnSteer(CodexTurnSteerEventRequest), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -375,6 +379,25 @@ pub(crate) struct CodexTurnEventRequest { pub(crate) event_params: CodexTurnEventParams, } +#[derive(Serialize)] +pub(crate) struct CodexTurnSteerEventParams { + pub(crate) thread_id: String, + pub(crate) expected_turn_id: Option, + pub(crate) accepted_turn_id: Option, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) num_input_images: usize, + pub(crate) result: TurnSteerResult, + pub(crate) rejection_reason: Option, + pub(crate) created_at: u64, +} + +#[derive(Serialize)] +pub(crate) struct CodexTurnSteerEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexTurnSteerEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexPluginMetadata { pub(crate) plugin_id: Option, @@ -497,6 +520,25 @@ pub(crate) fn codex_plugin_used_metadata( } } +pub(crate) fn codex_turn_steer_event_params( + app_server_client: CodexAppServerClientMetadata, + runtime: CodexRuntimeMetadata, + tracking: &TrackEventsContext, + turn_steer: CodexTurnSteerEvent, +) -> CodexTurnSteerEventParams { + CodexTurnSteerEventParams { + thread_id: tracking.thread_id.clone(), + expected_turn_id: turn_steer.expected_turn_id, + accepted_turn_id: turn_steer.accepted_turn_id, + app_server_client, + runtime, + num_input_images: turn_steer.num_input_images, + result: turn_steer.result, + rejection_reason: turn_steer.rejection_reason, + created_at: turn_steer.created_at, + } +} + pub(crate) fn thread_source_name(thread_source: &SessionSource) -> Option<&'static str> { match thread_source { SessionSource::Cli | SessionSource::VSCode | SessionSource::Exec => Some("user"), diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 2ed0fba7ff3..5590fbfa6dc 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -4,6 +4,7 @@ use crate::events::GuardianReviewEventParams; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_plugin::PluginTelemetryMetadata; @@ -93,6 +94,74 @@ pub enum TurnStatus { Interrupted, } +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSteerResult { + Accepted, + Rejected, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSteerRejectionReason { + NoActiveTurn, + ExpectedTurnMismatch, + NonSteerableReview, + NonSteerableCompact, + EmptyInput, + InputTooLarge, +} + +#[derive(Clone)] +pub struct CodexTurnSteerEvent { + pub expected_turn_id: Option, + pub accepted_turn_id: Option, + pub num_input_images: usize, + pub result: TurnSteerResult, + pub rejection_reason: Option, + pub created_at: u64, +} + +#[derive(Clone, Copy, Debug)] +pub enum AnalyticsJsonRpcError { + TurnSteer(TurnSteerRequestError), + Input(InputError), +} + +#[derive(Clone, Copy, Debug)] +pub enum TurnSteerRequestError { + NoActiveTurn, + ExpectedTurnMismatch, + NonSteerableReview, + NonSteerableCompact, +} + +#[derive(Clone, Copy, Debug)] +pub enum InputError { + Empty, + TooLarge, +} + +impl From for TurnSteerRejectionReason { + fn from(error: TurnSteerRequestError) -> Self { + match error { + TurnSteerRequestError::NoActiveTurn => Self::NoActiveTurn, + TurnSteerRequestError::ExpectedTurnMismatch => Self::ExpectedTurnMismatch, + TurnSteerRequestError::NonSteerableReview => Self::NonSteerableReview, + TurnSteerRequestError::NonSteerableCompact => Self::NonSteerableCompact, + } + } +} + +impl From for TurnSteerRejectionReason { + fn from(error: InputError) -> Self { + match error { + InputError::Empty => Self::EmptyInput, + InputError::TooLarge => Self::InputTooLarge, + } + } +} + #[derive(Clone, Debug)] pub struct SkillInvocation { pub skill_name: String, @@ -208,6 +277,12 @@ pub(crate) enum AnalyticsFact { connection_id: u64, response: Box, }, + ErrorResponse { + connection_id: u64, + request_id: RequestId, + error: JSONRPCErrorError, + error_type: Option, + }, Notification(Box), // Facts that do not naturally exist on the app-server protocol surface, or // would require non-trivial protocol reshaping on this branch. diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index a93f1f80511..1a1a1231523 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -3,6 +3,9 @@ mod events; mod facts; mod reducer; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + pub use client::AnalyticsEventsClient; pub use events::AppServerRpcTransport; pub use events::GuardianApprovalRequestSource; @@ -16,14 +19,17 @@ pub use events::GuardianReviewSessionKind; pub use events::GuardianReviewTerminalStatus; pub use events::GuardianReviewUserAuthorization; pub use events::GuardianReviewedAction; +pub use facts::AnalyticsJsonRpcError; pub use facts::AppInvocation; pub use facts::CodexCompactionEvent; +pub use facts::CodexTurnSteerEvent; pub use facts::CompactionImplementation; pub use facts::CompactionPhase; pub use facts::CompactionReason; pub use facts::CompactionStatus; pub use facts::CompactionStrategy; pub use facts::CompactionTrigger; +pub use facts::InputError; pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; @@ -31,8 +37,18 @@ pub use facts::ThreadInitializationMode; pub use facts::TrackEventsContext; pub use facts::TurnResolvedConfigFact; pub use facts::TurnStatus; +pub use facts::TurnSteerRejectionReason; +pub use facts::TurnSteerRequestError; +pub use facts::TurnSteerResult; pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; #[cfg(test)] mod analytics_client_tests; + +pub fn now_unix_seconds() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index d49e94a653f..352ea1c8928 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -8,6 +8,7 @@ use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventParams; use crate::events::CodexTurnEventRequest; +use crate::events::CodexTurnSteerEventRequest; use crate::events::GuardianReviewEventParams; use crate::events::GuardianReviewEventPayload; use crate::events::GuardianReviewEventRequest; @@ -20,15 +21,18 @@ use crate::events::codex_app_metadata; use crate::events::codex_compaction_event_params; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; +use crate::events::codex_turn_steer_event_params; use crate::events::plugin_state_event_type; use crate::events::subagent_parent_thread_id; use crate::events::subagent_source_name; use crate::events::subagent_thread_started_event_request; use crate::events::thread_source_name; use crate::facts::AnalyticsFact; +use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexTurnSteerEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; @@ -36,15 +40,20 @@ use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::ThreadInitializationMode; +use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnSteerRejectionReason; +use crate::facts::TurnSteerResult; use crate::facts::TurnTokenUsageFact; +use crate::now_unix_seconds; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::TurnSteerResponse; use codex_app_server_protocol::UserInput; use codex_git_utils::collect_git_info; use codex_git_utils::get_git_repo_root; @@ -105,6 +114,7 @@ impl ThreadMetadataState { enum RequestState { TurnStart(PendingTurnStartState), + TurnSteer(PendingTurnSteerState), } struct PendingTurnStartState { @@ -112,6 +122,13 @@ struct PendingTurnStartState { num_input_images: usize, } +struct PendingTurnSteerState { + thread_id: String, + expected_turn_id: String, + num_input_images: usize, + created_at: u64, +} + #[derive(Clone)] struct CompletedTurnState { status: Option, @@ -128,6 +145,7 @@ struct TurnState { started_at: Option, token_usage: Option, completed: Option, + steer_count: usize, } impl AnalyticsReducer { @@ -161,6 +179,14 @@ impl AnalyticsReducer { } => { self.ingest_response(connection_id, *response, out); } + AnalyticsFact::ErrorResponse { + connection_id, + request_id, + error: _, + error_type, + } => { + self.ingest_error_response(connection_id, request_id, error_type, out); + } AnalyticsFact::Notification(notification) => { self.ingest_notification(*notification, out); } @@ -276,22 +302,29 @@ impl AnalyticsReducer { request_id: RequestId, request: ClientRequest, ) { - let ClientRequest::TurnStart { params, .. } = request else { - return; - }; - self.requests.insert( - (connection_id, request_id), - RequestState::TurnStart(PendingTurnStartState { - thread_id: params.thread_id, - num_input_images: params - .input - .iter() - .filter(|item| { - matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) - }) - .count(), - }), - ); + match request { + ClientRequest::TurnStart { params, .. } => { + self.requests.insert( + (connection_id, request_id), + RequestState::TurnStart(PendingTurnStartState { + thread_id: params.thread_id, + num_input_images: num_input_images(¶ms.input), + }), + ); + } + ClientRequest::TurnSteer { params, .. } => { + self.requests.insert( + (connection_id, request_id), + RequestState::TurnSteer(PendingTurnSteerState { + thread_id: params.thread_id, + expected_turn_id: params.expected_turn_id, + num_input_images: num_input_images(¶ms.input), + created_at: now_unix_seconds(), + }), + ); + } + _ => {} + } } fn ingest_turn_resolved_config( @@ -310,6 +343,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.thread_id = Some(thread_id); turn_state.num_input_images = Some(num_input_images); @@ -331,6 +365,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.thread_id = Some(input.thread_id); turn_state.token_usage = Some(input.token_usage); @@ -483,16 +518,73 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.connection_id = Some(connection_id); turn_state.thread_id = Some(pending_request.thread_id); turn_state.num_input_images = Some(pending_request.num_input_images); self.maybe_emit_turn_event(&turn_id, out); } + ClientResponse::TurnSteer { + request_id, + response, + } => { + self.ingest_turn_steer_response(connection_id, request_id, response, out); + } _ => {} } } + fn ingest_error_response( + &mut self, + connection_id: u64, + request_id: RequestId, + error_type: Option, + out: &mut Vec, + ) { + let Some(request) = self.requests.remove(&(connection_id, request_id)) else { + return; + }; + self.ingest_request_error_response(connection_id, request, error_type, out); + } + + fn ingest_request_error_response( + &mut self, + connection_id: u64, + request: RequestState, + error_type: Option, + out: &mut Vec, + ) { + match request { + RequestState::TurnStart(_) => {} + RequestState::TurnSteer(pending_request) => { + self.ingest_turn_steer_error_response( + connection_id, + pending_request, + error_type, + out, + ); + } + } + } + + fn ingest_turn_steer_error_response( + &mut self, + connection_id: u64, + pending_request: PendingTurnSteerState, + error_type: Option, + out: &mut Vec, + ) { + self.emit_turn_steer_event( + connection_id, + pending_request, + /*accepted_turn_id*/ None, + TurnSteerResult::Rejected, + rejection_reason_from_error_type(error_type), + out, + ); + } + fn ingest_notification( &mut self, notification: ServerNotification, @@ -508,6 +600,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.started_at = notification .turn @@ -526,6 +619,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.completed = Some(CompletedTurnState { status: analytics_turn_status(notification.turn.status), @@ -628,6 +722,70 @@ impl AnalyticsReducer { ))); } + fn ingest_turn_steer_response( + &mut self, + connection_id: u64, + request_id: RequestId, + response: TurnSteerResponse, + out: &mut Vec, + ) { + let Some(RequestState::TurnSteer(pending_request)) = + self.requests.remove(&(connection_id, request_id)) + else { + return; + }; + if let Some(turn_state) = self.turns.get_mut(&response.turn_id) { + turn_state.steer_count += 1; + } + self.emit_turn_steer_event( + connection_id, + pending_request, + Some(response.turn_id), + TurnSteerResult::Accepted, + /*rejection_reason*/ None, + out, + ); + } + + fn emit_turn_steer_event( + &mut self, + connection_id: u64, + pending_request: PendingTurnSteerState, + accepted_turn_id: Option, + result: TurnSteerResult, + rejection_reason: Option, + out: &mut Vec, + ) { + let Some(connection_state) = self.connections.get(&connection_id) else { + return; + }; + let tracking = TrackEventsContext { + model_slug: String::new(), + thread_id: pending_request.thread_id, + turn_id: accepted_turn_id + .as_deref() + .unwrap_or(pending_request.expected_turn_id.as_str()) + .to_string(), + }; + let turn_steer = CodexTurnSteerEvent { + expected_turn_id: Some(pending_request.expected_turn_id), + accepted_turn_id, + num_input_images: pending_request.num_input_images, + result, + rejection_reason, + created_at: pending_request.created_at, + }; + out.push(TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest { + event_type: "codex_turn_steer_event", + event_params: codex_turn_steer_event_params( + connection_state.app_server_client.clone(), + connection_state.runtime.clone(), + &tracking, + turn_steer, + ), + })); + } + fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec) { let Some(turn_state) = self.turns.get(turn_id) else { return; @@ -726,7 +884,7 @@ fn codex_turn_event_params( is_first_turn, status: completed.status, turn_error: completed.turn_error, - steer_count: None, + steer_count: Some(turn_state.steer_count), total_tool_call_count: None, shell_command_count: None, file_change_count: None, @@ -795,6 +953,22 @@ fn analytics_turn_status(status: codex_app_server_protocol::TurnStatus) -> Optio } } +fn num_input_images(input: &[UserInput]) -> usize { + input + .iter() + .filter(|item| matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })) + .count() +} + +fn rejection_reason_from_error_type( + error_type: Option, +) -> Option { + match error_type? { + AnalyticsJsonRpcError::TurnSteer(error) => Some(error.into()), + AnalyticsJsonRpcError::Input(error) => Some(error.into()), + } +} + pub(crate) fn skill_id_for_local_skill( repo_url: Option<&str>, repo_root: Option<&Path>, diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index ebc6a1aab9c..e7ba062dad7 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -22,6 +22,9 @@ use chrono::DateTime; use chrono::SecondsFormat; use chrono::Utc; use codex_analytics::AnalyticsEventsClient; +use codex_analytics::AnalyticsJsonRpcError; +use codex_analytics::InputError; +use codex_analytics::TurnSteerRequestError; use codex_app_server_protocol::Account; use codex_app_server_protocol::AccountLoginCompletedNotification; use codex_app_server_protocol::AccountUpdatedNotification; @@ -36,7 +39,7 @@ use codex_app_server_protocol::CancelLoginAccountResponse; use codex_app_server_protocol::CancelLoginAccountStatus; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; -use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo; +use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::CollaborationModeListParams; use codex_app_server_protocol::CollaborationModeListResponse; use codex_app_server_protocol::CommandExecParams; @@ -504,6 +507,22 @@ impl CodexMessageProcessor { } } + fn track_error_response( + &self, + request_id: &ConnectionRequestId, + error: &JSONRPCErrorError, + error_type: Option, + ) { + if self.config.features.enabled(Feature::GeneralAnalytics) { + self.analytics_events_client.track_error_response( + request_id.connection_id.0, + request_id.request_id.clone(), + error.clone(), + error_type, + ); + } + } + async fn load_thread( &self, thread_id: &str, @@ -6569,12 +6588,18 @@ impl CodexMessageProcessor { app_server_client_version: Option, ) { if let Err(error) = Self::validate_v2_input_limit(¶ms.input) { + self.track_error_response( + &request_id, + &error, + Some(AnalyticsJsonRpcError::Input(InputError::TooLarge)), + ); self.outgoing.send_error(request_id, error).await; return; } let (_, thread) = match self.load_thread(¶ms.thread_id).await { Ok(v) => v, Err(error) => { + self.track_error_response(&request_id, &error, /*error_type*/ None); self.outgoing.send_error(request_id, error).await; return; } @@ -6586,6 +6611,7 @@ impl CodexMessageProcessor { ) .await { + self.track_error_response(&request_id, &error, /*error_type*/ None); self.outgoing.send_error(request_id, error).await; return; } @@ -6686,6 +6712,7 @@ impl CodexMessageProcessor { message: format!("failed to start turn: {err}"), data: None, }; + self.track_error_response(&request_id, &error, /*error_type*/ None); self.outgoing.send_error(request_id, error).await; } } @@ -6710,6 +6737,7 @@ impl CodexMessageProcessor { let (_, thread) = match self.load_thread(¶ms.thread_id).await { Ok(v) => v, Err(error) => { + self.track_error_response(&request_id, &error, /*error_type*/ None); self.outgoing.send_error(request_id, error).await; return; } @@ -6727,6 +6755,11 @@ impl CodexMessageProcessor { .record_request_turn_id(&request_id, ¶ms.expected_turn_id) .await; if let Err(error) = Self::validate_v2_input_limit(¶ms.input) { + self.track_error_response( + &request_id, + &error, + Some(AnalyticsJsonRpcError::Input(InputError::TooLarge)), + ); self.outgoing.send_error(request_id, error).await; return; } @@ -6747,36 +6780,51 @@ impl CodexMessageProcessor { { Ok(turn_id) => { let response = TurnSteerResponse { turn_id }; + if self.config.features.enabled(Feature::GeneralAnalytics) { + self.analytics_events_client.track_response( + request_id.connection_id.0, + ClientResponse::TurnSteer { + request_id: request_id.request_id.clone(), + response: response.clone(), + }, + ); + } self.outgoing.send_response(request_id, response).await; } Err(err) => { - let (code, message, data) = match err { + let (code, message, data, error_type) = match err { SteerInputError::NoActiveTurn(_) => ( INVALID_REQUEST_ERROR_CODE, "no active turn to steer".to_string(), None, + Some(AnalyticsJsonRpcError::TurnSteer( + TurnSteerRequestError::NoActiveTurn, + )), ), SteerInputError::ExpectedTurnMismatch { expected, actual } => ( INVALID_REQUEST_ERROR_CODE, format!("expected active turn id `{expected}` but found `{actual}`"), None, + Some(AnalyticsJsonRpcError::TurnSteer( + TurnSteerRequestError::ExpectedTurnMismatch, + )), ), SteerInputError::ActiveTurnNotSteerable { turn_kind } => { - let message = match turn_kind { - codex_protocol::protocol::NonSteerableTurnKind::Review => { - "cannot steer a review turn".to_string() - } - codex_protocol::protocol::NonSteerableTurnKind::Compact => { - "cannot steer a compact turn".to_string() - } + let (message, turn_steer_error) = match turn_kind { + codex_protocol::protocol::NonSteerableTurnKind::Review => ( + "cannot steer a review turn".to_string(), + TurnSteerRequestError::NonSteerableReview, + ), + codex_protocol::protocol::NonSteerableTurnKind::Compact => ( + "cannot steer a compact turn".to_string(), + TurnSteerRequestError::NonSteerableCompact, + ), }; let error = TurnError { message: message.clone(), - codex_error_info: Some( - AppServerCodexErrorInfo::ActiveTurnNotSteerable { - turn_kind: turn_kind.into(), - }, - ), + codex_error_info: Some(CodexErrorInfo::ActiveTurnNotSteerable { + turn_kind: turn_kind.into(), + }), additional_details: None, }; let data = match serde_json::to_value(error) { @@ -6789,12 +6837,18 @@ impl CodexMessageProcessor { None } }; - (INVALID_REQUEST_ERROR_CODE, message, data) + ( + INVALID_REQUEST_ERROR_CODE, + message, + data, + Some(AnalyticsJsonRpcError::TurnSteer(turn_steer_error)), + ) } SteerInputError::EmptyInput => ( INVALID_REQUEST_ERROR_CODE, "input must not be empty".to_string(), None, + Some(AnalyticsJsonRpcError::Input(InputError::Empty)), ), }; let error = JSONRPCErrorError { @@ -6802,6 +6856,7 @@ impl CodexMessageProcessor { message, data, }; + self.track_error_response(&request_id, &error, error_type); self.outgoing.send_error(request_id, error).await; } } diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 4c47ec15aa6..18ee4e36b6b 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -678,7 +678,8 @@ impl MessageProcessor { return; } if self.config.features.enabled(Feature::GeneralAnalytics) - && let ClientRequest::TurnStart { request_id, .. } = &codex_request + && let ClientRequest::TurnStart { request_id, .. } + | ClientRequest::TurnSteer { request_id, .. } = &codex_request { self.analytics_events_client.track_request( connection_id.0, diff --git a/codex-rs/app-server/tests/suite/v2/turn_steer.rs b/codex-rs/app-server/tests/suite/v2/turn_steer.rs index e66dfc9e0b0..16e28d6cc5f 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_steer.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_steer.rs @@ -6,6 +6,7 @@ use app_test_support::create_mock_responses_server_sequence; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::JSONRPCError; @@ -23,6 +24,9 @@ use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use tempfile::TempDir; use tokio::time::timeout; +use super::analytics::enable_analytics_capture; +use super::analytics::wait_for_analytics_event; + const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] @@ -32,7 +36,12 @@ async fn turn_steer_requires_active_turn() -> Result<()> { std::fs::create_dir(&codex_home)?; let server = create_mock_responses_server_sequence(vec![]).await; - create_config_toml(&codex_home, &server.uri())?; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, &codex_home).await?; let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -52,7 +61,7 @@ async fn turn_steer_requires_active_turn() -> Result<()> { let steer_req = mcp .send_turn_steer_request(TurnSteerParams { - thread_id: thread.id, + thread_id: thread.id.clone(), input: vec![V2UserInput::Text { text: "steer".to_string(), text_elements: Vec::new(), @@ -68,6 +77,21 @@ async fn turn_steer_requires_active_turn() -> Result<()> { .await??; assert_eq!(steer_err.error.code, -32600); + let event = + wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["result"], "rejected"); + assert_eq!(event["event_params"]["num_input_images"], 0); + assert_eq!( + event["event_params"]["expected_turn_id"], + "turn-does-not-exist" + ); + assert_eq!( + event["event_params"]["accepted_turn_id"], + serde_json::Value::Null + ); + assert_eq!(event["event_params"]["rejection_reason"], "no_active_turn"); + Ok(()) } @@ -96,7 +120,12 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> { "call_sleep", )?]) .await; - create_config_toml(&codex_home, &server.uri())?; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, &codex_home).await?; let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -200,7 +229,12 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { "call_sleep", )?]) .await; - create_config_toml(&codex_home, &server.uri())?; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, &codex_home).await?; let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -261,31 +295,20 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { let steer: TurnSteerResponse = to_response::(steer_resp)?; assert_eq!(steer.turn_id, turn.id); + let event = + wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["result"], "accepted"); + assert_eq!(event["event_params"]["num_input_images"], 0); + assert_eq!(event["event_params"]["expected_turn_id"], turn.id); + assert_eq!(event["event_params"]["accepted_turn_id"], turn.id); + assert_eq!( + event["event_params"]["rejection_reason"], + serde_json::Value::Null + ); + mcp.interrupt_turn_and_wait_for_aborted(thread.id, steer.turn_id, DEFAULT_READ_TIMEOUT) .await?; Ok(()) } - -fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> { - let config_toml = codex_home.join("config.toml"); - std::fs::write( - config_toml, - format!( - r#" -model = "mock-model" -approval_policy = "never" -sandbox_mode = "danger-full-access" - -model_provider = "mock_provider" - -[model_providers.mock_provider] -name = "Mock provider for test" -base_url = "{server_uri}/v1" -wire_api = "responses" -request_max_retries = 0 -stream_max_retries = 0 -"# - ), - ) -} diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index e132ce0213a..59fb45454d2 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -1,7 +1,5 @@ use std::sync::Arc; use std::time::Instant; -use std::time::SystemTime; -use std::time::UNIX_EPOCH; use crate::Prompt; use crate::client::ModelClientSession; @@ -19,6 +17,7 @@ use codex_analytics::CompactionReason; use codex_analytics::CompactionStatus; use codex_analytics::CompactionStrategy; use codex_analytics::CompactionTrigger; +use codex_analytics::now_unix_seconds; use codex_features::Feature; use codex_model_provider_info::ModelProviderInfo; use codex_protocol::error::CodexErr; @@ -372,13 +371,6 @@ pub(crate) fn compaction_status_from_result(result: &CodexResult) -> Compa } } -fn now_unix_seconds() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_secs()) - .unwrap_or_default() -} - pub fn content_items_to_text(content: &[ContentItem]) -> Option { let mut pieces = Vec::new(); for item in content { From fd4d5412abac2717e58c01449461a6d7fa75db0a Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Fri, 10 Apr 2026 21:27:14 -0700 Subject: [PATCH 4/4] [codex-analytics] denormalize thread metadata onto turn events --- .../analytics/src/analytics_client_tests.rs | 28 +++++++++++ codex-rs/analytics/src/events.rs | 20 ++++++++ codex-rs/analytics/src/reducer.rs | 48 ++++++++++++++++++- .../app-server/tests/suite/v2/turn_start.rs | 11 +++++ 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index e57c1ed375c..9320030354e 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -455,6 +455,18 @@ async fn ingest_turn_prerequisites( ) { if include_initialize { ingest_initialize(reducer, out).await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_thread_start_response( + "thread-2", /*ephemeral*/ false, "gpt-5", + )), + }, + out, + ) + .await; + out.clear(); } reducer @@ -1442,6 +1454,10 @@ fn turn_event_serializes_expected_shape() { runtime: sample_runtime_metadata(), submission_type: None, ephemeral: false, + thread_source: Some("user".to_string()), + initialization_mode: ThreadInitializationMode::New, + subagent_source: None, + parent_thread_id: None, model: Some("gpt-5".to_string()), model_provider: "openai".to_string(), sandbox_policy: Some("read_only"), @@ -1499,6 +1515,10 @@ fn turn_event_serializes_expected_shape() { "runtime_arch": "aarch64" }, "ephemeral": false, + "thread_source": "user", + "initialization_mode": "new", + "subagent_source": null, + "parent_thread_id": null, "model": "gpt-5", "model_provider": "openai", "sandbox_policy": "read_only", @@ -1598,6 +1618,10 @@ async fn accepted_turn_steer_emits_expected_event() { payload["event_params"]["runtime"]["codex_rs_version"], json!("0.1.0") ); + assert_eq!(payload["event_params"]["thread_source"], json!("user")); + assert_eq!(payload["event_params"]["initialization_mode"], json!("new")); + assert_eq!(payload["event_params"]["subagent_source"], json!(null)); + assert_eq!(payload["event_params"]["parent_thread_id"], json!(null)); assert!(payload["event_params"].get("product_client_id").is_none()); } @@ -1626,6 +1650,10 @@ async fn rejected_turn_steer_uses_request_connection_metadata() { payload["event_params"]["runtime"]["codex_rs_version"], json!("0.1.0") ); + assert_eq!(payload["event_params"]["thread_source"], json!("user")); + assert_eq!(payload["event_params"]["initialization_mode"], json!("new")); + assert_eq!(payload["event_params"]["subagent_source"], json!(null)); + assert_eq!(payload["event_params"]["parent_thread_id"], json!(null)); assert_eq!(payload["event_params"]["result"], json!("rejected")); assert_eq!( payload["event_params"]["rejection_reason"], diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 3f149b493ae..c4ab4225f82 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -86,6 +86,13 @@ pub(crate) struct CodexRuntimeMetadata { pub(crate) runtime_arch: String, } +pub(crate) struct ThreadMetadata { + pub(crate) thread_source: Option<&'static str>, + pub(crate) initialization_mode: ThreadInitializationMode, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, +} + #[derive(Serialize)] pub(crate) struct ThreadInitializedEventParams { pub(crate) thread_id: String, @@ -339,6 +346,10 @@ pub(crate) struct CodexTurnEventParams { pub(crate) app_server_client: CodexAppServerClientMetadata, pub(crate) runtime: CodexRuntimeMetadata, pub(crate) ephemeral: bool, + pub(crate) thread_source: Option, + pub(crate) initialization_mode: ThreadInitializationMode, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, pub(crate) model: Option, pub(crate) model_provider: String, pub(crate) sandbox_policy: Option<&'static str>, @@ -386,6 +397,10 @@ pub(crate) struct CodexTurnSteerEventParams { pub(crate) accepted_turn_id: Option, pub(crate) app_server_client: CodexAppServerClientMetadata, pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) thread_source: Option, + pub(crate) initialization_mode: ThreadInitializationMode, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, pub(crate) num_input_images: usize, pub(crate) result: TurnSteerResult, pub(crate) rejection_reason: Option, @@ -524,6 +539,7 @@ pub(crate) fn codex_turn_steer_event_params( app_server_client: CodexAppServerClientMetadata, runtime: CodexRuntimeMetadata, tracking: &TrackEventsContext, + thread_metadata: ThreadMetadata, turn_steer: CodexTurnSteerEvent, ) -> CodexTurnSteerEventParams { CodexTurnSteerEventParams { @@ -532,6 +548,10 @@ pub(crate) fn codex_turn_steer_event_params( accepted_turn_id: turn_steer.accepted_turn_id, app_server_client, runtime, + thread_source: thread_metadata.thread_source.map(str::to_string), + initialization_mode: thread_metadata.initialization_mode, + subagent_source: thread_metadata.subagent_source, + parent_thread_id: thread_metadata.parent_thread_id, num_input_images: turn_steer.num_input_images, result: turn_steer.result, rejection_reason: turn_steer.rejection_reason, diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 352ea1c8928..33843a30bc7 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -16,6 +16,7 @@ use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; +use crate::events::ThreadMetadata; use crate::events::TrackEventRequest; use crate::events::codex_app_metadata; use crate::events::codex_compaction_event_params; @@ -86,12 +87,16 @@ struct ConnectionState { #[derive(Clone)] struct ThreadMetadataState { thread_source: Option<&'static str>, + initialization_mode: ThreadInitializationMode, subagent_source: Option, parent_thread_id: Option, } impl ThreadMetadataState { - fn from_session_source(session_source: &SessionSource) -> Self { + fn from_thread_lifecycle( + session_source: &SessionSource, + initialization_mode: ThreadInitializationMode, + ) -> Self { let (subagent_source, parent_thread_id) = match session_source { SessionSource::SubAgent(subagent_source) => ( Some(subagent_source_name(subagent_source)), @@ -106,6 +111,7 @@ impl ThreadMetadataState { }; Self { thread_source: thread_source_name(session_source), + initialization_mode, subagent_source, parent_thread_id, } @@ -657,7 +663,8 @@ impl AnalyticsReducer { let Some(connection_state) = self.connections.get(&connection_id) else { return; }; - let thread_metadata = ThreadMetadataState::from_session_source(&thread_source); + let thread_metadata = + ThreadMetadataState::from_thread_lifecycle(&thread_source, initialization_mode); self.thread_connections .insert(thread_id.clone(), connection_id); self.thread_metadata @@ -759,6 +766,13 @@ impl AnalyticsReducer { let Some(connection_state) = self.connections.get(&connection_id) else { return; }; + let Some(thread_metadata) = self.thread_metadata.get(&pending_request.thread_id) else { + tracing::warn!( + thread_id = %pending_request.thread_id, + "dropping turn steer analytics event: missing thread lifecycle metadata" + ); + return; + }; let tracking = TrackEventsContext { model_slug: String::new(), thread_id: pending_request.thread_id, @@ -781,6 +795,12 @@ impl AnalyticsReducer { connection_state.app_server_client.clone(), connection_state.runtime.clone(), &tracking, + ThreadMetadata { + thread_source: thread_metadata.thread_source, + initialization_mode: thread_metadata.initialization_mode, + subagent_source: thread_metadata.subagent_source.clone(), + parent_thread_id: thread_metadata.parent_thread_id.clone(), + }, turn_steer, ), })); @@ -807,6 +827,24 @@ impl AnalyticsReducer { ) }); let Some((app_server_client, runtime)) = connection_metadata else { + if let Some(connection_id) = turn_state.connection_id { + tracing::warn!( + turn_id, + connection_id, + "dropping turn analytics event: missing connection metadata" + ); + } + return; + }; + let Some(thread_id) = turn_state.thread_id.as_ref() else { + return; + }; + let Some(thread_metadata) = self.thread_metadata.get(thread_id) else { + tracing::warn!( + thread_id, + turn_id, + "dropping turn analytics event: missing thread lifecycle metadata" + ); return; }; out.push(TrackEventRequest::TurnEvent(Box::new( @@ -817,6 +855,7 @@ impl AnalyticsReducer { runtime, turn_id.to_string(), turn_state, + thread_metadata, ), }, ))); @@ -829,6 +868,7 @@ fn codex_turn_event_params( runtime: CodexRuntimeMetadata, turn_id: String, turn_state: &TurnState, + thread_metadata: &ThreadMetadataState, ) -> CodexTurnEventParams { let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = ( turn_state.thread_id.clone(), @@ -867,6 +907,10 @@ fn codex_turn_event_params( runtime, submission_type, ephemeral, + thread_source: thread_metadata.thread_source.map(str::to_string), + initialization_mode: thread_metadata.initialization_mode, + subagent_source: thread_metadata.subagent_source.clone(), + parent_thread_id: thread_metadata.parent_thread_id.clone(), model: Some(model), model_provider, sandbox_policy: Some(sandbox_policy_mode(&sandbox_policy)), diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index ea3ba02b468..694d0a6eefe 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -395,6 +395,17 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert_eq!(event["event_params"]["model"], "mock-model"); assert_eq!(event["event_params"]["model_provider"], "mock_provider"); assert_eq!(event["event_params"]["sandbox_policy"], "read_only"); + assert_eq!(event["event_params"]["ephemeral"], false); + assert_eq!(event["event_params"]["thread_source"], "user"); + assert_eq!(event["event_params"]["initialization_mode"], "new"); + assert_eq!( + event["event_params"]["subagent_source"], + serde_json::Value::Null + ); + assert_eq!( + event["event_params"]["parent_thread_id"], + serde_json::Value::Null + ); assert_eq!(event["event_params"]["num_input_images"], 1); assert_eq!(event["event_params"]["status"], "completed"); assert!(event["event_params"]["started_at"].as_u64().is_some());