From f1ec573e26b70595e144989057c6224359be7d6a Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Mon, 23 Feb 2026 11:25:19 -0800 Subject: [PATCH 01/64] Unify rollout reconstruction and resume/fork hydration --- codex-rs/core/src/codex.rs | 354 +++++++++++++++++++------------------ 1 file changed, 184 insertions(+), 170 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 6d583e46ed7..ea137a6c66e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -875,6 +875,13 @@ pub(crate) struct SessionSettingsUpdate { pub(crate) personality: Option, } +#[derive(Debug)] +struct RolloutReconstruction { + history: Vec, + previous_model: Option, + reference_context_item: Option, +} + impl Session { /// Builds the `x-codex-beta-features` header value for this session. /// @@ -1720,21 +1727,14 @@ impl Session { let rollout_items = resumed_history.history; let restored_tool_selection = Self::extract_mcp_tool_selection_from_rollout(&rollout_items); - let (previous_regular_turn_context_item, crossed_compaction_after_turn) = - Self::last_rollout_regular_turn_context_lookup(&rollout_items); - let previous_model = - previous_regular_turn_context_item.map(|ctx| ctx.model.clone()); + let reconstructed_rollout = self + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + let previous_model = reconstructed_rollout.previous_model.clone(); let curr = turn_context.model_info.slug.as_str(); - let reference_context_item = if !crossed_compaction_after_turn { - previous_regular_turn_context_item.cloned() - } else { - // Keep the baseline empty when compaction may have stripped the referenced - // context diffs so the first resumed regular turn fully reinjects context. - None - }; { let mut state = self.state.lock().await; - state.set_reference_context_item(reference_context_item); + state.set_reference_context_item(reconstructed_rollout.reference_context_item); } self.set_previous_model(previous_model.clone()).await; @@ -1754,9 +1754,7 @@ impl Session { } // Always add response items to conversation history - let reconstructed_history = self - .reconstruct_history_from_rollout(&turn_context, &rollout_items) - .await; + let reconstructed_history = reconstructed_rollout.history; if !reconstructed_history.is_empty() { self.record_into_history(&reconstructed_history, &turn_context) .await; @@ -1779,16 +1777,14 @@ impl Session { InitialHistory::Forked(rollout_items) => { let restored_tool_selection = Self::extract_mcp_tool_selection_from_rollout(&rollout_items); - let (previous_regular_turn_context_item, _) = - Self::last_rollout_regular_turn_context_lookup(&rollout_items); - let previous_model = - previous_regular_turn_context_item.map(|ctx| ctx.model.clone()); + let reconstructed_rollout = self + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + let previous_model = reconstructed_rollout.previous_model.clone(); self.set_previous_model(previous_model).await; // Always add response items to conversation history - let reconstructed_history = self - .reconstruct_history_from_rollout(&turn_context, &rollout_items) - .await; + let reconstructed_history = reconstructed_rollout.history; if !reconstructed_history.is_empty() { self.record_into_history(&reconstructed_history, &turn_context) .await; @@ -1827,150 +1823,6 @@ impl Session { } } - /// Returns `(last_turn_context_item, crossed_compaction_after_turn)` from the - /// rollback-adjusted rollout view. - /// - /// This relies on the invariant that only regular turns persist `TurnContextItem`. - /// `ThreadRolledBack` markers are applied so resume/fork uses the post-rollback history view. - /// - /// Returns `(None, false)` when no persisted `TurnContextItem` can be found. - /// - /// Older/minimal rollouts may only contain `RolloutItem::TurnContext` entries without turn - /// lifecycle events. In that case we fall back to the last `TurnContextItem` (plus whether a - /// later `Compacted` item appears in rollout order). - // TODO(ccunningham): Simplify this lookup by sharing rollout traversal/rollback application - // with `reconstruct_history_from_rollout` so resume/fork baseline hydration does not need a - // second bespoke rollout scan. - fn last_rollout_regular_turn_context_lookup( - rollout_items: &[RolloutItem], - ) -> (Option<&TurnContextItem>, bool) { - // Reverse scan over rollout items. `ThreadRolledBack(num_turns)` is naturally handled by - // skipping the next `num_turns` completed turn spans we encounter while walking backward. - // - // "Active turn" here means: we have seen `TurnComplete`/`TurnAborted` and are currently - // scanning backward through that completed turn until its matching `TurnStarted`. - let mut turns_to_skip_due_to_rollback = 0usize; - let mut saw_surviving_compaction_after_candidate = false; - let mut saw_turn_lifecycle_event = false; - let mut active_turn_id: Option<&str> = None; - let mut active_turn_saw_user_message = false; - let mut active_turn_context: Option<&TurnContextItem> = None; - let mut active_turn_contains_compaction = false; - - for item in rollout_items.iter().rev() { - match item { - RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { - // Rollbacks count completed turns, not `TurnContextItem`s. We must continue - // ignoring all items inside each skipped turn until we reach its - // corresponding `TurnStarted`. - let num_turns = usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); - turns_to_skip_due_to_rollback = - turns_to_skip_due_to_rollback.saturating_add(num_turns); - } - RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { - saw_turn_lifecycle_event = true; - // Enter the reverse "turn span" for this completed turn. - active_turn_id = Some(event.turn_id.as_str()); - active_turn_saw_user_message = false; - active_turn_context = None; - active_turn_contains_compaction = false; - } - RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { - saw_turn_lifecycle_event = true; - // Same reverse-turn handling as `TurnComplete`. Some aborted turns may not - // have a turn id; in that case we cannot match `TurnContextItem`s to them. - active_turn_id = event.turn_id.as_deref(); - active_turn_saw_user_message = false; - active_turn_context = None; - active_turn_contains_compaction = false; - } - RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { - if active_turn_id.is_some() { - active_turn_saw_user_message = true; - } - } - RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - saw_turn_lifecycle_event = true; - if active_turn_id == Some(event.turn_id.as_str()) { - let active_turn_is_rolled_back = - active_turn_saw_user_message && turns_to_skip_due_to_rollback > 0; - if active_turn_is_rolled_back { - // `ThreadRolledBack(num_turns)` counts user turns, so only consume a - // skip once we've confirmed this reverse-scanned turn span contains a - // user message. Standalone task turns must not consume rollback skips. - turns_to_skip_due_to_rollback -= 1; - } - if !active_turn_is_rolled_back { - if let Some(context_item) = active_turn_context { - return ( - Some(context_item), - saw_surviving_compaction_after_candidate, - ); - } - // No `TurnContextItem` in this surviving turn; keep scanning older - // turns, but remember if this turn compacted so the eventual - // candidate reports "compaction happened after it". - if active_turn_contains_compaction { - saw_surviving_compaction_after_candidate = true; - } - } - active_turn_id = None; - active_turn_saw_user_message = false; - active_turn_context = None; - active_turn_contains_compaction = false; - } - } - RolloutItem::TurnContext(ctx) => { - // Capture the latest turn context seen in this reverse-scanned turn span. If - // the turn later proves to be rolled back, we discard it when we hit the - // matching `TurnStarted`. Older rollouts may have lifecycle events but omit - // `TurnContextItem.turn_id`; accept those as belonging to the active turn - // span for resume/fork hydration. - if let Some(active_id) = active_turn_id - && ctx - .turn_id - .as_deref() - .is_none_or(|turn_id| turn_id == active_id) - { - // Reverse scan sees the latest `TurnContextItem` for the turn first. - active_turn_context.get_or_insert(ctx); - } - } - RolloutItem::Compacted(_) => { - if active_turn_id.is_some() { - // Compaction inside the currently scanned turn is only "after" the - // eventual candidate if this turn has no `TurnContextItem` and we keep - // scanning into older turns. - active_turn_contains_compaction = true; - } else { - saw_surviving_compaction_after_candidate = true; - } - } - _ => {} - } - } - - // Legacy/minimal rollouts may only persist `TurnContextItem`/`Compacted` without turn - // lifecycle events. Fall back to the last `TurnContextItem` in rollout order so - // resume/fork can still hydrate `previous_model` and detect compaction-after-baseline. - if !saw_turn_lifecycle_event { - let mut saw_compaction_after_last_turn_context = false; - for item in rollout_items.iter().rev() { - match item { - RolloutItem::Compacted(_) => { - saw_compaction_after_last_turn_context = true; - } - RolloutItem::TurnContext(ctx) => { - return (Some(ctx), saw_compaction_after_last_turn_context); - } - _ => {} - } - } - } - - (None, false) - } - fn last_token_info_from_rollout(rollout_items: &[RolloutItem]) -> Option { rollout_items.iter().rev().find_map(|item| match item { RolloutItem::EventMsg(EventMsg::TokenCount(ev)) => ev.info.clone(), @@ -2883,8 +2735,35 @@ impl Session { &self, turn_context: &TurnContext, rollout_items: &[RolloutItem], - ) -> Vec { + ) -> RolloutReconstruction { + #[derive(Debug)] + struct ActiveRolloutTurn { + turn_id: String, + saw_user_message: bool, + turn_context_item: Option, + contains_compaction: bool, + } + + #[derive(Debug)] + struct ReplayedRolloutTurn { + saw_user_message: bool, + turn_context_item: Option, + contains_compaction: bool, + } + + #[derive(Debug)] + enum RolloutReplayMetaSegment { + Turn(ReplayedRolloutTurn), + CompactionOutsideTurn, + } + let mut history = ContextManager::new(); + let mut saw_turn_lifecycle_event = false; + let mut active_turn: Option = None; + let mut replayed_segments = Vec::new(); + let mut legacy_last_turn_context_item: Option = None; + let mut legacy_saw_compaction_after_last_turn_context = false; + for item in rollout_items { match item { RolloutItem::ResponseItem(response_item) => { @@ -2905,14 +2784,145 @@ impl Session { ); history.replace(rebuilt); } + if let Some(active_turn) = active_turn.as_mut() { + active_turn.contains_compaction = true; + } else { + replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + } + if legacy_last_turn_context_item.is_some() { + legacy_saw_compaction_after_last_turn_context = true; + } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { history.drop_last_n_user_turns(rollback.num_turns); + let mut turns_to_drop = + usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); + if turns_to_drop > 0 { + let mut idx = replayed_segments.len(); + while idx > 0 && turns_to_drop > 0 { + idx -= 1; + if let RolloutReplayMetaSegment::Turn(turn) = &replayed_segments[idx] + && turn.saw_user_message + { + replayed_segments.remove(idx); + turns_to_drop -= 1; + } + } + } + } + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + saw_turn_lifecycle_event = true; + active_turn = Some(ActiveRolloutTurn { + turn_id: event.turn_id.clone(), + saw_user_message: false, + turn_context_item: None, + contains_compaction: false, + }); + } + RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { + saw_turn_lifecycle_event = true; + if active_turn + .as_ref() + .is_some_and(|turn| turn.turn_id == event.turn_id) + && let Some(active_turn) = active_turn.take() + { + replayed_segments.push(RolloutReplayMetaSegment::Turn( + ReplayedRolloutTurn { + saw_user_message: active_turn.saw_user_message, + turn_context_item: active_turn.turn_context_item, + contains_compaction: active_turn.contains_compaction, + }, + )); + } + } + RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { + saw_turn_lifecycle_event = true; + if let Some(aborted_turn_id) = event.turn_id.as_deref() + && active_turn + .as_ref() + .is_some_and(|turn| turn.turn_id == aborted_turn_id) + && let Some(active_turn) = active_turn.take() + { + replayed_segments.push(RolloutReplayMetaSegment::Turn( + ReplayedRolloutTurn { + saw_user_message: active_turn.saw_user_message, + turn_context_item: active_turn.turn_context_item, + contains_compaction: active_turn.contains_compaction, + }, + )); + } + } + RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { + if let Some(active_turn) = active_turn.as_mut() { + active_turn.saw_user_message = true; + } + } + RolloutItem::TurnContext(ctx) => { + if let Some(active_turn) = active_turn.as_mut() + && ctx + .turn_id + .as_deref() + .is_none_or(|turn_id| turn_id == active_turn.turn_id) + { + // Keep the latest `TurnContextItem` in rollout order for the turn. + active_turn.turn_context_item = Some(ctx.clone()); + } + legacy_last_turn_context_item = Some(ctx.clone()); + legacy_saw_compaction_after_last_turn_context = false; } _ => {} } } - history.raw_items().to_vec() + + let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { + let mut crossed_compaction_after_turn = false; + let mut previous_regular_turn_context_item = None; + + for segment in replayed_segments.iter().rev() { + match segment { + RolloutReplayMetaSegment::CompactionOutsideTurn => { + crossed_compaction_after_turn = true; + } + RolloutReplayMetaSegment::Turn(turn) => { + if let Some(turn_context_item) = &turn.turn_context_item { + previous_regular_turn_context_item = Some(turn_context_item.clone()); + break; + } + if turn.contains_compaction { + crossed_compaction_after_turn = true; + } + } + } + } + + let previous_model = previous_regular_turn_context_item + .as_ref() + .map(|ctx| ctx.model.clone()); + let reference_context_item = if crossed_compaction_after_turn { + // Keep the baseline empty when compaction may have stripped the referenced + // context diffs so the first resumed regular turn fully reinjects context. + None + } else { + previous_regular_turn_context_item + }; + (previous_model, reference_context_item) + } else { + let previous_model = legacy_last_turn_context_item + .as_ref() + .map(|ctx| ctx.model.clone()); + let reference_context_item = if legacy_saw_compaction_after_last_turn_context { + None + } else { + legacy_last_turn_context_item + }; + (previous_model, reference_context_item) + }; + + RolloutReconstruction { + history: history.raw_items().to_vec(), + previous_model, + reference_context_item, + } } /// Append ResponseItems to the in-memory conversation history only. @@ -7219,7 +7229,7 @@ mod tests { .reconstruct_history_from_rollout(reconstruction_turn.as_ref(), &rollout_items) .await; - assert_eq!(expected, reconstructed); + assert_eq!(expected, reconstructed.history); } #[tokio::test] @@ -7255,7 +7265,7 @@ mod tests { .reconstruct_history_from_rollout(&turn_context, &rollout_items) .await; - assert_eq!(reconstructed, replacement_history); + assert_eq!(reconstructed.history, replacement_history); } #[tokio::test] @@ -7487,6 +7497,10 @@ mod tests { })) .await; + assert_eq!( + session.previous_model().await, + Some(turn_context.model_info.slug.clone()) + ); assert!(session.reference_context_item().await.is_none()); } From e519dec1e1cdfe10b151ee4eaf577e0207a1620d Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Mon, 23 Feb 2026 12:17:08 -0800 Subject: [PATCH 02/64] Handle missing abort turn IDs in rollout replay --- codex-rs/core/src/codex.rs | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index ea137a6c66e..cba53a6b007 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2850,6 +2850,14 @@ impl Session { contains_compaction: active_turn.contains_compaction, }, )); + } else if let Some(active_turn) = active_turn.take() + && active_turn.contains_compaction + { + // Older lifecycle events may omit `turn_id` on abort. Match the prior + // reverse-scan behavior by dropping the active turn span (so we do not + // attribute its `TurnContextItem`) but preserving any compaction as a + // surviving compaction outside a matched turn. + replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); } } RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { @@ -7504,6 +7512,98 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_turn_for_compaction_accounting() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let aborted_turn_id = "aborted-turn-without-id".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: aborted_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "aborted".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::EventMsg(EventMsg::TurnAborted( + codex_protocol::protocol::TurnAbortedEvent { + turn_id: None, + reason: TurnAbortReason::Interrupted, + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); + } + #[tokio::test] async fn resumed_history_injects_initial_context_on_first_context_update_only() { let (session, turn_context) = make_session_and_context().await; From 6a4dfc4c0b47cc2bd8f74b74058c9e06aea9ff63 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Mon, 23 Feb 2026 12:18:42 -0800 Subject: [PATCH 03/64] Box rollout replay meta turn variant for clippy --- codex-rs/core/src/codex.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cba53a6b007..0cc5c6f5011 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2753,7 +2753,7 @@ impl Session { #[derive(Debug)] enum RolloutReplayMetaSegment { - Turn(ReplayedRolloutTurn), + Turn(Box), CompactionOutsideTurn, } @@ -2826,13 +2826,13 @@ impl Session { .is_some_and(|turn| turn.turn_id == event.turn_id) && let Some(active_turn) = active_turn.take() { - replayed_segments.push(RolloutReplayMetaSegment::Turn( + replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( ReplayedRolloutTurn { saw_user_message: active_turn.saw_user_message, turn_context_item: active_turn.turn_context_item, contains_compaction: active_turn.contains_compaction, }, - )); + ))); } } RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { @@ -2843,13 +2843,13 @@ impl Session { .is_some_and(|turn| turn.turn_id == aborted_turn_id) && let Some(active_turn) = active_turn.take() { - replayed_segments.push(RolloutReplayMetaSegment::Turn( + replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( ReplayedRolloutTurn { saw_user_message: active_turn.saw_user_message, turn_context_item: active_turn.turn_context_item, contains_compaction: active_turn.contains_compaction, }, - )); + ))); } else if let Some(active_turn) = active_turn.take() && active_turn.contains_compaction { From 098717f64524535b8dbbba0c24b69f7ab889beb8 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 11:40:28 -0800 Subject: [PATCH 04/64] Document rollout replay metadata reconstruction --- codex-rs/core/src/codex.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 0cc5c6f5011..cca77ab8722 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2736,6 +2736,31 @@ impl Session { turn_context: &TurnContext, rollout_items: &[RolloutItem], ) -> RolloutReconstruction { + // Replay rollout items once and compute two things in lockstep: + // 1) reconstructed conversation history (via `ContextManager`) + // 2) resume/fork hydration metadata (`previous_model` and + // `reference_context_item`) + // + // The metadata part needs rollback-aware accounting over "turn spans" and + // compaction placement: + // - `ActiveRolloutTurn` tracks the in-progress turn span while we walk forward + // through lifecycle events (`TurnStarted` ... `TurnComplete`/`TurnAborted`). + // - `ReplayedRolloutTurn` is the finalized per-turn metadata we keep after a + // turn ends (whether it had a user message, a `TurnContextItem`, and/or a + // compaction). + // - `RolloutReplayMetaSegment` stores the finalized sequence we later + // rollback-adjust and reverse-scan to find the last surviving regular turn + // context. `CompactionOutsideTurn` is a marker for compaction that happened + // outside any matched turn span; this matters because surviving compaction + // after the last surviving `TurnContextItem` must null + // `reference_context_item` while still preserving `previous_model`. + // + // `ThreadRolledBack` updates both: + // - history: drop user turns from reconstructed response items + // - metadata segments: remove finalized turn spans that consumed those user turns + // + // This keeps resume/fork baseline hydration consistent with the same replay + // logic used to rebuild history, instead of maintaining a second bespoke scan. #[derive(Debug)] struct ActiveRolloutTurn { turn_id: String, From 92c85b2c89dbefa72ab4dd13e96a38d6ce38c2b5 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 11:43:59 -0800 Subject: [PATCH 05/64] Clarify compaction-after-turn replay comment --- codex-rs/core/src/codex.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cca77ab8722..08b2a6306e2 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2752,7 +2752,8 @@ impl Session { // rollback-adjust and reverse-scan to find the last surviving regular turn // context. `CompactionOutsideTurn` is a marker for compaction that happened // outside any matched turn span; this matters because surviving compaction - // after the last surviving `TurnContextItem` must null + // after the last surviving turn span (not merely later than that turn's + // persisted `TurnContextItem` within the same turn) must null // `reference_context_item` while still preserving `previous_model`. // // `ThreadRolledBack` updates both: From 60456cb841559a8be292f9c8650ef6fdb407abe3 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 14:32:14 -0800 Subject: [PATCH 06/64] Make preturn vs mid-turn compaction replay explicit --- codex-rs/core/src/codex.rs | 123 ++++++++++++++++++++++++++++++++----- 1 file changed, 109 insertions(+), 14 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 08b2a6306e2..fecbb506115 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2746,15 +2746,21 @@ impl Session { // - `ActiveRolloutTurn` tracks the in-progress turn span while we walk forward // through lifecycle events (`TurnStarted` ... `TurnComplete`/`TurnAborted`). // - `ReplayedRolloutTurn` is the finalized per-turn metadata we keep after a - // turn ends (whether it had a user message, a `TurnContextItem`, and/or a - // compaction). + // turn ends (whether it had a user message, a `TurnContextItem`, and whether + // any compaction in that span happened before or after the first + // `TurnContextItem` for that turn). // - `RolloutReplayMetaSegment` stores the finalized sequence we later // rollback-adjust and reverse-scan to find the last surviving regular turn // context. `CompactionOutsideTurn` is a marker for compaction that happened - // outside any matched turn span; this matters because surviving compaction - // after the last surviving turn span (not merely later than that turn's - // persisted `TurnContextItem` within the same turn) must null - // `reference_context_item` while still preserving `previous_model`. + // outside any matched turn span. + // + // Explicit replay rule: + // - compaction before the first `TurnContextItem` in a turn span is treated as + // preturn compaction for that turn and invalidates + // `reference_context_item` on resume + // - compaction after the first `TurnContextItem` in the same turn span is + // treated as mid-turn compaction and does not invalidate that turn's own + // `reference_context_item` // // `ThreadRolledBack` updates both: // - history: drop user turns from reconstructed response items @@ -2767,14 +2773,16 @@ impl Session { turn_id: String, saw_user_message: bool, turn_context_item: Option, - contains_compaction: bool, + has_preturn_compaction: bool, + has_midturn_compaction: bool, } #[derive(Debug)] struct ReplayedRolloutTurn { saw_user_message: bool, turn_context_item: Option, - contains_compaction: bool, + has_preturn_compaction: bool, + has_midturn_compaction: bool, } #[derive(Debug)] @@ -2811,7 +2819,11 @@ impl Session { history.replace(rebuilt); } if let Some(active_turn) = active_turn.as_mut() { - active_turn.contains_compaction = true; + if active_turn.turn_context_item.is_none() { + active_turn.has_preturn_compaction = true; + } else { + active_turn.has_midturn_compaction = true; + } } else { replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); } @@ -2842,7 +2854,8 @@ impl Session { turn_id: event.turn_id.clone(), saw_user_message: false, turn_context_item: None, - contains_compaction: false, + has_preturn_compaction: false, + has_midturn_compaction: false, }); } RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { @@ -2856,7 +2869,8 @@ impl Session { ReplayedRolloutTurn { saw_user_message: active_turn.saw_user_message, turn_context_item: active_turn.turn_context_item, - contains_compaction: active_turn.contains_compaction, + has_preturn_compaction: active_turn.has_preturn_compaction, + has_midturn_compaction: active_turn.has_midturn_compaction, }, ))); } @@ -2873,11 +2887,13 @@ impl Session { ReplayedRolloutTurn { saw_user_message: active_turn.saw_user_message, turn_context_item: active_turn.turn_context_item, - contains_compaction: active_turn.contains_compaction, + has_preturn_compaction: active_turn.has_preturn_compaction, + has_midturn_compaction: active_turn.has_midturn_compaction, }, ))); } else if let Some(active_turn) = active_turn.take() - && active_turn.contains_compaction + && (active_turn.has_preturn_compaction + || active_turn.has_midturn_compaction) { // Older lifecycle events may omit `turn_id` on abort. Match the prior // reverse-scan behavior by dropping the active turn span (so we do not @@ -2919,10 +2935,13 @@ impl Session { } RolloutReplayMetaSegment::Turn(turn) => { if let Some(turn_context_item) = &turn.turn_context_item { + if turn.has_preturn_compaction { + crossed_compaction_after_turn = true; + } previous_regular_turn_context_item = Some(turn_context_item.clone()); break; } - if turn.contains_compaction { + if turn.has_preturn_compaction || turn.has_midturn_compaction { crossed_compaction_after_turn = true; } } @@ -6250,6 +6269,12 @@ async fn try_run_sampling_request( // Persist one TurnContext marker per sampling request (not just per user turn) so rollout // analysis can reconstruct API-turn boundaries. `run_turn` persists model-visible context // diffs/full reinjection earlier in the same regular turn before reaching this path. + // + // Replay invariant for resume/fork hydration: + // compaction events in a turn span that happen before this first persisted TurnContextItem + // are treated as preturn compaction for that turn and invalidate + // `reference_context_item`, while compaction after this point in the same turn span is + // treated as mid-turn compaction and does not invalidate that turn's own baseline. let rollout_item = RolloutItem::TurnContext(turn_context.to_turn_context_item()); feedback_tags!( @@ -7538,6 +7563,76 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_nulls_reference_context_item_for_preturn_compaction_in_same_turn() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + // Compaction before the first TurnContextItem in this turn is treated as preturn. + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); + } + #[tokio::test] async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_turn_for_compaction_accounting() { From d884e02b102b427480ba6ae14d5b590c4a491770 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 16:19:02 -0800 Subject: [PATCH 07/64] Clarify CompactionOutsideTurn as legacy fallback --- codex-rs/core/src/codex.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index fecbb506115..b395d0d6768 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2788,6 +2788,12 @@ impl Session { #[derive(Debug)] enum RolloutReplayMetaSegment { Turn(Box), + // Unexpected for modern rollouts, where compaction should occur inside + // a matched turn span (`TurnStarted` ... `TurnComplete`/`TurnAborted`). + // + // We keep this as a minimal fallback for legacy/incomplete lifecycle + // data: treat as "compaction happened after older baseline" and prefer + // conservative baseline invalidation over complex reconstruction. CompactionOutsideTurn, } @@ -2895,10 +2901,9 @@ impl Session { && (active_turn.has_preturn_compaction || active_turn.has_midturn_compaction) { - // Older lifecycle events may omit `turn_id` on abort. Match the prior - // reverse-scan behavior by dropping the active turn span (so we do not - // attribute its `TurnContextItem`) but preserving any compaction as a - // surviving compaction outside a matched turn. + // Legacy/incomplete lifecycle events may omit `turn_id` on abort. + // Keep fallback handling minimal: drop this ambiguous turn span and + // preserve only a conservative "outside-turn compaction" marker. replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); } } From 252106ae401b4e64813e2bfd62725cedd3ce38dc Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 16:19:41 -0800 Subject: [PATCH 08/64] Clarify outside-turn compaction replay semantics --- codex-rs/core/src/codex.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index b395d0d6768..02f0f892cf6 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2761,6 +2761,8 @@ impl Session { // - compaction after the first `TurnContextItem` in the same turn span is // treated as mid-turn compaction and does not invalidate that turn's own // `reference_context_item` + // - compaction outside any matched turn span is treated conservatively as + // preturn-equivalent for baseline hydration (invalidate older baseline) // // `ThreadRolledBack` updates both: // - history: drop user turns from reconstructed response items From b2fb999f15aac1526498984553a9c8ae76725de8 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 16:50:07 -0800 Subject: [PATCH 09/64] Simplify legacy compaction fallback accounting --- codex-rs/core/src/codex.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 02f0f892cf6..655390f0536 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2803,8 +2803,6 @@ impl Session { let mut saw_turn_lifecycle_event = false; let mut active_turn: Option = None; let mut replayed_segments = Vec::new(); - let mut legacy_last_turn_context_item: Option = None; - let mut legacy_saw_compaction_after_last_turn_context = false; for item in rollout_items { match item { @@ -2835,9 +2833,6 @@ impl Session { } else { replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); } - if legacy_last_turn_context_item.is_some() { - legacy_saw_compaction_after_last_turn_context = true; - } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { history.drop_last_n_user_turns(rollback.num_turns); @@ -2924,8 +2919,6 @@ impl Session { // Keep the latest `TurnContextItem` in rollout order for the turn. active_turn.turn_context_item = Some(ctx.clone()); } - legacy_last_turn_context_item = Some(ctx.clone()); - legacy_saw_compaction_after_last_turn_context = false; } _ => {} } @@ -2967,6 +2960,24 @@ impl Session { }; (previous_model, reference_context_item) } else { + // Legacy/minimal fallback (no lifecycle events): use the last persisted + // `TurnContextItem` in rollout order and conservatively null baseline when a + // later `Compacted` item exists. + let mut legacy_last_turn_context_item: Option = None; + let mut legacy_saw_compaction_after_last_turn_context = false; + for item in rollout_items.iter().rev() { + match item { + RolloutItem::Compacted(_) => { + legacy_saw_compaction_after_last_turn_context = true; + } + RolloutItem::TurnContext(ctx) => { + legacy_last_turn_context_item = Some(ctx.clone()); + break; + } + _ => {} + } + } + let previous_model = legacy_last_turn_context_item .as_ref() .map(|ctx| ctx.model.clone()); From 69a171cf7eb282616719189963f8894aa66f0d5c Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 17:39:25 -0800 Subject: [PATCH 10/64] Rename compaction baseline-clear replay flag --- codex-rs/core/src/codex.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 655390f0536..428720ab0ff 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2925,24 +2925,24 @@ impl Session { } let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { - let mut crossed_compaction_after_turn = false; + let mut compaction_cleared_reference_context_item = false; let mut previous_regular_turn_context_item = None; for segment in replayed_segments.iter().rev() { match segment { RolloutReplayMetaSegment::CompactionOutsideTurn => { - crossed_compaction_after_turn = true; + compaction_cleared_reference_context_item = true; } RolloutReplayMetaSegment::Turn(turn) => { if let Some(turn_context_item) = &turn.turn_context_item { if turn.has_preturn_compaction { - crossed_compaction_after_turn = true; + compaction_cleared_reference_context_item = true; } previous_regular_turn_context_item = Some(turn_context_item.clone()); break; } if turn.has_preturn_compaction || turn.has_midturn_compaction { - crossed_compaction_after_turn = true; + compaction_cleared_reference_context_item = true; } } } @@ -2951,7 +2951,7 @@ impl Session { let previous_model = previous_regular_turn_context_item .as_ref() .map(|ctx| ctx.model.clone()); - let reference_context_item = if crossed_compaction_after_turn { + let reference_context_item = if compaction_cleared_reference_context_item { // Keep the baseline empty when compaction may have stripped the referenced // context diffs so the first resumed regular turn fully reinjects context. None From 53f012b62e6101838a6d125df3fd32bab58c6726 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 17:40:51 -0800 Subject: [PATCH 11/64] Document compacted-turn fallback baseline invalidation --- codex-rs/core/src/codex.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 428720ab0ff..f097d3e4546 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2942,6 +2942,9 @@ impl Session { break; } if turn.has_preturn_compaction || turn.has_midturn_compaction { + // This later surviving turn compacted (for example via `/compact` or + // auto-compaction) but did not persist a replacement TurnContextItem, + // so conservatively invalidate any older baseline we might select. compaction_cleared_reference_context_item = true; } } From f746620f1e724b2257d3df07ad6ba6b669ae8864 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 18:52:06 -0800 Subject: [PATCH 12/64] Handle trailing incomplete compacted turns in replay --- codex-rs/core/src/codex.rs | 95 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index f097d3e4546..778839c370a 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2924,6 +2924,15 @@ impl Session { } } + if let Some(active_turn) = active_turn.take() + && (active_turn.has_preturn_compaction || active_turn.has_midturn_compaction) + { + // Trailing incomplete turn (started but never completed/aborted). Keep fallback + // handling minimal and conservative: preserve only compaction impact so older + // baseline context is invalidated on resume. + replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + } + let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { let mut compaction_cleared_reference_context_item = false; let mut previous_regular_turn_context_item = None; @@ -7746,6 +7755,92 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clears_reference_context_item() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let incomplete_turn_id = "trailing-incomplete-turn".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "incomplete".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); + } + #[tokio::test] async fn resumed_history_injects_initial_context_on_first_context_update_only() { let (session, turn_context) = make_session_and_context().await; From f5005ae2b92f4151e8fb896f061ec505de5021b1 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 19:26:59 -0800 Subject: [PATCH 13/64] Preserve compacted incomplete turn replacement on replay --- codex-rs/core/src/codex.rs | 105 +++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 778839c370a..b64442936c3 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2853,6 +2853,15 @@ impl Session { } RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { saw_turn_lifecycle_event = true; + if let Some(active_turn) = active_turn.take() + && (active_turn.has_preturn_compaction + || active_turn.has_midturn_compaction) + { + // Incomplete prior turn replaced by a newer TurnStarted. Keep fallback + // handling minimal and conservative: preserve only compaction impact so + // older baseline context is invalidated on resume. + replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + } active_turn = Some(ActiveRolloutTurn { turn_id: event.turn_id.clone(), saw_user_message: false, @@ -7841,6 +7850,102 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clears_reference_context_item() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let compacted_incomplete_turn_id = "compacted-incomplete-turn".to_string(); + let replacing_turn_id = "replacing-turn".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: compacted_incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "compacted".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + // A newer TurnStarted replaces the incomplete compacted turn without a matching + // completion/abort for the old one. + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: replacing_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); + } + #[tokio::test] async fn resumed_history_injects_initial_context_on_first_context_update_only() { let (session, turn_context) = make_session_and_context().await; From 0e091a877b9d813e72d980d704ab350921bcea8d Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 19:35:45 -0800 Subject: [PATCH 14/64] Align rollback replay with incomplete user turns --- codex-rs/core/src/codex.rs | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index b64442936c3..ff5db6d5f0d 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2838,6 +2838,17 @@ impl Session { history.drop_last_n_user_turns(rollback.num_turns); let mut turns_to_drop = usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); + if turns_to_drop > 0 + && active_turn + .as_ref() + .is_some_and(|turn| turn.saw_user_message) + { + // Match `drop_last_n_user_turns`: an unfinished active turn that has + // already emitted a user message is the newest user turn and should be + // dropped before we trim older finalized turn spans. + active_turn = None; + turns_to_drop -= 1; + } if turns_to_drop > 0 { let mut idx = replayed_segments.len(); while idx > 0 && turns_to_drop > 0 { @@ -7552,6 +7563,84 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_rollback_drops_incomplete_user_turn_compaction_metadata() + { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let incomplete_turn_id = "incomplete-compacted-user-turn".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "rolled back".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(previous_context_item)) + .expect("serialize expected reference context item") + ); + } + #[tokio::test] async fn record_initial_history_resumed_seeds_reference_context_item_without_compaction() { let (session, turn_context) = make_session_and_context().await; From 922aaf8795acf5325c2a82dadcdec1caf7c3a76e Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 19:51:27 -0800 Subject: [PATCH 15/64] Finalize incomplete replay turns --- codex-rs/core/src/codex.rs | 105 +++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 33 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index ff5db6d5f0d..31d2951f156 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2751,8 +2751,9 @@ impl Session { // `TurnContextItem` for that turn). // - `RolloutReplayMetaSegment` stores the finalized sequence we later // rollback-adjust and reverse-scan to find the last surviving regular turn - // context. `CompactionOutsideTurn` is a marker for compaction that happened - // outside any matched turn span. + // context. Replaced/trailing incomplete turns are finalized as ordinary + // `Turn(...)` segments; `CompactionOutsideTurn` is only for compaction that + // happened outside any matched turn span. // // Explicit replay rule: // - compaction before the first `TurnContextItem` in a turn span is treated as @@ -2803,6 +2804,17 @@ impl Session { let mut saw_turn_lifecycle_event = false; let mut active_turn: Option = None; let mut replayed_segments = Vec::new(); + let push_replayed_turn = |replayed_segments: &mut Vec, + active_turn: ActiveRolloutTurn| { + replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( + ReplayedRolloutTurn { + saw_user_message: active_turn.saw_user_message, + turn_context_item: active_turn.turn_context_item, + has_preturn_compaction: active_turn.has_preturn_compaction, + has_midturn_compaction: active_turn.has_midturn_compaction, + }, + ))); + }; for item in rollout_items { match item { @@ -2864,14 +2876,10 @@ impl Session { } RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { saw_turn_lifecycle_event = true; - if let Some(active_turn) = active_turn.take() - && (active_turn.has_preturn_compaction - || active_turn.has_midturn_compaction) - { - // Incomplete prior turn replaced by a newer TurnStarted. Keep fallback - // handling minimal and conservative: preserve only compaction impact so - // older baseline context is invalidated on resume. - replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + if let Some(active_turn) = active_turn.take() { + // Treat a replaced incomplete turn as ended at the point the next turn + // starts so replay preserves any `TurnContextItem` it already emitted. + push_replayed_turn(&mut replayed_segments, active_turn); } active_turn = Some(ActiveRolloutTurn { turn_id: event.turn_id.clone(), @@ -2888,14 +2896,7 @@ impl Session { .is_some_and(|turn| turn.turn_id == event.turn_id) && let Some(active_turn) = active_turn.take() { - replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( - ReplayedRolloutTurn { - saw_user_message: active_turn.saw_user_message, - turn_context_item: active_turn.turn_context_item, - has_preturn_compaction: active_turn.has_preturn_compaction, - has_midturn_compaction: active_turn.has_midturn_compaction, - }, - ))); + push_replayed_turn(&mut replayed_segments, active_turn); } } RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { @@ -2906,14 +2907,7 @@ impl Session { .is_some_and(|turn| turn.turn_id == aborted_turn_id) && let Some(active_turn) = active_turn.take() { - replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( - ReplayedRolloutTurn { - saw_user_message: active_turn.saw_user_message, - turn_context_item: active_turn.turn_context_item, - has_preturn_compaction: active_turn.has_preturn_compaction, - has_midturn_compaction: active_turn.has_midturn_compaction, - }, - ))); + push_replayed_turn(&mut replayed_segments, active_turn); } else if let Some(active_turn) = active_turn.take() && (active_turn.has_preturn_compaction || active_turn.has_midturn_compaction) @@ -2944,13 +2938,10 @@ impl Session { } } - if let Some(active_turn) = active_turn.take() - && (active_turn.has_preturn_compaction || active_turn.has_midturn_compaction) - { - // Trailing incomplete turn (started but never completed/aborted). Keep fallback - // handling minimal and conservative: preserve only compaction impact so older - // baseline context is invalidated on resume. - replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + if let Some(active_turn) = active_turn.take() { + // Treat a trailing incomplete turn as ended at EOF so replay preserves any + // `TurnContextItem` it already emitted before the rollout was truncated. + push_replayed_turn(&mut replayed_segments, active_turn); } let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { @@ -7939,6 +7930,54 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_trailing_incomplete_turn_preserves_turn_context_item() { + let (session, turn_context) = make_session_and_context().await; + let current_context_item = turn_context.to_turn_context_item(); + let current_turn_id = current_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: current_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "incomplete".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(current_context_item.clone()), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(current_context_item)) + .expect("serialize expected reference context item") + ); + } + #[tokio::test] async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clears_reference_context_item() { From b0e59da7ed5358ab1dc8f75ddca2eb6b9b200190 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 20:03:46 -0800 Subject: [PATCH 16/64] comment --- codex-rs/core/src/codex.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 31d2951f156..cc91aa6dc82 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2752,8 +2752,7 @@ impl Session { // - `RolloutReplayMetaSegment` stores the finalized sequence we later // rollback-adjust and reverse-scan to find the last surviving regular turn // context. Replaced/trailing incomplete turns are finalized as ordinary - // `Turn(...)` segments; `CompactionOutsideTurn` is only for compaction that - // happened outside any matched turn span. + // `Turn(...)` segments. // // Explicit replay rule: // - compaction before the first `TurnContextItem` in a turn span is treated as From 904093e1db23e47a376a9669277584a4c7e1810b Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 20:27:06 -0800 Subject: [PATCH 17/64] Add rollback replay equivalence tests --- codex-rs/core/src/codex.rs | 335 +++++++++++++++++++++++++++++++++++++ 1 file changed, 335 insertions(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cc91aa6dc82..ca9820d3330 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -6738,6 +6738,18 @@ mod tests { } } + fn assistant_message(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: text.to_string(), + }], + end_turn: None, + phase: None, + } + } + fn skill_message(text: &str) -> ResponseItem { ResponseItem::Message { id: None, @@ -7490,6 +7502,329 @@ mod tests { ); } + #[tokio::test] + async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_completed_turns() { + let (session, turn_context) = make_session_and_context().await; + let first_context_item = turn_context.to_turn_context_item(); + let first_turn_id = first_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let mut rolled_back_context_item = first_context_item.clone(); + rolled_back_context_item.turn_id = Some("rolled-back-turn".to_string()); + rolled_back_context_item.model = "rolled-back-model".to_string(); + let rolled_back_turn_id = rolled_back_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let turn_one_user = user_message("turn 1 user"); + let turn_one_assistant = assistant_message("turn 1 assistant"); + let turn_two_user = user_message("turn 2 user"); + let turn_two_assistant = assistant_message("turn 2 assistant"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: first_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 1 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(first_context_item.clone()), + RolloutItem::ResponseItem(turn_one_user.clone()), + RolloutItem::ResponseItem(turn_one_assistant.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: first_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: rolled_back_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 2 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(rolled_back_context_item), + RolloutItem::ResponseItem(turn_two_user), + RolloutItem::ResponseItem(turn_two_assistant), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: rolled_back_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![turn_one_user, turn_one_assistant] + ); + assert_eq!( + reconstructed.previous_model, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(reconstructed.reference_context_item) + .expect("serialize reconstructed reference context item"), + serde_json::to_value(Some(first_context_item)) + .expect("serialize expected reference context item") + ); + } + + #[tokio::test] + async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_incomplete_turn() { + let (session, turn_context) = make_session_and_context().await; + let first_context_item = turn_context.to_turn_context_item(); + let first_turn_id = first_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let incomplete_turn_id = "incomplete-rolled-back-turn".to_string(); + let turn_one_user = user_message("turn 1 user"); + let turn_one_assistant = assistant_message("turn 1 assistant"); + let turn_two_user = user_message("turn 2 user"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: first_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 1 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(first_context_item.clone()), + RolloutItem::ResponseItem(turn_one_user.clone()), + RolloutItem::ResponseItem(turn_one_assistant.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: first_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 2 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::ResponseItem(turn_two_user), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![turn_one_user, turn_one_assistant] + ); + assert_eq!( + reconstructed.previous_model, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(reconstructed.reference_context_item) + .expect("serialize reconstructed reference context item"), + serde_json::to_value(Some(first_context_item)) + .expect("serialize expected reference context item") + ); + } + + #[tokio::test] + async fn reconstruct_history_rollback_skips_non_user_turns_for_history_and_metadata() { + let (session, turn_context) = make_session_and_context().await; + let first_context_item = turn_context.to_turn_context_item(); + let first_turn_id = first_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let second_turn_id = "rolled-back-user-turn".to_string(); + let standalone_turn_id = "standalone-turn".to_string(); + let turn_one_user = user_message("turn 1 user"); + let turn_one_assistant = assistant_message("turn 1 assistant"); + let turn_two_user = user_message("turn 2 user"); + let turn_two_assistant = assistant_message("turn 2 assistant"); + let standalone_assistant = assistant_message("standalone assistant"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: first_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 1 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(first_context_item.clone()), + RolloutItem::ResponseItem(turn_one_user.clone()), + RolloutItem::ResponseItem(turn_one_assistant.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: first_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: second_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 2 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::ResponseItem(turn_two_user), + RolloutItem::ResponseItem(turn_two_assistant), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: second_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: standalone_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::ResponseItem(standalone_assistant), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: standalone_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![turn_one_user, turn_one_assistant] + ); + assert_eq!( + reconstructed.previous_model, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(reconstructed.reference_context_item) + .expect("serialize reconstructed reference context item"), + serde_json::to_value(Some(first_context_item)) + .expect("serialize expected reference context item") + ); + } + + #[tokio::test] + async fn reconstruct_history_rollback_clears_history_and_metadata_when_exceeding_user_turns() { + let (session, turn_context) = make_session_and_context().await; + let only_context_item = turn_context.to_turn_context_item(); + let only_turn_id = only_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: only_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "only user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(only_context_item), + RolloutItem::ResponseItem(user_message("only user")), + RolloutItem::ResponseItem(assistant_message("only assistant")), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: only_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 99 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!(reconstructed.history, Vec::new()); + assert_eq!(reconstructed.previous_model, None); + assert!(reconstructed.reference_context_item.is_none()); + } + #[tokio::test] async fn record_initial_history_resumed_rollback_skips_only_user_turns() { let (session, turn_context) = make_session_and_context().await; From 25d5ca8967b900a8de234b4ab0b18e057c148b6d Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 20:47:32 -0800 Subject: [PATCH 18/64] Ignore unmatched aborts during replay --- codex-rs/core/src/codex.rs | 144 +++++++++++++++++++++++++++++++++---- 1 file changed, 129 insertions(+), 15 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index ca9820d3330..203be1cfbc1 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2900,21 +2900,33 @@ impl Session { } RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { saw_turn_lifecycle_event = true; - if let Some(aborted_turn_id) = event.turn_id.as_deref() - && active_turn - .as_ref() - .is_some_and(|turn| turn.turn_id == aborted_turn_id) - && let Some(active_turn) = active_turn.take() - { - push_replayed_turn(&mut replayed_segments, active_turn); - } else if let Some(active_turn) = active_turn.take() - && (active_turn.has_preturn_compaction - || active_turn.has_midturn_compaction) - { - // Legacy/incomplete lifecycle events may omit `turn_id` on abort. - // Keep fallback handling minimal: drop this ambiguous turn span and - // preserve only a conservative "outside-turn compaction" marker. - replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + match event.turn_id.as_deref() { + Some(aborted_turn_id) + if active_turn + .as_ref() + .is_some_and(|turn| turn.turn_id == aborted_turn_id) => + { + if let Some(active_turn) = active_turn.take() { + push_replayed_turn(&mut replayed_segments, active_turn); + } + } + Some(_) => { + // Ignore aborts for some other turn and keep the current active turn + // alive so later `TurnContext`/`TurnComplete` events still apply. + } + None => { + if let Some(active_turn) = active_turn.take() + && (active_turn.has_preturn_compaction + || active_turn.has_midturn_compaction) + { + // Legacy/incomplete lifecycle events may omit `turn_id` on + // abort. Keep fallback handling minimal: drop this ambiguous + // turn span and preserve only a conservative "outside-turn + // compaction" marker. + replayed_segments + .push(RolloutReplayMetaSegment::CompactionOutsideTurn); + } + } } } RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { @@ -8178,6 +8190,108 @@ mod tests { assert!(session.reference_context_item().await.is_none()); } + #[tokio::test] + async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_for_later_turn_context() + { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let current_model = "current-rollout-model"; + let current_turn_id = "current-turn".to_string(); + let unmatched_abort_turn_id = "other-turn".to_string(); + let current_context_item = TurnContextItem { + turn_id: Some(current_turn_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: current_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: current_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "current".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::EventMsg(EventMsg::TurnAborted( + codex_protocol::protocol::TurnAbortedEvent { + turn_id: Some(unmatched_abort_turn_id), + reason: TurnAbortReason::Interrupted, + }, + )), + RolloutItem::TurnContext(current_context_item.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: current_turn_id, + last_agent_message: None, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(current_model.to_string()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(current_context_item)) + .expect("serialize expected reference context item") + ); + } + #[tokio::test] async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clears_reference_context_item() { From fc1e2452c4db27863dbf160ff970e8e06f8b796b Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Tue, 24 Feb 2026 22:38:54 -0800 Subject: [PATCH 19/64] Move rollout reconstruction out of codex.rs --- codex-rs/core/src/codex.rs | 308 +---------------- .../core/src/codex/rollout_reconstruction.rs | 309 ++++++++++++++++++ 2 files changed, 311 insertions(+), 306 deletions(-) create mode 100644 codex-rs/core/src/codex/rollout_reconstruction.rs diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 203be1cfbc1..cf7b46c8c13 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -156,6 +156,8 @@ use crate::error::Result as CodexResult; use crate::exec::StreamOutput; use codex_config::CONFIG_TOML_FILE; +mod rollout_reconstruction; + #[derive(Debug, PartialEq)] pub enum SteerInputError { NoActiveTurn(Vec), @@ -875,13 +877,6 @@ pub(crate) struct SessionSettingsUpdate { pub(crate) personality: Option, } -#[derive(Debug)] -struct RolloutReconstruction { - history: Vec, - previous_model: Option, - reference_context_item: Option, -} - impl Session { /// Builds the `x-codex-beta-features` header value for this session. /// @@ -2731,305 +2726,6 @@ impl Session { self.send_raw_response_items(turn_context, items).await; } - async fn reconstruct_history_from_rollout( - &self, - turn_context: &TurnContext, - rollout_items: &[RolloutItem], - ) -> RolloutReconstruction { - // Replay rollout items once and compute two things in lockstep: - // 1) reconstructed conversation history (via `ContextManager`) - // 2) resume/fork hydration metadata (`previous_model` and - // `reference_context_item`) - // - // The metadata part needs rollback-aware accounting over "turn spans" and - // compaction placement: - // - `ActiveRolloutTurn` tracks the in-progress turn span while we walk forward - // through lifecycle events (`TurnStarted` ... `TurnComplete`/`TurnAborted`). - // - `ReplayedRolloutTurn` is the finalized per-turn metadata we keep after a - // turn ends (whether it had a user message, a `TurnContextItem`, and whether - // any compaction in that span happened before or after the first - // `TurnContextItem` for that turn). - // - `RolloutReplayMetaSegment` stores the finalized sequence we later - // rollback-adjust and reverse-scan to find the last surviving regular turn - // context. Replaced/trailing incomplete turns are finalized as ordinary - // `Turn(...)` segments. - // - // Explicit replay rule: - // - compaction before the first `TurnContextItem` in a turn span is treated as - // preturn compaction for that turn and invalidates - // `reference_context_item` on resume - // - compaction after the first `TurnContextItem` in the same turn span is - // treated as mid-turn compaction and does not invalidate that turn's own - // `reference_context_item` - // - compaction outside any matched turn span is treated conservatively as - // preturn-equivalent for baseline hydration (invalidate older baseline) - // - // `ThreadRolledBack` updates both: - // - history: drop user turns from reconstructed response items - // - metadata segments: remove finalized turn spans that consumed those user turns - // - // This keeps resume/fork baseline hydration consistent with the same replay - // logic used to rebuild history, instead of maintaining a second bespoke scan. - #[derive(Debug)] - struct ActiveRolloutTurn { - turn_id: String, - saw_user_message: bool, - turn_context_item: Option, - has_preturn_compaction: bool, - has_midturn_compaction: bool, - } - - #[derive(Debug)] - struct ReplayedRolloutTurn { - saw_user_message: bool, - turn_context_item: Option, - has_preturn_compaction: bool, - has_midturn_compaction: bool, - } - - #[derive(Debug)] - enum RolloutReplayMetaSegment { - Turn(Box), - // Unexpected for modern rollouts, where compaction should occur inside - // a matched turn span (`TurnStarted` ... `TurnComplete`/`TurnAborted`). - // - // We keep this as a minimal fallback for legacy/incomplete lifecycle - // data: treat as "compaction happened after older baseline" and prefer - // conservative baseline invalidation over complex reconstruction. - CompactionOutsideTurn, - } - - let mut history = ContextManager::new(); - let mut saw_turn_lifecycle_event = false; - let mut active_turn: Option = None; - let mut replayed_segments = Vec::new(); - let push_replayed_turn = |replayed_segments: &mut Vec, - active_turn: ActiveRolloutTurn| { - replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( - ReplayedRolloutTurn { - saw_user_message: active_turn.saw_user_message, - turn_context_item: active_turn.turn_context_item, - has_preturn_compaction: active_turn.has_preturn_compaction, - has_midturn_compaction: active_turn.has_midturn_compaction, - }, - ))); - }; - - for item in rollout_items { - match item { - RolloutItem::ResponseItem(response_item) => { - history.record_items( - std::iter::once(response_item), - turn_context.truncation_policy, - ); - } - RolloutItem::Compacted(compacted) => { - if let Some(replacement) = &compacted.replacement_history { - history.replace(replacement.clone()); - } else { - let user_messages = collect_user_messages(history.raw_items()); - let rebuilt = compact::build_compacted_history( - self.build_initial_context(turn_context, None).await, - &user_messages, - &compacted.message, - ); - history.replace(rebuilt); - } - if let Some(active_turn) = active_turn.as_mut() { - if active_turn.turn_context_item.is_none() { - active_turn.has_preturn_compaction = true; - } else { - active_turn.has_midturn_compaction = true; - } - } else { - replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); - } - } - RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { - history.drop_last_n_user_turns(rollback.num_turns); - let mut turns_to_drop = - usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); - if turns_to_drop > 0 - && active_turn - .as_ref() - .is_some_and(|turn| turn.saw_user_message) - { - // Match `drop_last_n_user_turns`: an unfinished active turn that has - // already emitted a user message is the newest user turn and should be - // dropped before we trim older finalized turn spans. - active_turn = None; - turns_to_drop -= 1; - } - if turns_to_drop > 0 { - let mut idx = replayed_segments.len(); - while idx > 0 && turns_to_drop > 0 { - idx -= 1; - if let RolloutReplayMetaSegment::Turn(turn) = &replayed_segments[idx] - && turn.saw_user_message - { - replayed_segments.remove(idx); - turns_to_drop -= 1; - } - } - } - } - RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - saw_turn_lifecycle_event = true; - if let Some(active_turn) = active_turn.take() { - // Treat a replaced incomplete turn as ended at the point the next turn - // starts so replay preserves any `TurnContextItem` it already emitted. - push_replayed_turn(&mut replayed_segments, active_turn); - } - active_turn = Some(ActiveRolloutTurn { - turn_id: event.turn_id.clone(), - saw_user_message: false, - turn_context_item: None, - has_preturn_compaction: false, - has_midturn_compaction: false, - }); - } - RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { - saw_turn_lifecycle_event = true; - if active_turn - .as_ref() - .is_some_and(|turn| turn.turn_id == event.turn_id) - && let Some(active_turn) = active_turn.take() - { - push_replayed_turn(&mut replayed_segments, active_turn); - } - } - RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { - saw_turn_lifecycle_event = true; - match event.turn_id.as_deref() { - Some(aborted_turn_id) - if active_turn - .as_ref() - .is_some_and(|turn| turn.turn_id == aborted_turn_id) => - { - if let Some(active_turn) = active_turn.take() { - push_replayed_turn(&mut replayed_segments, active_turn); - } - } - Some(_) => { - // Ignore aborts for some other turn and keep the current active turn - // alive so later `TurnContext`/`TurnComplete` events still apply. - } - None => { - if let Some(active_turn) = active_turn.take() - && (active_turn.has_preturn_compaction - || active_turn.has_midturn_compaction) - { - // Legacy/incomplete lifecycle events may omit `turn_id` on - // abort. Keep fallback handling minimal: drop this ambiguous - // turn span and preserve only a conservative "outside-turn - // compaction" marker. - replayed_segments - .push(RolloutReplayMetaSegment::CompactionOutsideTurn); - } - } - } - } - RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { - if let Some(active_turn) = active_turn.as_mut() { - active_turn.saw_user_message = true; - } - } - RolloutItem::TurnContext(ctx) => { - if let Some(active_turn) = active_turn.as_mut() - && ctx - .turn_id - .as_deref() - .is_none_or(|turn_id| turn_id == active_turn.turn_id) - { - // Keep the latest `TurnContextItem` in rollout order for the turn. - active_turn.turn_context_item = Some(ctx.clone()); - } - } - _ => {} - } - } - - if let Some(active_turn) = active_turn.take() { - // Treat a trailing incomplete turn as ended at EOF so replay preserves any - // `TurnContextItem` it already emitted before the rollout was truncated. - push_replayed_turn(&mut replayed_segments, active_turn); - } - - let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { - let mut compaction_cleared_reference_context_item = false; - let mut previous_regular_turn_context_item = None; - - for segment in replayed_segments.iter().rev() { - match segment { - RolloutReplayMetaSegment::CompactionOutsideTurn => { - compaction_cleared_reference_context_item = true; - } - RolloutReplayMetaSegment::Turn(turn) => { - if let Some(turn_context_item) = &turn.turn_context_item { - if turn.has_preturn_compaction { - compaction_cleared_reference_context_item = true; - } - previous_regular_turn_context_item = Some(turn_context_item.clone()); - break; - } - if turn.has_preturn_compaction || turn.has_midturn_compaction { - // This later surviving turn compacted (for example via `/compact` or - // auto-compaction) but did not persist a replacement TurnContextItem, - // so conservatively invalidate any older baseline we might select. - compaction_cleared_reference_context_item = true; - } - } - } - } - - let previous_model = previous_regular_turn_context_item - .as_ref() - .map(|ctx| ctx.model.clone()); - let reference_context_item = if compaction_cleared_reference_context_item { - // Keep the baseline empty when compaction may have stripped the referenced - // context diffs so the first resumed regular turn fully reinjects context. - None - } else { - previous_regular_turn_context_item - }; - (previous_model, reference_context_item) - } else { - // Legacy/minimal fallback (no lifecycle events): use the last persisted - // `TurnContextItem` in rollout order and conservatively null baseline when a - // later `Compacted` item exists. - let mut legacy_last_turn_context_item: Option = None; - let mut legacy_saw_compaction_after_last_turn_context = false; - for item in rollout_items.iter().rev() { - match item { - RolloutItem::Compacted(_) => { - legacy_saw_compaction_after_last_turn_context = true; - } - RolloutItem::TurnContext(ctx) => { - legacy_last_turn_context_item = Some(ctx.clone()); - break; - } - _ => {} - } - } - - let previous_model = legacy_last_turn_context_item - .as_ref() - .map(|ctx| ctx.model.clone()); - let reference_context_item = if legacy_saw_compaction_after_last_turn_context { - None - } else { - legacy_last_turn_context_item - }; - (previous_model, reference_context_item) - }; - - RolloutReconstruction { - history: history.raw_items().to_vec(), - previous_model, - reference_context_item, - } - } - /// Append ResponseItems to the in-memory conversation history only. pub(crate) async fn record_into_history( &self, diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs new file mode 100644 index 00000000000..d6fb27ae99d --- /dev/null +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -0,0 +1,309 @@ +use super::*; + +#[derive(Debug)] +pub(super) struct RolloutReconstruction { + pub(super) history: Vec, + pub(super) previous_model: Option, + pub(super) reference_context_item: Option, +} + +impl Session { + pub(super) async fn reconstruct_history_from_rollout( + &self, + turn_context: &TurnContext, + rollout_items: &[RolloutItem], + ) -> RolloutReconstruction { + // Replay rollout items once and compute two things in lockstep: + // 1) reconstructed conversation history (via `ContextManager`) + // 2) resume/fork hydration metadata (`previous_model` and + // `reference_context_item`) + // + // The metadata part needs rollback-aware accounting over "turn spans" and + // compaction placement: + // - `ActiveRolloutTurn` tracks the in-progress turn span while we walk forward + // through lifecycle events (`TurnStarted` ... `TurnComplete`/`TurnAborted`). + // - `ReplayedRolloutTurn` is the finalized per-turn metadata we keep after a + // turn ends (whether it had a user message, a `TurnContextItem`, and whether + // any compaction in that span happened before or after the first + // `TurnContextItem` for that turn). + // - `RolloutReplayMetaSegment` stores the finalized sequence we later + // rollback-adjust and reverse-scan to find the last surviving regular turn + // context. Replaced/trailing incomplete turns are finalized as ordinary + // `Turn(...)` segments. + // + // Explicit replay rule: + // - compaction before the first `TurnContextItem` in a turn span is treated as + // preturn compaction for that turn and invalidates + // `reference_context_item` on resume + // - compaction after the first `TurnContextItem` in the same turn span is + // treated as mid-turn compaction and does not invalidate that turn's own + // `reference_context_item` + // - compaction outside any matched turn span is treated conservatively as + // preturn-equivalent for baseline hydration (invalidate older baseline) + // + // `ThreadRolledBack` updates both: + // - history: drop user turns from reconstructed response items + // - metadata segments: remove finalized turn spans that consumed those user turns + // + // This keeps resume/fork baseline hydration consistent with the same replay + // logic used to rebuild history, instead of maintaining a second bespoke scan. + #[derive(Debug)] + struct ActiveRolloutTurn { + turn_id: String, + saw_user_message: bool, + turn_context_item: Option, + has_preturn_compaction: bool, + has_midturn_compaction: bool, + } + + #[derive(Debug)] + struct ReplayedRolloutTurn { + saw_user_message: bool, + turn_context_item: Option, + has_preturn_compaction: bool, + has_midturn_compaction: bool, + } + + #[derive(Debug)] + enum RolloutReplayMetaSegment { + Turn(Box), + // Unexpected for modern rollouts, where compaction should occur inside + // a matched turn span (`TurnStarted` ... `TurnComplete`/`TurnAborted`). + // + // We keep this as a minimal fallback for legacy/incomplete lifecycle + // data: treat as "compaction happened after older baseline" and prefer + // conservative baseline invalidation over complex reconstruction. + CompactionOutsideTurn, + } + + let mut history = ContextManager::new(); + let mut saw_turn_lifecycle_event = false; + let mut active_turn: Option = None; + let mut replayed_segments = Vec::new(); + let push_replayed_turn = |replayed_segments: &mut Vec, + active_turn: ActiveRolloutTurn| { + replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( + ReplayedRolloutTurn { + saw_user_message: active_turn.saw_user_message, + turn_context_item: active_turn.turn_context_item, + has_preturn_compaction: active_turn.has_preturn_compaction, + has_midturn_compaction: active_turn.has_midturn_compaction, + }, + ))); + }; + + for item in rollout_items { + match item { + RolloutItem::ResponseItem(response_item) => { + history.record_items( + std::iter::once(response_item), + turn_context.truncation_policy, + ); + } + RolloutItem::Compacted(compacted) => { + if let Some(replacement) = &compacted.replacement_history { + history.replace(replacement.clone()); + } else { + let user_messages = collect_user_messages(history.raw_items()); + let rebuilt = compact::build_compacted_history( + self.build_initial_context(turn_context).await, + &user_messages, + &compacted.message, + ); + history.replace(rebuilt); + } + if let Some(active_turn) = active_turn.as_mut() { + if active_turn.turn_context_item.is_none() { + active_turn.has_preturn_compaction = true; + } else { + active_turn.has_midturn_compaction = true; + } + } else { + replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + } + } + RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { + history.drop_last_n_user_turns(rollback.num_turns); + let mut turns_to_drop = + usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); + if turns_to_drop > 0 + && active_turn + .as_ref() + .is_some_and(|turn| turn.saw_user_message) + { + // Match `drop_last_n_user_turns`: an unfinished active turn that has + // already emitted a user message is the newest user turn and should be + // dropped before we trim older finalized turn spans. + active_turn = None; + turns_to_drop -= 1; + } + if turns_to_drop > 0 { + let mut idx = replayed_segments.len(); + while idx > 0 && turns_to_drop > 0 { + idx -= 1; + if let RolloutReplayMetaSegment::Turn(turn) = &replayed_segments[idx] + && turn.saw_user_message + { + replayed_segments.remove(idx); + turns_to_drop -= 1; + } + } + } + } + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + saw_turn_lifecycle_event = true; + if let Some(active_turn) = active_turn.take() { + // Treat a replaced incomplete turn as ended at the point the next turn + // starts so replay preserves any `TurnContextItem` it already emitted. + push_replayed_turn(&mut replayed_segments, active_turn); + } + active_turn = Some(ActiveRolloutTurn { + turn_id: event.turn_id.clone(), + saw_user_message: false, + turn_context_item: None, + has_preturn_compaction: false, + has_midturn_compaction: false, + }); + } + RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { + saw_turn_lifecycle_event = true; + if active_turn + .as_ref() + .is_some_and(|turn| turn.turn_id == event.turn_id) + && let Some(active_turn) = active_turn.take() + { + push_replayed_turn(&mut replayed_segments, active_turn); + } + } + RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { + saw_turn_lifecycle_event = true; + match event.turn_id.as_deref() { + Some(aborted_turn_id) + if active_turn + .as_ref() + .is_some_and(|turn| turn.turn_id == aborted_turn_id) => + { + if let Some(active_turn) = active_turn.take() { + push_replayed_turn(&mut replayed_segments, active_turn); + } + } + Some(_) => { + // Ignore aborts for some other turn and keep the current active turn + // alive so later `TurnContext`/`TurnComplete` events still apply. + } + None => { + if let Some(active_turn) = active_turn.take() + && (active_turn.has_preturn_compaction + || active_turn.has_midturn_compaction) + { + // Legacy/incomplete lifecycle events may omit `turn_id` on + // abort. Keep fallback handling minimal: drop this ambiguous + // turn span and preserve only a conservative "outside-turn + // compaction" marker. + replayed_segments + .push(RolloutReplayMetaSegment::CompactionOutsideTurn); + } + } + } + } + RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { + if let Some(active_turn) = active_turn.as_mut() { + active_turn.saw_user_message = true; + } + } + RolloutItem::TurnContext(ctx) => { + if let Some(active_turn) = active_turn.as_mut() + && ctx + .turn_id + .as_deref() + .is_none_or(|turn_id| turn_id == active_turn.turn_id) + { + // Keep the latest `TurnContextItem` in rollout order for the turn. + active_turn.turn_context_item = Some(ctx.clone()); + } + } + _ => {} + } + } + + if let Some(active_turn) = active_turn.take() { + // Treat a trailing incomplete turn as ended at EOF so replay preserves any + // `TurnContextItem` it already emitted before the rollout was truncated. + push_replayed_turn(&mut replayed_segments, active_turn); + } + + let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { + let mut compaction_cleared_reference_context_item = false; + let mut previous_regular_turn_context_item = None; + + for segment in replayed_segments.iter().rev() { + match segment { + RolloutReplayMetaSegment::CompactionOutsideTurn => { + compaction_cleared_reference_context_item = true; + } + RolloutReplayMetaSegment::Turn(turn) => { + if let Some(turn_context_item) = &turn.turn_context_item { + if turn.has_preturn_compaction { + compaction_cleared_reference_context_item = true; + } + previous_regular_turn_context_item = Some(turn_context_item.clone()); + break; + } + if turn.has_preturn_compaction || turn.has_midturn_compaction { + // This later surviving turn compacted (for example via `/compact` or + // auto-compaction) but did not persist a replacement TurnContextItem, + // so conservatively invalidate any older baseline we might select. + compaction_cleared_reference_context_item = true; + } + } + } + } + + let previous_model = previous_regular_turn_context_item + .as_ref() + .map(|ctx| ctx.model.clone()); + let reference_context_item = if compaction_cleared_reference_context_item { + // Keep the baseline empty when compaction may have stripped the referenced + // context diffs so the first resumed regular turn fully reinjects context. + None + } else { + previous_regular_turn_context_item + }; + (previous_model, reference_context_item) + } else { + // Legacy/minimal fallback (no lifecycle events): use the last persisted + // `TurnContextItem` in rollout order and conservatively null baseline when a + // later `Compacted` item exists. + let mut legacy_last_turn_context_item: Option = None; + let mut legacy_saw_compaction_after_last_turn_context = false; + for item in rollout_items.iter().rev() { + match item { + RolloutItem::Compacted(_) => { + legacy_saw_compaction_after_last_turn_context = true; + } + RolloutItem::TurnContext(ctx) => { + legacy_last_turn_context_item = Some(ctx.clone()); + break; + } + _ => {} + } + } + + let previous_model = legacy_last_turn_context_item + .as_ref() + .map(|ctx| ctx.model.clone()); + let reference_context_item = if legacy_saw_compaction_after_last_turn_context { + None + } else { + legacy_last_turn_context_item + }; + (previous_model, reference_context_item) + }; + + RolloutReconstruction { + history: history.raw_items().to_vec(), + previous_model, + reference_context_item, + } + } +} From b6270c289f94b1d4a3416fba281c794b839d536b Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 11:07:22 -0800 Subject: [PATCH 20/64] Persist TurnContext only when model-visible context changes --- codex-rs/core/src/codex.rs | 57 ++++--- .../core/src/codex/rollout_reconstruction.rs | 160 +++++++----------- codex-rs/core/src/compact.rs | 12 +- codex-rs/core/src/compact_remote.rs | 16 +- codex-rs/core/tests/suite/compact.rs | 14 +- 5 files changed, 125 insertions(+), 134 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cf7b46c8c13..160333fb972 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2969,7 +2969,7 @@ impl Session { state.reference_context_item() } - /// Persist the latest turn context snapshot and emit any required model-visible context updates. + /// Persist the latest turn context snapshot only when we emit model-visible context updates. /// /// When the reference snapshot is missing, this injects full initial context. Otherwise, it /// emits only settings diff items. @@ -2977,9 +2977,10 @@ impl Session { /// If full context is injected and a model switch occurred, this prepends the /// `` developer message so model-specific instructions are not lost. /// - /// Invariant: this is the only runtime path that writes a non-`None` - /// `reference_context_item`. Non-regular tasks intentionally do not update that - /// baseline; `reference_context_item` tracks the latest regular model turn. + /// This is the normal runtime path that establishes a new `reference_context_item`. + /// Mid-turn compaction is the other path that can re-establish that baseline when it + /// reinjects full initial context into replacement history. Other non-regular tasks + /// intentionally do not update the baseline. pub(crate) async fn record_context_updates_and_set_reference_context_item( &self, turn_context: &TurnContext, @@ -3001,10 +3002,13 @@ impl Session { if !context_items.is_empty() { self.record_conversation_items(turn_context, &context_items) .await; - } + let turn_context_item = turn_context.to_turn_context_item(); + self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) + .await; - let mut state = self.state.lock().await; - state.set_reference_context_item(Some(turn_context.to_turn_context_item())); + let mut state = self.state.lock().await; + state.set_reference_context_item(Some(turn_context_item)); + } } pub(crate) async fn update_token_usage_info( @@ -6014,17 +6018,6 @@ async fn try_run_sampling_request( prompt: &Prompt, cancellation_token: CancellationToken, ) -> CodexResult { - // Persist one TurnContext marker per sampling request (not just per user turn) so rollout - // analysis can reconstruct API-turn boundaries. `run_turn` persists model-visible context - // diffs/full reinjection earlier in the same regular turn before reaching this path. - // - // Replay invariant for resume/fork hydration: - // compaction events in a turn span that happen before this first persisted TurnContextItem - // are treated as preturn compaction for that turn and invalidate - // `reference_context_item`, while compaction after this point in the same turn span is - // treated as mid-turn compaction and does not invalidate that turn's own baseline. - let rollout_item = RolloutItem::TurnContext(turn_context.to_turn_context_item()); - feedback_tags!( model = turn_context.model_info.slug.clone(), approval_policy = turn_context.approval_policy.value(), @@ -6033,8 +6026,6 @@ async fn try_run_sampling_request( auth_mode = sess.services.auth_manager.auth_mode(), features = sess.features.enabled_features(), ); - - sess.persist_rollout_items(&[rollout_item]).await; let mut stream = client_session .stream( prompt, @@ -7725,7 +7716,7 @@ mod tests { } #[tokio::test] - async fn record_initial_history_resumed_nulls_reference_context_item_for_preturn_compaction_in_same_turn() + async fn record_initial_history_resumed_turn_context_after_compaction_reestablishes_reference_context_item() { let (session, turn_context) = make_session_and_context().await; let previous_model = "previous-rollout-model"; @@ -7765,7 +7756,7 @@ mod tests { text_elements: Vec::new(), }, )), - // Compaction before the first TurnContextItem in this turn is treated as preturn. + // Compaction clears baseline until a later TurnContextItem re-establishes it. RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), @@ -7791,7 +7782,27 @@ mod tests { session.previous_model().await, Some(previous_model.to_string()) ); - assert!(session.reference_context_item().await.is_none()); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + })) + .expect("serialize expected reference context item") + ); } #[tokio::test] diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index d6fb27ae99d..e92b37ba7cf 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -18,62 +18,47 @@ impl Session { // 2) resume/fork hydration metadata (`previous_model` and // `reference_context_item`) // - // The metadata part needs rollback-aware accounting over "turn spans" and - // compaction placement: - // - `ActiveRolloutTurn` tracks the in-progress turn span while we walk forward - // through lifecycle events (`TurnStarted` ... `TurnComplete`/`TurnAborted`). - // - `ReplayedRolloutTurn` is the finalized per-turn metadata we keep after a - // turn ends (whether it had a user message, a `TurnContextItem`, and whether - // any compaction in that span happened before or after the first - // `TurnContextItem` for that turn). - // - `RolloutReplayMetaSegment` stores the finalized sequence we later - // rollback-adjust and reverse-scan to find the last surviving regular turn - // context. Replaced/trailing incomplete turns are finalized as ordinary - // `Turn(...)` segments. + // `TurnContextItem` is now only persisted when the current user turn actually + // emits model-visible context items (during normal context updates or when + // mid-turn compaction reinjects full initial context). That means replay only + // needs to answer: + // - which surviving user turn last established a model baseline + // - whether any later compaction cleared that baseline before resume // - // Explicit replay rule: - // - compaction before the first `TurnContextItem` in a turn span is treated as - // preturn compaction for that turn and invalidates - // `reference_context_item` on resume - // - compaction after the first `TurnContextItem` in the same turn span is - // treated as mid-turn compaction and does not invalidate that turn's own - // `reference_context_item` - // - compaction outside any matched turn span is treated conservatively as - // preturn-equivalent for baseline hydration (invalidate older baseline) + // We model that with: + // - one active turn span while walking lifecycle events + // - one finalized metadata segment per surviving user turn + // - conservative `Compaction` markers for standalone compaction outside user turns // // `ThreadRolledBack` updates both: // - history: drop user turns from reconstructed response items - // - metadata segments: remove finalized turn spans that consumed those user turns + // - metadata segments: drop the same number of surviving user-turn segments // - // This keeps resume/fork baseline hydration consistent with the same replay - // logic used to rebuild history, instead of maintaining a second bespoke scan. + // This keeps history reconstruction and resume/fork hydration on the same replay. #[derive(Debug)] struct ActiveRolloutTurn { turn_id: String, saw_user_message: bool, - turn_context_item: Option, - has_preturn_compaction: bool, - has_midturn_compaction: bool, + previous_model: Option, + reference_context_item: Option, + cleared_reference_context_item: bool, } #[derive(Debug)] - struct ReplayedRolloutTurn { - saw_user_message: bool, - turn_context_item: Option, - has_preturn_compaction: bool, - has_midturn_compaction: bool, + struct ReplayedUserTurn { + previous_model: Option, + reference_context_item: Option, + cleared_reference_context_item: bool, } #[derive(Debug)] enum RolloutReplayMetaSegment { - Turn(Box), - // Unexpected for modern rollouts, where compaction should occur inside - // a matched turn span (`TurnStarted` ... `TurnComplete`/`TurnAborted`). - // - // We keep this as a minimal fallback for legacy/incomplete lifecycle - // data: treat as "compaction happened after older baseline" and prefer - // conservative baseline invalidation over complex reconstruction. - CompactionOutsideTurn, + UserTurn(Box), + // Unexpected for modern rollouts, where compaction should normally happen inside a + // turn span. We keep this as a minimal fallback for standalone/legacy compaction so + // later resume conservatively clears the baseline until another `TurnContextItem` + // re-establishes it. + Compaction, } let mut history = ContextManager::new(); @@ -82,14 +67,17 @@ impl Session { let mut replayed_segments = Vec::new(); let push_replayed_turn = |replayed_segments: &mut Vec, active_turn: ActiveRolloutTurn| { - replayed_segments.push(RolloutReplayMetaSegment::Turn(Box::new( - ReplayedRolloutTurn { - saw_user_message: active_turn.saw_user_message, - turn_context_item: active_turn.turn_context_item, - has_preturn_compaction: active_turn.has_preturn_compaction, - has_midturn_compaction: active_turn.has_midturn_compaction, - }, - ))); + if active_turn.saw_user_message { + replayed_segments.push(RolloutReplayMetaSegment::UserTurn(Box::new( + ReplayedUserTurn { + previous_model: active_turn.previous_model, + reference_context_item: active_turn.reference_context_item, + cleared_reference_context_item: active_turn.cleared_reference_context_item, + }, + ))); + } else if active_turn.cleared_reference_context_item { + replayed_segments.push(RolloutReplayMetaSegment::Compaction); + } }; for item in rollout_items { @@ -113,13 +101,10 @@ impl Session { history.replace(rebuilt); } if let Some(active_turn) = active_turn.as_mut() { - if active_turn.turn_context_item.is_none() { - active_turn.has_preturn_compaction = true; - } else { - active_turn.has_midturn_compaction = true; - } + active_turn.reference_context_item = None; + active_turn.cleared_reference_context_item = true; } else { - replayed_segments.push(RolloutReplayMetaSegment::CompactionOutsideTurn); + replayed_segments.push(RolloutReplayMetaSegment::Compaction); } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { @@ -133,7 +118,7 @@ impl Session { { // Match `drop_last_n_user_turns`: an unfinished active turn that has // already emitted a user message is the newest user turn and should be - // dropped before we trim older finalized turn spans. + // dropped before we trim older finalized user-turn segments. active_turn = None; turns_to_drop -= 1; } @@ -141,9 +126,7 @@ impl Session { let mut idx = replayed_segments.len(); while idx > 0 && turns_to_drop > 0 { idx -= 1; - if let RolloutReplayMetaSegment::Turn(turn) = &replayed_segments[idx] - && turn.saw_user_message - { + if let RolloutReplayMetaSegment::UserTurn(_) = &replayed_segments[idx] { replayed_segments.remove(idx); turns_to_drop -= 1; } @@ -153,16 +136,14 @@ impl Session { RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { saw_turn_lifecycle_event = true; if let Some(active_turn) = active_turn.take() { - // Treat a replaced incomplete turn as ended at the point the next turn - // starts so replay preserves any `TurnContextItem` it already emitted. push_replayed_turn(&mut replayed_segments, active_turn); } active_turn = Some(ActiveRolloutTurn { turn_id: event.turn_id.clone(), saw_user_message: false, - turn_context_item: None, - has_preturn_compaction: false, - has_midturn_compaction: false, + previous_model: None, + reference_context_item: None, + cleared_reference_context_item: false, }); } RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { @@ -192,16 +173,8 @@ impl Session { // alive so later `TurnContext`/`TurnComplete` events still apply. } None => { - if let Some(active_turn) = active_turn.take() - && (active_turn.has_preturn_compaction - || active_turn.has_midturn_compaction) - { - // Legacy/incomplete lifecycle events may omit `turn_id` on - // abort. Keep fallback handling minimal: drop this ambiguous - // turn span and preserve only a conservative "outside-turn - // compaction" marker. - replayed_segments - .push(RolloutReplayMetaSegment::CompactionOutsideTurn); + if let Some(active_turn) = active_turn.take() { + push_replayed_turn(&mut replayed_segments, active_turn); } } } @@ -218,8 +191,9 @@ impl Session { .as_deref() .is_none_or(|turn_id| turn_id == active_turn.turn_id) { - // Keep the latest `TurnContextItem` in rollout order for the turn. - active_turn.turn_context_item = Some(ctx.clone()); + active_turn.previous_model = Some(ctx.model.clone()); + active_turn.reference_context_item = Some(ctx.clone()); + active_turn.cleared_reference_context_item = false; } } _ => {} @@ -227,52 +201,40 @@ impl Session { } if let Some(active_turn) = active_turn.take() { - // Treat a trailing incomplete turn as ended at EOF so replay preserves any - // `TurnContextItem` it already emitted before the rollout was truncated. push_replayed_turn(&mut replayed_segments, active_turn); } let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { let mut compaction_cleared_reference_context_item = false; - let mut previous_regular_turn_context_item = None; + let mut previous_model = None; + let mut reference_context_item = None; for segment in replayed_segments.iter().rev() { match segment { - RolloutReplayMetaSegment::CompactionOutsideTurn => { + RolloutReplayMetaSegment::Compaction => { compaction_cleared_reference_context_item = true; } - RolloutReplayMetaSegment::Turn(turn) => { - if let Some(turn_context_item) = &turn.turn_context_item { - if turn.has_preturn_compaction { - compaction_cleared_reference_context_item = true; + RolloutReplayMetaSegment::UserTurn(turn) => { + if let Some(turn_previous_model) = &turn.previous_model { + previous_model = Some(turn_previous_model.clone()); + if !compaction_cleared_reference_context_item + && !turn.cleared_reference_context_item + { + reference_context_item = turn.reference_context_item.clone(); } - previous_regular_turn_context_item = Some(turn_context_item.clone()); break; } - if turn.has_preturn_compaction || turn.has_midturn_compaction { - // This later surviving turn compacted (for example via `/compact` or - // auto-compaction) but did not persist a replacement TurnContextItem, - // so conservatively invalidate any older baseline we might select. + if turn.cleared_reference_context_item { compaction_cleared_reference_context_item = true; } } } } - let previous_model = previous_regular_turn_context_item - .as_ref() - .map(|ctx| ctx.model.clone()); - let reference_context_item = if compaction_cleared_reference_context_item { - // Keep the baseline empty when compaction may have stripped the referenced - // context diffs so the first resumed regular turn fully reinjects context. - None - } else { - previous_regular_turn_context_item - }; (previous_model, reference_context_item) } else { // Legacy/minimal fallback (no lifecycle events): use the last persisted - // `TurnContextItem` in rollout order and conservatively null baseline when a + // `TurnContextItem` in rollout order and conservatively clear the baseline when a // later `Compacted` item exists. let mut legacy_last_turn_context_item: Option = None; let mut legacy_saw_compaction_after_last_turn_context = false; diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 7470a148958..0441f9c4969 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -224,7 +224,7 @@ async fn run_compact_task_inner( InitialContextInjection::DoNotInject => None, InitialContextInjection::BeforeLastUserMessage => Some(turn_context.to_turn_context_item()), }; - sess.replace_history(new_history.clone(), reference_context_item) + sess.replace_history(new_history.clone(), reference_context_item.clone()) .await; sess.recompute_token_usage(&turn_context).await; @@ -232,7 +232,15 @@ async fn run_compact_task_inner( message: summary_text.clone(), replacement_history: Some(new_history), }); - sess.persist_rollout_items(&[rollout_item]).await; + let rollout_items = if let Some(turn_context_item) = reference_context_item { + // Mid-turn compaction re-injected initial context into the replacement history, so + // persist a fresh `TurnContextItem` after `Compacted` to re-establish the baseline for + // resume/fork replay. + vec![rollout_item, RolloutItem::TurnContext(turn_context_item)] + } else { + vec![rollout_item] + }; + sess.persist_rollout_items(&rollout_items).await; sess.emit_turn_item_completed(&turn_context, compaction_item) .await; diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index cc5f5164c39..3019a4ad8b6 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -159,7 +159,7 @@ async fn run_remote_compact_task_inner_impl( InitialContextInjection::DoNotInject => None, InitialContextInjection::BeforeLastUserMessage => Some(turn_context.to_turn_context_item()), }; - sess.replace_history(new_history.clone(), reference_context_item) + sess.replace_history(new_history.clone(), reference_context_item.clone()) .await; sess.recompute_token_usage(turn_context).await; @@ -167,8 +167,18 @@ async fn run_remote_compact_task_inner_impl( message: String::new(), replacement_history: Some(new_history), }; - sess.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) - .await; + let rollout_items = if let Some(turn_context_item) = reference_context_item { + // Mid-turn compaction re-injected initial context into the replacement history, so + // persist a fresh `TurnContextItem` after `Compacted` to re-establish the baseline for + // resume/fork replay. + vec![ + RolloutItem::Compacted(compacted_item), + RolloutItem::TurnContext(turn_context_item), + ] + } else { + vec![RolloutItem::Compacted(compacted_item)] + }; + sess.persist_rollout_items(&rollout_items).await; sess.emit_turn_item_completed(turn_context, compaction_item) .await; diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 9e376ce494e..2c3ac45625c 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -362,7 +362,7 @@ async fn summarize_context_three_requests_and_instructions() { codex.submit(Op::Shutdown).await.unwrap(); wait_for_event(&codex, |ev| matches!(ev, EventMsg::ShutdownComplete)).await; - // Verify rollout contains regular sampling TurnContext entries and a Compacted entry. + // Verify rollout contains user-turn TurnContext entries and a Compacted entry. println!("rollout path: {}", rollout_path.display()); let text = std::fs::read_to_string(&rollout_path).unwrap_or_else(|e| { panic!( @@ -393,9 +393,9 @@ async fn summarize_context_three_requests_and_instructions() { } } - assert!( - regular_turn_context_count == 2, - "expected two regular sampling TurnContext entries in rollout" + assert_eq!( + regular_turn_context_count, 1, + "manual compact should not emit its own TurnContext entry" ); assert!( saw_compacted_summary, @@ -2080,9 +2080,9 @@ async fn auto_compact_persists_rollout_entries() { } } - assert!( - turn_context_count >= 2, - "expected at least two turn context entries, got {turn_context_count}" + assert_eq!( + turn_context_count, 1, + "pre-sampling compaction should not emit its own TurnContext entry" ); } From 4b3dee63743fa38336b86ad1cd20b6e037730c1e Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 11:54:35 -0800 Subject: [PATCH 21/64] Move rollout reconstruction tests out of codex.rs --- codex-rs/core/src/codex.rs | 1149 +--------------- .../src/codex/rollout_reconstruction_tests.rs | 1162 +++++++++++++++++ 2 files changed, 1164 insertions(+), 1147 deletions(-) create mode 100644 codex-rs/core/src/codex/rollout_reconstruction_tests.rs diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 160333fb972..1ac7834671d 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -157,6 +157,8 @@ use crate::exec::StreamOutput; use codex_config::CONFIG_TOML_FILE; mod rollout_reconstruction; +#[cfg(test)] +mod rollout_reconstruction_tests; #[derive(Debug, PartialEq)] pub enum SteerInputError { @@ -6437,18 +6439,6 @@ mod tests { } } - fn assistant_message(text: &str) -> ResponseItem { - ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: text.to_string(), - }], - end_turn: None, - phase: None, - } - } - fn skill_message(text: &str) -> ResponseItem { ResponseItem::Message { id: None, @@ -7095,1141 +7085,6 @@ mod tests { assert_eq!(expected, history.raw_items()); } - #[tokio::test] - async fn record_initial_history_resumed_hydrates_previous_model() { - let (session, turn_context) = make_session_and_context().await; - let previous_model = "previous-rollout-model"; - let previous_context_item = TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - current_date: turn_context.current_date.clone(), - timezone: turn_context.timezone.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - let rollout_items = vec![RolloutItem::TurnContext(previous_context_item)]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); - } - - #[tokio::test] - async fn record_initial_history_resumed_hydrates_previous_model_from_lifecycle_turn_with_missing_turn_context_id() - { - let (session, turn_context) = make_session_and_context().await; - let previous_model = "previous-rollout-model"; - let mut previous_context_item = TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - current_date: turn_context.current_date.clone(), - timezone: turn_context.timezone.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - let turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - previous_context_item.turn_id = None; - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id, - last_agent_message: None, - }, - )), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); - } - - #[tokio::test] - async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_completed_turns() { - let (session, turn_context) = make_session_and_context().await; - let first_context_item = turn_context.to_turn_context_item(); - let first_turn_id = first_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let mut rolled_back_context_item = first_context_item.clone(); - rolled_back_context_item.turn_id = Some("rolled-back-turn".to_string()); - rolled_back_context_item.model = "rolled-back-model".to_string(); - let rolled_back_turn_id = rolled_back_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let turn_one_user = user_message("turn 1 user"); - let turn_one_assistant = assistant_message("turn 1 assistant"); - let turn_two_user = user_message("turn 2 user"); - let turn_two_assistant = assistant_message("turn 2 assistant"); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: first_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "turn 1 user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(first_context_item.clone()), - RolloutItem::ResponseItem(turn_one_user.clone()), - RolloutItem::ResponseItem(turn_one_assistant.clone()), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: first_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: rolled_back_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "turn 2 user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(rolled_back_context_item), - RolloutItem::ResponseItem(turn_two_user), - RolloutItem::ResponseItem(turn_two_assistant), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: rolled_back_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::ThreadRolledBack( - codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, - )), - ]; - - let reconstructed = session - .reconstruct_history_from_rollout(&turn_context, &rollout_items) - .await; - - assert_eq!( - reconstructed.history, - vec![turn_one_user, turn_one_assistant] - ); - assert_eq!( - reconstructed.previous_model, - Some(turn_context.model_info.slug.clone()) - ); - assert_eq!( - serde_json::to_value(reconstructed.reference_context_item) - .expect("serialize reconstructed reference context item"), - serde_json::to_value(Some(first_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_incomplete_turn() { - let (session, turn_context) = make_session_and_context().await; - let first_context_item = turn_context.to_turn_context_item(); - let first_turn_id = first_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let incomplete_turn_id = "incomplete-rolled-back-turn".to_string(); - let turn_one_user = user_message("turn 1 user"); - let turn_one_assistant = assistant_message("turn 1 assistant"); - let turn_two_user = user_message("turn 2 user"); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: first_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "turn 1 user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(first_context_item.clone()), - RolloutItem::ResponseItem(turn_one_user.clone()), - RolloutItem::ResponseItem(turn_one_assistant.clone()), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: first_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: incomplete_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "turn 2 user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::ResponseItem(turn_two_user), - RolloutItem::EventMsg(EventMsg::ThreadRolledBack( - codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, - )), - ]; - - let reconstructed = session - .reconstruct_history_from_rollout(&turn_context, &rollout_items) - .await; - - assert_eq!( - reconstructed.history, - vec![turn_one_user, turn_one_assistant] - ); - assert_eq!( - reconstructed.previous_model, - Some(turn_context.model_info.slug.clone()) - ); - assert_eq!( - serde_json::to_value(reconstructed.reference_context_item) - .expect("serialize reconstructed reference context item"), - serde_json::to_value(Some(first_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn reconstruct_history_rollback_skips_non_user_turns_for_history_and_metadata() { - let (session, turn_context) = make_session_and_context().await; - let first_context_item = turn_context.to_turn_context_item(); - let first_turn_id = first_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let second_turn_id = "rolled-back-user-turn".to_string(); - let standalone_turn_id = "standalone-turn".to_string(); - let turn_one_user = user_message("turn 1 user"); - let turn_one_assistant = assistant_message("turn 1 assistant"); - let turn_two_user = user_message("turn 2 user"); - let turn_two_assistant = assistant_message("turn 2 assistant"); - let standalone_assistant = assistant_message("standalone assistant"); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: first_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "turn 1 user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(first_context_item.clone()), - RolloutItem::ResponseItem(turn_one_user.clone()), - RolloutItem::ResponseItem(turn_one_assistant.clone()), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: first_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: second_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "turn 2 user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::ResponseItem(turn_two_user), - RolloutItem::ResponseItem(turn_two_assistant), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: second_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: standalone_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::ResponseItem(standalone_assistant), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: standalone_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::ThreadRolledBack( - codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, - )), - ]; - - let reconstructed = session - .reconstruct_history_from_rollout(&turn_context, &rollout_items) - .await; - - assert_eq!( - reconstructed.history, - vec![turn_one_user, turn_one_assistant] - ); - assert_eq!( - reconstructed.previous_model, - Some(turn_context.model_info.slug.clone()) - ); - assert_eq!( - serde_json::to_value(reconstructed.reference_context_item) - .expect("serialize reconstructed reference context item"), - serde_json::to_value(Some(first_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn reconstruct_history_rollback_clears_history_and_metadata_when_exceeding_user_turns() { - let (session, turn_context) = make_session_and_context().await; - let only_context_item = turn_context.to_turn_context_item(); - let only_turn_id = only_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: only_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "only user".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(only_context_item), - RolloutItem::ResponseItem(user_message("only user")), - RolloutItem::ResponseItem(assistant_message("only assistant")), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: only_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::ThreadRolledBack( - codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 99 }, - )), - ]; - - let reconstructed = session - .reconstruct_history_from_rollout(&turn_context, &rollout_items) - .await; - - assert_eq!(reconstructed.history, Vec::new()); - assert_eq!(reconstructed.previous_model, None); - assert!(reconstructed.reference_context_item.is_none()); - } - - #[tokio::test] - async fn record_initial_history_resumed_rollback_skips_only_user_turns() { - let (session, turn_context) = make_session_and_context().await; - let previous_context_item = turn_context.to_turn_context_item(); - let user_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let standalone_turn_id = "standalone-task-turn".to_string(); - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: user_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: user_turn_id, - last_agent_message: None, - }, - )), - // Standalone task turn (no UserMessage) should not consume rollback skips. - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: standalone_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: standalone_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::ThreadRolledBack( - codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, - )), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!(session.previous_model().await, None); - assert!(session.reference_context_item().await.is_none()); - } - - #[tokio::test] - async fn record_initial_history_resumed_rollback_drops_incomplete_user_turn_compaction_metadata() - { - let (session, turn_context) = make_session_and_context().await; - let previous_context_item = turn_context.to_turn_context_item(); - let previous_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let incomplete_turn_id = "incomplete-compacted-user-turn".to_string(); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: previous_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item.clone()), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: previous_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: incomplete_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "rolled back".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::Compacted(CompactedItem { - message: String::new(), - replacement_history: Some(Vec::new()), - }), - RolloutItem::EventMsg(EventMsg::ThreadRolledBack( - codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, - )), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(turn_context.model_info.slug.clone()) - ); - assert_eq!( - serde_json::to_value(session.reference_context_item().await) - .expect("serialize seeded reference context item"), - serde_json::to_value(Some(previous_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn record_initial_history_resumed_seeds_reference_context_item_without_compaction() { - let (session, turn_context) = make_session_and_context().await; - let previous_context_item = turn_context.to_turn_context_item(); - let rollout_items = vec![RolloutItem::TurnContext(previous_context_item.clone())]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - serde_json::to_value(session.reference_context_item().await) - .expect("serialize seeded reference context item"), - serde_json::to_value(Some(previous_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn record_initial_history_resumed_does_not_seed_reference_context_item_after_compaction() - { - let (session, turn_context) = make_session_and_context().await; - let previous_context_item = turn_context.to_turn_context_item(); - let rollout_items = vec![ - RolloutItem::TurnContext(previous_context_item), - RolloutItem::Compacted(CompactedItem { - message: String::new(), - replacement_history: Some(Vec::new()), - }), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(turn_context.model_info.slug.clone()) - ); - assert!(session.reference_context_item().await.is_none()); - } - - #[tokio::test] - async fn record_initial_history_resumed_turn_context_after_compaction_reestablishes_reference_context_item() - { - let (session, turn_context) = make_session_and_context().await; - let previous_model = "previous-rollout-model"; - let previous_context_item = TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - let previous_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: previous_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - // Compaction clears baseline until a later TurnContextItem re-establishes it. - RolloutItem::Compacted(CompactedItem { - message: String::new(), - replacement_history: Some(Vec::new()), - }), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: previous_turn_id, - last_agent_message: None, - }, - )), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); - assert_eq!( - serde_json::to_value(session.reference_context_item().await) - .expect("serialize seeded reference context item"), - serde_json::to_value(Some(TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - })) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_turn_for_compaction_accounting() - { - let (session, turn_context) = make_session_and_context().await; - let previous_model = "previous-rollout-model"; - let previous_context_item = TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - let previous_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let aborted_turn_id = "aborted-turn-without-id".to_string(); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: previous_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: previous_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: aborted_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "aborted".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::EventMsg(EventMsg::TurnAborted( - codex_protocol::protocol::TurnAbortedEvent { - turn_id: None, - reason: TurnAbortReason::Interrupted, - }, - )), - RolloutItem::Compacted(CompactedItem { - message: String::new(), - replacement_history: Some(Vec::new()), - }), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); - assert!(session.reference_context_item().await.is_none()); - } - - #[tokio::test] - async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_for_later_turn_context() - { - let (session, turn_context) = make_session_and_context().await; - let previous_context_item = turn_context.to_turn_context_item(); - let previous_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let current_model = "current-rollout-model"; - let current_turn_id = "current-turn".to_string(); - let unmatched_abort_turn_id = "other-turn".to_string(); - let current_context_item = TurnContextItem { - turn_id: Some(current_turn_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: current_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: previous_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: previous_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: current_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "current".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::EventMsg(EventMsg::TurnAborted( - codex_protocol::protocol::TurnAbortedEvent { - turn_id: Some(unmatched_abort_turn_id), - reason: TurnAbortReason::Interrupted, - }, - )), - RolloutItem::TurnContext(current_context_item.clone()), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: current_turn_id, - last_agent_message: None, - }, - )), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(current_model.to_string()) - ); - assert_eq!( - serde_json::to_value(session.reference_context_item().await) - .expect("serialize seeded reference context item"), - serde_json::to_value(Some(current_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clears_reference_context_item() - { - let (session, turn_context) = make_session_and_context().await; - let previous_model = "previous-rollout-model"; - let previous_context_item = TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - let previous_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let incomplete_turn_id = "trailing-incomplete-turn".to_string(); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: previous_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: previous_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: incomplete_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "incomplete".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::Compacted(CompactedItem { - message: String::new(), - replacement_history: Some(Vec::new()), - }), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); - assert!(session.reference_context_item().await.is_none()); - } - - #[tokio::test] - async fn record_initial_history_resumed_trailing_incomplete_turn_preserves_turn_context_item() { - let (session, turn_context) = make_session_and_context().await; - let current_context_item = turn_context.to_turn_context_item(); - let current_turn_id = current_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: current_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "incomplete".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(current_context_item.clone()), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(turn_context.model_info.slug.clone()) - ); - assert_eq!( - serde_json::to_value(session.reference_context_item().await) - .expect("serialize seeded reference context item"), - serde_json::to_value(Some(current_context_item)) - .expect("serialize expected reference context item") - ); - } - - #[tokio::test] - async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clears_reference_context_item() - { - let (session, turn_context) = make_session_and_context().await; - let previous_model = "previous-rollout-model"; - let previous_context_item = TurnContextItem { - turn_id: Some(turn_context.sub_id.clone()), - cwd: turn_context.cwd.clone(), - approval_policy: turn_context.approval_policy.value(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - network: None, - model: previous_model.to_string(), - personality: turn_context.personality, - collaboration_mode: Some(turn_context.collaboration_mode.clone()), - effort: turn_context.reasoning_effort, - summary: turn_context.reasoning_summary, - user_instructions: None, - developer_instructions: None, - final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), - }; - let previous_turn_id = previous_context_item - .turn_id - .clone() - .expect("turn context should have turn_id"); - let compacted_incomplete_turn_id = "compacted-incomplete-turn".to_string(); - let replacing_turn_id = "replacing-turn".to_string(); - - let rollout_items = vec![ - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: previous_turn_id.clone(), - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "seed".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::TurnContext(previous_context_item), - RolloutItem::EventMsg(EventMsg::TurnComplete( - codex_protocol::protocol::TurnCompleteEvent { - turn_id: previous_turn_id, - last_agent_message: None, - }, - )), - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: compacted_incomplete_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - RolloutItem::EventMsg(EventMsg::UserMessage( - codex_protocol::protocol::UserMessageEvent { - message: "compacted".to_string(), - images: None, - local_images: Vec::new(), - text_elements: Vec::new(), - }, - )), - RolloutItem::Compacted(CompactedItem { - message: String::new(), - replacement_history: Some(Vec::new()), - }), - // A newer TurnStarted replaces the incomplete compacted turn without a matching - // completion/abort for the old one. - RolloutItem::EventMsg(EventMsg::TurnStarted( - codex_protocol::protocol::TurnStartedEvent { - turn_id: replacing_turn_id, - model_context_window: Some(128_000), - collaboration_mode_kind: ModeKind::Default, - }, - )), - ]; - - session - .record_initial_history(InitialHistory::Resumed(ResumedHistory { - conversation_id: ThreadId::default(), - history: rollout_items, - rollout_path: PathBuf::from("/tmp/resume.jsonl"), - })) - .await; - - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); - assert!(session.reference_context_item().await.is_none()); - } - - #[tokio::test] async fn resumed_history_injects_initial_context_on_first_context_update_only() { let (session, turn_context) = make_session_and_context().await; let (rollout_items, mut expected) = sample_rollout(&session, &turn_context).await; diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs new file mode 100644 index 00000000000..88b6f74731a --- /dev/null +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -0,0 +1,1162 @@ +use super::*; + +use crate::protocol::CompactedItem; +use crate::protocol::InitialHistory; +use crate::protocol::ResumedHistory; +use codex_protocol::ThreadId; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use pretty_assertions::assert_eq; +use std::path::PathBuf; + +fn user_message(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: text.to_string(), + }], + end_turn: None, + phase: None, + } +} + +fn assistant_message(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: text.to_string(), + }], + end_turn: None, + phase: None, + } +} + +#[tokio::test] +async fn record_initial_history_resumed_hydrates_previous_model() { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let rollout_items = vec![RolloutItem::TurnContext(previous_context_item)]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); +} + +#[tokio::test] +async fn record_initial_history_resumed_hydrates_previous_model_from_lifecycle_turn_with_missing_turn_context_id() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let mut previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + previous_context_item.turn_id = None; + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id, + last_agent_message: None, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); +} + +#[tokio::test] +async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_completed_turns() { + let (session, turn_context) = make_session_and_context().await; + let first_context_item = turn_context.to_turn_context_item(); + let first_turn_id = first_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let mut rolled_back_context_item = first_context_item.clone(); + rolled_back_context_item.turn_id = Some("rolled-back-turn".to_string()); + rolled_back_context_item.model = "rolled-back-model".to_string(); + let rolled_back_turn_id = rolled_back_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let turn_one_user = user_message("turn 1 user"); + let turn_one_assistant = assistant_message("turn 1 assistant"); + let turn_two_user = user_message("turn 2 user"); + let turn_two_assistant = assistant_message("turn 2 assistant"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: first_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 1 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(first_context_item.clone()), + RolloutItem::ResponseItem(turn_one_user.clone()), + RolloutItem::ResponseItem(turn_one_assistant.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: first_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: rolled_back_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 2 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(rolled_back_context_item), + RolloutItem::ResponseItem(turn_two_user), + RolloutItem::ResponseItem(turn_two_assistant), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: rolled_back_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![turn_one_user, turn_one_assistant] + ); + assert_eq!( + reconstructed.previous_model, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(reconstructed.reference_context_item) + .expect("serialize reconstructed reference context item"), + serde_json::to_value(Some(first_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_incomplete_turn() { + let (session, turn_context) = make_session_and_context().await; + let first_context_item = turn_context.to_turn_context_item(); + let first_turn_id = first_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let incomplete_turn_id = "incomplete-rolled-back-turn".to_string(); + let turn_one_user = user_message("turn 1 user"); + let turn_one_assistant = assistant_message("turn 1 assistant"); + let turn_two_user = user_message("turn 2 user"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: first_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 1 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(first_context_item.clone()), + RolloutItem::ResponseItem(turn_one_user.clone()), + RolloutItem::ResponseItem(turn_one_assistant.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: first_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 2 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::ResponseItem(turn_two_user), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![turn_one_user, turn_one_assistant] + ); + assert_eq!( + reconstructed.previous_model, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(reconstructed.reference_context_item) + .expect("serialize reconstructed reference context item"), + serde_json::to_value(Some(first_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn reconstruct_history_rollback_skips_non_user_turns_for_history_and_metadata() { + let (session, turn_context) = make_session_and_context().await; + let first_context_item = turn_context.to_turn_context_item(); + let first_turn_id = first_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let second_turn_id = "rolled-back-user-turn".to_string(); + let standalone_turn_id = "standalone-turn".to_string(); + let turn_one_user = user_message("turn 1 user"); + let turn_one_assistant = assistant_message("turn 1 assistant"); + let turn_two_user = user_message("turn 2 user"); + let turn_two_assistant = assistant_message("turn 2 assistant"); + let standalone_assistant = assistant_message("standalone assistant"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: first_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 1 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(first_context_item.clone()), + RolloutItem::ResponseItem(turn_one_user.clone()), + RolloutItem::ResponseItem(turn_one_assistant.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: first_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: second_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "turn 2 user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::ResponseItem(turn_two_user), + RolloutItem::ResponseItem(turn_two_assistant), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: second_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: standalone_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::ResponseItem(standalone_assistant), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: standalone_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![turn_one_user, turn_one_assistant] + ); + assert_eq!( + reconstructed.previous_model, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(reconstructed.reference_context_item) + .expect("serialize reconstructed reference context item"), + serde_json::to_value(Some(first_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn reconstruct_history_rollback_clears_history_and_metadata_when_exceeding_user_turns() { + let (session, turn_context) = make_session_and_context().await; + let only_context_item = turn_context.to_turn_context_item(); + let only_turn_id = only_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: only_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "only user".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(only_context_item), + RolloutItem::ResponseItem(user_message("only user")), + RolloutItem::ResponseItem(assistant_message("only assistant")), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: only_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 99 }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!(reconstructed.history, Vec::new()); + assert_eq!(reconstructed.previous_model, None); + assert!(reconstructed.reference_context_item.is_none()); +} + +#[tokio::test] +async fn record_initial_history_resumed_rollback_skips_only_user_turns() { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let user_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let standalone_turn_id = "standalone-task-turn".to_string(); + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: user_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: user_turn_id, + last_agent_message: None, + }, + )), + // Standalone task turn (no UserMessage) should not consume rollback skips. + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: standalone_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: standalone_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!(session.previous_model().await, None); + assert!(session.reference_context_item().await.is_none()); +} + +#[tokio::test] +async fn record_initial_history_resumed_rollback_drops_incomplete_user_turn_compaction_metadata() { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let incomplete_turn_id = "incomplete-compacted-user-turn".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "rolled back".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + RolloutItem::EventMsg(EventMsg::ThreadRolledBack( + codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(previous_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn record_initial_history_resumed_seeds_reference_context_item_without_compaction() { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let rollout_items = vec![RolloutItem::TurnContext(previous_context_item.clone())]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(previous_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn record_initial_history_resumed_does_not_seed_reference_context_item_after_compaction() { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let rollout_items = vec![ + RolloutItem::TurnContext(previous_context_item), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(turn_context.model_info.slug.clone()) + ); + assert!(session.reference_context_item().await.is_none()); +} + +#[tokio::test] +async fn record_initial_history_resumed_turn_context_after_compaction_reestablishes_reference_context_item() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + // Compaction clears baseline until a later TurnContextItem re-establishes it. + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + })) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_turn_for_compaction_accounting() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let aborted_turn_id = "aborted-turn-without-id".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: aborted_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "aborted".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::EventMsg(EventMsg::TurnAborted( + codex_protocol::protocol::TurnAbortedEvent { + turn_id: None, + reason: TurnAbortReason::Interrupted, + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); +} + +#[tokio::test] +async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_for_later_turn_context() + { + let (session, turn_context) = make_session_and_context().await; + let previous_context_item = turn_context.to_turn_context_item(); + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let current_model = "current-rollout-model"; + let current_turn_id = "current-turn".to_string(); + let unmatched_abort_turn_id = "other-turn".to_string(); + let current_context_item = TurnContextItem { + turn_id: Some(current_turn_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: current_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: current_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "current".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::EventMsg(EventMsg::TurnAborted( + codex_protocol::protocol::TurnAbortedEvent { + turn_id: Some(unmatched_abort_turn_id), + reason: TurnAbortReason::Interrupted, + }, + )), + RolloutItem::TurnContext(current_context_item.clone()), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: current_turn_id, + last_agent_message: None, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(current_model.to_string()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(current_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clears_reference_context_item() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let incomplete_turn_id = "trailing-incomplete-turn".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "incomplete".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); +} + +#[tokio::test] +async fn record_initial_history_resumed_trailing_incomplete_turn_preserves_turn_context_item() { + let (session, turn_context) = make_session_and_context().await; + let current_context_item = turn_context.to_turn_context_item(); + let current_turn_id = current_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: current_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "incomplete".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(current_context_item.clone()), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(turn_context.model_info.slug.clone()) + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize seeded reference context item"), + serde_json::to_value(Some(current_context_item)) + .expect("serialize expected reference context item") + ); +} + +#[tokio::test] +async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clears_reference_context_item() + { + let (session, turn_context) = make_session_and_context().await; + let previous_model = "previous-rollout-model"; + let previous_context_item = TurnContextItem { + turn_id: Some(turn_context.sub_id.clone()), + cwd: turn_context.cwd.clone(), + approval_policy: turn_context.approval_policy.value(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + network: None, + model: previous_model.to_string(), + personality: turn_context.personality, + collaboration_mode: Some(turn_context.collaboration_mode.clone()), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: Some(turn_context.truncation_policy.into()), + }; + let previous_turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let compacted_incomplete_turn_id = "compacted-incomplete-turn".to_string(); + let replacing_turn_id = "replacing-turn".to_string(); + + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: previous_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: previous_turn_id, + last_agent_message: None, + }, + )), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: compacted_incomplete_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "compacted".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(Vec::new()), + }), + // A newer TurnStarted replaces the incomplete compacted turn without a matching + // completion/abort for the old one. + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: replacing_turn_id, + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + ]; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: rollout_items, + rollout_path: PathBuf::from("/tmp/resume.jsonl"), + })) + .await; + + assert_eq!( + session.previous_model().await, + Some(previous_model.to_string()) + ); + assert!(session.reference_context_item().await.is_none()); +} From 51dacc558541e26310d40cc5821a00047b61135d Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 12:10:27 -0800 Subject: [PATCH 22/64] Make rollout reconstruction comment timeless --- codex-rs/core/src/codex/rollout_reconstruction.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index e92b37ba7cf..0391f4235bc 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -18,8 +18,8 @@ impl Session { // 2) resume/fork hydration metadata (`previous_model` and // `reference_context_item`) // - // `TurnContextItem` is now only persisted when the current user turn actually - // emits model-visible context items (during normal context updates or when + // A `TurnContextItem` appears in rollout only for user turns that + // emit model-visible context items (during normal context updates or when // mid-turn compaction reinjects full initial context). That means replay only // needs to answer: // - which surviving user turn last established a model baseline From dd6a25fdf3d49424c5ea39b0f8ac9901d1321d56 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 12:12:35 -0800 Subject: [PATCH 23/64] Clarify replay comment for reference context --- codex-rs/core/src/codex/rollout_reconstruction.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 0391f4235bc..d1c183dbe64 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -22,8 +22,11 @@ impl Session { // emit model-visible context items (during normal context updates or when // mid-turn compaction reinjects full initial context). That means replay only // needs to answer: - // - which surviving user turn last established a model baseline - // - whether any later compaction cleared that baseline before resume + // - which surviving user turn last established the + // `reference_context_item` baseline for context diffing + // - which `previous_model` that same turn carried + // - whether any later compaction cleared that + // `reference_context_item` before resume // // We model that with: // - one active turn span while walking lifecycle events From 912fefefce01e54a58ea56f964f6cbf18d90ec7c Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 12:14:51 -0800 Subject: [PATCH 24/64] Trim legacy detail from rollout replay comment --- codex-rs/core/src/codex/rollout_reconstruction.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index d1c183dbe64..be36a4bf7a6 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -31,7 +31,6 @@ impl Session { // We model that with: // - one active turn span while walking lifecycle events // - one finalized metadata segment per surviving user turn - // - conservative `Compaction` markers for standalone compaction outside user turns // // `ThreadRolledBack` updates both: // - history: drop user turns from reconstructed response items From b9da1e7dc3e02dd72ab0ec2c9b367cbc569dfdee Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 13:42:02 -0800 Subject: [PATCH 25/64] Rename replay compaction marker for clarity --- codex-rs/core/src/codex/rollout_reconstruction.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index be36a4bf7a6..3c84548f5da 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -56,11 +56,10 @@ impl Session { #[derive(Debug)] enum RolloutReplayMetaSegment { UserTurn(Box), - // Unexpected for modern rollouts, where compaction should normally happen inside a - // turn span. We keep this as a minimal fallback for standalone/legacy compaction so - // later resume conservatively clears the baseline until another `TurnContextItem` - // re-establishes it. - Compaction, + // A later segment cleared any older `reference_context_item` without producing a new + // surviving user-turn baseline. This can come from standalone compaction turns or + // legacy/unmatched compaction replay. + ReferenceContextCleared, } let mut history = ContextManager::new(); @@ -78,7 +77,7 @@ impl Session { }, ))); } else if active_turn.cleared_reference_context_item { - replayed_segments.push(RolloutReplayMetaSegment::Compaction); + replayed_segments.push(RolloutReplayMetaSegment::ReferenceContextCleared); } }; @@ -106,7 +105,7 @@ impl Session { active_turn.reference_context_item = None; active_turn.cleared_reference_context_item = true; } else { - replayed_segments.push(RolloutReplayMetaSegment::Compaction); + replayed_segments.push(RolloutReplayMetaSegment::ReferenceContextCleared); } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { @@ -213,7 +212,7 @@ impl Session { for segment in replayed_segments.iter().rev() { match segment { - RolloutReplayMetaSegment::Compaction => { + RolloutReplayMetaSegment::ReferenceContextCleared => { compaction_cleared_reference_context_item = true; } RolloutReplayMetaSegment::UserTurn(turn) => { From 687e137347671bb647c4470a18532a0822ce3b35 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 15:17:00 -0800 Subject: [PATCH 26/64] Simplify rollout replay metadata resolution --- .../core/src/codex/rollout_reconstruction.rs | 81 ++++++------------- 1 file changed, 25 insertions(+), 56 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 3c84548f5da..19352641b21 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -31,6 +31,8 @@ impl Session { // We model that with: // - one active turn span while walking lifecycle events // - one finalized metadata segment per surviving user turn + // - legacy `TurnContextItem` / `Compacted` entries contributing directly to that same + // segment stream when no matching turn span exists // // `ThreadRolledBack` updates both: // - history: drop user turns from reconstructed response items @@ -63,7 +65,6 @@ impl Session { } let mut history = ContextManager::new(); - let mut saw_turn_lifecycle_event = false; let mut active_turn: Option = None; let mut replayed_segments = Vec::new(); let push_replayed_turn = |replayed_segments: &mut Vec, @@ -135,7 +136,6 @@ impl Session { } } RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - saw_turn_lifecycle_event = true; if let Some(active_turn) = active_turn.take() { push_replayed_turn(&mut replayed_segments, active_turn); } @@ -148,7 +148,6 @@ impl Session { }); } RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { - saw_turn_lifecycle_event = true; if active_turn .as_ref() .is_some_and(|turn| turn.turn_id == event.turn_id) @@ -158,7 +157,6 @@ impl Session { } } RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { - saw_turn_lifecycle_event = true; match event.turn_id.as_deref() { Some(aborted_turn_id) if active_turn @@ -195,6 +193,14 @@ impl Session { active_turn.previous_model = Some(ctx.model.clone()); active_turn.reference_context_item = Some(ctx.clone()); active_turn.cleared_reference_context_item = false; + } else { + replayed_segments.push(RolloutReplayMetaSegment::UserTurn(Box::new( + ReplayedUserTurn { + previous_model: Some(ctx.model.clone()), + reference_context_item: Some(ctx.clone()), + cleared_reference_context_item: false, + }, + ))); } } _ => {} @@ -205,63 +211,26 @@ impl Session { push_replayed_turn(&mut replayed_segments, active_turn); } - let (previous_model, reference_context_item) = if saw_turn_lifecycle_event { - let mut compaction_cleared_reference_context_item = false; - let mut previous_model = None; - let mut reference_context_item = None; - - for segment in replayed_segments.iter().rev() { - match segment { - RolloutReplayMetaSegment::ReferenceContextCleared => { - compaction_cleared_reference_context_item = true; - } - RolloutReplayMetaSegment::UserTurn(turn) => { - if let Some(turn_previous_model) = &turn.previous_model { - previous_model = Some(turn_previous_model.clone()); - if !compaction_cleared_reference_context_item - && !turn.cleared_reference_context_item - { - reference_context_item = turn.reference_context_item.clone(); - } - break; - } - if turn.cleared_reference_context_item { - compaction_cleared_reference_context_item = true; - } - } + let mut previous_model = None; + let mut reference_context_item = None; + for segment in replayed_segments { + match segment { + RolloutReplayMetaSegment::ReferenceContextCleared => { + reference_context_item = None; } - } - - (previous_model, reference_context_item) - } else { - // Legacy/minimal fallback (no lifecycle events): use the last persisted - // `TurnContextItem` in rollout order and conservatively clear the baseline when a - // later `Compacted` item exists. - let mut legacy_last_turn_context_item: Option = None; - let mut legacy_saw_compaction_after_last_turn_context = false; - for item in rollout_items.iter().rev() { - match item { - RolloutItem::Compacted(_) => { - legacy_saw_compaction_after_last_turn_context = true; + RolloutReplayMetaSegment::UserTurn(turn) => { + if let Some(turn_previous_model) = turn.previous_model { + previous_model = Some(turn_previous_model); + } + if turn.cleared_reference_context_item { + reference_context_item = None; } - RolloutItem::TurnContext(ctx) => { - legacy_last_turn_context_item = Some(ctx.clone()); - break; + if let Some(turn_reference_context_item) = turn.reference_context_item { + reference_context_item = Some(turn_reference_context_item); } - _ => {} } } - - let previous_model = legacy_last_turn_context_item - .as_ref() - .map(|ctx| ctx.model.clone()); - let reference_context_item = if legacy_saw_compaction_after_last_turn_context { - None - } else { - legacy_last_turn_context_item - }; - (previous_model, reference_context_item) - }; + } RolloutReconstruction { history: history.raw_items().to_vec(), From 6113ae8184940de5947ffde8b6a5eb203304f812 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 16:14:20 -0800 Subject: [PATCH 27/64] Advance reference context baseline without prompt diffs --- codex-rs/core/src/codex.rs | 54 ++++++++++++++++--- .../core/src/codex/rollout_reconstruction.rs | 8 +-- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 1ac7834671d..5384028e3b8 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -3004,13 +3004,17 @@ impl Session { if !context_items.is_empty() { self.record_conversation_items(turn_context, &context_items) .await; - let turn_context_item = turn_context.to_turn_context_item(); - self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) - .await; - - let mut state = self.state.lock().await; - state.set_reference_context_item(Some(turn_context_item)); } + + // Advance the diff baseline even when this turn emitted no model-visible context items. + // A model/context change can still matter for later turns, and resume replay must see the + // same latest `TurnContextItem` that runtime diffing used. + let turn_context_item = turn_context.to_turn_context_item(); + self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) + .await; + + let mut state = self.state.lock().await; + state.set_reference_context_item(Some(turn_context_item)); } pub(crate) async fn update_token_usage_info( @@ -8541,6 +8545,44 @@ mod tests { assert_eq!(history.raw_items().to_vec(), expected_history); } + #[tokio::test] + async fn record_context_updates_and_set_reference_context_item_updates_baseline_without_emitting_diffs( + ) { + let (session, previous_context) = make_session_and_context().await; + let next_model = if previous_context.model_info.slug == "gpt-5.1" { + "gpt-5" + } else { + "gpt-5.1" + }; + let turn_context = previous_context + .with_model(next_model.to_string(), &session.services.models_manager) + .await; + let previous_context_item = previous_context.to_turn_context_item(); + { + let mut state = session.state.lock().await; + state.set_reference_context_item(Some(previous_context_item.clone())); + } + + let update_items = + session.build_settings_update_items(Some(&previous_context_item), None, &turn_context); + assert_eq!(update_items, Vec::new()); + + session + .record_context_updates_and_set_reference_context_item(&turn_context, None) + .await; + + assert_eq!( + session.clone_history().await.raw_items().to_vec(), + Vec::new() + ); + assert_eq!( + serde_json::to_value(session.reference_context_item().await) + .expect("serialize current context item"), + serde_json::to_value(Some(turn_context.to_turn_context_item())) + .expect("serialize expected context item") + ); + } + #[tokio::test] async fn build_initial_context_prepends_model_switch_message() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 19352641b21..c681c8a97fa 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -18,10 +18,10 @@ impl Session { // 2) resume/fork hydration metadata (`previous_model` and // `reference_context_item`) // - // A `TurnContextItem` appears in rollout only for user turns that - // emit model-visible context items (during normal context updates or when - // mid-turn compaction reinjects full initial context). That means replay only - // needs to answer: + // A `TurnContextItem` appears in rollout for user turns that establish or advance the + // context-diff baseline, whether or not that turn emitted model-visible context items. + // Mid-turn compaction can also re-establish the baseline by reinjecting full initial + // context. That means replay only needs to answer: // - which surviving user turn last established the // `reference_context_item` baseline for context diffing // - which `previous_model` that same turn carried From 491a5327c4c077189bb756e33f3f00a3b3eb0092 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 16:44:53 -0800 Subject: [PATCH 28/64] Narrow TurnContext rollout persistence --- codex-rs/core/src/codex.rs | 30 +++++++++++++++++++++--------- codex-rs/protocol/src/protocol.rs | 8 ++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5384028e3b8..c97eb2e528e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2971,7 +2971,8 @@ impl Session { state.reference_context_item() } - /// Persist the latest turn context snapshot only when we emit model-visible context updates. + /// Persist the latest turn context snapshot for the first real user turn and for + /// steady-state turns that emit model-visible context updates. /// /// When the reference snapshot is missing, this injects full initial context. Otherwise, it /// emits only settings diff items. @@ -2988,7 +2989,10 @@ impl Session { turn_context: &TurnContext, previous_user_turn_model: Option<&str>, ) { - let reference_context_item = self.reference_context_item().await; + let reference_context_item = { + let state = self.state.lock().await; + state.reference_context_item() + }; let should_inject_full_context = reference_context_item.is_none(); let context_items = if should_inject_full_context { self.build_initial_context(turn_context, previous_user_turn_model) @@ -3001,18 +3005,26 @@ impl Session { turn_context, ) }; + let turn_context_item = turn_context.to_turn_context_item(); if !context_items.is_empty() { self.record_conversation_items(turn_context, &context_items) .await; } + if previous_user_turn_model.is_none() + || (!should_inject_full_context && !context_items.is_empty()) + { + // Keep rollout TurnContext entries to: + // - the first real user turn (to recover `previous_model` on resume) + // - steady-state turns that emitted explicit context diffs + // Full reinjection after compaction is tracked in runtime state only; resume will + // conservatively fall back to a missing baseline and reinject again if needed. + self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) + .await; + } - // Advance the diff baseline even when this turn emitted no model-visible context items. - // A model/context change can still matter for later turns, and resume replay must see the - // same latest `TurnContextItem` that runtime diffing used. - let turn_context_item = turn_context.to_turn_context_item(); - self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) - .await; - + // Advance the in-memory diff baseline even when this turn emitted no model-visible + // context items. This keeps later runtime diffing aligned with the current turn state + // without forcing rollout `TurnContextItem` churn after compaction/no-op turns. let mut state = self.state.lock().await; state.set_reference_context_item(Some(turn_context_item)); } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index b5aaf47ea65..9a631ea0bd0 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2124,10 +2124,10 @@ pub struct TurnContextNetworkItem { pub denied_domains: Vec, } -/// Persist only when the same turn also persists the corresponding -/// model-visible context updates (diffs or full reinjection), so -/// resume/fork does not use a `reference_context_item` whose context -/// was never actually visible to the model. +/// Persist for the first real user turn in a thread and for later turns that +/// persist corresponding model-visible context updates, so resume/fork can +/// recover `previous_model` without treating post-compaction reinjection as a +/// durable `reference_context_item` baseline. #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct TurnContextItem { #[serde(default, skip_serializing_if = "Option::is_none")] From aa404bdd06902dc2064eccf87a87684c541ba088 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 16:45:05 -0800 Subject: [PATCH 29/64] Clarify replay TurnContext comment --- codex-rs/core/src/codex/rollout_reconstruction.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index c681c8a97fa..c34c277f337 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -18,10 +18,9 @@ impl Session { // 2) resume/fork hydration metadata (`previous_model` and // `reference_context_item`) // - // A `TurnContextItem` appears in rollout for user turns that establish or advance the - // context-diff baseline, whether or not that turn emitted model-visible context items. - // Mid-turn compaction can also re-establish the baseline by reinjecting full initial - // context. That means replay only needs to answer: + // A `TurnContextItem` appears in rollout only for user turns that emit model-visible + // context items. Mid-turn compaction can also re-establish the baseline by reinjecting + // full initial context. That means replay only needs to answer: // - which surviving user turn last established the // `reference_context_item` baseline for context diffing // - which `previous_model` that same turn carried From 12744e7107897a7854b470f79eeceb1e6c232ebe Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 16:50:30 -0800 Subject: [PATCH 30/64] Make rollout metadata resolution tail-oriented --- .../core/src/codex/rollout_reconstruction.rs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index c34c277f337..412c6ef09f3 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -13,7 +13,7 @@ impl Session { turn_context: &TurnContext, rollout_items: &[RolloutItem], ) -> RolloutReconstruction { - // Replay rollout items once and compute two things in lockstep: + // Replay rollout items once and compute two things: // 1) reconstructed conversation history (via `ContextManager`) // 2) resume/fork hydration metadata (`previous_model` and // `reference_context_item`) @@ -37,7 +37,11 @@ impl Session { // - history: drop user turns from reconstructed response items // - metadata segments: drop the same number of surviving user-turn segments // - // This keeps history reconstruction and resume/fork hydration on the same replay. + // History replay must stay forward because `Compacted { replacement_history: None }` + // rebuilds from the current history snapshot. Resume metadata resolution, however, is + // intentionally tail-oriented: once we have finalized metadata segments in rollout order, + // we scan them newest-to-oldest. That keeps the metadata rules easy to migrate to a + // future reverse rollout reader without changing the forward history replay above. #[derive(Debug)] struct ActiveRolloutTurn { turn_id: String, @@ -212,20 +216,24 @@ impl Session { let mut previous_model = None; let mut reference_context_item = None; - for segment in replayed_segments { + let mut reference_context_item_cleared = false; + for segment in replayed_segments.iter().rev() { match segment { RolloutReplayMetaSegment::ReferenceContextCleared => { - reference_context_item = None; + reference_context_item_cleared = true; } RolloutReplayMetaSegment::UserTurn(turn) => { - if let Some(turn_previous_model) = turn.previous_model { - previous_model = Some(turn_previous_model); - } if turn.cleared_reference_context_item { - reference_context_item = None; + reference_context_item_cleared = true; + } + if !reference_context_item_cleared + && let Some(turn_reference_context_item) = &turn.reference_context_item + { + reference_context_item = Some(turn_reference_context_item.clone()); } - if let Some(turn_reference_context_item) = turn.reference_context_item { - reference_context_item = Some(turn_reference_context_item); + if let Some(turn_previous_model) = &turn.previous_model { + previous_model = Some(turn_previous_model.clone()); + break; } } } From 1a0b2f39b471038ce418b0e143a2bd45bc6d60f5 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 17:12:02 -0800 Subject: [PATCH 31/64] Make rollout replay reverse-scan friendly --- .../core/src/codex/rollout_reconstruction.rs | 553 +++++++++++------- 1 file changed, 341 insertions(+), 212 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 412c6ef09f3..549b00ac962 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -7,240 +7,369 @@ pub(super) struct RolloutReconstruction { pub(super) reference_context_item: Option, } -impl Session { - pub(super) async fn reconstruct_history_from_rollout( - &self, - turn_context: &TurnContext, - rollout_items: &[RolloutItem], - ) -> RolloutReconstruction { - // Replay rollout items once and compute two things: - // 1) reconstructed conversation history (via `ContextManager`) - // 2) resume/fork hydration metadata (`previous_model` and - // `reference_context_item`) - // - // A `TurnContextItem` appears in rollout only for user turns that emit model-visible - // context items. Mid-turn compaction can also re-establish the baseline by reinjecting - // full initial context. That means replay only needs to answer: - // - which surviving user turn last established the - // `reference_context_item` baseline for context diffing - // - which `previous_model` that same turn carried - // - whether any later compaction cleared that - // `reference_context_item` before resume - // - // We model that with: - // - one active turn span while walking lifecycle events - // - one finalized metadata segment per surviving user turn - // - legacy `TurnContextItem` / `Compacted` entries contributing directly to that same - // segment stream when no matching turn span exists - // - // `ThreadRolledBack` updates both: - // - history: drop user turns from reconstructed response items - // - metadata segments: drop the same number of surviving user-turn segments - // - // History replay must stay forward because `Compacted { replacement_history: None }` - // rebuilds from the current history snapshot. Resume metadata resolution, however, is - // intentionally tail-oriented: once we have finalized metadata segments in rollout order, - // we scan them newest-to-oldest. That keeps the metadata rules easy to migrate to a - // future reverse rollout reader without changing the forward history replay above. - #[derive(Debug)] - struct ActiveRolloutTurn { - turn_id: String, - saw_user_message: bool, - previous_model: Option, - reference_context_item: Option, - cleared_reference_context_item: bool, +#[derive(Debug)] +struct HistoryCheckpoint { + prefix_len: usize, + replacement_history: Option>, + message: String, +} + +#[derive(Debug, Default)] +struct ReverseHistoryCollector { + rollback_user_turns_to_skip: usize, + kept_items_rev: Vec, + pending_items_rev: Vec, + pending_keep_start: usize, +} + +impl ReverseHistoryCollector { + fn record_response_item(&mut self, item: &ResponseItem) { + if self.rollback_user_turns_to_skip == 0 { + self.kept_items_rev.push(item.clone()); + return; } - #[derive(Debug)] - struct ReplayedUserTurn { - previous_model: Option, - reference_context_item: Option, - cleared_reference_context_item: bool, + self.pending_items_rev.push(item.clone()); + if matches!(item, ResponseItem::Message { role, .. } if role == "user") { + self.rollback_user_turns_to_skip -= 1; + self.pending_keep_start = self.pending_items_rev.len(); + if self.rollback_user_turns_to_skip == 0 { + self.pending_items_rev.clear(); + self.pending_keep_start = 0; + } } + } - #[derive(Debug)] - enum RolloutReplayMetaSegment { - UserTurn(Box), - // A later segment cleared any older `reference_context_item` without producing a new - // surviving user-turn baseline. This can come from standalone compaction turns or - // legacy/unmatched compaction replay. - ReferenceContextCleared, + fn record_rollback(&mut self, num_turns: u32) { + self.rollback_user_turns_to_skip = self + .rollback_user_turns_to_skip + .saturating_add(usize::try_from(num_turns).unwrap_or(usize::MAX)); + } + + fn finish_with_base_history( + mut self, + base_history: Vec, + truncation_policy: TruncationPolicy, + ) -> Vec { + for item in base_history.iter().rev() { + self.record_response_item(item); } + let mut surviving_items = if self.rollback_user_turns_to_skip > 0 { + self.pending_items_rev[self.pending_keep_start..] + .iter() + .rev() + .cloned() + .collect() + } else { + Vec::new() + }; + surviving_items.extend(self.kept_items_rev.into_iter().rev()); + let mut history = ContextManager::new(); - let mut active_turn: Option = None; - let mut replayed_segments = Vec::new(); - let push_replayed_turn = |replayed_segments: &mut Vec, - active_turn: ActiveRolloutTurn| { - if active_turn.saw_user_message { - replayed_segments.push(RolloutReplayMetaSegment::UserTurn(Box::new( - ReplayedUserTurn { - previous_model: active_turn.previous_model, - reference_context_item: active_turn.reference_context_item, - cleared_reference_context_item: active_turn.cleared_reference_context_item, - }, - ))); - } else if active_turn.cleared_reference_context_item { - replayed_segments.push(RolloutReplayMetaSegment::ReferenceContextCleared); + history.record_items(surviving_items.iter(), truncation_policy); + history.raw_items().to_vec() + } +} + +#[derive(Debug, Default)] +struct ReverseMetadataTurn { + turn_id: Option, + saw_user_message: bool, + previous_model: Option, + reference_context_item: Option, + reference_context_item_cleared: bool, +} + +#[derive(Debug, Default)] +struct ReverseMetadataState { + rollback_user_turns_to_skip: usize, + active_turn: Option, + previous_model: Option, + reference_context_item: Option, + reference_context_item_cleared: bool, +} + +impl ReverseMetadataState { + fn record_rollback(&mut self, num_turns: u32) { + self.rollback_user_turns_to_skip = self + .rollback_user_turns_to_skip + .saturating_add(usize::try_from(num_turns).unwrap_or(usize::MAX)); + } + + fn record_turn_end(&mut self, turn_id: Option<&str>) { + match (&mut self.active_turn, turn_id) { + (None, Some(turn_id)) => { + self.active_turn = Some(ReverseMetadataTurn { + turn_id: Some(turn_id.to_string()), + ..Default::default() + }); } - }; + (None, None) => { + self.active_turn = Some(ReverseMetadataTurn::default()); + } + (Some(active_turn), Some(turn_id)) if active_turn.turn_id.is_none() => { + active_turn.turn_id = Some(turn_id.to_string()); + } + (Some(active_turn), Some(turn_id)) + if active_turn.turn_id.as_deref() == Some(turn_id) => {} + (Some(_), Some(_)) => { + // Ignore unmatched end markers for some other turn; they should not consume the + // newer turn we are currently walking backwards through. + } + (Some(_), None) => {} + } + } - for item in rollout_items { - match item { - RolloutItem::ResponseItem(response_item) => { - history.record_items( - std::iter::once(response_item), - turn_context.truncation_policy, - ); - } - RolloutItem::Compacted(compacted) => { - if let Some(replacement) = &compacted.replacement_history { - history.replace(replacement.clone()); - } else { - let user_messages = collect_user_messages(history.raw_items()); - let rebuilt = compact::build_compacted_history( - self.build_initial_context(turn_context).await, - &user_messages, - &compacted.message, - ); - history.replace(rebuilt); - } - if let Some(active_turn) = active_turn.as_mut() { - active_turn.reference_context_item = None; - active_turn.cleared_reference_context_item = true; - } else { - replayed_segments.push(RolloutReplayMetaSegment::ReferenceContextCleared); - } + fn record_turn_start(&mut self, turn_id: &str) { + if self + .active_turn + .as_ref() + .is_some_and(|turn| turn.turn_id.as_deref().is_none_or(|id| id == turn_id)) + && let Some(turn) = self.active_turn.take() + { + self.finalize_turn(turn); + } + } + + fn ensure_active_turn(&mut self) -> &mut ReverseMetadataTurn { + self.active_turn + .get_or_insert_with(ReverseMetadataTurn::default) + } + + fn record_user_message(&mut self) { + self.ensure_active_turn().saw_user_message = true; + } + + fn record_turn_context(&mut self, ctx: &TurnContextItem) { + if self.active_turn.is_none() { + let turn = self.ensure_active_turn(); + turn.saw_user_message = true; + turn.previous_model = Some(ctx.model.clone()); + turn.reference_context_item = Some(ctx.clone()); + turn.reference_context_item_cleared = false; + return; + } + + if self.active_turn.as_ref().is_some_and(|turn| { + turn.turn_id.as_deref().is_none_or(|turn_id| { + ctx.turn_id + .as_deref() + .is_none_or(|ctx_turn_id| ctx_turn_id == turn_id) + }) + }) { + let turn = self.ensure_active_turn(); + if turn.turn_id.is_none() { + turn.saw_user_message = true; + } + if turn.previous_model.is_none() { + turn.previous_model = Some(ctx.model.clone()); + } + if !turn.reference_context_item_cleared && turn.reference_context_item.is_none() { + turn.reference_context_item = Some(ctx.clone()); + } + return; + } + + self.finalize_turn(ReverseMetadataTurn { + turn_id: ctx.turn_id.clone(), + saw_user_message: true, + previous_model: Some(ctx.model.clone()), + reference_context_item: Some(ctx.clone()), + reference_context_item_cleared: false, + }); + } + + fn record_compaction(&mut self) { + let turn = self.ensure_active_turn(); + if turn.reference_context_item.is_none() { + turn.reference_context_item_cleared = true; + } + } + + fn finalize_turn(&mut self, turn: ReverseMetadataTurn) { + if turn.saw_user_message { + if self.rollback_user_turns_to_skip > 0 { + self.rollback_user_turns_to_skip -= 1; + return; + } + + if turn.reference_context_item_cleared { + self.reference_context_item = None; + self.reference_context_item_cleared = true; + } + + if self.previous_model.is_none() + && let Some(previous_model) = turn.previous_model + { + self.previous_model = Some(previous_model); + if !self.reference_context_item_cleared { + self.reference_context_item = turn.reference_context_item; } - RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { - history.drop_last_n_user_turns(rollback.num_turns); - let mut turns_to_drop = - usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); - if turns_to_drop > 0 - && active_turn - .as_ref() - .is_some_and(|turn| turn.saw_user_message) - { - // Match `drop_last_n_user_turns`: an unfinished active turn that has - // already emitted a user message is the newest user turn and should be - // dropped before we trim older finalized user-turn segments. - active_turn = None; - turns_to_drop -= 1; - } - if turns_to_drop > 0 { - let mut idx = replayed_segments.len(); - while idx > 0 && turns_to_drop > 0 { - idx -= 1; - if let RolloutReplayMetaSegment::UserTurn(_) = &replayed_segments[idx] { - replayed_segments.remove(idx); - turns_to_drop -= 1; - } - } - } + } + } else if turn.reference_context_item_cleared { + self.reference_context_item = None; + self.reference_context_item_cleared = true; + } + } + + fn finish(mut self) -> (Option, Option) { + if let Some(turn) = self.active_turn.take() { + self.finalize_turn(turn); + } + (self.previous_model, self.reference_context_item) + } + + fn resolved_previous_model(&self) -> bool { + self.previous_model.is_some() + } +} + +#[derive(Debug)] +struct TailScan { + history_collector: ReverseHistoryCollector, + history_checkpoint: Option, + previous_model: Option, + reference_context_item: Option, +} + +fn scan_rollout_tail(rollout_items: &[RolloutItem]) -> TailScan { + let mut history_collector = ReverseHistoryCollector::default(); + let mut history_checkpoint = None; + let mut metadata = ReverseMetadataState::default(); + + for (index, item) in rollout_items.iter().enumerate().rev() { + match item { + RolloutItem::ResponseItem(response_item) => { + if history_checkpoint.is_none() { + history_collector.record_response_item(response_item); } - RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - if let Some(active_turn) = active_turn.take() { - push_replayed_turn(&mut replayed_segments, active_turn); - } - active_turn = Some(ActiveRolloutTurn { - turn_id: event.turn_id.clone(), - saw_user_message: false, - previous_model: None, - reference_context_item: None, - cleared_reference_context_item: false, + } + RolloutItem::Compacted(compacted) => { + if history_checkpoint.is_none() { + history_checkpoint = Some(HistoryCheckpoint { + prefix_len: index, + replacement_history: compacted.replacement_history.clone(), + message: compacted.message.clone(), }); } - RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { - if active_turn - .as_ref() - .is_some_and(|turn| turn.turn_id == event.turn_id) - && let Some(active_turn) = active_turn.take() - { - push_replayed_turn(&mut replayed_segments, active_turn); - } - } - RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { - match event.turn_id.as_deref() { - Some(aborted_turn_id) - if active_turn - .as_ref() - .is_some_and(|turn| turn.turn_id == aborted_turn_id) => - { - if let Some(active_turn) = active_turn.take() { - push_replayed_turn(&mut replayed_segments, active_turn); - } - } - Some(_) => { - // Ignore aborts for some other turn and keep the current active turn - // alive so later `TurnContext`/`TurnComplete` events still apply. - } - None => { - if let Some(active_turn) = active_turn.take() { - push_replayed_turn(&mut replayed_segments, active_turn); - } - } - } - } - RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { - if let Some(active_turn) = active_turn.as_mut() { - active_turn.saw_user_message = true; - } - } - RolloutItem::TurnContext(ctx) => { - if let Some(active_turn) = active_turn.as_mut() - && ctx - .turn_id - .as_deref() - .is_none_or(|turn_id| turn_id == active_turn.turn_id) - { - active_turn.previous_model = Some(ctx.model.clone()); - active_turn.reference_context_item = Some(ctx.clone()); - active_turn.cleared_reference_context_item = false; - } else { - replayed_segments.push(RolloutReplayMetaSegment::UserTurn(Box::new( - ReplayedUserTurn { - previous_model: Some(ctx.model.clone()), - reference_context_item: Some(ctx.clone()), - cleared_reference_context_item: false, - }, - ))); - } + metadata.record_compaction(); + } + RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { + if history_checkpoint.is_none() { + history_collector.record_rollback(rollback.num_turns); } - _ => {} + metadata.record_rollback(rollback.num_turns); } + RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { + metadata.record_turn_end(Some(&event.turn_id)); + } + RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { + metadata.record_turn_end(event.turn_id.as_deref()); + } + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + metadata.record_turn_start(&event.turn_id); + } + RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { + metadata.record_user_message(); + } + RolloutItem::TurnContext(ctx) => { + metadata.record_turn_context(ctx); + } + _ => {} } - if let Some(active_turn) = active_turn.take() { - push_replayed_turn(&mut replayed_segments, active_turn); + if history_checkpoint.is_some() && metadata.resolved_previous_model() { + break; } + } - let mut previous_model = None; - let mut reference_context_item = None; - let mut reference_context_item_cleared = false; - for segment in replayed_segments.iter().rev() { - match segment { - RolloutReplayMetaSegment::ReferenceContextCleared => { - reference_context_item_cleared = true; - } - RolloutReplayMetaSegment::UserTurn(turn) => { - if turn.cleared_reference_context_item { - reference_context_item_cleared = true; - } - if !reference_context_item_cleared - && let Some(turn_reference_context_item) = &turn.reference_context_item - { - reference_context_item = Some(turn_reference_context_item.clone()); - } - if let Some(turn_previous_model) = &turn.previous_model { - previous_model = Some(turn_previous_model.clone()); - break; - } - } - } + let (previous_model, reference_context_item) = metadata.finish(); + TailScan { + history_collector, + history_checkpoint, + previous_model, + reference_context_item, + } +} + +fn reconstruct_history_from_tail_scan( + initial_context: &[ResponseItem], + truncation_policy: TruncationPolicy, + rollout_items: &[RolloutItem], + tail_scan: TailScan, +) -> Vec { + let base_history = match tail_scan.history_checkpoint { + Some(HistoryCheckpoint { + prefix_len: _, + replacement_history: Some(replacement_history), + .. + }) => replacement_history, + Some(HistoryCheckpoint { + prefix_len, + replacement_history: None, + message, + }) => { + let history_before = reconstruct_history_from_rollout_items( + initial_context, + truncation_policy, + &rollout_items[..prefix_len], + ); + let user_messages = collect_user_messages(&history_before); + compact::build_compacted_history(initial_context.to_vec(), &user_messages, &message) } + None => Vec::new(), + }; + + tail_scan + .history_collector + .finish_with_base_history(base_history, truncation_policy) +} + +fn reconstruct_history_from_rollout_items( + initial_context: &[ResponseItem], + truncation_policy: TruncationPolicy, + rollout_items: &[RolloutItem], +) -> Vec { + let tail_scan = scan_rollout_tail(rollout_items); + reconstruct_history_from_tail_scan(initial_context, truncation_policy, rollout_items, tail_scan) +} + +impl Session { + pub(super) async fn reconstruct_history_from_rollout( + &self, + turn_context: &TurnContext, + rollout_items: &[RolloutItem], + ) -> RolloutReconstruction { + // Read the rollout from the tail inward. + // + // The reverse scan does two things at once: + // - resolve resume metadata from the newest surviving user turn after applying + // `ThreadRolledBack` as a simple "skip N user turns" counter + // - capture the raw history suffix after the newest `Compacted` checkpoint + // + // Once that tail scan finds a `Compacted` item, older raw `ResponseItem`s no longer need + // to be read directly. `replacement_history: Some(...)` already contains the full base + // snapshot at that point, while `replacement_history: None` rebuilds that snapshot by + // recursively reconstructing the rollout prefix before the compaction and passing its user + // messages into `build_compacted_history`. + // + // This keeps replay aligned with the eventual reverse-file reader we want to build: the + // tail scan identifies the newest surviving baseline information and the newest history + // checkpoint, then recursive prefix rebuild handles only the compacted prefix when needed. + let initial_context = self.build_initial_context(turn_context).await; + let tail_scan = scan_rollout_tail(rollout_items); + let previous_model = tail_scan.previous_model.clone(); + let reference_context_item = tail_scan.reference_context_item.clone(); + let history = reconstruct_history_from_tail_scan( + &initial_context, + turn_context.truncation_policy, + rollout_items, + tail_scan, + ); RolloutReconstruction { - history: history.raw_items().to_vec(), + history, previous_model, reference_context_item, } From bee596b357fb0fb7174676d42f22e48b42cfaa18 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 17:30:35 -0800 Subject: [PATCH 32/64] Document reverse replay rollback buffer --- codex-rs/core/src/codex/rollout_reconstruction.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 549b00ac962..b06cf2d4495 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -18,6 +18,11 @@ struct HistoryCheckpoint { struct ReverseHistoryCollector { rollback_user_turns_to_skip: usize, kept_items_rev: Vec, + // While scanning newest-to-oldest, we cannot tell whether a response item survives rollback + // until we either hit the user message that anchors its turn or exhaust the rollback skip + // count. These items are buffered here so reverse replay matches + // `ContextManager::drop_last_n_user_turns`, which drops everything from the Nth-last user + // message onward but still preserves items before the first surviving user message. pending_items_rev: Vec, pending_keep_start: usize, } From 931fcb2a175b032ad790d96f1b0888c49fd6e940 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 17:42:54 -0800 Subject: [PATCH 33/64] Document reverse replay collector fields --- codex-rs/core/src/codex/rollout_reconstruction.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index b06cf2d4495..3a695aa9663 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -16,7 +16,10 @@ struct HistoryCheckpoint { #[derive(Debug, Default)] struct ReverseHistoryCollector { + // Number of newest user turns that still need to be removed while scanning backwards. rollback_user_turns_to_skip: usize, + // Items already known to survive rollback because we have either consumed all rollback skips + // or scanned past the user message that anchors the dropped turns. kept_items_rev: Vec, // While scanning newest-to-oldest, we cannot tell whether a response item survives rollback // until we either hit the user message that anchors its turn or exhaust the rollback skip @@ -24,6 +27,9 @@ struct ReverseHistoryCollector { // `ContextManager::drop_last_n_user_turns`, which drops everything from the Nth-last user // message onward but still preserves items before the first surviving user message. pending_items_rev: Vec, + // Split point inside `pending_items_rev` for the overflow case where rollback drops more user + // turns than exist after the current checkpoint. The suffix starting here is the buffered + // prefix that still survives because it occurred before the first dropped user message. pending_keep_start: usize, } From 6d0338012a2a6128499f4cbd6dbda51fa029320c Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 17:44:20 -0800 Subject: [PATCH 34/64] Document rollout reconstruction return type --- codex-rs/core/src/codex/rollout_reconstruction.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 3a695aa9663..af04de82e62 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -1,5 +1,7 @@ use super::*; +// Return value of `Session::reconstruct_history_from_rollout`, bundling the rebuilt history with +// the resume/fork hydration metadata derived from the same replay. #[derive(Debug)] pub(super) struct RolloutReconstruction { pub(super) history: Vec, From 01141fd494ec0795a14ddb458249535b0ce6b27e Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 18:16:45 -0800 Subject: [PATCH 35/64] Add rollout lazy loading design note --- ...ollout-reconstruction-lazy-loading-plan.md | 279 ++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 docs/rollout-reconstruction-lazy-loading-plan.md diff --git a/docs/rollout-reconstruction-lazy-loading-plan.md b/docs/rollout-reconstruction-lazy-loading-plan.md new file mode 100644 index 00000000000..a3b8d601f73 --- /dev/null +++ b/docs/rollout-reconstruction-lazy-loading-plan.md @@ -0,0 +1,279 @@ +# Rollout Reconstruction Lazy Reverse Loading Plan + +## Summary + +The current rollout reconstruction code has the right intent, but not the right long-term shape for lazy reverse loading. + +Today, [`reconstruct_history_from_rollout`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L351-L389) still takes a fully materialized slice of rollout items, and the history path still assumes in-memory prefix rereads through [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17), [`scan_rollout_tail`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L249-L307), and [`reconstruct_history_from_tail_scan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L309-L340). + +The target design should instead: + +- consume rollout items through a lazy reverse source that can load earlier items on demand +- compute [`previous_model`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) and [`reference_context_item`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) eagerly during initial reconstruction +- return a lazy history object that can later accept additional backtracking and resume loading older rollout items without restarting reconstruction from scratch +- keep the replay state alive in memory for the rest of the process + +## Why The Current Shape Falls Short + +The current refactor improved the direction of travel, but it still has two in-memory assumptions that work against the future design. + +First, [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) stores a slice index (`prefix_len`) rather than an opaque position in the rollout source. That is useful for the current implementation, but it ties compaction replay to already-loaded rollout data. + +Second, the `replacement_history: None` path in [`reconstruct_history_from_tail_scan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L321-L333) recursively slices the already-loaded rollout prefix and rebuilds from there. That is logically correct, but operationally it assumes that the full prefix is already resident in memory. + +The current code is therefore better than a pure forward replay, but it is still a one-shot reconstruction function rather than a persistent lazy replay object. + +## Key Design Principle + +Almost all rollout replay logic should operate as a stateful lazy reverse consumer over rollout items. + +That means the replay code should: + +- read newest-to-oldest +- stop as soon as it has enough information for the current request +- retain enough state to later continue consuming older items when more history is needed + +The only part that should remain eagerly resolved is resume metadata: + +- [`previous_model`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) +- [`reference_context_item`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) + +Those values should be fully computed during initial construction so that the lazy history object only needs to manage history materialization and future backtracking. + +## Proposed API Shape + +[`reconstruct_history_from_rollout`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L351-L389) should stop taking `&[RolloutItem]` and instead take a lazy reverse source. + +A plain Rust `Iterator` is not the right abstraction here because we need: + +- async loading +- reverse pagination +- stop and resume +- opaque cursors for loading older data later + +A better shape is a custom async source trait: + +```rust +#[async_trait] +trait ReverseRolloutSource { + type Cursor: Clone + Send + Sync + 'static; + + async fn load_earlier( + &mut self, + before: Option, + limit: usize, + ) -> CodexResult>; +} + +struct ReverseRolloutChunk { + items_newest_to_oldest: Vec<(C, RolloutItem)>, + reached_start: bool, +} +``` + +The exact naming is flexible, but the important properties are: + +- items are yielded newest-to-oldest +- the caller can ask for more older items later +- the source owns the pagination details + +## Proposed Return Type Shape + +[`RolloutReconstruction`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L5-L10) should keep eager metadata, but replace eager `Vec` history with a lazy history object. + +A target shape would look like: + +```rust +pub(super) struct RolloutReconstruction { + pub(super) history: LazyReconstructedHistory, + pub(super) previous_model: Option, + pub(super) reference_context_item: Option, +} +``` + +This keeps metadata simple while moving only history into the lazy segment. + +## Lazy History State + +The lazy history object should be persistent and own both: + +- the reverse rollout source +- the in-memory replay state accumulated so far + +A useful shape is: + +```rust +struct LazyReconstructedHistory { + source: S, + earliest_loaded_cursor: Option, + reached_start: bool, + rollback_debt: usize, + replay: HistoryReplayState, +} +``` + +Where `rollback_debt` is the persisted version of the same idea currently used by [`ReverseHistoryCollector`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L19-L85) and [`ReverseMetadataState`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L97-L239): a count of newest user turns that still need to be skipped. + +That is the right semantic model because it lines up with [`drop_last_n_user_turns`](../codex-rs/core/src/context_manager/history.rs#L201-L230): rollback is fundamentally “skip N newest user turns,” not “cut to a precomputed absolute boundary.” + +## Replace `HistoryCheckpoint` With A Deferred Base + +The current [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) concept is useful, but it should evolve into a lazy base description rather than a slice-based checkpoint. + +A target shape is: + +```rust +enum HistoryBase { + Unknown, + StartOfFile, + Replacement(Vec), + Compacted { + summary_text: String, + before: Option, + }, +} + +struct HistoryReplayState { + base: HistoryBase, + suffix: ReverseHistoryCollector, +} +``` + +This is the key change. + +Instead of immediately rebuilding history when we hit `Compacted { replacement_history: None }`, we should record: + +- the compaction summary text +- where to resume reading earlier rollout data if we later need to materialize that prefix + +That lets lazy history retain a deferred base instead of forcing immediate recursive prefix reconstruction. + +## Reverse Consumption Rules + +The existing reverse metadata consumer in [`ReverseMetadataState`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L97-L239) is broadly the right shape and should mostly survive. + +The lazy reverse history consumer should follow these rules: + +1. `ResponseItem` + - feed into a reverse suffix collector, equivalent to the current [`ReverseHistoryCollector`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L19-L85) + - apply rollback debt the same way as today + +2. `ThreadRolledBack` + - increment rollback debt + - do not immediately rebuild history + +3. `Compacted { replacement_history: Some(...) }` + - set `base = Replacement(...)` + - stop scanning once metadata is also resolved for the current operation + +4. `Compacted { replacement_history: None }` + - set `base = Compacted { summary_text, before }` + - stop scanning once metadata is also resolved for the current operation + +5. beginning of file with no compaction base found + - set `base = StartOfFile` + +That means the replay logic only reads as much rollout as the current operation requires. + +## Materializing History + +The lazy history object should expose a method that materializes a `Vec` only when needed. + +A target interface could be: + +```rust +impl LazyReconstructedHistory { + async fn materialize( + &mut self, + initial_context: &[ResponseItem], + truncation_policy: TruncationPolicy, + ) -> CodexResult>; +} +``` + +The materialization rules would be: + +- `Replacement(history)` returns that history as the base +- `StartOfFile` uses an empty base +- `Compacted { summary_text, before }` recursively materializes the older prefix before `before`, collects user messages from that prefix, and then calls [`build_compacted_history`](../codex-rs/core/src/compact.rs#L379-L390) +- after the base is ready, append the surviving suffix collected by the reverse history collector + +This keeps the current compaction semantics, but moves prefix loading behind a lazy cursor instead of an in-memory slice. + +## Supporting Future Backtracking + +The lazy history object should also expose an operation for additional rollback requests that go before the current loaded boundary. + +A target interface could be: + +```rust +impl LazyReconstructedHistory { + async fn apply_backtracking(&mut self, additional_user_turns: u32) -> CodexResult<()>; +} +``` + +This method should: + +- add to the stored rollback debt +- keep reading earlier rollout items until either: + - the debt is satisfiable from loaded data, or + - the source reaches the beginning of the rollout +- update the existing replay state rather than reconstructing from scratch + +That is the critical affordance missing from the current design. + +## Recommended Refactor Sequence + +A staged migration should keep the semantic rules stable while improving the structure. + +1. Introduce a reverse rollout source abstraction. + - keep the current in-memory slice source as the first implementation + - make the replay logic consume that abstraction instead of `&[RolloutItem]` + +2. Replace [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) with a deferred base enum. + - remove `prefix_len` + - store an opaque “before this point” cursor instead + +3. Split eager metadata from lazy history. + - keep [`previous_model`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) and [`reference_context_item`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) eager + - make `history` lazy and persistent + +4. Replace [`scan_rollout_tail`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L249-L307) with a persistent reverse consumer. + - consume chunks incrementally + - stop when current caller has enough data + - preserve replay state for later backfill + +5. Add `apply_backtracking` to the lazy history object. + - use rollback debt rather than forcing immediate full rematerialization + +6. Only after that, add a real reverse file reader. + - the in-memory behavior and API should already match the desired loading semantics + +## What Can Stay From The Current PR + +The following pieces are directionally correct and worth keeping: + +- [`ReverseMetadataState`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L97-L239) +- [`ReverseHistoryCollector`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L19-L85) +- rollback as user-turn skip debt rather than an eager cut index + +The following pieces are temporary and should be removed in the next structural pass: + +- [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) as currently defined +- [`TailScan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L241-L247) +- [`scan_rollout_tail`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L249-L307) as a one-shot helper over `&[RolloutItem]` +- recursive prefix slicing in [`reconstruct_history_from_tail_scan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L321-L333) + +## Bottom Line + +The main architectural change is this: + +- stop treating rollout reconstruction as a one-shot function over a fully loaded slice +- start treating it as a persistent reverse replay state machine with eager metadata and lazy history + +That gives us: + +- fast initial resume/fork hydration +- a clear stop condition for reverse loading +- a straightforward path to real lazy file reads +- support for future backtracking requests that extend beyond the currently loaded history boundary From 5900705bfee1bd2aac488ca30f45b9a4f3272f09 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 18:33:33 -0800 Subject: [PATCH 36/64] Remove rollout lazy loading design note --- ...ollout-reconstruction-lazy-loading-plan.md | 279 ------------------ 1 file changed, 279 deletions(-) delete mode 100644 docs/rollout-reconstruction-lazy-loading-plan.md diff --git a/docs/rollout-reconstruction-lazy-loading-plan.md b/docs/rollout-reconstruction-lazy-loading-plan.md deleted file mode 100644 index a3b8d601f73..00000000000 --- a/docs/rollout-reconstruction-lazy-loading-plan.md +++ /dev/null @@ -1,279 +0,0 @@ -# Rollout Reconstruction Lazy Reverse Loading Plan - -## Summary - -The current rollout reconstruction code has the right intent, but not the right long-term shape for lazy reverse loading. - -Today, [`reconstruct_history_from_rollout`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L351-L389) still takes a fully materialized slice of rollout items, and the history path still assumes in-memory prefix rereads through [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17), [`scan_rollout_tail`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L249-L307), and [`reconstruct_history_from_tail_scan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L309-L340). - -The target design should instead: - -- consume rollout items through a lazy reverse source that can load earlier items on demand -- compute [`previous_model`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) and [`reference_context_item`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) eagerly during initial reconstruction -- return a lazy history object that can later accept additional backtracking and resume loading older rollout items without restarting reconstruction from scratch -- keep the replay state alive in memory for the rest of the process - -## Why The Current Shape Falls Short - -The current refactor improved the direction of travel, but it still has two in-memory assumptions that work against the future design. - -First, [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) stores a slice index (`prefix_len`) rather than an opaque position in the rollout source. That is useful for the current implementation, but it ties compaction replay to already-loaded rollout data. - -Second, the `replacement_history: None` path in [`reconstruct_history_from_tail_scan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L321-L333) recursively slices the already-loaded rollout prefix and rebuilds from there. That is logically correct, but operationally it assumes that the full prefix is already resident in memory. - -The current code is therefore better than a pure forward replay, but it is still a one-shot reconstruction function rather than a persistent lazy replay object. - -## Key Design Principle - -Almost all rollout replay logic should operate as a stateful lazy reverse consumer over rollout items. - -That means the replay code should: - -- read newest-to-oldest -- stop as soon as it has enough information for the current request -- retain enough state to later continue consuming older items when more history is needed - -The only part that should remain eagerly resolved is resume metadata: - -- [`previous_model`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) -- [`reference_context_item`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) - -Those values should be fully computed during initial construction so that the lazy history object only needs to manage history materialization and future backtracking. - -## Proposed API Shape - -[`reconstruct_history_from_rollout`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L351-L389) should stop taking `&[RolloutItem]` and instead take a lazy reverse source. - -A plain Rust `Iterator` is not the right abstraction here because we need: - -- async loading -- reverse pagination -- stop and resume -- opaque cursors for loading older data later - -A better shape is a custom async source trait: - -```rust -#[async_trait] -trait ReverseRolloutSource { - type Cursor: Clone + Send + Sync + 'static; - - async fn load_earlier( - &mut self, - before: Option, - limit: usize, - ) -> CodexResult>; -} - -struct ReverseRolloutChunk { - items_newest_to_oldest: Vec<(C, RolloutItem)>, - reached_start: bool, -} -``` - -The exact naming is flexible, but the important properties are: - -- items are yielded newest-to-oldest -- the caller can ask for more older items later -- the source owns the pagination details - -## Proposed Return Type Shape - -[`RolloutReconstruction`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L5-L10) should keep eager metadata, but replace eager `Vec` history with a lazy history object. - -A target shape would look like: - -```rust -pub(super) struct RolloutReconstruction { - pub(super) history: LazyReconstructedHistory, - pub(super) previous_model: Option, - pub(super) reference_context_item: Option, -} -``` - -This keeps metadata simple while moving only history into the lazy segment. - -## Lazy History State - -The lazy history object should be persistent and own both: - -- the reverse rollout source -- the in-memory replay state accumulated so far - -A useful shape is: - -```rust -struct LazyReconstructedHistory { - source: S, - earliest_loaded_cursor: Option, - reached_start: bool, - rollback_debt: usize, - replay: HistoryReplayState, -} -``` - -Where `rollback_debt` is the persisted version of the same idea currently used by [`ReverseHistoryCollector`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L19-L85) and [`ReverseMetadataState`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L97-L239): a count of newest user turns that still need to be skipped. - -That is the right semantic model because it lines up with [`drop_last_n_user_turns`](../codex-rs/core/src/context_manager/history.rs#L201-L230): rollback is fundamentally “skip N newest user turns,” not “cut to a precomputed absolute boundary.” - -## Replace `HistoryCheckpoint` With A Deferred Base - -The current [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) concept is useful, but it should evolve into a lazy base description rather than a slice-based checkpoint. - -A target shape is: - -```rust -enum HistoryBase { - Unknown, - StartOfFile, - Replacement(Vec), - Compacted { - summary_text: String, - before: Option, - }, -} - -struct HistoryReplayState { - base: HistoryBase, - suffix: ReverseHistoryCollector, -} -``` - -This is the key change. - -Instead of immediately rebuilding history when we hit `Compacted { replacement_history: None }`, we should record: - -- the compaction summary text -- where to resume reading earlier rollout data if we later need to materialize that prefix - -That lets lazy history retain a deferred base instead of forcing immediate recursive prefix reconstruction. - -## Reverse Consumption Rules - -The existing reverse metadata consumer in [`ReverseMetadataState`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L97-L239) is broadly the right shape and should mostly survive. - -The lazy reverse history consumer should follow these rules: - -1. `ResponseItem` - - feed into a reverse suffix collector, equivalent to the current [`ReverseHistoryCollector`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L19-L85) - - apply rollback debt the same way as today - -2. `ThreadRolledBack` - - increment rollback debt - - do not immediately rebuild history - -3. `Compacted { replacement_history: Some(...) }` - - set `base = Replacement(...)` - - stop scanning once metadata is also resolved for the current operation - -4. `Compacted { replacement_history: None }` - - set `base = Compacted { summary_text, before }` - - stop scanning once metadata is also resolved for the current operation - -5. beginning of file with no compaction base found - - set `base = StartOfFile` - -That means the replay logic only reads as much rollout as the current operation requires. - -## Materializing History - -The lazy history object should expose a method that materializes a `Vec` only when needed. - -A target interface could be: - -```rust -impl LazyReconstructedHistory { - async fn materialize( - &mut self, - initial_context: &[ResponseItem], - truncation_policy: TruncationPolicy, - ) -> CodexResult>; -} -``` - -The materialization rules would be: - -- `Replacement(history)` returns that history as the base -- `StartOfFile` uses an empty base -- `Compacted { summary_text, before }` recursively materializes the older prefix before `before`, collects user messages from that prefix, and then calls [`build_compacted_history`](../codex-rs/core/src/compact.rs#L379-L390) -- after the base is ready, append the surviving suffix collected by the reverse history collector - -This keeps the current compaction semantics, but moves prefix loading behind a lazy cursor instead of an in-memory slice. - -## Supporting Future Backtracking - -The lazy history object should also expose an operation for additional rollback requests that go before the current loaded boundary. - -A target interface could be: - -```rust -impl LazyReconstructedHistory { - async fn apply_backtracking(&mut self, additional_user_turns: u32) -> CodexResult<()>; -} -``` - -This method should: - -- add to the stored rollback debt -- keep reading earlier rollout items until either: - - the debt is satisfiable from loaded data, or - - the source reaches the beginning of the rollout -- update the existing replay state rather than reconstructing from scratch - -That is the critical affordance missing from the current design. - -## Recommended Refactor Sequence - -A staged migration should keep the semantic rules stable while improving the structure. - -1. Introduce a reverse rollout source abstraction. - - keep the current in-memory slice source as the first implementation - - make the replay logic consume that abstraction instead of `&[RolloutItem]` - -2. Replace [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) with a deferred base enum. - - remove `prefix_len` - - store an opaque “before this point” cursor instead - -3. Split eager metadata from lazy history. - - keep [`previous_model`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) and [`reference_context_item`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L8-L9) eager - - make `history` lazy and persistent - -4. Replace [`scan_rollout_tail`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L249-L307) with a persistent reverse consumer. - - consume chunks incrementally - - stop when current caller has enough data - - preserve replay state for later backfill - -5. Add `apply_backtracking` to the lazy history object. - - use rollback debt rather than forcing immediate full rematerialization - -6. Only after that, add a real reverse file reader. - - the in-memory behavior and API should already match the desired loading semantics - -## What Can Stay From The Current PR - -The following pieces are directionally correct and worth keeping: - -- [`ReverseMetadataState`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L97-L239) -- [`ReverseHistoryCollector`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L19-L85) -- rollback as user-turn skip debt rather than an eager cut index - -The following pieces are temporary and should be removed in the next structural pass: - -- [`HistoryCheckpoint`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L12-L17) as currently defined -- [`TailScan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L241-L247) -- [`scan_rollout_tail`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L249-L307) as a one-shot helper over `&[RolloutItem]` -- recursive prefix slicing in [`reconstruct_history_from_tail_scan`](../codex-rs/core/src/codex/rollout_reconstruction.rs#L321-L333) - -## Bottom Line - -The main architectural change is this: - -- stop treating rollout reconstruction as a one-shot function over a fully loaded slice -- start treating it as a persistent reverse replay state machine with eager metadata and lazy history - -That gives us: - -- fast initial resume/fork hydration -- a clear stop condition for reverse loading -- a straightforward path to real lazy file reads -- support for future backtracking requests that extend beyond the currently loaded history boundary From e94181f30ac26af0b42dde174ed482e07329d95c Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 21:17:14 -0800 Subject: [PATCH 37/64] Simplify unified rollout reconstruction --- .../core/src/codex/rollout_reconstruction.rs | 541 +++++++----------- 1 file changed, 195 insertions(+), 346 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index af04de82e62..f758c845251 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -10,83 +10,23 @@ pub(super) struct RolloutReconstruction { } #[derive(Debug)] -struct HistoryCheckpoint { - prefix_len: usize, - replacement_history: Option>, - message: String, -} - -#[derive(Debug, Default)] -struct ReverseHistoryCollector { - // Number of newest user turns that still need to be removed while scanning backwards. - rollback_user_turns_to_skip: usize, - // Items already known to survive rollback because we have either consumed all rollback skips - // or scanned past the user message that anchors the dropped turns. - kept_items_rev: Vec, - // While scanning newest-to-oldest, we cannot tell whether a response item survives rollback - // until we either hit the user message that anchors its turn or exhaust the rollback skip - // count. These items are buffered here so reverse replay matches - // `ContextManager::drop_last_n_user_turns`, which drops everything from the Nth-last user - // message onward but still preserves items before the first surviving user message. - pending_items_rev: Vec, - // Split point inside `pending_items_rev` for the overflow case where rollback drops more user - // turns than exist after the current checkpoint. The suffix starting here is the buffered - // prefix that still survives because it occurred before the first dropped user message. - pending_keep_start: usize, +struct ReplayedUserTurn { + previous_model: Option, + reference_context_item: Option, + reference_context_item_cleared: bool, } -impl ReverseHistoryCollector { - fn record_response_item(&mut self, item: &ResponseItem) { - if self.rollback_user_turns_to_skip == 0 { - self.kept_items_rev.push(item.clone()); - return; - } - - self.pending_items_rev.push(item.clone()); - if matches!(item, ResponseItem::Message { role, .. } if role == "user") { - self.rollback_user_turns_to_skip -= 1; - self.pending_keep_start = self.pending_items_rev.len(); - if self.rollback_user_turns_to_skip == 0 { - self.pending_items_rev.clear(); - self.pending_keep_start = 0; - } - } - } - - fn record_rollback(&mut self, num_turns: u32) { - self.rollback_user_turns_to_skip = self - .rollback_user_turns_to_skip - .saturating_add(usize::try_from(num_turns).unwrap_or(usize::MAX)); - } - - fn finish_with_base_history( - mut self, - base_history: Vec, - truncation_policy: TruncationPolicy, - ) -> Vec { - for item in base_history.iter().rev() { - self.record_response_item(item); - } - - let mut surviving_items = if self.rollback_user_turns_to_skip > 0 { - self.pending_items_rev[self.pending_keep_start..] - .iter() - .rev() - .cloned() - .collect() - } else { - Vec::new() - }; - surviving_items.extend(self.kept_items_rev.into_iter().rev()); - - let mut history = ContextManager::new(); - history.record_items(surviving_items.iter(), truncation_policy); - history.raw_items().to_vec() - } +#[derive(Debug)] +enum ReplayMetadataSegment { + UserTurn(Box), + // Unexpected for modern rollouts, where compaction should normally happen inside a user turn. + // Keep this as a minimal legacy/incomplete-rollout fallback so later resume conservatively + // clears the baseline until another `TurnContextItem` re-establishes it. + ReferenceContextCleared, } #[derive(Debug, Default)] -struct ReverseMetadataTurn { +struct ActiveUserTurn { turn_id: Option, saw_user_message: bool, previous_model: Option, @@ -94,295 +34,204 @@ struct ReverseMetadataTurn { reference_context_item_cleared: bool, } -#[derive(Debug, Default)] -struct ReverseMetadataState { - rollback_user_turns_to_skip: usize, - active_turn: Option, - previous_model: Option, - reference_context_item: Option, - reference_context_item_cleared: bool, -} - -impl ReverseMetadataState { - fn record_rollback(&mut self, num_turns: u32) { - self.rollback_user_turns_to_skip = self - .rollback_user_turns_to_skip - .saturating_add(usize::try_from(num_turns).unwrap_or(usize::MAX)); - } - - fn record_turn_end(&mut self, turn_id: Option<&str>) { - match (&mut self.active_turn, turn_id) { - (None, Some(turn_id)) => { - self.active_turn = Some(ReverseMetadataTurn { - turn_id: Some(turn_id.to_string()), - ..Default::default() - }); - } - (None, None) => { - self.active_turn = Some(ReverseMetadataTurn::default()); - } - (Some(active_turn), Some(turn_id)) if active_turn.turn_id.is_none() => { - active_turn.turn_id = Some(turn_id.to_string()); - } - (Some(active_turn), Some(turn_id)) - if active_turn.turn_id.as_deref() == Some(turn_id) => {} - (Some(_), Some(_)) => { - // Ignore unmatched end markers for some other turn; they should not consume the - // newer turn we are currently walking backwards through. - } - (Some(_), None) => {} - } - } - - fn record_turn_start(&mut self, turn_id: &str) { - if self - .active_turn - .as_ref() - .is_some_and(|turn| turn.turn_id.as_deref().is_none_or(|id| id == turn_id)) - && let Some(turn) = self.active_turn.take() - { - self.finalize_turn(turn); - } - } - - fn ensure_active_turn(&mut self) -> &mut ReverseMetadataTurn { - self.active_turn - .get_or_insert_with(ReverseMetadataTurn::default) - } - - fn record_user_message(&mut self) { - self.ensure_active_turn().saw_user_message = true; - } - - fn record_turn_context(&mut self, ctx: &TurnContextItem) { - if self.active_turn.is_none() { - let turn = self.ensure_active_turn(); - turn.saw_user_message = true; - turn.previous_model = Some(ctx.model.clone()); - turn.reference_context_item = Some(ctx.clone()); - turn.reference_context_item_cleared = false; - return; - } - - if self.active_turn.as_ref().is_some_and(|turn| { - turn.turn_id.as_deref().is_none_or(|turn_id| { - ctx.turn_id - .as_deref() - .is_none_or(|ctx_turn_id| ctx_turn_id == turn_id) - }) - }) { - let turn = self.ensure_active_turn(); - if turn.turn_id.is_none() { - turn.saw_user_message = true; - } - if turn.previous_model.is_none() { - turn.previous_model = Some(ctx.model.clone()); - } - if !turn.reference_context_item_cleared && turn.reference_context_item.is_none() { - turn.reference_context_item = Some(ctx.clone()); - } - return; - } - - self.finalize_turn(ReverseMetadataTurn { - turn_id: ctx.turn_id.clone(), - saw_user_message: true, - previous_model: Some(ctx.model.clone()), - reference_context_item: Some(ctx.clone()), - reference_context_item_cleared: false, - }); - } - - fn record_compaction(&mut self) { - let turn = self.ensure_active_turn(); - if turn.reference_context_item.is_none() { - turn.reference_context_item_cleared = true; - } - } - - fn finalize_turn(&mut self, turn: ReverseMetadataTurn) { - if turn.saw_user_message { - if self.rollback_user_turns_to_skip > 0 { - self.rollback_user_turns_to_skip -= 1; - return; - } - - if turn.reference_context_item_cleared { - self.reference_context_item = None; - self.reference_context_item_cleared = true; - } - - if self.previous_model.is_none() - && let Some(previous_model) = turn.previous_model - { - self.previous_model = Some(previous_model); - if !self.reference_context_item_cleared { - self.reference_context_item = turn.reference_context_item; +impl Session { + pub(super) async fn reconstruct_history_from_rollout( + &self, + turn_context: &TurnContext, + rollout_items: &[RolloutItem], + ) -> RolloutReconstruction { + // Keep rollout replay eager and forward-only for now. + // + // This unifies mainline history replay with eager resume metadata hydration in one place, + // without yet introducing the lazy reverse-loading machinery we want later. History is + // rebuilt exactly as rollout recorded it, while metadata keeps only enough per-turn state + // to answer: + // - which surviving user turn last provided `previous_model` + // - which surviving `TurnContextItem` baseline, if any, remains after later compaction + let initial_context = self.build_initial_context(turn_context).await; + let mut history = ContextManager::new(); + let mut replayed_segments = Vec::new(); + let mut active_turn: Option = None; + + let finalize_active_turn = + |replayed_segments: &mut Vec, active_turn: ActiveUserTurn| { + if active_turn.saw_user_message { + replayed_segments.push(ReplayMetadataSegment::UserTurn(Box::new( + ReplayedUserTurn { + previous_model: active_turn.previous_model, + reference_context_item: active_turn.reference_context_item, + reference_context_item_cleared: active_turn + .reference_context_item_cleared, + }, + ))); + } else if active_turn.reference_context_item_cleared { + replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); } + }; + + let drop_last_n_user_turns = |replayed_segments: &mut Vec, + num_turns: u32| { + let mut remaining = usize::try_from(num_turns).unwrap_or(usize::MAX); + while remaining > 0 { + let Some(index) = replayed_segments + .iter() + .rposition(|segment| matches!(segment, ReplayMetadataSegment::UserTurn(_))) + else { + break; + }; + replayed_segments.remove(index); + remaining -= 1; } - } else if turn.reference_context_item_cleared { - self.reference_context_item = None; - self.reference_context_item_cleared = true; - } - } - - fn finish(mut self) -> (Option, Option) { - if let Some(turn) = self.active_turn.take() { - self.finalize_turn(turn); - } - (self.previous_model, self.reference_context_item) - } - - fn resolved_previous_model(&self) -> bool { - self.previous_model.is_some() - } -} - -#[derive(Debug)] -struct TailScan { - history_collector: ReverseHistoryCollector, - history_checkpoint: Option, - previous_model: Option, - reference_context_item: Option, -} - -fn scan_rollout_tail(rollout_items: &[RolloutItem]) -> TailScan { - let mut history_collector = ReverseHistoryCollector::default(); - let mut history_checkpoint = None; - let mut metadata = ReverseMetadataState::default(); + }; - for (index, item) in rollout_items.iter().enumerate().rev() { - match item { - RolloutItem::ResponseItem(response_item) => { - if history_checkpoint.is_none() { - history_collector.record_response_item(response_item); + for item in rollout_items { + match item { + RolloutItem::ResponseItem(response_item) => { + history.record_items( + std::iter::once(response_item), + turn_context.truncation_policy, + ); } - } - RolloutItem::Compacted(compacted) => { - if history_checkpoint.is_none() { - history_checkpoint = Some(HistoryCheckpoint { - prefix_len: index, - replacement_history: compacted.replacement_history.clone(), - message: compacted.message.clone(), + RolloutItem::Compacted(compacted) => { + if let Some(replacement_history) = &compacted.replacement_history { + history.replace(replacement_history.clone()); + } else { + let user_messages = collect_user_messages(history.raw_items()); + let rebuilt = compact::build_compacted_history( + initial_context.clone(), + &user_messages, + &compacted.message, + ); + history.replace(rebuilt); + } + + if let Some(active_turn) = active_turn.as_mut() { + active_turn.reference_context_item = None; + active_turn.reference_context_item_cleared = true; + } else { + replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); + } + } + RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { + history.drop_last_n_user_turns(rollback.num_turns); + + let mut remaining = usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); + if remaining > 0 + && active_turn + .as_ref() + .is_some_and(|active_turn| active_turn.saw_user_message) + { + active_turn = None; + remaining -= 1; + } + if remaining > 0 { + drop_last_n_user_turns( + &mut replayed_segments, + u32::try_from(remaining).unwrap_or(u32::MAX), + ); + } + } + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + if let Some(active_turn) = active_turn.take() { + finalize_active_turn(&mut replayed_segments, active_turn); + } + active_turn = Some(ActiveUserTurn { + turn_id: Some(event.turn_id.clone()), + ..Default::default() }); } - metadata.record_compaction(); - } - RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { - if history_checkpoint.is_none() { - history_collector.record_rollback(rollback.num_turns); + RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { + if active_turn.as_ref().is_some_and(|active_turn| { + active_turn + .turn_id + .as_deref() + .is_none_or(|turn_id| turn_id == event.turn_id) + }) && let Some(active_turn) = active_turn.take() + { + finalize_active_turn(&mut replayed_segments, active_turn); + } } - metadata.record_rollback(rollback.num_turns); - } - RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { - metadata.record_turn_end(Some(&event.turn_id)); - } - RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { - metadata.record_turn_end(event.turn_id.as_deref()); - } - RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - metadata.record_turn_start(&event.turn_id); - } - RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { - metadata.record_user_message(); - } - RolloutItem::TurnContext(ctx) => { - metadata.record_turn_context(ctx); + RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { + match event.turn_id.as_deref() { + Some(turn_id) + if active_turn.as_ref().is_some_and(|active_turn| { + active_turn + .turn_id + .as_deref() + .is_none_or(|active_turn_id| active_turn_id == turn_id) + }) => + { + if let Some(active_turn) = active_turn.take() { + finalize_active_turn(&mut replayed_segments, active_turn); + } + } + Some(_) => {} + None => { + if let Some(active_turn) = active_turn.take() { + finalize_active_turn(&mut replayed_segments, active_turn); + } + } + } + } + RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { + if let Some(active_turn) = active_turn.as_mut() { + active_turn.saw_user_message = true; + } + } + RolloutItem::TurnContext(ctx) => { + if let Some(active_turn) = active_turn.as_mut() { + if active_turn.turn_id.as_deref().is_none_or(|turn_id| { + ctx.turn_id + .as_deref() + .is_none_or(|ctx_turn_id| ctx_turn_id == turn_id) + }) { + if active_turn.previous_model.is_none() { + active_turn.previous_model = Some(ctx.model.clone()); + } + active_turn.reference_context_item = Some(ctx.clone()); + active_turn.reference_context_item_cleared = false; + } + } else { + replayed_segments.push(ReplayMetadataSegment::UserTurn(Box::new( + ReplayedUserTurn { + previous_model: Some(ctx.model.clone()), + reference_context_item: Some(ctx.clone()), + reference_context_item_cleared: false, + }, + ))); + } + } + _ => {} } - _ => {} } - if history_checkpoint.is_some() && metadata.resolved_previous_model() { - break; + if let Some(active_turn) = active_turn.take() { + finalize_active_turn(&mut replayed_segments, active_turn); } - } - let (previous_model, reference_context_item) = metadata.finish(); - TailScan { - history_collector, - history_checkpoint, - previous_model, - reference_context_item, - } -} - -fn reconstruct_history_from_tail_scan( - initial_context: &[ResponseItem], - truncation_policy: TruncationPolicy, - rollout_items: &[RolloutItem], - tail_scan: TailScan, -) -> Vec { - let base_history = match tail_scan.history_checkpoint { - Some(HistoryCheckpoint { - prefix_len: _, - replacement_history: Some(replacement_history), - .. - }) => replacement_history, - Some(HistoryCheckpoint { - prefix_len, - replacement_history: None, - message, - }) => { - let history_before = reconstruct_history_from_rollout_items( - initial_context, - truncation_policy, - &rollout_items[..prefix_len], - ); - let user_messages = collect_user_messages(&history_before); - compact::build_compacted_history(initial_context.to_vec(), &user_messages, &message) + let mut previous_model = None; + let mut reference_context_item = None; + for segment in replayed_segments { + match segment { + ReplayMetadataSegment::UserTurn(turn) => { + if let Some(turn_previous_model) = turn.previous_model { + previous_model = Some(turn_previous_model); + } + if turn.reference_context_item_cleared { + reference_context_item = None; + } + if let Some(turn_reference_context_item) = turn.reference_context_item { + reference_context_item = Some(turn_reference_context_item); + } + } + ReplayMetadataSegment::ReferenceContextCleared => { + reference_context_item = None; + } + } } - None => Vec::new(), - }; - - tail_scan - .history_collector - .finish_with_base_history(base_history, truncation_policy) -} - -fn reconstruct_history_from_rollout_items( - initial_context: &[ResponseItem], - truncation_policy: TruncationPolicy, - rollout_items: &[RolloutItem], -) -> Vec { - let tail_scan = scan_rollout_tail(rollout_items); - reconstruct_history_from_tail_scan(initial_context, truncation_policy, rollout_items, tail_scan) -} - -impl Session { - pub(super) async fn reconstruct_history_from_rollout( - &self, - turn_context: &TurnContext, - rollout_items: &[RolloutItem], - ) -> RolloutReconstruction { - // Read the rollout from the tail inward. - // - // The reverse scan does two things at once: - // - resolve resume metadata from the newest surviving user turn after applying - // `ThreadRolledBack` as a simple "skip N user turns" counter - // - capture the raw history suffix after the newest `Compacted` checkpoint - // - // Once that tail scan finds a `Compacted` item, older raw `ResponseItem`s no longer need - // to be read directly. `replacement_history: Some(...)` already contains the full base - // snapshot at that point, while `replacement_history: None` rebuilds that snapshot by - // recursively reconstructing the rollout prefix before the compaction and passing its user - // messages into `build_compacted_history`. - // - // This keeps replay aligned with the eventual reverse-file reader we want to build: the - // tail scan identifies the newest surviving baseline information and the newest history - // checkpoint, then recursive prefix rebuild handles only the compacted prefix when needed. - let initial_context = self.build_initial_context(turn_context).await; - let tail_scan = scan_rollout_tail(rollout_items); - let previous_model = tail_scan.previous_model.clone(); - let reference_context_item = tail_scan.reference_context_item.clone(); - let history = reconstruct_history_from_tail_scan( - &initial_context, - turn_context.truncation_policy, - rollout_items, - tail_scan, - ); RolloutReconstruction { - history, + history: history.raw_items().to_vec(), previous_model, reference_context_item, } From c52dcf0dd2af68ef8c4b15f5c6ff52b4cd8aac07 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Wed, 25 Feb 2026 22:46:31 -0800 Subject: [PATCH 38/64] Persist TurnContext after full-context reinjection --- codex-rs/core/src/codex.rs | 92 +++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c97eb2e528e..c5c7a8e169f 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -3010,14 +3010,15 @@ impl Session { self.record_conversation_items(turn_context, &context_items) .await; } - if previous_user_turn_model.is_none() - || (!should_inject_full_context && !context_items.is_empty()) + if should_inject_full_context + || previous_user_turn_model.is_none() + || !context_items.is_empty() { - // Keep rollout TurnContext entries to: + // Keep rollout TurnContext entries for any turn that establishes or changes the + // persisted model-visible baseline: // - the first real user turn (to recover `previous_model` on resume) + // - full-context reinjection after the baseline was cleared // - steady-state turns that emitted explicit context diffs - // Full reinjection after compaction is tracked in runtime state only; resume will - // conservatively fall back to a missing baseline and reinject again if needed. self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) .await; } @@ -6405,6 +6406,10 @@ mod tests { use crate::protocol::TokenCountEvent; use crate::protocol::TokenUsage; use crate::protocol::TokenUsageInfo; + use crate::protocol::UserMessageEvent; + use crate::rollout::policy::EventPersistenceMode; + use crate::rollout::recorder::RolloutRecorder; + use crate::rollout::recorder::RolloutRecorderParams; use crate::state::TaskKind; use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; @@ -8613,6 +8618,83 @@ mod tests { assert!(text.contains("")); } + #[tokio::test] + async fn record_context_updates_and_set_reference_context_item_persists_full_reinjection_to_rollout( + ) { + let (session, previous_context) = make_session_and_context().await; + let next_model = if previous_context.model_info.slug == "gpt-5.1" { + "gpt-5" + } else { + "gpt-5.1" + }; + let turn_context = previous_context + .with_model(next_model.to_string(), &session.services.models_manager) + .await; + let config = session.get_config().await; + let recorder = RolloutRecorder::new( + config.as_ref(), + RolloutRecorderParams::new( + ThreadId::default(), + None, + SessionSource::Exec, + BaseInstructions::default(), + Vec::new(), + EventPersistenceMode::Limited, + ), + None, + None, + ) + .await + .expect("create rollout recorder"); + let rollout_path = recorder.rollout_path().to_path_buf(); + { + let mut rollout = session.services.rollout.lock().await; + *rollout = Some(recorder); + } + + session + .persist_rollout_items(&[RolloutItem::EventMsg(EventMsg::UserMessage( + UserMessageEvent { + message: "seed rollout".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + ))]) + .await; + { + let mut state = session.state.lock().await; + state.set_reference_context_item(None); + } + + session + .record_context_updates_and_set_reference_context_item( + &turn_context, + Some(previous_context.model_info.slug.as_str()), + ) + .await; + session.ensure_rollout_materialized().await; + session.flush_rollout().await; + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read rollout history") + else { + panic!("expected resumed rollout history"); + }; + let persisted_turn_context = resumed.history.iter().find_map(|item| match item { + RolloutItem::TurnContext(ctx) => Some(ctx.clone()), + _ => None, + }); + + assert_eq!( + serde_json::to_value(persisted_turn_context) + .expect("serialize persisted turn context item"), + serde_json::to_value(Some(turn_context.to_turn_context_item())) + .expect("serialize expected turn context item") + ); + } + #[tokio::test] async fn run_user_shell_command_does_not_set_reference_context_item() { let (session, _turn_context, rx) = make_session_and_context_with_rx().await; From 54b1e2720c97896217596ca77a939936980df2d1 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 13:06:43 -0800 Subject: [PATCH 39/64] Make replay reference-context state explicit --- .../core/src/codex/rollout_reconstruction.rs | 56 ++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index f758c845251..c76d38f6060 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -12,8 +12,7 @@ pub(super) struct RolloutReconstruction { #[derive(Debug)] struct ReplayedUserTurn { previous_model: Option, - reference_context_item: Option, - reference_context_item_cleared: bool, + reference_context_item: ReferenceContextReplayState, } #[derive(Debug)] @@ -30,8 +29,23 @@ struct ActiveUserTurn { turn_id: Option, saw_user_message: bool, previous_model: Option, - reference_context_item: Option, - reference_context_item_cleared: bool, + reference_context_item: ReferenceContextReplayState, +} + +#[derive(Debug, Default)] +enum ReferenceContextReplayState { + /// No `TurnContextItem` has been seen for this replay span yet. + /// + /// This differs from `Cleared`: `NeverSet` means there is no evidence this turn ever + /// established a baseline, while `Cleared` means a baseline existed and a later compaction + /// invalidated it. Only the latter must emit an explicit clearing segment for resume/fork + /// hydration. + #[default] + NeverSet, + /// A previously established baseline was invalidated by later compaction. + Cleared, + /// The latest baseline established by this replay span. + LatestSet(TurnContextItem), } impl Session { @@ -48,7 +62,7 @@ impl Session { // to answer: // - which surviving user turn last provided `previous_model` // - which surviving `TurnContextItem` baseline, if any, remains after later compaction - let initial_context = self.build_initial_context(turn_context).await; + let initial_context = self.build_initial_context(turn_context, None).await; let mut history = ContextManager::new(); let mut replayed_segments = Vec::new(); let mut active_turn: Option = None; @@ -60,11 +74,12 @@ impl Session { ReplayedUserTurn { previous_model: active_turn.previous_model, reference_context_item: active_turn.reference_context_item, - reference_context_item_cleared: active_turn - .reference_context_item_cleared, }, ))); - } else if active_turn.reference_context_item_cleared { + } else if matches!( + active_turn.reference_context_item, + ReferenceContextReplayState::Cleared + ) { replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); } }; @@ -106,8 +121,7 @@ impl Session { } if let Some(active_turn) = active_turn.as_mut() { - active_turn.reference_context_item = None; - active_turn.reference_context_item_cleared = true; + active_turn.reference_context_item = ReferenceContextReplayState::Cleared; } else { replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); } @@ -188,15 +202,16 @@ impl Session { if active_turn.previous_model.is_none() { active_turn.previous_model = Some(ctx.model.clone()); } - active_turn.reference_context_item = Some(ctx.clone()); - active_turn.reference_context_item_cleared = false; + active_turn.reference_context_item = + ReferenceContextReplayState::LatestSet(ctx.clone()); } } else { replayed_segments.push(ReplayMetadataSegment::UserTurn(Box::new( ReplayedUserTurn { previous_model: Some(ctx.model.clone()), - reference_context_item: Some(ctx.clone()), - reference_context_item_cleared: false, + reference_context_item: ReferenceContextReplayState::LatestSet( + ctx.clone(), + ), }, ))); } @@ -217,11 +232,14 @@ impl Session { if let Some(turn_previous_model) = turn.previous_model { previous_model = Some(turn_previous_model); } - if turn.reference_context_item_cleared { - reference_context_item = None; - } - if let Some(turn_reference_context_item) = turn.reference_context_item { - reference_context_item = Some(turn_reference_context_item); + match turn.reference_context_item { + ReferenceContextReplayState::NeverSet => {} + ReferenceContextReplayState::Cleared => { + reference_context_item = None; + } + ReferenceContextReplayState::LatestSet(turn_reference_context_item) => { + reference_context_item = Some(turn_reference_context_item); + } } } ReplayMetadataSegment::ReferenceContextCleared => { From 1d32c1e63f7900555d7fe74584879fd636a2f0d4 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 13:21:13 -0800 Subject: [PATCH 40/64] Rename replay reference-context enum --- .../core/src/codex/rollout_reconstruction.rs | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index c76d38f6060..45bc55dfcd1 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -9,10 +9,31 @@ pub(super) struct RolloutReconstruction { pub(super) reference_context_item: Option, } +#[derive(Debug)] +enum TurnReferenceContextItem { + /// No `TurnContextItem` has been seen for this replay span yet. + /// + /// This differs from `Cleared`: `NeverSet` means there is no evidence this turn ever + /// established a baseline, while `Cleared` means a baseline existed and a later compaction + /// invalidated it. Only the latter must emit an explicit clearing segment for resume/fork + /// hydration. + NeverSet, + /// A previously established baseline was invalidated by later compaction. + Cleared, + /// The latest baseline established by this replay span. + LatestSet(TurnContextItem), +} + +impl Default for TurnReferenceContextItem { + fn default() -> Self { + Self::NeverSet + } +} + #[derive(Debug)] struct ReplayedUserTurn { previous_model: Option, - reference_context_item: ReferenceContextReplayState, + reference_context_item: TurnReferenceContextItem, } #[derive(Debug)] @@ -29,23 +50,7 @@ struct ActiveUserTurn { turn_id: Option, saw_user_message: bool, previous_model: Option, - reference_context_item: ReferenceContextReplayState, -} - -#[derive(Debug, Default)] -enum ReferenceContextReplayState { - /// No `TurnContextItem` has been seen for this replay span yet. - /// - /// This differs from `Cleared`: `NeverSet` means there is no evidence this turn ever - /// established a baseline, while `Cleared` means a baseline existed and a later compaction - /// invalidated it. Only the latter must emit an explicit clearing segment for resume/fork - /// hydration. - #[default] - NeverSet, - /// A previously established baseline was invalidated by later compaction. - Cleared, - /// The latest baseline established by this replay span. - LatestSet(TurnContextItem), + reference_context_item: TurnReferenceContextItem, } impl Session { @@ -78,7 +83,7 @@ impl Session { ))); } else if matches!( active_turn.reference_context_item, - ReferenceContextReplayState::Cleared + TurnReferenceContextItem::Cleared ) { replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); } @@ -121,7 +126,7 @@ impl Session { } if let Some(active_turn) = active_turn.as_mut() { - active_turn.reference_context_item = ReferenceContextReplayState::Cleared; + active_turn.reference_context_item = TurnReferenceContextItem::Cleared; } else { replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); } @@ -203,13 +208,13 @@ impl Session { active_turn.previous_model = Some(ctx.model.clone()); } active_turn.reference_context_item = - ReferenceContextReplayState::LatestSet(ctx.clone()); + TurnReferenceContextItem::LatestSet(ctx.clone()); } } else { replayed_segments.push(ReplayMetadataSegment::UserTurn(Box::new( ReplayedUserTurn { previous_model: Some(ctx.model.clone()), - reference_context_item: ReferenceContextReplayState::LatestSet( + reference_context_item: TurnReferenceContextItem::LatestSet( ctx.clone(), ), }, @@ -233,11 +238,11 @@ impl Session { previous_model = Some(turn_previous_model); } match turn.reference_context_item { - ReferenceContextReplayState::NeverSet => {} - ReferenceContextReplayState::Cleared => { + TurnReferenceContextItem::NeverSet => {} + TurnReferenceContextItem::Cleared => { reference_context_item = None; } - ReferenceContextReplayState::LatestSet(turn_reference_context_item) => { + TurnReferenceContextItem::LatestSet(turn_reference_context_item) => { reference_context_item = Some(turn_reference_context_item); } } From edcb754be99e015f575bc4444fa4fbffa322253f Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 13:28:18 -0800 Subject: [PATCH 41/64] Rename replay turn reference-context state --- codex-rs/core/src/codex.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c5c7a8e169f..4ae3a7a3492 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -8563,8 +8563,8 @@ mod tests { } #[tokio::test] - async fn record_context_updates_and_set_reference_context_item_updates_baseline_without_emitting_diffs( - ) { + async fn record_context_updates_and_set_reference_context_item_updates_baseline_without_emitting_diffs() + { let (session, previous_context) = make_session_and_context().await; let next_model = if previous_context.model_info.slug == "gpt-5.1" { "gpt-5" @@ -8619,8 +8619,8 @@ mod tests { } #[tokio::test] - async fn record_context_updates_and_set_reference_context_item_persists_full_reinjection_to_rollout( - ) { + async fn record_context_updates_and_set_reference_context_item_persists_full_reinjection_to_rollout() + { let (session, previous_context) = make_session_and_context().await; let next_model = if previous_context.model_info.slug == "gpt-5.1" { "gpt-5" From 3cd2b57d1486df205b049f9180baa9cf2c841fc4 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 14:13:23 -0800 Subject: [PATCH 42/64] Refactor rollout reconstruction around reverse replay --- .../core/src/codex/rollout_reconstruction.rs | 338 ++++++++---------- 1 file changed, 158 insertions(+), 180 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 45bc55dfcd1..f4a55bb5ef9 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -9,7 +9,7 @@ pub(super) struct RolloutReconstruction { pub(super) reference_context_item: Option, } -#[derive(Debug)] +#[derive(Debug, Default)] enum TurnReferenceContextItem { /// No `TurnContextItem` has been seen for this replay span yet. /// @@ -17,40 +17,21 @@ enum TurnReferenceContextItem { /// established a baseline, while `Cleared` means a baseline existed and a later compaction /// invalidated it. Only the latter must emit an explicit clearing segment for resume/fork /// hydration. + #[default] NeverSet, /// A previously established baseline was invalidated by later compaction. Cleared, /// The latest baseline established by this replay span. - LatestSet(TurnContextItem), -} - -impl Default for TurnReferenceContextItem { - fn default() -> Self { - Self::NeverSet - } -} - -#[derive(Debug)] -struct ReplayedUserTurn { - previous_model: Option, - reference_context_item: TurnReferenceContextItem, -} - -#[derive(Debug)] -enum ReplayMetadataSegment { - UserTurn(Box), - // Unexpected for modern rollouts, where compaction should normally happen inside a user turn. - // Keep this as a minimal legacy/incomplete-rollout fallback so later resume conservatively - // clears the baseline until another `TurnContextItem` re-establishes it. - ReferenceContextCleared, + LatestSet(Box), } #[derive(Debug, Default)] -struct ActiveUserTurn { +struct ActiveReplaySegment { turn_id: Option, - saw_user_message: bool, + counts_as_user_turn: bool, previous_model: Option, reference_context_item: TurnReferenceContextItem, + replacement_history_index: Option, } impl Session { @@ -59,204 +40,201 @@ impl Session { turn_context: &TurnContext, rollout_items: &[RolloutItem], ) -> RolloutReconstruction { - // Keep rollout replay eager and forward-only for now. - // - // This unifies mainline history replay with eager resume metadata hydration in one place, - // without yet introducing the lazy reverse-loading machinery we want later. History is - // rebuilt exactly as rollout recorded it, while metadata keeps only enough per-turn state - // to answer: - // - which surviving user turn last provided `previous_model` - // - which surviving `TurnContextItem` baseline, if any, remains after later compaction - let initial_context = self.build_initial_context(turn_context, None).await; - let mut history = ContextManager::new(); - let mut replayed_segments = Vec::new(); - let mut active_turn: Option = None; + // Replay metadata should already match the shape of the future lazy reverse loader, even + // while history materialization still uses an eager bridge. Scan newest-to-oldest, + // stopping once a surviving replacement-history checkpoint and the required resume metadata + // are both known; then replay only that suffix forward to preserve exact history semantics. + let mut replay_start_index = None; + let mut previous_model = None; + let mut reference_context_item = TurnReferenceContextItem::NeverSet; + let mut pending_rollback_turns = 0usize; + let mut active_segment: Option = None; - let finalize_active_turn = - |replayed_segments: &mut Vec, active_turn: ActiveUserTurn| { - if active_turn.saw_user_message { - replayed_segments.push(ReplayMetadataSegment::UserTurn(Box::new( - ReplayedUserTurn { - previous_model: active_turn.previous_model, - reference_context_item: active_turn.reference_context_item, - }, - ))); - } else if matches!( - active_turn.reference_context_item, - TurnReferenceContextItem::Cleared - ) { - replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); + let finalize_active_segment = + |active_segment: ActiveReplaySegment, + replay_start_index: &mut Option, + previous_model: &mut Option, + reference_context_item: &mut TurnReferenceContextItem, + pending_rollback_turns: &mut usize| { + if *pending_rollback_turns > 0 { + if active_segment.counts_as_user_turn { + *pending_rollback_turns -= 1; + } + return; } - }; - let drop_last_n_user_turns = |replayed_segments: &mut Vec, - num_turns: u32| { - let mut remaining = usize::try_from(num_turns).unwrap_or(usize::MAX); - while remaining > 0 { - let Some(index) = replayed_segments - .iter() - .rposition(|segment| matches!(segment, ReplayMetadataSegment::UserTurn(_))) - else { - break; - }; - replayed_segments.remove(index); - remaining -= 1; - } - }; + if replay_start_index.is_none() + && let Some(replacement_history_index) = + active_segment.replacement_history_index + { + *replay_start_index = Some(replacement_history_index); + } - for item in rollout_items { - match item { - RolloutItem::ResponseItem(response_item) => { - history.record_items( - std::iter::once(response_item), - turn_context.truncation_policy, - ); + if previous_model.is_none() && active_segment.counts_as_user_turn { + *previous_model = active_segment.previous_model; } - RolloutItem::Compacted(compacted) => { - if let Some(replacement_history) = &compacted.replacement_history { - history.replace(replacement_history.clone()); - } else { - let user_messages = collect_user_messages(history.raw_items()); - let rebuilt = compact::build_compacted_history( - initial_context.clone(), - &user_messages, - &compacted.message, - ); - history.replace(rebuilt); - } - if let Some(active_turn) = active_turn.as_mut() { - active_turn.reference_context_item = TurnReferenceContextItem::Cleared; - } else { - replayed_segments.push(ReplayMetadataSegment::ReferenceContextCleared); - } + if matches!(reference_context_item, TurnReferenceContextItem::NeverSet) + && (active_segment.counts_as_user_turn + || matches!( + active_segment.reference_context_item, + TurnReferenceContextItem::Cleared + )) + { + *reference_context_item = active_segment.reference_context_item; } - RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { - history.drop_last_n_user_turns(rollback.num_turns); + }; - let mut remaining = usize::try_from(rollback.num_turns).unwrap_or(usize::MAX); - if remaining > 0 - && active_turn - .as_ref() - .is_some_and(|active_turn| active_turn.saw_user_message) - { - active_turn = None; - remaining -= 1; + for (index, item) in rollout_items.iter().enumerate().rev() { + match item { + RolloutItem::Compacted(compacted) => { + let active_segment = + active_segment.get_or_insert_with(ActiveReplaySegment::default); + if matches!( + active_segment.reference_context_item, + TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared + ) { + active_segment.reference_context_item = TurnReferenceContextItem::Cleared; } - if remaining > 0 { - drop_last_n_user_turns( - &mut replayed_segments, - u32::try_from(remaining).unwrap_or(u32::MAX), - ); + if active_segment.replacement_history_index.is_none() + && compacted.replacement_history.is_some() + { + active_segment.replacement_history_index = Some(index); } } - RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - if let Some(active_turn) = active_turn.take() { - finalize_active_turn(&mut replayed_segments, active_turn); - } - active_turn = Some(ActiveUserTurn { - turn_id: Some(event.turn_id.clone()), - ..Default::default() - }); + RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { + pending_rollback_turns = pending_rollback_turns + .saturating_add(usize::try_from(rollback.num_turns).unwrap_or(usize::MAX)); } - RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { - if active_turn.as_ref().is_some_and(|active_turn| { - active_turn + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + if active_segment.as_ref().is_some_and(|active_segment| { + active_segment .turn_id .as_deref() .is_none_or(|turn_id| turn_id == event.turn_id) - }) && let Some(active_turn) = active_turn.take() + }) && let Some(active_segment) = active_segment.take() { - finalize_active_turn(&mut replayed_segments, active_turn); + finalize_active_segment( + active_segment, + &mut replay_start_index, + &mut previous_model, + &mut reference_context_item, + &mut pending_rollback_turns, + ); + } + } + RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { + let active_segment = + active_segment.get_or_insert_with(ActiveReplaySegment::default); + if active_segment.turn_id.is_none() { + active_segment.turn_id = Some(event.turn_id.clone()); } } RolloutItem::EventMsg(EventMsg::TurnAborted(event)) => { - match event.turn_id.as_deref() { - Some(turn_id) - if active_turn.as_ref().is_some_and(|active_turn| { - active_turn - .turn_id - .as_deref() - .is_none_or(|active_turn_id| active_turn_id == turn_id) - }) => + if let Some(active_segment) = active_segment.as_mut() { + if active_segment.turn_id.is_none() + && let Some(turn_id) = &event.turn_id { - if let Some(active_turn) = active_turn.take() { - finalize_active_turn(&mut replayed_segments, active_turn); - } - } - Some(_) => {} - None => { - if let Some(active_turn) = active_turn.take() { - finalize_active_turn(&mut replayed_segments, active_turn); - } + active_segment.turn_id = Some(turn_id.clone()); } + } else if let Some(turn_id) = &event.turn_id { + active_segment = Some(ActiveReplaySegment { + turn_id: Some(turn_id.clone()), + ..Default::default() + }); } } RolloutItem::EventMsg(EventMsg::UserMessage(_)) => { - if let Some(active_turn) = active_turn.as_mut() { - active_turn.saw_user_message = true; - } + let active_segment = + active_segment.get_or_insert_with(ActiveReplaySegment::default); + active_segment.counts_as_user_turn = true; } RolloutItem::TurnContext(ctx) => { - if let Some(active_turn) = active_turn.as_mut() { - if active_turn.turn_id.as_deref().is_none_or(|turn_id| { - ctx.turn_id - .as_deref() - .is_none_or(|ctx_turn_id| ctx_turn_id == turn_id) - }) { - if active_turn.previous_model.is_none() { - active_turn.previous_model = Some(ctx.model.clone()); - } - active_turn.reference_context_item = - TurnReferenceContextItem::LatestSet(ctx.clone()); + let active_segment = + active_segment.get_or_insert_with(ActiveReplaySegment::default); + if active_segment.turn_id.is_none() { + active_segment.turn_id = ctx.turn_id.clone(); + active_segment.counts_as_user_turn = true; + } + if active_segment.turn_id.as_deref().is_none_or(|turn_id| { + ctx.turn_id + .as_deref() + .is_none_or(|ctx_turn_id| ctx_turn_id == turn_id) + }) { + active_segment.previous_model = Some(ctx.model.clone()); + if matches!( + active_segment.reference_context_item, + TurnReferenceContextItem::NeverSet + ) { + active_segment.reference_context_item = + TurnReferenceContextItem::LatestSet(Box::new(ctx.clone())); } - } else { - replayed_segments.push(ReplayMetadataSegment::UserTurn(Box::new( - ReplayedUserTurn { - previous_model: Some(ctx.model.clone()), - reference_context_item: TurnReferenceContextItem::LatestSet( - ctx.clone(), - ), - }, - ))); } } - _ => {} + RolloutItem::ResponseItem(_) + | RolloutItem::EventMsg(_) + | RolloutItem::SessionMeta(_) => {} + } + + if replay_start_index.is_some() + && previous_model.is_some() + && !matches!(reference_context_item, TurnReferenceContextItem::NeverSet) + { + break; } } - if let Some(active_turn) = active_turn.take() { - finalize_active_turn(&mut replayed_segments, active_turn); + if let Some(active_segment) = active_segment.take() { + finalize_active_segment( + active_segment, + &mut replay_start_index, + &mut previous_model, + &mut reference_context_item, + &mut pending_rollback_turns, + ); } - let mut previous_model = None; - let mut reference_context_item = None; - for segment in replayed_segments { - match segment { - ReplayMetadataSegment::UserTurn(turn) => { - if let Some(turn_previous_model) = turn.previous_model { - previous_model = Some(turn_previous_model); - } - match turn.reference_context_item { - TurnReferenceContextItem::NeverSet => {} - TurnReferenceContextItem::Cleared => { - reference_context_item = None; - } - TurnReferenceContextItem::LatestSet(turn_reference_context_item) => { - reference_context_item = Some(turn_reference_context_item); - } + let initial_context = self.build_initial_context(turn_context, None).await; + let mut history = ContextManager::new(); + for item in &rollout_items[replay_start_index.unwrap_or(0)..] { + match item { + RolloutItem::ResponseItem(response_item) => { + history.record_items( + std::iter::once(response_item), + turn_context.truncation_policy, + ); + } + RolloutItem::Compacted(compacted) => { + if let Some(replacement_history) = &compacted.replacement_history { + history.replace(replacement_history.clone()); + } else { + let user_messages = collect_user_messages(history.raw_items()); + let rebuilt = compact::build_compacted_history( + initial_context.clone(), + &user_messages, + &compacted.message, + ); + history.replace(rebuilt); } } - ReplayMetadataSegment::ReferenceContextCleared => { - reference_context_item = None; + RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { + history.drop_last_n_user_turns(rollback.num_turns); } + RolloutItem::EventMsg(_) + | RolloutItem::TurnContext(_) + | RolloutItem::SessionMeta(_) => {} } } RolloutReconstruction { history: history.raw_items().to_vec(), previous_model, - reference_context_item, + reference_context_item: match reference_context_item { + TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared => None, + TurnReferenceContextItem::LatestSet(turn_reference_context_item) => { + Some(*turn_reference_context_item) + } + }, } } } From 87362f252aab559a68f49cecf994a2dfa0ce6edb Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 15:40:53 -0800 Subject: [PATCH 43/64] Document reverse replay state in rollout reconstruction --- .../core/src/codex/rollout_reconstruction.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index f4a55bb5ef9..d736a91e94e 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -44,10 +44,17 @@ impl Session { // while history materialization still uses an eager bridge. Scan newest-to-oldest, // stopping once a surviving replacement-history checkpoint and the required resume metadata // are both known; then replay only that suffix forward to preserve exact history semantics. + // Index of the earliest rollout item that still needs forward materialization. Once reverse + // replay finds a surviving `replacement_history` checkpoint, everything before that index + // is obsolete for rebuilt history. let mut replay_start_index = None; let mut previous_model = None; let mut reference_context_item = TurnReferenceContextItem::NeverSet; + // Rollback is "drop the newest N user turns". While scanning in reverse, that becomes + // "skip the next N user-turn segments we finalize". let mut pending_rollback_turns = 0usize; + // Reverse replay accumulates rollout items into the newest in-progress turn segment until + // we hit its matching `TurnStarted`, at which point the segment can be finalized. let mut active_segment: Option = None; let finalize_active_segment = @@ -56,6 +63,8 @@ impl Session { previous_model: &mut Option, reference_context_item: &mut TurnReferenceContextItem, pending_rollback_turns: &mut usize| { + // Thread rollback always targets the newest surviving user turns, so consume that + // skip budget before letting this segment contribute metadata or a compaction base. if *pending_rollback_turns > 0 { if active_segment.counts_as_user_turn { *pending_rollback_turns -= 1; @@ -63,6 +72,8 @@ impl Session { return; } + // A surviving replacement-history checkpoint is a complete history base. Once we + // know the newest surviving one, older rollout items do not affect rebuilt history. if replay_start_index.is_none() && let Some(replacement_history_index) = active_segment.replacement_history_index @@ -70,10 +81,13 @@ impl Session { *replay_start_index = Some(replacement_history_index); } + // `previous_model` comes from the newest surviving user turn that established one. if previous_model.is_none() && active_segment.counts_as_user_turn { *previous_model = active_segment.previous_model; } + // `reference_context_item` comes from the newest surviving user turn baseline, or + // from a surviving compaction that explicitly cleared that baseline. if matches!(reference_context_item, TurnReferenceContextItem::NeverSet) && (active_segment.counts_as_user_turn || matches!( @@ -90,6 +104,8 @@ impl Session { RolloutItem::Compacted(compacted) => { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); + // Looking backward, compaction clears any older baseline unless a newer + // `TurnContextItem` in this same segment has already re-established it. if matches!( active_segment.reference_context_item, TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared @@ -107,6 +123,7 @@ impl Session { .saturating_add(usize::try_from(rollback.num_turns).unwrap_or(usize::MAX)); } RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + // `TurnStarted` is the oldest boundary of the active reverse segment. if active_segment.as_ref().is_some_and(|active_segment| { active_segment .turn_id @@ -126,6 +143,8 @@ impl Session { RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); + // Reverse replay often sees `TurnComplete` before any turn-scoped metadata. + // Capture the turn id early so later `TurnContext` / abort items can match it. if active_segment.turn_id.is_none() { active_segment.turn_id = Some(event.turn_id.clone()); } @@ -152,6 +171,8 @@ impl Session { RolloutItem::TurnContext(ctx) => { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); + // Legacy rollouts can omit lifecycle ids, so a bare `TurnContextItem` still + // establishes a user-turn segment and its metadata by itself. if active_segment.turn_id.is_none() { active_segment.turn_id = ctx.turn_id.clone(); active_segment.counts_as_user_turn = true; @@ -180,6 +201,8 @@ impl Session { && previous_model.is_some() && !matches!(reference_context_item, TurnReferenceContextItem::NeverSet) { + // At this point we have both eager resume metadata values and a history checkpoint + // for the surviving suffix, so older rollout items cannot affect this result. break; } } @@ -196,6 +219,9 @@ impl Session { let initial_context = self.build_initial_context(turn_context, None).await; let mut history = ContextManager::new(); + // Temporary eager bridge: rebuild exact history semantics from only the surviving suffix + // discovered by reverse replay. This keeps the history result stable while the control + // flow moves toward the future lazy reverse loader design. for item in &rollout_items[replay_start_index.unwrap_or(0)..] { match item { RolloutItem::ResponseItem(response_item) => { From e11a7f27b3e8a6d3bdbbc6cd3509443781848f03 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 15:43:52 -0800 Subject: [PATCH 44/64] Simplify rollout reconstruction turn id matching --- .../core/src/codex/rollout_reconstruction.rs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index d736a91e94e..14578bfb91b 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -34,6 +34,11 @@ struct ActiveReplaySegment { replacement_history_index: Option, } +fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool { + active_turn_id + .is_none_or(|turn_id| item_turn_id.is_none_or(|item_turn_id| item_turn_id == turn_id)) +} + impl Session { pub(super) async fn reconstruct_history_from_rollout( &self, @@ -108,7 +113,7 @@ impl Session { // `TurnContextItem` in this same segment has already re-established it. if matches!( active_segment.reference_context_item, - TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared + TurnReferenceContextItem::NeverSet ) { active_segment.reference_context_item = TurnReferenceContextItem::Cleared; } @@ -125,10 +130,10 @@ impl Session { RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { // `TurnStarted` is the oldest boundary of the active reverse segment. if active_segment.as_ref().is_some_and(|active_segment| { - active_segment - .turn_id - .as_deref() - .is_none_or(|turn_id| turn_id == event.turn_id) + turn_ids_are_compatible( + active_segment.turn_id.as_deref(), + Some(event.turn_id.as_str()), + ) }) && let Some(active_segment) = active_segment.take() { finalize_active_segment( @@ -177,11 +182,10 @@ impl Session { active_segment.turn_id = ctx.turn_id.clone(); active_segment.counts_as_user_turn = true; } - if active_segment.turn_id.as_deref().is_none_or(|turn_id| { - ctx.turn_id - .as_deref() - .is_none_or(|ctx_turn_id| ctx_turn_id == turn_id) - }) { + if turn_ids_are_compatible( + active_segment.turn_id.as_deref(), + ctx.turn_id.as_deref(), + ) { active_segment.previous_model = Some(ctx.model.clone()); if matches!( active_segment.reference_context_item, From 9710a985fbb67a3f0f1a967c24601a576ea50b3f Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 15:45:20 -0800 Subject: [PATCH 45/64] Name reconstructed reference context binding --- codex-rs/core/src/codex/rollout_reconstruction.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 14578bfb91b..9d864ad1a0b 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -256,15 +256,17 @@ impl Session { } } + let reference_context_item = match reference_context_item { + TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared => None, + TurnReferenceContextItem::LatestSet(turn_reference_context_item) => { + Some(*turn_reference_context_item) + } + }; + RolloutReconstruction { history: history.raw_items().to_vec(), previous_model, - reference_context_item: match reference_context_item { - TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared => None, - TurnReferenceContextItem::LatestSet(turn_reference_context_item) => { - Some(*turn_reference_context_item) - } - }, + reference_context_item, } } } From cc25272ca5f087cb8f0a062b40fca02b6d2a5d14 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 16:07:43 -0800 Subject: [PATCH 46/64] Fix rebased rollout reconstruction tests --- .../src/codex/rollout_reconstruction_tests.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 88b6f74731a..1288dd1ea75 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -40,6 +40,8 @@ async fn record_initial_history_resumed_hydrates_previous_model() { let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -77,6 +79,8 @@ async fn record_initial_history_resumed_hydrates_previous_model_from_lifecycle_t let mut previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -655,6 +659,8 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -720,6 +726,8 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis serde_json::to_value(Some(TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -745,6 +753,8 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -844,6 +854,8 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo let current_context_item = TurnContextItem { turn_id: Some(current_turn_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -939,6 +951,8 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -1073,6 +1087,8 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, From 023cedaa25c056edf87e8be2c04190c84c67f966 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 16:19:03 -0800 Subject: [PATCH 47/64] Remove index-based rollout replay bridge --- .../core/src/codex/rollout_reconstruction.rs | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 9d864ad1a0b..405b7a46c10 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -31,7 +31,7 @@ struct ActiveReplaySegment { counts_as_user_turn: bool, previous_model: Option, reference_context_item: TurnReferenceContextItem, - replacement_history_index: Option, + base_replacement_history: Option>, } fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool { @@ -48,23 +48,24 @@ impl Session { // Replay metadata should already match the shape of the future lazy reverse loader, even // while history materialization still uses an eager bridge. Scan newest-to-oldest, // stopping once a surviving replacement-history checkpoint and the required resume metadata - // are both known; then replay only that suffix forward to preserve exact history semantics. - // Index of the earliest rollout item that still needs forward materialization. Once reverse - // replay finds a surviving `replacement_history` checkpoint, everything before that index - // is obsolete for rebuilt history. - let mut replay_start_index = None; + // are both known; then replay only the buffered surviving tail forward to preserve exact + // history semantics. + let mut base_replacement_history = None; let mut previous_model = None; let mut reference_context_item = TurnReferenceContextItem::NeverSet; // Rollback is "drop the newest N user turns". While scanning in reverse, that becomes // "skip the next N user-turn segments we finalize". let mut pending_rollback_turns = 0usize; + // Rollout items that survive after the newest replacement-history checkpoint, stored in + // reverse scan order so they can later be replayed forward on top of that checkpoint. + let mut rollout_suffix_items_rev = Vec::new(); // Reverse replay accumulates rollout items into the newest in-progress turn segment until // we hit its matching `TurnStarted`, at which point the segment can be finalized. let mut active_segment: Option = None; let finalize_active_segment = |active_segment: ActiveReplaySegment, - replay_start_index: &mut Option, + base_replacement_history: &mut Option>, previous_model: &mut Option, reference_context_item: &mut TurnReferenceContextItem, pending_rollback_turns: &mut usize| { @@ -79,11 +80,11 @@ impl Session { // A surviving replacement-history checkpoint is a complete history base. Once we // know the newest surviving one, older rollout items do not affect rebuilt history. - if replay_start_index.is_none() - && let Some(replacement_history_index) = - active_segment.replacement_history_index + if base_replacement_history.is_none() + && let Some(segment_base_replacement_history) = + active_segment.base_replacement_history { - *replay_start_index = Some(replacement_history_index); + *base_replacement_history = Some(segment_base_replacement_history); } // `previous_model` comes from the newest surviving user turn that established one. @@ -104,7 +105,8 @@ impl Session { } }; - for (index, item) in rollout_items.iter().enumerate().rev() { + for item in rollout_items.iter().rev() { + let mut should_buffer_item = true; match item { RolloutItem::Compacted(compacted) => { let active_segment = @@ -117,10 +119,11 @@ impl Session { ) { active_segment.reference_context_item = TurnReferenceContextItem::Cleared; } - if active_segment.replacement_history_index.is_none() - && compacted.replacement_history.is_some() + if active_segment.base_replacement_history.is_none() + && let Some(replacement_history) = &compacted.replacement_history { - active_segment.replacement_history_index = Some(index); + active_segment.base_replacement_history = Some(replacement_history.clone()); + should_buffer_item = false; } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { @@ -138,7 +141,7 @@ impl Session { { finalize_active_segment( active_segment, - &mut replay_start_index, + &mut base_replacement_history, &mut previous_model, &mut reference_context_item, &mut pending_rollback_turns, @@ -201,20 +204,25 @@ impl Session { | RolloutItem::SessionMeta(_) => {} } - if replay_start_index.is_some() + if base_replacement_history.is_some() && previous_model.is_some() && !matches!(reference_context_item, TurnReferenceContextItem::NeverSet) { - // At this point we have both eager resume metadata values and a history checkpoint - // for the surviving suffix, so older rollout items cannot affect this result. + // At this point we have both eager resume metadata values and the replacement- + // history base for the surviving tail, so older rollout items cannot affect this + // result. break; } + + if should_buffer_item { + rollout_suffix_items_rev.push(item.clone()); + } } if let Some(active_segment) = active_segment.take() { finalize_active_segment( active_segment, - &mut replay_start_index, + &mut base_replacement_history, &mut previous_model, &mut reference_context_item, &mut pending_rollback_turns, @@ -223,10 +231,14 @@ impl Session { let initial_context = self.build_initial_context(turn_context, None).await; let mut history = ContextManager::new(); - // Temporary eager bridge: rebuild exact history semantics from only the surviving suffix + if let Some(base_replacement_history) = base_replacement_history { + history.replace(base_replacement_history); + } + // Temporary eager bridge: rebuild exact history semantics from the buffered surviving tail // discovered by reverse replay. This keeps the history result stable while the control - // flow moves toward the future lazy reverse loader design. - for item in &rollout_items[replay_start_index.unwrap_or(0)..] { + // flow moves toward the future lazy reverse loader design without depending on absolute + // rollout indices. + for item in rollout_suffix_items_rev.iter().rev() { match item { RolloutItem::ResponseItem(response_item) => { history.record_items( From 7fa5958192cbaaad78b4e80bd45d0b96bd56b0a1 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 16:21:23 -0800 Subject: [PATCH 48/64] Rename replay baseline state variant --- codex-rs/core/src/codex/rollout_reconstruction.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 405b7a46c10..38f89127ee5 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -22,7 +22,7 @@ enum TurnReferenceContextItem { /// A previously established baseline was invalidated by later compaction. Cleared, /// The latest baseline established by this replay span. - LatestSet(Box), + Latest(Box), } #[derive(Debug, Default)] @@ -195,7 +195,7 @@ impl Session { TurnReferenceContextItem::NeverSet ) { active_segment.reference_context_item = - TurnReferenceContextItem::LatestSet(Box::new(ctx.clone())); + TurnReferenceContextItem::Latest(Box::new(ctx.clone())); } } } @@ -270,7 +270,7 @@ impl Session { let reference_context_item = match reference_context_item { TurnReferenceContextItem::NeverSet | TurnReferenceContextItem::Cleared => None, - TurnReferenceContextItem::LatestSet(turn_reference_context_item) => { + TurnReferenceContextItem::Latest(turn_reference_context_item) => { Some(*turn_reference_context_item) } }; From 1f67e04d040e3b48061d5ae08aeb3166a66de72e Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 16:47:11 -0800 Subject: [PATCH 49/64] Clarify rollout suffix replay naming --- codex-rs/core/src/codex/rollout_reconstruction.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 38f89127ee5..38be26fa5b7 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -106,7 +106,7 @@ impl Session { }; for item in rollout_items.iter().rev() { - let mut should_buffer_item = true; + let mut should_push_to_rollout_suffix = true; match item { RolloutItem::Compacted(compacted) => { let active_segment = @@ -123,7 +123,7 @@ impl Session { && let Some(replacement_history) = &compacted.replacement_history { active_segment.base_replacement_history = Some(replacement_history.clone()); - should_buffer_item = false; + should_push_to_rollout_suffix = false; } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { @@ -214,7 +214,7 @@ impl Session { break; } - if should_buffer_item { + if should_push_to_rollout_suffix { rollout_suffix_items_rev.push(item.clone()); } } From 41fe131e1bd9ea645d8169f6335ee6c896ca204f Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 16:51:07 -0800 Subject: [PATCH 50/64] Require user messages for replayed user turns --- .../core/src/codex/rollout_reconstruction.rs | 5 ++--- .../src/codex/rollout_reconstruction_tests.rs | 22 +++++-------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 38be26fa5b7..5d39cf9175f 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -179,11 +179,10 @@ impl Session { RolloutItem::TurnContext(ctx) => { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); - // Legacy rollouts can omit lifecycle ids, so a bare `TurnContextItem` still - // establishes a user-turn segment and its metadata by itself. + // `TurnContextItem` can attach metadata to an existing segment, but only a + // real `UserMessage` event should make the segment count as a user turn. if active_segment.turn_id.is_none() { active_segment.turn_id = ctx.turn_id.clone(); - active_segment.counts_as_user_turn = true; } if turn_ids_are_compatible( active_segment.turn_id.as_deref(), diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 1288dd1ea75..5765bda145b 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -34,7 +34,7 @@ fn assistant_message(text: &str) -> ResponseItem { } #[tokio::test] -async fn record_initial_history_resumed_hydrates_previous_model() { +async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previous_model() { let (session, turn_context) = make_session_and_context().await; let previous_model = "previous-rollout-model"; let previous_context_item = TurnContextItem { @@ -65,10 +65,8 @@ async fn record_initial_history_resumed_hydrates_previous_model() { })) .await; - assert_eq!( - session.previous_model().await, - Some(previous_model.to_string()) - ); + assert_eq!(session.previous_model().await, None); + assert!(session.reference_context_item().await.is_none()); } #[tokio::test] @@ -603,7 +601,7 @@ async fn record_initial_history_resumed_rollback_drops_incomplete_user_turn_comp } #[tokio::test] -async fn record_initial_history_resumed_seeds_reference_context_item_without_compaction() { +async fn record_initial_history_resumed_bare_turn_context_does_not_seed_reference_context_item() { let (session, turn_context) = make_session_and_context().await; let previous_context_item = turn_context.to_turn_context_item(); let rollout_items = vec![RolloutItem::TurnContext(previous_context_item.clone())]; @@ -616,12 +614,7 @@ async fn record_initial_history_resumed_seeds_reference_context_item_without_com })) .await; - assert_eq!( - serde_json::to_value(session.reference_context_item().await) - .expect("serialize seeded reference context item"), - serde_json::to_value(Some(previous_context_item)) - .expect("serialize expected reference context item") - ); + assert!(session.reference_context_item().await.is_none()); } #[tokio::test] @@ -644,10 +637,7 @@ async fn record_initial_history_resumed_does_not_seed_reference_context_item_aft })) .await; - assert_eq!( - session.previous_model().await, - Some(turn_context.model_info.slug.clone()) - ); + assert_eq!(session.previous_model().await, None); assert!(session.reference_context_item().await.is_none()); } From a128d2070d8606fc22b1b269ce9ffbae3682d98a Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 16:52:42 -0800 Subject: [PATCH 51/64] Stop buffering rollout items before replacement history --- codex-rs/core/src/codex/rollout_reconstruction.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 5d39cf9175f..ed5f71db45f 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -106,7 +106,6 @@ impl Session { }; for item in rollout_items.iter().rev() { - let mut should_push_to_rollout_suffix = true; match item { RolloutItem::Compacted(compacted) => { let active_segment = @@ -123,7 +122,6 @@ impl Session { && let Some(replacement_history) = &compacted.replacement_history { active_segment.base_replacement_history = Some(replacement_history.clone()); - should_push_to_rollout_suffix = false; } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { @@ -213,7 +211,7 @@ impl Session { break; } - if should_push_to_rollout_suffix { + if base_replacement_history.is_none() { rollout_suffix_items_rev.push(item.clone()); } } From 1e7fc3294dcb0549184351bd21705082aa8a3fe5 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 17:02:46 -0800 Subject: [PATCH 52/64] Reorder reverse replay match cases --- .../core/src/codex/rollout_reconstruction.rs | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index ed5f71db45f..e4b2772188b 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -128,24 +128,6 @@ impl Session { pending_rollback_turns = pending_rollback_turns .saturating_add(usize::try_from(rollback.num_turns).unwrap_or(usize::MAX)); } - RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { - // `TurnStarted` is the oldest boundary of the active reverse segment. - if active_segment.as_ref().is_some_and(|active_segment| { - turn_ids_are_compatible( - active_segment.turn_id.as_deref(), - Some(event.turn_id.as_str()), - ) - }) && let Some(active_segment) = active_segment.take() - { - finalize_active_segment( - active_segment, - &mut base_replacement_history, - &mut previous_model, - &mut reference_context_item, - &mut pending_rollback_turns, - ); - } - } RolloutItem::EventMsg(EventMsg::TurnComplete(event)) => { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); @@ -196,6 +178,24 @@ impl Session { } } } + RolloutItem::EventMsg(EventMsg::TurnStarted(event)) => { + // `TurnStarted` is the oldest boundary of the active reverse segment. + if active_segment.as_ref().is_some_and(|active_segment| { + turn_ids_are_compatible( + active_segment.turn_id.as_deref(), + Some(event.turn_id.as_str()), + ) + }) && let Some(active_segment) = active_segment.take() + { + finalize_active_segment( + active_segment, + &mut base_replacement_history, + &mut previous_model, + &mut reference_context_item, + &mut pending_rollback_turns, + ); + } + } RolloutItem::ResponseItem(_) | RolloutItem::EventMsg(_) | RolloutItem::SessionMeta(_) => {} From addaed87ab832b882e0a38420a14f213187c52ed Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 17:07:10 -0800 Subject: [PATCH 53/64] Clarify TurnContextItem persistence docs --- codex-rs/protocol/src/protocol.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 9a631ea0bd0..af963b74295 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2125,8 +2125,8 @@ pub struct TurnContextNetworkItem { } /// Persist for the first real user turn in a thread and for later turns that -/// persist corresponding model-visible context updates, so resume/fork can -/// recover `previous_model` without treating post-compaction reinjection as a +/// emit model-visible context updates, so resume/fork can recover +/// `previous_model` without treating every in-memory baseline advance as a /// durable `reference_context_item` baseline. #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct TurnContextItem { From e9828f6dc9497b5e48d9b26a6b3b047b52f6bad9 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 17:15:36 -0800 Subject: [PATCH 54/64] Persist TurnContextItem for every user turn --- codex-rs/core/src/codex.rs | 61 +++++++++++++++++++++++-------- codex-rs/protocol/src/protocol.rs | 7 ++-- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 4ae3a7a3492..4e75119e055 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -3010,22 +3010,13 @@ impl Session { self.record_conversation_items(turn_context, &context_items) .await; } - if should_inject_full_context - || previous_user_turn_model.is_none() - || !context_items.is_empty() - { - // Keep rollout TurnContext entries for any turn that establishes or changes the - // persisted model-visible baseline: - // - the first real user turn (to recover `previous_model` on resume) - // - full-context reinjection after the baseline was cleared - // - steady-state turns that emitted explicit context diffs - self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) - .await; - } + // Persist one `TurnContextItem` per real user turn so resume/lazy replay can recover the + // latest durable baseline even when this turn emitted no model-visible context diffs. + self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item.clone())]) + .await; // Advance the in-memory diff baseline even when this turn emitted no model-visible - // context items. This keeps later runtime diffing aligned with the current turn state - // without forcing rollout `TurnContextItem` churn after compaction/no-op turns. + // context items. This keeps later runtime diffing aligned with the current turn state. let mut state = self.state.lock().await; state.set_reference_context_item(Some(turn_context_item)); } @@ -8563,7 +8554,7 @@ mod tests { } #[tokio::test] - async fn record_context_updates_and_set_reference_context_item_updates_baseline_without_emitting_diffs() + async fn record_context_updates_and_set_reference_context_item_persists_baseline_without_emitting_diffs() { let (session, previous_context) = make_session_and_context().await; let next_model = if previous_context.model_info.slug == "gpt-5.1" { @@ -8579,6 +8570,27 @@ mod tests { let mut state = session.state.lock().await; state.set_reference_context_item(Some(previous_context_item.clone())); } + let config = session.get_config().await; + let recorder = RolloutRecorder::new( + config.as_ref(), + RolloutRecorderParams::new( + ThreadId::default(), + None, + SessionSource::Exec, + BaseInstructions::default(), + Vec::new(), + EventPersistenceMode::Limited, + ), + None, + None, + ) + .await + .expect("create rollout recorder"); + let rollout_path = recorder.rollout_path().to_path_buf(); + { + let mut rollout = session.services.rollout.lock().await; + *rollout = Some(recorder); + } let update_items = session.build_settings_update_items(Some(&previous_context_item), None, &turn_context); @@ -8598,6 +8610,25 @@ mod tests { serde_json::to_value(Some(turn_context.to_turn_context_item())) .expect("serialize expected context item") ); + session.ensure_rollout_materialized().await; + session.flush_rollout().await; + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read rollout history") + else { + panic!("expected resumed rollout history"); + }; + let persisted_turn_context = resumed.history.iter().find_map(|item| match item { + RolloutItem::TurnContext(ctx) => Some(ctx.clone()), + _ => None, + }); + assert_eq!( + serde_json::to_value(persisted_turn_context) + .expect("serialize persisted turn context item"), + serde_json::to_value(Some(turn_context.to_turn_context_item())) + .expect("serialize expected turn context item") + ); } #[tokio::test] diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index af963b74295..932d0ae534a 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2124,10 +2124,9 @@ pub struct TurnContextNetworkItem { pub denied_domains: Vec, } -/// Persist for the first real user turn in a thread and for later turns that -/// emit model-visible context updates, so resume/fork can recover -/// `previous_model` without treating every in-memory baseline advance as a -/// durable `reference_context_item` baseline. +/// Persist once per real user turn after computing that turn's model-visible +/// context updates, so resume/fork replay can recover the latest durable +/// baseline even when the turn emitted no context diffs. #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct TurnContextItem { #[serde(default, skip_serializing_if = "Option::is_none")] From be8c4653a9c36be4a13ba596865f02b131767aa5 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 17:16:44 -0800 Subject: [PATCH 55/64] Restore resumed history test attribute --- codex-rs/core/src/codex.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 4e75119e055..84c55bae457 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -7097,6 +7097,7 @@ mod tests { assert_eq!(expected, history.raw_items()); } + #[tokio::test] async fn resumed_history_injects_initial_context_on_first_context_update_only() { let (session, turn_context) = make_session_and_context().await; let (rollout_items, mut expected) = sample_rollout(&session, &turn_context).await; From 9b62fe1ba91da91ca9ce4612a09cf80269491109 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 17:19:41 -0800 Subject: [PATCH 56/64] Mention compaction in TurnContextItem docs --- codex-rs/protocol/src/protocol.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 932d0ae534a..880e6a24fe5 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2125,8 +2125,9 @@ pub struct TurnContextNetworkItem { } /// Persist once per real user turn after computing that turn's model-visible -/// context updates, so resume/fork replay can recover the latest durable -/// baseline even when the turn emitted no context diffs. +/// context updates, and again after mid-turn compaction when replacement +/// history re-establishes full context, so resume/fork replay can recover the +/// latest durable baseline. #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct TurnContextItem { #[serde(default, skip_serializing_if = "Option::is_none")] From aa7b019b85932c25280b4a35547fe2b46d108288 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 18:00:46 -0800 Subject: [PATCH 57/64] Borrow replay suffix instead of cloning rollout --- .../core/src/codex/rollout_reconstruction.rs | 117 +++++++++--------- 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index e4b2772188b..d2fa9e2a3e0 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -26,12 +26,12 @@ enum TurnReferenceContextItem { } #[derive(Debug, Default)] -struct ActiveReplaySegment { +struct ActiveReplaySegment<'a> { turn_id: Option, counts_as_user_turn: bool, previous_model: Option, reference_context_item: TurnReferenceContextItem, - base_replacement_history: Option>, + base_replacement_history: Option<&'a [ResponseItem]>, } fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool { @@ -39,6 +39,48 @@ fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&s .is_none_or(|turn_id| item_turn_id.is_none_or(|item_turn_id| item_turn_id == turn_id)) } +fn finalize_active_segment<'a>( + active_segment: ActiveReplaySegment<'a>, + base_replacement_history: &mut Option<&'a [ResponseItem]>, + previous_model: &mut Option, + reference_context_item: &mut TurnReferenceContextItem, + pending_rollback_turns: &mut usize, +) { + // Thread rollback always targets the newest surviving user turns, so consume that + // skip budget before letting this segment contribute metadata or a compaction base. + if *pending_rollback_turns > 0 { + if active_segment.counts_as_user_turn { + *pending_rollback_turns -= 1; + } + return; + } + + // A surviving replacement-history checkpoint is a complete history base. Once we + // know the newest surviving one, older rollout items do not affect rebuilt history. + if base_replacement_history.is_none() + && let Some(segment_base_replacement_history) = active_segment.base_replacement_history + { + *base_replacement_history = Some(segment_base_replacement_history); + } + + // `previous_model` comes from the newest surviving user turn that established one. + if previous_model.is_none() && active_segment.counts_as_user_turn { + *previous_model = active_segment.previous_model; + } + + // `reference_context_item` comes from the newest surviving user turn baseline, or + // from a surviving compaction that explicitly cleared that baseline. + if matches!(reference_context_item, TurnReferenceContextItem::NeverSet) + && (active_segment.counts_as_user_turn + || matches!( + active_segment.reference_context_item, + TurnReferenceContextItem::Cleared + )) + { + *reference_context_item = active_segment.reference_context_item; + } +} + impl Session { pub(super) async fn reconstruct_history_from_rollout( &self, @@ -50,62 +92,20 @@ impl Session { // stopping once a surviving replacement-history checkpoint and the required resume metadata // are both known; then replay only the buffered surviving tail forward to preserve exact // history semantics. - let mut base_replacement_history = None; + let mut base_replacement_history: Option<&[ResponseItem]> = None; let mut previous_model = None; let mut reference_context_item = TurnReferenceContextItem::NeverSet; // Rollback is "drop the newest N user turns". While scanning in reverse, that becomes // "skip the next N user-turn segments we finalize". let mut pending_rollback_turns = 0usize; - // Rollout items that survive after the newest replacement-history checkpoint, stored in - // reverse scan order so they can later be replayed forward on top of that checkpoint. - let mut rollout_suffix_items_rev = Vec::new(); + // Borrowed suffix of rollout items newer than the newest surviving replacement-history + // checkpoint. If no such checkpoint exists, this remains the full rollout. + let mut rollout_suffix = rollout_items; // Reverse replay accumulates rollout items into the newest in-progress turn segment until // we hit its matching `TurnStarted`, at which point the segment can be finalized. - let mut active_segment: Option = None; - - let finalize_active_segment = - |active_segment: ActiveReplaySegment, - base_replacement_history: &mut Option>, - previous_model: &mut Option, - reference_context_item: &mut TurnReferenceContextItem, - pending_rollback_turns: &mut usize| { - // Thread rollback always targets the newest surviving user turns, so consume that - // skip budget before letting this segment contribute metadata or a compaction base. - if *pending_rollback_turns > 0 { - if active_segment.counts_as_user_turn { - *pending_rollback_turns -= 1; - } - return; - } - - // A surviving replacement-history checkpoint is a complete history base. Once we - // know the newest surviving one, older rollout items do not affect rebuilt history. - if base_replacement_history.is_none() - && let Some(segment_base_replacement_history) = - active_segment.base_replacement_history - { - *base_replacement_history = Some(segment_base_replacement_history); - } - - // `previous_model` comes from the newest surviving user turn that established one. - if previous_model.is_none() && active_segment.counts_as_user_turn { - *previous_model = active_segment.previous_model; - } + let mut active_segment: Option> = None; - // `reference_context_item` comes from the newest surviving user turn baseline, or - // from a surviving compaction that explicitly cleared that baseline. - if matches!(reference_context_item, TurnReferenceContextItem::NeverSet) - && (active_segment.counts_as_user_turn - || matches!( - active_segment.reference_context_item, - TurnReferenceContextItem::Cleared - )) - { - *reference_context_item = active_segment.reference_context_item; - } - }; - - for item in rollout_items.iter().rev() { + for (index, item) in rollout_items.iter().enumerate().rev() { match item { RolloutItem::Compacted(compacted) => { let active_segment = @@ -121,7 +121,8 @@ impl Session { if active_segment.base_replacement_history.is_none() && let Some(replacement_history) = &compacted.replacement_history { - active_segment.base_replacement_history = Some(replacement_history.clone()); + active_segment.base_replacement_history = Some(replacement_history); + rollout_suffix = &rollout_items[index + 1..]; } } RolloutItem::EventMsg(EventMsg::ThreadRolledBack(rollback)) => { @@ -210,10 +211,6 @@ impl Session { // result. break; } - - if base_replacement_history.is_none() { - rollout_suffix_items_rev.push(item.clone()); - } } if let Some(active_segment) = active_segment.take() { @@ -229,13 +226,13 @@ impl Session { let initial_context = self.build_initial_context(turn_context, None).await; let mut history = ContextManager::new(); if let Some(base_replacement_history) = base_replacement_history { - history.replace(base_replacement_history); + history.replace(base_replacement_history.to_vec()); } - // Temporary eager bridge: rebuild exact history semantics from the buffered surviving tail + // Temporary eager bridge: rebuild exact history semantics from the borrowed rollout suffix // discovered by reverse replay. This keeps the history result stable while the control - // flow moves toward the future lazy reverse loader design without depending on absolute - // rollout indices. - for item in rollout_suffix_items_rev.iter().rev() { + // flow moves toward the future lazy reverse loader design without cloning the full rollout + // in the common no-compaction case. + for item in rollout_suffix { match item { RolloutItem::ResponseItem(response_item) => { history.record_items( From ff0efdbbee8e96b2f9d4ddc213c33a5b671bff94 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 18:13:18 -0800 Subject: [PATCH 58/64] Fix rollout tests for TurnContext persistence --- codex-rs/core/src/codex.rs | 29 +++++++++++++++++++++++++++- codex-rs/core/tests/suite/compact.rs | 8 ++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 84c55bae457..ce8b35849cf 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -7310,7 +7310,34 @@ mod tests { final_output_json_schema: None, truncation_policy: Some(turn_context.truncation_policy.into()), }; - let rollout_items = vec![RolloutItem::TurnContext(previous_context_item)]; + let turn_id = previous_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let rollout_items = vec![ + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "forked seed".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(previous_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id, + last_agent_message: None, + }, + )), + ]; session .record_initial_history(InitialHistory::Forked(rollout_items)) diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 2c3ac45625c..618861f33e3 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -394,8 +394,8 @@ async fn summarize_context_three_requests_and_instructions() { } assert_eq!( - regular_turn_context_count, 1, - "manual compact should not emit its own TurnContext entry" + regular_turn_context_count, 2, + "rollout should contain one TurnContext entry per real user turn" ); assert!( saw_compacted_summary, @@ -2081,8 +2081,8 @@ async fn auto_compact_persists_rollout_entries() { } assert_eq!( - turn_context_count, 1, - "pre-sampling compaction should not emit its own TurnContext entry" + turn_context_count, 3, + "rollout should contain one TurnContext entry per real user turn" ); } From 1159a5251193bdb42421301523c4c5fa38221362 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Thu, 26 Feb 2026 21:02:32 -0800 Subject: [PATCH 59/64] Clarify rollout reconstruction materialization comment --- codex-rs/core/src/codex/rollout_reconstruction.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index d2fa9e2a3e0..64ae58fd37e 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -228,10 +228,9 @@ impl Session { if let Some(base_replacement_history) = base_replacement_history { history.replace(base_replacement_history.to_vec()); } - // Temporary eager bridge: rebuild exact history semantics from the borrowed rollout suffix - // discovered by reverse replay. This keeps the history result stable while the control - // flow moves toward the future lazy reverse loader design without cloning the full rollout - // in the common no-compaction case. + // Materialize exact history semantics from the replay-derived suffix. The eventual lazy + // design should keep this same replay shape, but drive it from a resumable reverse source + // instead of an eagerly loaded `&[RolloutItem]`. for item in rollout_suffix { match item { RolloutItem::ResponseItem(response_item) => { From c6d346a829fe888b7f2184bb10abefda341dd625 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Fri, 27 Feb 2026 11:54:53 -0800 Subject: [PATCH 60/64] Simplify legacy rollout compaction rebuild --- .../core/src/codex/rollout_reconstruction.rs | 16 +++- .../src/codex/rollout_reconstruction_tests.rs | 73 +++++++++++++++++++ codex-rs/core/src/compact.rs | 11 ++- 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 64ae58fd37e..76537dc895f 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -223,8 +223,8 @@ impl Session { ); } - let initial_context = self.build_initial_context(turn_context, None).await; let mut history = ContextManager::new(); + let mut saw_legacy_compaction_without_replacement_history = false; if let Some(base_replacement_history) = base_replacement_history { history.replace(base_replacement_history.to_vec()); } @@ -243,9 +243,16 @@ impl Session { if let Some(replacement_history) = &compacted.replacement_history { history.replace(replacement_history.clone()); } else { + saw_legacy_compaction_without_replacement_history = true; + // Legacy rollouts without `replacement_history` should rebuild the + // historical TurnContext at the correct insertion point from persisted + // `TurnContextItem`s. These are rare enough that we currently just clear + // `reference_context_item`, reinject canonical context at the end of the + // resumed conversation, and accept the temporary out-of-distribution + // prompt shape. let user_messages = collect_user_messages(history.raw_items()); let rebuilt = compact::build_compacted_history( - initial_context.clone(), + Vec::new(), &user_messages, &compacted.message, ); @@ -267,6 +274,11 @@ impl Session { Some(*turn_reference_context_item) } }; + let reference_context_item = if saw_legacy_compaction_without_replacement_history { + None + } else { + reference_context_item + }; RolloutReconstruction { history: history.raw_items().to_vec(), diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 5765bda145b..d2cbca79068 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -641,6 +641,79 @@ async fn record_initial_history_resumed_does_not_seed_reference_context_item_aft assert!(session.reference_context_item().await.is_none()); } +#[tokio::test] +async fn reconstruct_history_legacy_compaction_without_replacement_history_does_not_inject_current_initial_context() + { + let (session, turn_context) = make_session_and_context().await; + let rollout_items = vec![ + RolloutItem::ResponseItem(user_message("before compact")), + RolloutItem::ResponseItem(assistant_message("assistant reply")), + RolloutItem::Compacted(CompactedItem { + message: "legacy summary".to_string(), + replacement_history: None, + }), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert_eq!( + reconstructed.history, + vec![ + user_message("before compact"), + user_message("legacy summary"), + ] + ); + assert!(reconstructed.reference_context_item.is_none()); +} + +#[tokio::test] +async fn reconstruct_history_legacy_compaction_without_replacement_history_clears_later_reference_context_item() + { + let (session, turn_context) = make_session_and_context().await; + let current_context_item = turn_context.to_turn_context_item(); + let current_turn_id = current_context_item + .turn_id + .clone() + .expect("turn context should have turn_id"); + let rollout_items = vec![ + RolloutItem::ResponseItem(user_message("before compact")), + RolloutItem::Compacted(CompactedItem { + message: "legacy summary".to_string(), + replacement_history: None, + }), + RolloutItem::EventMsg(EventMsg::TurnStarted( + codex_protocol::protocol::TurnStartedEvent { + turn_id: current_turn_id.clone(), + model_context_window: Some(128_000), + collaboration_mode_kind: ModeKind::Default, + }, + )), + RolloutItem::EventMsg(EventMsg::UserMessage( + codex_protocol::protocol::UserMessageEvent { + message: "after legacy compact".to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + }, + )), + RolloutItem::TurnContext(current_context_item), + RolloutItem::EventMsg(EventMsg::TurnComplete( + codex_protocol::protocol::TurnCompleteEvent { + turn_id: current_turn_id, + last_agent_message: None, + }, + )), + ]; + + let reconstructed = session + .reconstruct_history_from_rollout(&turn_context, &rollout_items) + .await; + + assert!(reconstructed.reference_context_item.is_none()); +} + #[tokio::test] async fn record_initial_history_resumed_turn_context_after_compaction_reestablishes_reference_context_item() { diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 0441f9c4969..f0003f71541 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -232,15 +232,14 @@ async fn run_compact_task_inner( message: summary_text.clone(), replacement_history: Some(new_history), }); - let rollout_items = if let Some(turn_context_item) = reference_context_item { + sess.persist_rollout_items(&[rollout_item]).await; + if let Some(turn_context_item) = reference_context_item { // Mid-turn compaction re-injected initial context into the replacement history, so // persist a fresh `TurnContextItem` after `Compacted` to re-establish the baseline for // resume/fork replay. - vec![rollout_item, RolloutItem::TurnContext(turn_context_item)] - } else { - vec![rollout_item] - }; - sess.persist_rollout_items(&rollout_items).await; + sess.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item)]) + .await; + } sess.emit_turn_item_completed(&turn_context, compaction_item) .await; From 5f8e9ddc0df05c35907895f91f6e0b710e191878 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Fri, 27 Feb 2026 12:06:17 -0800 Subject: [PATCH 61/64] Clarify rollback replay comment --- codex-rs/core/src/codex/rollout_reconstruction.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 76537dc895f..1895c006f26 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -46,8 +46,9 @@ fn finalize_active_segment<'a>( reference_context_item: &mut TurnReferenceContextItem, pending_rollback_turns: &mut usize, ) { - // Thread rollback always targets the newest surviving user turns, so consume that - // skip budget before letting this segment contribute metadata or a compaction base. + // Thread rollback drops the newest surviving real user-message boundaries. In replay, that + // means skipping the next finalized segments that contain a non-contextual + // `EventMsg::UserMessage`. if *pending_rollback_turns > 0 { if active_segment.counts_as_user_turn { *pending_rollback_turns -= 1; From 4aa32da026162b93fbda97c504432b7910ad521e Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Fri, 27 Feb 2026 12:08:52 -0800 Subject: [PATCH 62/64] Align rollout reconstruction fixture with legacy compaction --- codex-rs/core/src/codex.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index ce8b35849cf..1b7d105d542 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -9220,8 +9220,7 @@ mod tests { .clone() .for_prompt(&reconstruction_turn.model_info.input_modalities); let user_messages1 = collect_user_messages(&snapshot1); - let rebuilt1 = - compact::build_compacted_history(initial_context.clone(), &user_messages1, summary1); + let rebuilt1 = compact::build_compacted_history(Vec::new(), &user_messages1, summary1); live_history.replace(rebuilt1); rollout_items.push(RolloutItem::Compacted(CompactedItem { message: summary1.to_string(), @@ -9263,8 +9262,7 @@ mod tests { .clone() .for_prompt(&reconstruction_turn.model_info.input_modalities); let user_messages2 = collect_user_messages(&snapshot2); - let rebuilt2 = - compact::build_compacted_history(initial_context.clone(), &user_messages2, summary2); + let rebuilt2 = compact::build_compacted_history(Vec::new(), &user_messages2, summary2); live_history.replace(rebuilt2); rollout_items.push(RolloutItem::Compacted(CompactedItem { message: summary2.to_string(), From 5d6ca57ea930570417f2dc97c4993d8423fbeaa8 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Fri, 27 Feb 2026 13:24:53 -0800 Subject: [PATCH 63/64] Unify compaction history replacement persistence --- codex-rs/core/src/codex.rs | 18 ++++++++++++++++++ codex-rs/core/src/compact.rs | 20 +++++--------------- codex-rs/core/src/compact_remote.rs | 22 ++++------------------ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 1b7d105d542..a8a2bf35f48 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -194,6 +194,7 @@ use crate::protocol::AgentReasoningSectionBreakEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; use crate::protocol::AskForApproval; use crate::protocol::BackgroundEventEvent; +use crate::protocol::CompactedItem; use crate::protocol::DeprecationNoticeEvent; use crate::protocol::ErrorEvent; use crate::protocol::Event; @@ -2805,6 +2806,23 @@ impl Session { state.replace_history(items, reference_context_item); } + pub(crate) async fn replace_compacted_history( + &self, + items: Vec, + reference_context_item: Option, + compacted_item: CompactedItem, + ) { + self.replace_history(items, reference_context_item.clone()) + .await; + + self.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) + .await; + if let Some(turn_context_item) = reference_context_item { + self.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item)]) + .await; + } + } + async fn persist_rollout_response_items(&self, items: &[ResponseItem]) { let rollout_items: Vec = items .iter() diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index f0003f71541..2885a885fd2 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -22,7 +22,6 @@ use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::RolloutItem; use codex_protocol::user_input::UserInput; use futures::prelude::*; use tracing::error; @@ -224,23 +223,14 @@ async fn run_compact_task_inner( InitialContextInjection::DoNotInject => None, InitialContextInjection::BeforeLastUserMessage => Some(turn_context.to_turn_context_item()), }; - sess.replace_history(new_history.clone(), reference_context_item.clone()) + let compacted_item = CompactedItem { + message: summary_text.clone(), + replacement_history: Some(new_history.clone()), + }; + sess.replace_compacted_history(new_history, reference_context_item, compacted_item) .await; sess.recompute_token_usage(&turn_context).await; - let rollout_item = RolloutItem::Compacted(CompactedItem { - message: summary_text.clone(), - replacement_history: Some(new_history), - }); - sess.persist_rollout_items(&[rollout_item]).await; - if let Some(turn_context_item) = reference_context_item { - // Mid-turn compaction re-injected initial context into the replacement history, so - // persist a fresh `TurnContextItem` after `Compacted` to re-establish the baseline for - // resume/fork replay. - sess.persist_rollout_items(&[RolloutItem::TurnContext(turn_context_item)]) - .await; - } - sess.emit_turn_item_completed(&turn_context, compaction_item) .await; let warning = EventMsg::Warning(WarningEvent { diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 3019a4ad8b6..8029ba1bb7e 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -13,7 +13,6 @@ use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::protocol::CompactedItem; use crate::protocol::EventMsg; -use crate::protocol::RolloutItem; use crate::protocol::TurnStartedEvent; use codex_protocol::items::ContextCompactionItem; use codex_protocol::items::TurnItem; @@ -159,26 +158,13 @@ async fn run_remote_compact_task_inner_impl( InitialContextInjection::DoNotInject => None, InitialContextInjection::BeforeLastUserMessage => Some(turn_context.to_turn_context_item()), }; - sess.replace_history(new_history.clone(), reference_context_item.clone()) - .await; - sess.recompute_token_usage(turn_context).await; - let compacted_item = CompactedItem { message: String::new(), - replacement_history: Some(new_history), + replacement_history: Some(new_history.clone()), }; - let rollout_items = if let Some(turn_context_item) = reference_context_item { - // Mid-turn compaction re-injected initial context into the replacement history, so - // persist a fresh `TurnContextItem` after `Compacted` to re-establish the baseline for - // resume/fork replay. - vec![ - RolloutItem::Compacted(compacted_item), - RolloutItem::TurnContext(turn_context_item), - ] - } else { - vec![RolloutItem::Compacted(compacted_item)] - }; - sess.persist_rollout_items(&rollout_items).await; + sess.replace_compacted_history(new_history, reference_context_item, compacted_item) + .await; + sess.recompute_token_usage(turn_context).await; sess.emit_turn_item_completed(turn_context, compaction_item) .await; From 25e8ea4acceb07bca9276293581f66313138dc30 Mon Sep 17 00:00:00 2001 From: Charles Cunningham Date: Fri, 27 Feb 2026 13:26:54 -0800 Subject: [PATCH 64/64] Comment --- codex-rs/core/src/codex/rollout_reconstruction.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codex-rs/core/src/codex/rollout_reconstruction.rs b/codex-rs/core/src/codex/rollout_reconstruction.rs index 1895c006f26..1cfe5486693 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction.rs @@ -242,6 +242,8 @@ impl Session { } RolloutItem::Compacted(compacted) => { if let Some(replacement_history) = &compacted.replacement_history { + // This should actually never happen, because the reverse loop above (to build rollout_suffix) + // should stop before any compaction that has Some replacement_history history.replace(replacement_history.clone()); } else { saw_legacy_compaction_without_replacement_history = true; @@ -251,6 +253,8 @@ impl Session { // `reference_context_item`, reinject canonical context at the end of the // resumed conversation, and accept the temporary out-of-distribution // prompt shape. + // TODO(ccunningham): if we drop support for None replacement_history compaction items, + // we can get rid of this second loop entirely and just build `history` directly in the first loop. let user_messages = collect_user_messages(history.raw_items()); let rebuilt = compact::build_compacted_history( Vec::new(),