fix(server): Codex runs and delegated tasks settle when the shared app-server exits

apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts

@@ -4168,7 +4168,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
     );
   });
 
-  it.effect("Stop ends a background command no Codex process tracks any more", () =>
+  it.effect("settles a background command automatically after the Codex process exits", () =>
     Effect.scoped(
       Effect.gen(function* () {
         const fs = yield* FileSystem.FileSystem;
@@ -4244,27 +4244,19 @@ describe("CodexAdapterV2 post-settle continuation", () => {
           yield* worker.drain();
           yield* Fiber.join(settled);
           yield* worker.drain();
-          const before = yield* orchestrator.getThreadShell(threadId);
-          assert.deepEqual(
-            before?.pendingBackgroundTasks?.map((task) => task.kind),
-            ["command"],
-            "the thread still shows the command the gone process never finished",
-          );
-          const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
-          yield* orchestrator.dispatch({
-            type: "run.interrupt",
-            commandId: CommandId.make("stop-background-untracked"),
-            threadId,
-            runId: run.id,
-            holdQueue: true,
-          });
-          yield* worker.drain();
           const projection = yield* orchestrator.getThreadProjection(threadId);
           assert.deepEqual(
             projection.turnItems.flatMap((item) =>
               item.type === "command_execution" ? [item.status] : [],
             ),
-            ["interrupted"],
+            ["failed"],
+          );
+          const command = projection.turnItems.find((item) => item.type === "command_execution");
+          assert.isDefined(command);
+          assert.isNotNull(command?.completedAt);
+          assert.notInclude(
+            projection.turnItems.map((item) => item.type),
+            "run_interrupt_request",
           );
           assert.equal(projection.runs.at(-1)?.status, "completed");
           assert.deepEqual(

apps/server/src/orchestration-v2/EventSink.ts

@@ -68,6 +68,8 @@ export type EventSinkV2Error = typeof EventSinkV2Error.Type;
 
 export interface EventSinkV2Shape {
   readonly write: (input: {
+    /** Synchronous in-memory handoff after commit, before publication or pending cancellation. */
+    readonly onCommitted?: (events: ReadonlyArray<OrchestrationV2StoredEvent>) => void;
     /** Return no events if another thread owns this native session at commit time. */
     readonly unlessNativeThreadOwned?: Pick<
       OrchestrationV2ProviderThread,
@@ -409,6 +411,7 @@ const layerBase: Layer.Layer<
 
     const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* (
       input: Parameters<EventSinkV2Shape["writeWithEffects"]>[0],
+      onCommitted?: Parameters<EventSinkV2Shape["write"]>[0]["onCommitted"],
     ) {
       yield* Effect.annotateCurrentSpan({
         "orchestration_v2.command_id": input.commandId ?? null,
@@ -452,6 +455,7 @@ const layerBase: Layer.Layer<
         }).pipe(Effect.withTracerEnabled(false)),
         ({ committed, outcomes }) =>
           Effect.gen(function* () {
+            yield* Effect.sync(() => onCommitted?.(committed));
             if (input.effects.length > 0) {
               yield* effectOutbox.notifyAvailable(input.effects.length);
             }
@@ -879,7 +883,7 @@ const layerBase: Layer.Layer<
 
     return EventSinkV2.of({
       write: (input) =>
-        writeEffect({ ...input, effects: [] }).pipe(
+        writeEffect({ ...input, effects: [] }, input.onCommitted).pipe(
           Effect.mapError(
             (cause) =>
               new EventSinkWriteError({

apps/server/src/orchestration-v2/FoundationPersistence.test.ts

@@ -763,7 +763,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
     }),
   );
 
-  it.effect("replays shared provider-session payloads across every bound thread", () =>
+  it.effect("replays provider-session payloads independently for each bound thread", () =>
     Effect.gen(function* () {
       const eventSink = yield* EventSink.EventSinkV2;
       const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
@@ -819,12 +819,20 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
         ],
       });
 
-      assert.equal(
-        (yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.cwd,
-        secondSession.cwd,
-      );
+      const assertBindings = Effect.gen(function* () {
+        assert.deepEqual(
+          (yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions,
+          [firstSession],
+        );
+        assert.deepEqual(
+          (yield* projectionStore.getThreadProjection(secondThreadId)).providerSessions,
+          [secondSession],
+        );
+      });
+      yield* assertBindings;
       assert.isTrue((yield* maintenance.verify).valid);
       assert.isTrue((yield* maintenance.rebuild).valid);
+      yield* assertBindings;
     }),
   );
 
@@ -3666,3 +3674,88 @@ it.effect.each(["cancel transaction", "fail transaction", "cancel publication"]
       );
     }).pipe(Effect.provide(layerDatabase)),
 );
+
+it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] as const)(
+  "hands off committed ownership only after a successful transaction during %s",
+  (scenario) =>
+    Effect.gen(function* () {
+      const paused = yield* Deferred.make<void>();
+      const resume = yield* Deferred.make<void>();
+      const pause = Deferred.succeed(paused, undefined).pipe(
+        Effect.andThen(Deferred.await(resume)),
+      );
+      const committed = scenario === "cancel publication";
+      const stores = Layer.mergeAll(
+        layerStoresProvided,
+        Layer.effect(
+          EventStore.EventStoreV2,
+          Effect.gen(function* () {
+            const delegate = yield* EventStore.EventStoreV2;
+            return EventStore.EventStoreV2.of({
+              ...delegate,
+              publishCommitted: (events) =>
+                (committed ? pause : Effect.void).pipe(
+                  Effect.andThen(delegate.publishCommitted(events)),
+                ),
+            });
+          }),
+        ).pipe(Layer.provide(layerEventStoreProvided)),
+        Layer.effect(
+          EffectOutbox.EffectOutboxV2,
+          Effect.gen(function* () {
+            const delegate = yield* EffectOutbox.EffectOutboxV2;
+            return EffectOutbox.EffectOutboxV2.of({
+              ...delegate,
+              enqueue: (effects) =>
+                delegate
+                  .enqueue(effects)
+                  .pipe(
+                    Effect.andThen(committed ? Effect.void : pause),
+                    Effect.andThen(
+                      scenario === "fail transaction"
+                        ? Effect.die("Injected rollback")
+                        : Effect.void,
+                    ),
+                  ),
+            });
+          }),
+        ).pipe(Layer.provide(layerEffectOutboxProvided)),
+        layerCommandReceiptStoreProvided,
+        ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
+        TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
+      );
+      yield* Effect.gen(function* () {
+        const sink = yield* EventSink.EventSinkV2;
+        const projections = yield* ProjectionStore.ProjectionStoreV2;
+        const eventStore = yield* EventStore.EventStoreV2;
+        const now = yield* DateTime.now;
+        const thread = makeThread(ThreadId.make("thread:commit-handoff"), now);
+        const events = [threadCreatedEvent({ id: "event:commit-handoff", thread, now })];
+        const handedOff: Array<OrchestrationV2DomainEvent> = [];
+        const writer = yield* sink
+          .write({
+            events,
+            onCommitted: (stored) => {
+              handedOff.push(...stored.map((entry) => entry.event));
+            },
+          })
+          .pipe(Effect.forkScoped);
+        yield* Deferred.await(paused);
+        // The handoff precedes publication, and never sees transaction work that can roll back.
+        assert.deepEqual(handedOff, committed ? events : []);
+        if (scenario !== "fail transaction") writer.interruptUnsafe();
+        if (scenario !== "cancel transaction") yield* Deferred.succeed(resume, undefined);
+        assert.equal((yield* Fiber.await(writer))._tag, "Failure");
+        assert.deepEqual(handedOff, committed ? events : []);
+        assert.equal(
+          (yield* projections.getThreadShell(thread.id))?.id ?? null,
+          committed ? thread.id : null,
+        );
+        const stored = yield* eventStore.read({ threadId: thread.id }).pipe(Stream.runCollect);
+        assert.deepEqual(
+          Array.from(stored, (entry) => entry.event),
+          committed ? events : [],
+        );
+      }).pipe(Effect.provide(EventSink.layerFromStores.pipe(Layer.provideMerge(stores))));
+    }).pipe(Effect.provide(layerDatabase)),
+);

apps/server/src/orchestration-v2/Orchestrator.ts

@@ -10438,7 +10438,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
             .map((node) => [node.id, node]),
         ),
         turnItems: new Map(),
-        childTurnItems: new Map(
+        ownedTurnItems: new Map(
           input.projection.turnItems
             .filter(
               (item) =>

apps/server/src/orchestration-v2/ProjectionStore.test.ts

@@ -4775,6 +4775,44 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
         ],
       );
 
+      yield* projectionStore.apply({
+        id: EventId.make("event:projection-shared-provider-session:first-replaced"),
+        type: "provider-session.attached",
+        threadId: firstThreadId,
+        occurredAt: now,
+        payload: { ...session, status: "ready", lastError: null },
+      });
+      yield* projectionStore.apply({
+        id: EventId.make("event:projection-shared-provider-session:second-released"),
+        type: "provider-session.updated",
+        threadId: secondThreadId,
+        occurredAt: now,
+        payload: { ...session, status: "stopped", lastError: "old runtime stopped" },
+      });
+      assert.equal(
+        (yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.status,
+        "ready",
+      );
+      assert.equal(
+        (yield* projectionStore.getThreadProjection(secondThreadId)).providerSessions[0]?.status,
+        "stopped",
+      );
+      assert.deepEqual(
+        (yield* projectionStore.getShellSnapshot()).threads
+          .filter((thread) => thread.id === firstThreadId || thread.id === secondThreadId)
+          .map((thread) => thread.lastError),
+        [null, "old runtime stopped"],
+      );
+      assert.equal(
+        (yield* projectionStore.getRuntimeRecoveryProjection(firstThreadId)).providerSessions[0]
+          ?.status,
+        "ready",
+      );
+      assert.lengthOf(
+        (yield* projectionStore.getRuntimeRecoveryProjection(secondThreadId)).providerSessions,
+        0,
+      );
+
       yield* projectionStore.apply({
         id: EventId.make("event:projection-shared-provider-session:first-detached"),
         type: "provider-session.detached",