fix(server): bulk worktree launches no longer fail on Git races

apps/server/src/orchestration-v2/ThreadLaunchService.test.ts

@@ -34,6 +34,7 @@ import * as Effect from "effect/Effect";
 import * as Exit from "effect/Exit";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
+import * as Path from "effect/Path";
 import * as Ref from "effect/Ref";
 import * as Result from "effect/Result";
 import * as Stream from "effect/Stream";
@@ -1018,6 +1019,238 @@ it.effect("runs a Scratch scheduled task saved with a worktree in its own folder
   }).pipe(Effect.provide(harness.layer));
 });
 
+it.effect.each([false, true])(
+  "bulk launches preserve occupied work while creating independent worktrees (startFromOrigin: %s)",
+  (startFromOrigin) =>
+    Effect.gen(function* () {
+      const driver = yield* GitVcsDriver.GitVcsDriver;
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "launch-worktree-collisions-" });
+      const worktreesDirectory = yield* fs.makeTempDirectoryScoped({ prefix: "launch-worktrees-" });
+      const git = (args: ReadonlyArray<string>) =>
+        driver.execute({ operation: "ThreadLaunchService.test", cwd, args });
+      yield* git(["init", "--initial-branch=main"]);
+      yield* git([
+        "-c",
+        "user.name=Test",
+        "-c",
+        "user.email=test@example.com",
+        "commit",
+        "--allow-empty",
+        "-m",
+        "initial",
+      ]);
+      const localHead = (yield* git(["rev-parse", "HEAD"])).stdout.trim();
+      let remoteHead = localHead;
+      if (startFromOrigin) {
+        const remote = yield* fs.makeTempDirectoryScoped({ prefix: "launch-bulk-remote-" });
+        yield* git(["init", "--bare", remote]);
+        yield* git(["remote", "add", "origin", remote]);
+        yield* git(["push", "-u", "origin", "main"]);
+        yield* git([
+          "-c",
+          "user.name=Test",
+          "-c",
+          "user.email=test@example.com",
+          "commit",
+          "--allow-empty",
+          "-m",
+          "remote advance",
+        ]);
+        remoteHead = (yield* git(["rev-parse", "HEAD"])).stdout.trim();
+        yield* git(["push", "origin", "main"]);
+        yield* git(["reset", "--hard", localHead]);
+        yield* git(["update-ref", "-d", "refs/remotes/origin/main"]);
+      }
+      yield* git(["config", "branch.autoSetupMerge", "always"]);
+      const existingPath = path.join(worktreesDirectory, "existing");
+      yield* driver.createWorktree({
+        cwd,
+        refName: "main",
+        newRefName: "feature/shared",
+        path: existingPath,
+      });
+      yield* fs.writeFileString(path.join(existingPath, "keep.txt"), "existing uncommitted work");
+      const occupiedPath = path.join(worktreesDirectory, path.basename(cwd), "feature-shared-1");
+      yield* fs.makeDirectory(occupiedPath, { recursive: true });
+      yield* fs.writeFileString(path.join(occupiedPath, "keep.txt"), "unrelated directory");
+      const harness = makeHarness({
+        git: {
+          resolveWorktreeBaseRemote: (input) => driver.resolveWorktreeBaseRemote({ ...input, cwd }),
+        },
+        fetchWorktreeBaseCommit: (input) => driver.fetchWorktreeBaseCommit({ ...input, cwd }),
+        createWorktree: (input, options) =>
+          driver.createWorktree({ ...input, cwd }, { ...options, worktreesDirectory }),
+      });
+      yield* Effect.gen(function* () {
+        const launches = yield* ThreadLaunch.ThreadLaunchService;
+        const threads = yield* ThreadManagement.ThreadManagementService;
+        const launched = yield* Effect.all(
+          Array.from({ length: 6 }, (_, index) => {
+            const command = `command:launch:bulk-${index}`;
+            return launches
+              .launch(
+                launchInput({
+                  command,
+                  thread: `thread:launch:bulk-${index}`,
+                  message: "Independent work",
+                  workspace: {
+                    type: "worktree",
+                    baseRef: "main",
+                    branch: "feature/shared",
+                    startFromOrigin,
+                  },
+                }),
+              )
+              .pipe(
+                Effect.flatMap((result) =>
+                  threads.streamStoredEventsFrom({ threadId: result.threadId }).pipe(
+                    Stream.filter(
+                      (stored) =>
+                        (stored.commandId === CommandId.make(`${command}:release`) ||
+                          stored.commandId === CommandId.make(`${command}:fail`)) &&
+                        stored.event.type === "run.updated",
+                    ),
+                    Stream.runHead,
+                    Effect.andThen(threads.getThreadProjection(result.threadId)),
+                  ),
+                ),
+              );
+          }),
+          { concurrency: "unbounded" },
+        );
+        assert.equal(new Set(launched.map((projection) => projection.thread.branch)).size, 6);
+        assert.equal(new Set(launched.map((projection) => projection.thread.worktreePath)).size, 6);
+        for (const projection of launched) {
+          assert.equal(
+            projection.runs[0]?.status,
+            "starting",
+            projection.turnItems.find((item) => item.type === "error")?.failure.message,
+          );
+          assert.isTrue(yield* fs.exists(projection.thread.worktreePath!));
+          const base = yield* git(["config", `branch.${projection.thread.branch}.t3-merge-base`]);
+          assert.equal(base.stdout.trim(), "main");
+          const head = yield* driver.execute({
+            operation: "ThreadLaunchService.test",
+            cwd: projection.thread.worktreePath!,
+            args: ["rev-parse", "HEAD"],
+          });
+          assert.equal(head.stdout.trim(), remoteHead);
+          if (startFromOrigin) {
+            const comparison = yield* git([
+              "config",
+              `branch.${projection.thread.branch}.t3-merge-base-ref`,
+            ]);
+            assert.equal(comparison.stdout.trim(), "refs/remotes/origin/main");
+          } else {
+            const upstream = yield* git([
+              "rev-parse",
+              "--abbrev-ref",
+              `${projection.thread.branch}@{upstream}`,
+            ]);
+            assert.equal(upstream.stdout.trim(), "main");
+          }
+        }
+        assert.deepEqual(
+          launched.map((projection) => projection.thread.branch).toSorted(),
+          Array.from({ length: 6 }, (_, index) => `feature/shared-${index + 2}`).toSorted(),
+        );
+        assert.equal(
+          yield* fs.readFileString(path.join(existingPath, "keep.txt")),
+          "existing uncommitted work",
+        );
+        assert.equal(
+          yield* fs.readFileString(path.join(occupiedPath, "keep.txt")),
+          "unrelated directory",
+        );
+        assert.equal(harness.removeWorktree.mock.calls.length, 0);
+        assert.equal((yield* git(["rev-parse", "main"])).stdout.trim(), localHead);
+      }).pipe(Effect.provide(harness.layer));
+    }).pipe(
+      Effect.provide(
+        GitVcsDriver.layer.pipe(
+          Layer.provide(
+            ServerConfig.layerTest(process.cwd(), { prefix: "launch-collision-test-" }),
+          ),
+          Layer.provideMerge(NodeServices.layer),
+        ),
+      ),
+    ),
+);
+
+it.effect("bounds collision retries and recovers the same failed run on Retry", () => {
+  let fails = true;
+  let attempts = 0;
+  const harness = makeHarness({
+    createWorktree: (input) => {
+      if (++attempts > 103) return Effect.die("worktree collision retries did not stop");
+      return fails || input.newRefName === "my-feature"
+        ? Effect.fail(
+            new GitCommandError({
+              operation: "GitVcsDriver.createWorktree",
+              command: "git",
+              cwd: project.workspaceRoot,
+              detail: "git worktree add failed",
+              reason: "branch_already_exists",
+              exitCode: 128,
+            }),
+          )
+        : Effect.succeed({
+            worktree: { path: "/repo-worktrees/recovered", refName: input.newRefName! },
+          });
+    },
+  });
+  return Effect.gen(function* () {
+    const launches = yield* ThreadLaunch.ThreadLaunchService;
+    const threads = yield* ThreadManagement.ThreadManagementService;
+    const launched = yield* launches.launch(
+      launchInput({
+        command: "command:launch:taken-retry",
+        thread: "thread:launch:taken-retry",
+        message: "Retry me",
+        workspace: { type: "worktree", baseRef: "main", branch: "my-feature" },
+      }),
+    );
+    yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
+      Stream.filter(
+        (stored) =>
+          stored.commandId === CommandId.make("command:launch:taken-retry:fail") &&
+          stored.event.type === "run.updated",
+      ),
+      Stream.runHead,
+    );
+    const failed = yield* threads.getThreadProjection(launched.threadId);
+    assert.equal(failed.runs[0]?.status, "failed");
+    assert.isAbove(attempts, 1);
+    assert.isAtMost(attempts, 101);
+    assert.equal(harness.runSetup.mock.calls.length, 0);
+    assert.equal(harness.removeWorktree.mock.calls.length, 0);
+    fails = false;
+    yield* launches.retryPreparation({
+      commandId: CommandId.make("command:launch:taken-retry:retry"),
+      threadId: launched.threadId,
+      runId: failed.runs[0]!.id,
+    });
+    yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
+      Stream.filter(
+        (stored) =>
+          stored.commandId === CommandId.make("command:launch:taken-retry:retry:release") &&
+          stored.event.type === "run.updated",
+      ),
+      Stream.runHead,
+    );
+    const recovered = yield* threads.getThreadProjection(launched.threadId);
+    assert.equal(recovered.runs.length, 1);
+    assert.equal(recovered.runs[0]!.id, failed.runs[0]!.id);
+    assert.equal(recovered.messages.length, 1);
+    assert.equal(recovered.runs[0]?.status, "starting");
+    assert.equal(recovered.thread.branch, "my-feature-1");
+    assert.equal(recovered.thread.worktreePath, "/repo-worktrees/recovered");
+    assert.equal(recovered.turnItems.find((item) => item.type === "error")?.status, "cancelled");
+  }).pipe(Effect.provide(harness.layer));
+});
+
 it.effect.each(["stable", "upstream/release"])(
   "launches from %s with the fetched commit and remote PR base",
   (baseRef) =>
@@ -1336,6 +1569,53 @@ it.effect.each(["failed", "interrupted"] as const)(
   },
 );
 
+it.effect("removes a worktree that failed before the thread recorded it without retrying", () => {
+  const harness = makeHarness({
+    createWorktree: (_input, options) =>
+      (options?.progress?.onWorktreeClaimed?.("/repo-worktrees/partial") ?? Effect.void).pipe(
+        Effect.andThen(
+          Effect.fail(
+            new GitCommandError({
+              operation: "GitVcsDriver.createWorktree.configureBaseRef",
+              command: "git",
+              cwd: project.workspaceRoot,
+              detail: "checkout failed",
+              reason: "branch_already_exists",
+            }),
+          ),
+        ),
+      ),
+  });
+  return Effect.gen(function* () {
+    const launches = yield* ThreadLaunch.ThreadLaunchService;
+    const threads = yield* ThreadManagement.ThreadManagementService;
+    const launched = yield* launches.launch(
+      launchInput({
+        command: "command:launch:partial-worktree",
+        thread: "thread:launch:partial-worktree",
+        message: "Partial checkout",
+        workspace: { type: "worktree", baseRef: "main" },
+      }),
+    );
+    yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
+      Stream.filter(
+        (stored) =>
+          stored.commandId === CommandId.make("command:launch:partial-worktree:fail") &&
+          stored.event.type === "run.updated",
+      ),
+      Stream.runHead,
+    );
+    const projection = yield* threads.getThreadProjection(launched.threadId);
+    assert.equal(projection.runs[0]?.status, "failed");
+    assert.equal(projection.thread.worktreePath, null);
+    assert.equal(harness.createWorktree.mock.calls.length, 1);
+    assert.deepEqual(
+      harness.removeWorktree.mock.calls.map(([input]) => input.path),
+      ["/repo-worktrees/partial"],
+    );
+  }).pipe(Effect.provide(harness.layer));
+});
+
 it.effect("replays a server-allocated launch", () =>
   Effect.gen(function* () {
     const setupEntered = yield* Deferred.make<void>();

apps/server/src/orchestration-v2/ThreadLaunchService.ts

@@ -423,12 +423,12 @@ const make = Effect.gen(function* () {
           branch = flattenTemporaryWorktreeBranchName(branch);
         }
         yield* setupTracker.stageStatus(threadId, "checkout", "running");
-        const worktree = yield* git
-          .createWorktree(
+        const createWorktree = (newRefName: string) =>
+          git.createWorktree(
             {
               cwd: project.workspaceRoot,
               refName: startRef,
-              newRefName: branch!,
+              newRefName,
               baseRefName,
               path: null,
             },
@@ -442,8 +442,38 @@ const make = Effect.gen(function* () {
                   setupTracker.stage(threadId, "checkout", { percent: progress.percent }),
               },
             },
-          )
-          .pipe(Effect.mapError(mapError(input, "provision-worktree", threadId)));
+          );
+        const desiredBranch = branch!;
+        // Git claims the branch atomically. A preflight check alone would race
+        // with other launches, and retries must also escape leftover branches
+        // and directories from earlier attempts without touching those checkouts.
+        // Taken names get the same `-1`, `-2` suffixes renames use.
+        const worktree = yield* Effect.gen(function* () {
+          let candidate = desiredBranch;
+          for (let attempt = 1; ; attempt += 1) {
+            const result = yield* createWorktree(candidate).pipe(
+              Effect.catchTags({
+                GitCommandError: (error) =>
+                  attempt <= 100 &&
+                  createdWorktreePath === null &&
+                  (error.reason === "branch_already_exists" ||
+                    error.reason === "path_already_exists")
+                    ? Effect.succeed(null)
+                    : Effect.fail(error),
+              }),
+            );
+            if (result !== null) return result;
+            if (isTemporaryWorktreeBranch(desiredBranch)) {
+              const uuid = yield* randomUuidV4;
+              candidate = buildTemporaryWorktreeBranchName(() => uuid.replaceAll("-", ""));
+              if (!desiredBranch.includes("/")) {
+                candidate = flattenTemporaryWorktreeBranchName(candidate);
+              }
+            } else {
+              candidate = `${desiredBranch}-${attempt}`;
+            }
+          }
+        }).pipe(Effect.mapError(mapError(input, "provision-worktree", threadId)));
         worktreePath = worktree.worktree.path;
         branch = worktree.worktree.refName;
         createdWorktreePath = worktreePath;