diff --git a/apps/server/src/factory/gate.test.ts b/apps/server/src/factory/gate.test.ts index a7cbb0ece9a3..8577ca4f21db 100644 --- a/apps/server/src/factory/gate.test.ts +++ b/apps/server/src/factory/gate.test.ts @@ -24,12 +24,12 @@ import { gateRoot, layer as gateLayer, makeWorkspace, - sanitize, type CommandResult, type GateError, type GateInput, type GateOptions, } from "./gate.ts"; +import { sanitize } from "./text.ts"; import type { AppendIo } from "./ledger/appendFile.ts"; import { LedgerIoError } from "./ledger/errors.ts"; import { Ledger } from "./ledger/ledger.ts"; diff --git a/apps/server/src/factory/gate.ts b/apps/server/src/factory/gate.ts index 9982ee3cd757..8f4c29141ab9 100644 --- a/apps/server/src/factory/gate.ts +++ b/apps/server/src/factory/gate.ts @@ -16,6 +16,7 @@ import { Ledger } from "./ledger/ledger.ts"; import { readRun } from "./ledger/read.ts"; import { runIdToString, validateRunId, type RunId } from "./ledger/runId.ts"; import { FactoryProcess, type RunInput, type RunResult, type RunSpawnError } from "./process.ts"; +import { sanitize } from "./text.ts"; /** * The gate: a fresh checkout of a pull request's head merged with the base branch tip, the @@ -135,31 +136,6 @@ export type GateError = WorkspaceError | GateCommandSpawnError | GateRecordError // ---- text ----------------------------------------------------------------------------------- -const isControl = (cp: number) => cp <= 0x1f || (cp >= 0x7f && cp <= 0x9f); - -/** Characters that could fake a line break or hide text, besides the control characters. */ -const isInvisibleOrSeparator = (cp: number) => - cp === 0x2028 || - cp === 0x2029 || - (cp >= 0x200b && cp <= 0x200f) || - (cp >= 0x202a && cp <= 0x202e) || - (cp >= 0x2066 && cp <= 0x2069) || - cp === 0xfeff; - -/** - * Replace what could fake a line break, hide text or drive a terminal with a space. Newline and - * tab are kept only when `keepLayout` is set. - */ -export const sanitize = (text: string, keepLayout: boolean): string => { - let out = ""; - for (const char of text) { - const cp = char.codePointAt(0)!; - const layout = char === "\n" || char === "\t"; - out += (isControl(cp) && !(keepLayout && layout)) || isInvisibleOrSeparator(cp) ? " " : char; - } - return out; -}; - /** The command as one short line: sanitised, newlines and tabs turned into spaces, cut at 300 characters. */ const commandLine = (command: string): string => { const chars = Array.from(sanitize(command, false)); diff --git a/apps/server/src/factory/index.test.ts b/apps/server/src/factory/index.test.ts index 012787b69d36..b2f6e7c2d0df 100644 --- a/apps/server/src/factory/index.test.ts +++ b/apps/server/src/factory/index.test.ts @@ -1,22 +1,38 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; +import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import type * as PlatformError from "effect/PlatformError"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; import { describe } from "vite-plus/test"; import { timeLimitOnlyProjectConfig, type MachineConfig } from "./config.ts"; -import { FactoryRuntime, liveWorld } from "./index.ts"; +import { + factoryRuntime, + FactoryLoop, + FactoryRuntime, + liveWorld, + ownedThreads, + wakeOn, +} from "./index.ts"; +import { makeCause } from "./ledger/events.ts"; +import { Ledger } from "./ledger/ledger.ts"; import { scanMachineLog } from "./ledger/machineLog.ts"; +import { newRunId } from "./ledger/runId.ts"; import type { Repo } from "./pass.ts"; import { CUT_SHORT_COMMENT } from "./runner/messages.ts"; import { utcSecond } from "./runner/start.ts"; import { MARKER } from "./tracker/github.ts"; import { FactoryProcess, RunSpawnError, type RunInput, type RunResult } from "./process.ts"; +import { EventsError, FactoryEvents } from "./t3/events.ts"; import * as Paths from "./t3/paths.ts"; import * as Sessions from "./t3/sessions.ts"; @@ -40,6 +56,12 @@ const stubSessions = Sessions.FactorySessions.of({ }); const noSessions = Layer.succeed(Sessions.FactorySessions, stubSessions); +/** FactoryEvents that never says anything. */ +const quietEvents = Layer.succeed( + FactoryEvents, + FactoryEvents.of({ runStatusChanges: Stream.never }), +); + const messageText = (message: unknown): string => { const first = Array.isArray(message) ? message[0] : message; return typeof first === "string" ? first : String(first); @@ -90,6 +112,7 @@ const logsFor = ( ), Layer.provide(gh), Layer.provide(noSessions), + Layer.provide(quietEvents), ), ), ).pipe(Effect.provide(Logger.layer([capture], { mergeWithExisting: false }))); @@ -181,6 +204,172 @@ it.effect("logs each pass line as it is decided, so a pass that dies has said wh }), ); +// ---- the wake ---- + +describe("waking the pass when a factory thread's run changes status", () => { + /** Writes an unfinished run of issue `issue` whose session is `threadId`. */ + const seedRun = (factoryDir: string, issue: number, threadId: string, finished = false) => { + const out = Ledger.create(factoryDir, newRunId("a/b", String(issue), 1)); + try { + out.append( + { event: "claimed", data: { machine: "m", github_user: "u" } }, + makeCause("start", "seed"), + ); + out.append( + { + event: "session_launched", + data: { + thread_id: threadId, + worktree: "/t3/wt", + branch: "t3code/abc", + model_instance: "claudeAgent", + model: "claude-sonnet-5-5", + resumed_from: null, + }, + }, + makeCause("start", "seed"), + ); + if (finished) out.finish("stuck", null, null, makeCause("start", "seed")); + } finally { + out.close(); + } + }; + + /** A factory dir with a valid config and one unfinished run whose session is `thread-1`. */ + const seeded = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { join } = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "factory-wake-" }); + const factoryDir = join(baseDir, "factory"); + yield* fs.makeDirectory(factoryDir); + const machineConfigPath = join(factoryDir, "config.yml"); + yield* fs.writeFileString(machineConfigPath, VALID); + seedRun(factoryDir, 1, "thread-1"); + return { baseDir, factoryDir, machineConfigPath, worktreesDir: join(baseDir, "worktrees") }; + }); + + it.effect("owns the threads of unfinished runs' sessions, and no finished run's", () => + Effect.scoped( + Effect.gen(function* () { + const { factoryDir } = yield* seeded; + seedRun(factoryDir, 2, "thread-done", true); + assert.deepStrictEqual([...(yield* ownedThreads(factoryDir))], ["thread-1"]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a thread the factory does not own never wakes the pass", () => + Effect.gen(function* () { + const owned = yield* Ref.make>(new Set(["thread-1"])); + const wakes = yield* Ref.make(0); + yield* wakeOn( + owned, + Stream.fromIterable(["thread-other", "thread-1", "thread-other"]), + Ref.update(wakes, (n) => n + 1), + ); + assert.strictEqual(yield* Ref.get(wakes), 1); + }), + ); + + /** The factory's loop over the seeded dir, with `events` as T3's run status changes. */ + const loopOver = ( + dirs: Effect.Success, + events: Stream.Stream, + ) => + Layer.build( + factoryRuntime({ restartDelay: 0 }).pipe( + Layer.provide(Layer.succeed(Paths.FactoryPaths, dirs)), + Layer.provide(noGh), + Layer.provide(noSessions), + Layer.provide(Layer.succeed(FactoryEvents, FactoryEvents.of({ runStatusChanges: events }))), + ), + ).pipe(Effect.map((context) => Context.get(context, FactoryLoop))); + + const countOf = (factoryDir: string, event: string) => () => + scanMachineLog(factoryDir).filter((entry) => entry.event === event).length; + + it.effect("a thread launched after the start is owned once a pass has ended", () => + Effect.scoped( + Effect.gen(function* () { + const dirs = yield* seeded; + const queue = yield* Queue.unbounded(); + const loop = yield* loopOver(dirs, Stream.fromQueue(queue)); + const started = countOf(dirs.factoryDir, "pass_started"); + const finished = countOf(dirs.factoryDir, "pass_finished"); + // The poll is a minute away, so every pass here is the first or one a wake started. Each + // wait is bounded so a wake that never comes fails the test instead of hanging it. + const until = (done: () => boolean, each: Effect.Effect = Effect.void) => + Effect.gen(function* () { + while (!done()) { + yield* each; + yield* Effect.timeoutOption(loop.awaitPass, "1 second"); + } + }).pipe(Effect.timeoutOption("30 seconds"), Effect.map(Option.isSome)); + assert.isTrue(yield* until(() => finished() >= 1), "the first pass never ended"); + seedRun(dirs.factoryDir, 2, "thread-2"); + const before = finished(); + yield* loop.wake; + assert.isTrue(yield* until(() => finished() > before), "the woken pass never ended"); + // That pass read the new run's thread when it ended, so an event for the thread wakes the + // next one. It is offered until it does: one offered before the read is not a wake. + const after = started(); + const woke = yield* until(() => started() > after, Queue.offer(queue, "thread-2")); + assert.isTrue(woke, "the new run's thread never woke the pass"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("an owned thread starts the next pass, and a stream that failed is run again", () => + Effect.scoped( + Effect.gen(function* () { + const { baseDir, factoryDir, machineConfigPath, worktreesDir } = yield* seeded; + const queue = yield* Queue.unbounded(); + // The first run of the stream fails; the one after it reads the queue. + const failures = yield* Ref.make(1); + const events = Layer.succeed( + FactoryEvents, + FactoryEvents.of({ + runStatusChanges: Stream.unwrap( + Ref.modify(failures, (left): readonly [Stream.Stream, number] => + left > 0 + ? [Stream.fail(new EventsError({ cause: new Error("gone") })), left - 1] + : [Stream.fromQueue(queue), 0], + ), + ), + }), + ); + const loop = yield* Layer.build( + factoryRuntime({ restartDelay: 0 }).pipe( + Layer.provide( + Layer.succeed(Paths.FactoryPaths, { + baseDir, + worktreesDir, + factoryDir, + machineConfigPath, + }), + ), + Layer.provide(noGh), + Layer.provide(noSessions), + Layer.provide(events), + ), + ).pipe(Effect.map((context) => Context.get(context, FactoryLoop))); + const started = () => + scanMachineLog(factoryDir).filter((entry) => entry.event === "pass_started").length; + yield* Queue.offer(queue, "thread-other"); + yield* Queue.offer(queue, "thread-1"); + // The first pass, then the one the owned thread woke; the poll is a minute away. The wait is + // bounded so a wake that never comes fails here. The count stays `>=`: the offer can land + // while the first pass still runs, and a wake then is coalesced into that pass's latch. + const waited = yield* Effect.gen(function* () { + while (started() < 2) yield* loop.awaitPass; + }).pipe(Effect.timeoutOption("30 seconds")); + assert.isTrue(Option.isSome(waited), "the owned thread never woke the pass"); + assert.isAtLeast(started(), 2); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); + // ---- the live world over gh ---- describe("liveWorld", () => { diff --git a/apps/server/src/factory/index.ts b/apps/server/src/factory/index.ts index ede133af9142..7853df6b0064 100644 --- a/apps/server/src/factory/index.ts +++ b/apps/server/src/factory/index.ts @@ -4,10 +4,13 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as Context from "effect/Context"; +import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; +import * as Stream from "effect/Stream"; import { loadHandPause } from "./claims.ts"; import { @@ -20,25 +23,21 @@ import { type WatchedRepo, } from "./config.ts"; import { clearGateLocks, gateLockFile } from "./gate.ts"; -import { ledgerCall as ledger } from "./ledger/errors.ts"; +import { ledgerCall as ledger, type LedgerError } from "./ledger/errors.ts"; import { makeCause, type MachineEvent } from "./ledger/events.ts"; import { Ledger } from "./ledger/ledger.ts"; import { listUnfinishedRuns } from "./ledger/listRuns.ts"; +import { readRun } from "./ledger/read.ts"; import { MachineLog } from "./ledger/machineLog.ts"; import { runLoop, type Loop } from "./loop.ts"; import { runPass, type PassError, type PassWorld, type Repo } from "./pass.ts"; import * as Process from "./process.ts"; import { FactoryProcess } from "./process.ts"; +import { readProgress } from "./runner/progress.ts"; +import { advance, timeLimit, watch, type BuildingContext } from "./runner/building.ts"; import { CUT_SHORT_COMMENT } from "./runner/messages.ts"; -import { - isOurs, - resume, - start, - utcSecond, - type Sessions, - type StartContext, - type Tracker, -} from "./runner/start.ts"; +import { isOurs, resume, start, utcSecond, type StartContext } from "./runner/start.ts"; +import * as Events from "./t3/events.ts"; import * as Paths from "./t3/paths.ts"; import * as T3Sessions from "./t3/sessions.ts"; import { bindProcess, makeGitHub, type GitHubError, type IssueSummary } from "./tracker/github.ts"; @@ -114,7 +113,7 @@ export const liveWorld = ( machine: MachineConfig, dataDir: string, process: FactoryProcess["Service"], - sessions: Sessions, + sessions: T3Sessions.FactorySessions["Service"], clock: () => number, host: string, ): PassWorld => { @@ -122,7 +121,7 @@ export const liveWorld = ( Effect.provideService(effect, FactoryProcess, process); const githubFor = (watched: WatchedRepo) => makeGitHub({ repo: watched.repo, ghConfigDir: machine.ghConfigDir }); - const trackerFor = (watched: WatchedRepo): Tracker => bindProcess(githubFor(watched), process); + const trackerFor = (watched: WatchedRepo) => bindProcess(githubFor(watched), process); const tracker = (effect: Effect.Effect) => withProcess(effect).pipe(Effect.mapError(asPassError("tracker"))); const startContext = (repo: Repo): StartContext => ({ @@ -138,6 +137,16 @@ export const liveWorld = ( sessions, clock, }); + const buildingContext = (repo: Repo): BuildingContext => ({ + repo: repo.watched.repo, + trackerUser: repo.trackerUser, + machine: host, + project: repo.project, + dataDir, + tracker: trackerFor(repo.watched), + sessions, + clock, + }); return { project: (watched) => withProcess(projectFromBase(githubFor(watched))), trackerUser: (watched) => @@ -186,6 +195,9 @@ export const liveWorld = ( ), start: (repo: Repo, issue: number) => start(startContext(repo), issue), resume: (repo: Repo, issue: number) => resume(startContext(repo), issue), + timeLimit: (repo, run) => timeLimit(buildingContext(repo), run), + watch: (repo, run) => watch(buildingContext(repo), run), + advance: (repo, run) => advance(buildingContext(repo), run), gateRunning: (run) => NodeFS.existsSync(gateLockFile(dataDir, run)), recordError: (run, step, kind, message) => ledger(() => { @@ -221,45 +233,127 @@ const hostName = (): string => { } }; -const startFactory = Effect.gen(function* () { - const { factoryDir, machineConfigPath } = yield* Paths.FactoryPaths; - // Gates run inside this server, so a lock file left from an earlier run belongs to a dead gate. - const cleared = yield* clearGateLocks(factoryDir); - if (cleared > 0) yield* Effect.logInfo(`cleared ${cleared} gate lock(s)`); - const loaded = yield* Effect.result(loadMachineConfig(machineConfigPath)); - if (Result.isFailure(loaded)) { - for (const error of loaded.failure.errors) { - yield* Effect.logError(formatConfigError(error)); +/** + * The threads the unfinished runs under `dataDir` name as their sessions'. A run whose file + * cannot be read names none. + */ +export const ownedThreads = (dataDir: string): Effect.Effect, LedgerError> => + ledger(() => { + const ids = new Set(); + for (const info of listUnfinishedRuns(dataDir)) { + if (info.unreadable || !info.session) continue; + try { + const threadId = readProgress(readRun(dataDir, info.id).entries).threadId; + if (threadId !== null) ids.add(threadId); + } catch { + // Unreadable since it was listed: the next refresh reads it again. + } } - return idle; - } - if (Option.isNone(loaded.success)) { - yield* Effect.logInfo(`no factory config at ${machineConfigPath}`); - return idle; - } - const machine = loaded.success.value; - yield* Effect.logInfo("factory started"); - const process = yield* FactoryProcess; - const sessions = yield* T3Sessions.FactorySessions; - const world = liveWorld(machine, factoryDir, process, sessions, () => Date.now(), hostName()); - const onePass = Effect.gen(function* () { - // Each line is logged as it is decided, so a pass that hangs or dies has said what it did. - const context = yield* Effect.context(); - const log = Effect.runSyncWith(context); - return yield* runPass(world, machine, (line) => log(Effect.logInfo(line))); + return ids; + }); + +/** + * Reads the owned threads again into `owned`. A listing that fails is logged and keeps the set + * it had: the next refresh, or the next scheduled pass, covers what it missed. + */ +const refreshOwned = (dataDir: string, owned: Ref.Ref>) => + ownedThreads(dataDir).pipe( + Effect.flatMap((ids) => Ref.set(owned, ids)), + Effect.catch((error) => + Effect.logWarning(`factory cannot list the threads it owns: ${error.message}`), + ), + ); + +/** + * Runs one events stream to its end, waking for each thread in `owned`. A lookup is a set read, + * so the many events of threads the factory does not own cost nothing. The stream's own failure + * is the caller's to handle. + */ +export const wakeOn = ( + owned: Ref.Ref>, + events: Stream.Stream, + wake: Effect.Effect, +): Effect.Effect => + Stream.runForEach(events, (threadId) => + Effect.flatMap(Ref.get(owned), (ids) => (ids.has(threadId) ? wake : Effect.void)), + ); + +/** + * Wakes the loop when T3 says the latest run of a thread the factory owns changed status. A + * stream that ends or fails is run again after `restartDelay`; the status changes in the gap are + * not replayed, and the next pass reads them. + */ +const wakeOnOwnedThreads = ( + owned: Ref.Ref>, + loop: Loop, + restartDelay: Duration.Input, +) => + Effect.gen(function* () { + const events = yield* Events.FactoryEvents; + return yield* wakeOn(owned, events.runStatusChanges, loop.wake).pipe( + Effect.catch((error) => Effect.logWarning("factory events stream failed, restarting", error)), + Effect.andThen(Effect.sleep(restartDelay)), + Effect.forever, + ); + }); + +/** What the factory's runtime can be told, for a test that cannot wait. */ +export interface RuntimeOptions { + /** How long to wait before the events stream is run again. Default 5 seconds. */ + readonly restartDelay?: Duration.Input; +} + +const startFactory = (options: RuntimeOptions) => + Effect.gen(function* () { + const { factoryDir, machineConfigPath } = yield* Paths.FactoryPaths; + // Gates run inside this server, so a lock file left from an earlier run belongs to a dead gate. + const cleared = yield* clearGateLocks(factoryDir); + if (cleared > 0) yield* Effect.logInfo(`cleared ${cleared} gate lock(s)`); + const loaded = yield* Effect.result(loadMachineConfig(machineConfigPath)); + if (Result.isFailure(loaded)) { + for (const error of loaded.failure.errors) { + yield* Effect.logError(formatConfigError(error)); + } + return idle; + } + if (Option.isNone(loaded.success)) { + yield* Effect.logInfo(`no factory config at ${machineConfigPath}`); + return idle; + } + const machine = loaded.success.value; + yield* Effect.logInfo("factory started"); + const process = yield* FactoryProcess; + const sessions = yield* T3Sessions.FactorySessions; + const world = liveWorld(machine, factoryDir, process, sessions, () => Date.now(), hostName()); + // The threads of the unfinished runs, read at start and after every pass: a pass is what + // launches and finishes sessions, so between passes the set does not change. + const owned = yield* Ref.make>(new Set()); + yield* refreshOwned(factoryDir, owned); + const onePass = Effect.gen(function* () { + // Each line is logged as it is decided, so a pass that hangs or dies has said what it did. + const context = yield* Effect.context(); + const log = Effect.runSyncWith(context); + return yield* runPass(world, machine, (line) => log(Effect.logInfo(line))); + }).pipe(Effect.ensuring(refreshOwned(factoryDir, owned))); + const loop = yield* runLoop(onePass, machine.pollSeconds); + yield* Effect.forkScoped(wakeOnOwnedThreads(owned, loop, options.restartDelay ?? "5 seconds")); + return loop; }); - return yield* runLoop(onePass, machine.pollSeconds); -}); /** - * Needs FactoryPaths, FactoryProcess, FactorySessions and FileSystem; what index.test.ts builds - * with a test FactoryPaths, a fake FactoryProcess and a stub FactorySessions. + * Needs FactoryPaths, FactoryProcess, FactorySessions, FactoryEvents and FileSystem; what + * index.test.ts builds with a test FactoryPaths, a fake FactoryProcess, a stub FactorySessions + * and a FactoryEvents it feeds by hand. */ -export const FactoryRuntime = Layer.effect(FactoryLoop, startFactory); +export const factoryRuntime = (options: RuntimeOptions = {}) => + Layer.effect(FactoryLoop, startFactory(options)); + +export const FactoryRuntime = factoryRuntime(); /** The factory's root layer. server.ts merges it into the runtime; nothing else imports it. */ export const FactoryLayer = FactoryRuntime.pipe( Layer.provide(Paths.layer), Layer.provide(Process.layer), Layer.provide(T3Sessions.layerLive), + Layer.provide(Events.layerLive), ); diff --git a/apps/server/src/factory/ledger/listRuns.test.ts b/apps/server/src/factory/ledger/listRuns.test.ts index e1cdfdd46298..f4c56832e06a 100644 --- a/apps/server/src/factory/ledger/listRuns.test.ts +++ b/apps/server/src/factory/ledger/listRuns.test.ts @@ -66,9 +66,30 @@ describe("listUnfinishedRuns", () => { runAt(dataDir, rid(2), T0 + 10_000, "pr-open", true); runAt(dataDir, rid(4), T0 + 30_000, null); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(1), state: "triage", unreadable: false, confirmed: true, session: false }, - { id: rid(3), state: "building", unreadable: false, confirmed: true, session: false }, - { id: rid(4), state: null, unreadable: false, confirmed: true, session: false }, + { + id: rid(1), + state: "triage", + unreadable: false, + confirmed: true, + session: false, + prReported: false, + }, + { + id: rid(3), + state: "building", + unreadable: false, + confirmed: true, + session: false, + prReported: false, + }, + { + id: rid(4), + state: null, + unreadable: false, + confirmed: true, + session: false, + prReported: false, + }, ]); }); @@ -84,8 +105,22 @@ describe("listUnfinishedRuns", () => { // A directory named like a run file is a run whose file cannot be read. NodeFS.mkdirSync(NodePath.join(dataDir, RUNS_DIR, runIdFileName(rid(6)))); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(7), state: "building", unreadable: false, confirmed: true, session: false }, - { id: rid(6), state: null, unreadable: true, confirmed: false, session: false }, + { + id: rid(7), + state: "building", + unreadable: false, + confirmed: true, + session: false, + prReported: false, + }, + { + id: rid(6), + state: null, + unreadable: true, + confirmed: false, + session: false, + prReported: false, + }, ]); }); @@ -105,7 +140,14 @@ describe("listUnfinishedRuns", () => { `{"ts":"2026-10-04T10:13:16.123Z","ms":${T0},"event":"state_changed","data":{"from":"ready","to":"later"}}\n`, ); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(9), state: null, unreadable: false, confirmed: false, session: false }, + { + id: rid(9), + state: null, + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); }); @@ -113,7 +155,14 @@ describe("listUnfinishedRuns", () => { const dataDir = tempDir(); runAt(dataDir, rid(8), T0, "triage", false, false); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(8), state: "triage", unreadable: false, confirmed: false, session: false }, + { + id: rid(8), + state: "triage", + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); }); @@ -126,13 +175,21 @@ describe("listUnfinishedRuns", () => { // A file of another issue that cannot be read is not opened, so it is not listed. NodeFS.mkdirSync(NodePath.join(dataDir, RUNS_DIR, runIdFileName(rid(3)))); expect(listUnfinishedRuns(dataDir, { repo: "RawPhp/Softfac", issue: 1 })).toEqual([ - { id: rid(1), state: "triage", unreadable: false, confirmed: true, session: false }, + { + id: rid(1), + state: "triage", + unreadable: false, + confirmed: true, + session: false, + prReported: false, + }, { id: rid(1, "rawphp/softfac", 2), state: "building", unreadable: false, confirmed: true, session: false, + prReported: false, }, ]); }); @@ -197,7 +254,14 @@ describe("listUnfinishedRuns: finished runs are skipped by their tail", () => { const open = writeRun(dataDir, rid(2), stateLine("building")); vi.mocked(NodeFS.readFileSync).mockClear(); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(2), state: "building", unreadable: false, confirmed: false, session: false }, + { + id: rid(2), + state: "building", + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); expect(fullReads()).toContain(open); expect(fullReads()).not.toContain(finished); @@ -217,7 +281,14 @@ describe("listUnfinishedRuns: finished runs are skipped by their tail", () => { const dataDir = tempDir(); writeRun(dataDir, rid(1), stateLine("building") + finishedLine.slice(0, 40)); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(1), state: "building", unreadable: false, confirmed: false, session: false }, + { + id: rid(1), + state: "building", + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); }); @@ -230,7 +301,14 @@ describe("listUnfinishedRuns: finished runs are skipped by their tail", () => { ); vi.mocked(NodeFS.readFileSync).mockClear(); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(1), state: "building", unreadable: false, confirmed: false, session: false }, + { + id: rid(1), + state: "building", + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); expect(fullReads()).toContain(path); }); @@ -251,7 +329,14 @@ describe("listUnfinishedRuns: finished runs are skipped by their tail", () => { const dataDir = tempDir(); writeRun(dataDir, rid(1), stateLine("building") + "garbage\n"); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(1), state: "building", unreadable: false, confirmed: false, session: false }, + { + id: rid(1), + state: "building", + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); }); @@ -272,7 +357,13 @@ describe("listUnfinishedRuns: finished runs are skipped by their tail", () => { writeRun(dataDir, rid(1), "\n"); writeRun(dataDir, rid(2), "\n\n"); writeRun(dataDir, rid(3), "\n\r\n"); - const unfinished = { state: null, unreadable: false, confirmed: false, session: false }; + const unfinished = { + state: null, + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }; expect(listUnfinishedRuns(dataDir)).toEqual([ { id: rid(1), ...unfinished }, { id: rid(2), ...unfinished }, @@ -284,11 +375,31 @@ describe("listUnfinishedRuns: finished runs are skipped by their tail", () => { const dataDir = tempDir(); writeRun(dataDir, rid(1), stateLine("building") + "\n".repeat(9000)); expect(listUnfinishedRuns(dataDir)).toEqual([ - { id: rid(1), state: "building", unreadable: false, confirmed: false, session: false }, + { + id: rid(1), + state: "building", + unreadable: false, + confirmed: false, + session: false, + prReported: false, + }, ]); }); }); +describe("listUnfinishedRuns: a reported pull request", () => { + it("is listed from pr_opened until the run is building again", () => { + const dataDir = tempDir(); + runAt(dataDir, rid(1), T0, "building"); + const opened = handles.at(-1)!; + opened.append({ event: "pr_opened", data: { pr: 12, url: "https://x/pull/12" } }, cause); + expect(listUnfinishedRuns(dataDir).map((info) => info.prReported)).toEqual([true]); + opened.append({ event: "state_changed", data: { from: "building", to: "pr-open" } }, cause); + opened.append({ event: "state_changed", data: { from: "pr-open", to: "building" } }, cause); + expect(listUnfinishedRuns(dataDir).map((info) => info.prReported)).toEqual([false]); + }); +}); + describe("holdsSlot", () => { const run = (state: State | null, confirmed: boolean, unreadable = false) => ({ id: rid(1), @@ -296,6 +407,7 @@ describe("holdsSlot", () => { unreadable, confirmed, session: false, + prReported: false, }); it("every confirmed unfinished run holds a slot but one waiting in pr-open", () => { @@ -310,6 +422,10 @@ describe("holdsSlot", () => { expect(holdsSlot(run(null, false))).toBe(false); }); + it("a building run whose session reported its pull request holds no slot: it waits for the gate", () => { + expect(holdsSlot({ ...run("building", true), prReported: true })).toBe(false); + }); + it("a run whose file cannot be read holds a slot", () => { expect(holdsSlot(run(null, false, true))).toBe(true); expect(holdsSlot(run("pr-open", false, true))).toBe(true); diff --git a/apps/server/src/factory/ledger/listRuns.ts b/apps/server/src/factory/ledger/listRuns.ts index 3a0aa09124bc..7e597db0c60f 100644 --- a/apps/server/src/factory/ledger/listRuns.ts +++ b/apps/server/src/factory/ledger/listRuns.ts @@ -25,16 +25,22 @@ export interface RunInfo { * sent. False for a file that cannot be read. */ readonly session: boolean; + /** + * The session reported its pull request since the run last entered `building`: its work is + * done and waits for the gate. False for a file that cannot be read. + */ + readonly prReported: boolean; } /** * Whether a run holds one of the machine's `max_concurrent_runs` slots: it is not finished, it is - * not waiting on a pull request, and it is confirmed. A run in `pr-open` has no session to hold, - * and an unconfirmed run never gets one, so both are left out. A run whose file cannot be read - * counts. The caller has already left out finished runs. + * not waiting on a pull request, and it is confirmed. A run in `pr-open`, or one whose session + * reported its pull request and waits for the gate, has no session at work, and an unconfirmed + * run never gets one, so all three are left out. A run whose file cannot be read counts. The + * caller has already left out finished runs. */ export const holdsSlot = (run: RunInfo): boolean => - run.unreadable || (run.confirmed && run.state !== "pr-open"); + run.unreadable || (run.confirmed && run.state !== "pr-open" && !run.prReported); /** * Every run under `/runs/` that is not finished (a file whose last line is `finished` is @@ -76,9 +82,23 @@ export const listUnfinishedRuns = ( if (seen.finished) continue; const confirmed = seen.approvedAt !== null || seen.threadId !== null; const session = seen.threadId !== null || seen.launchSent; - info = { id, state: seen.state, unreadable: false, confirmed, session }; + info = { + id, + state: seen.state, + unreadable: false, + confirmed, + session, + prReported: seen.prReported, + }; } catch { - info = { id, state: null, unreadable: true, confirmed: false, session: false }; + info = { + id, + state: null, + unreadable: true, + confirmed: false, + session: false, + prReported: false, + }; } const rank = info.unreadable ? 2 : firstMs === null ? 1 : 0; runs.push({ firstMs, info, rank }); diff --git a/apps/server/src/factory/pass.test.ts b/apps/server/src/factory/pass.test.ts index 8fe0372fa1dd..4e8af067d97f 100644 --- a/apps/server/src/factory/pass.test.ts +++ b/apps/server/src/factory/pass.test.ts @@ -12,7 +12,7 @@ import type { HandPause } from "./claims.ts"; import { timeLimitOnlyProjectConfig, type MachineConfig, type ProjectConfig } from "./config.ts"; import type { State } from "./domain/lifecycle.ts"; import { LedgerIoError } from "./ledger/errors.ts"; -import type { ErrorKind, MachineEvent } from "./ledger/events.ts"; +import type { CauseStep, ErrorKind, MachineEvent } from "./ledger/events.ts"; import type { RunInfo } from "./ledger/listRuns.ts"; import { newRunId, runIdToString, type RunId } from "./ledger/runId.ts"; import { formatTs } from "./ledger/time.ts"; @@ -24,6 +24,13 @@ import { type Repo, type Report, } from "./pass.ts"; +import { + NotBuildingError, + type AdvanceOutcome, + type BuildingError, + type LimitOutcome, + type WatchOutcome, +} from "./runner/building.ts"; import { GateRunningError, type StartError, type StartOutcome } from "./runner/start.ts"; import { SessionsError } from "./t3/sessions.ts"; import { GitHubFailed, type Blocker } from "./tracker/github.ts"; @@ -86,6 +93,10 @@ class Fake implements PassWorld { starts: Record> = {}; /** Issues whose `start` leaves a `triage` run behind, whatever it answers. */ startLeavesRun: Array = []; + /** Scripted answers per `"#"`, popped per call; an exhausted script answers the default. */ + timeLimits: Record> = {}; + watches: Record> = {}; + advances: Record> = {}; pause: HandPause | null = null; machineLogFails = false; /** What the pass wrote to a run's ledger as an `error` line. */ @@ -99,7 +110,14 @@ class Fake implements PassWorld { events: Array = []; runIn(issue: number, state: State | null, repo = REPO, session = false) { - this.runs.push({ id: rid(issue, repo), state, unreadable: false, confirmed: true, session }); + this.runs.push({ + id: rid(issue, repo), + state, + unreadable: false, + confirmed: true, + session, + prReported: false, + }); } pushStart(key: string, answer: StartOutcome | StartError) { @@ -202,6 +220,7 @@ class Fake implements PassWorld { unreadable: false, confirmed: true, session: false, + prReported: false, }); } if ("kind" in answer && answer.kind === "started") { @@ -216,6 +235,7 @@ class Fake implements PassWorld { unreadable: false, confirmed: true, session: false, + prReported: false, }); return Effect.succeed(answer); } @@ -223,6 +243,43 @@ class Fake implements PassWorld { return Effect.fail(answer); } + private scripted( + table: Record>, + call: string, + repo: Repo, + issue: number, + fallback: T, + ) { + const key = `${repo.watched.repo}#${issue}`; + this.calls.push(`${call} ${key}`); + const answer = table[key]?.shift() ?? fallback; + return "kind" in answer ? Effect.succeed(answer as T) : Effect.fail(answer); + } + + timeLimit(repo: Repo, run: RunId) { + return this.scripted(this.timeLimits, "time_limit", repo, Number(run.issue), { + kind: "within", + }); + } + + watch(repo: Repo, run: RunId) { + return this.scripted(this.watches, "watch", repo, Number(run.issue), { kind: "unchanged" }); + } + + /** An advance that ends the run (`needs-info`, `stuck`) takes it off the list, as the ledger does. */ + advance(repo: Repo, run: RunId) { + const answered = this.scripted(this.advances, "advance", repo, Number(run.issue), { + kind: "running", + }); + return Effect.tap(answered, (outcome) => + Effect.sync(() => { + if (outcome.kind === "needs-info" || outcome.kind === "stuck") { + this.runs = this.runs.filter((r) => runIdToString(r.id) !== runIdToString(run)); + } + }), + ); + } + handPause() { return this.pause; } @@ -231,7 +288,7 @@ class Fake implements PassWorld { return this.now; } - recordError(run: RunId, step: "pass" | "start", kind: ErrorKind, message: string) { + recordError(run: RunId, step: CauseStep, kind: ErrorKind, message: string) { if (this.runErrorFails) { return Effect.fail( new LedgerIoError({ path: "/data/runs/x.jsonl", cause: new Error("finished") }), @@ -331,6 +388,24 @@ describe("claiming new issues", () => { }), ); + it.effect("a building run whose pull request waits for the gate uses no slot", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runs.push({ + id: rid(40), + state: BUILDING, + unreadable: false, + confirmed: true, + session: true, + prReported: true, + }); + fake.ready[REPO] = [1]; + fake.pushStart(`${REPO}#1`, started(1)); + yield* pass(fake, machine([REPO], 1)); + assert.deepEqual(starts(fake), [`start ${REPO}#1`]); + }), + ); + it.effect("a run in any watched repo uses a slot", () => Effect.gen(function* () { const fake = new Fake(); @@ -514,6 +589,7 @@ describe("claiming new issues", () => { unreadable: true, confirmed: false, session: false, + prReported: false, }); fake.ready[REPO] = [1]; fake.pushStart(`${REPO}#1`, started(1)); @@ -547,7 +623,10 @@ describe("a ready issue whose older run is gating", () => { fake.blockers[`${REPO}#14`] = blockerList([[REPO, 13]]); const { report, lines } = yield* pass(fake, machine([REPO], 1)); assert.deepEqual(starts(fake), []); - assert.equal(runLine(lines, "rawphp/softfac#14 "), "rawphp/softfac#14 ready: gate running"); + assert.equal( + runLine(lines, "rawphp/softfac#14 ready"), + "rawphp/softfac#14 ready: gate running", + ); assert.ok(!fake.calls.includes("blockers rawphp/softfac#14"), fake.calls.join("\n")); assert.deepEqual( fake.events.filter((e) => e.event === "claim_skipped"), @@ -685,6 +764,7 @@ describe("runs that are confirmed and still in triage", () => { unreadable: false, confirmed: true, session: false, + prReported: false, }); fake.pushStart(`${REPO}#13`, started(13)); yield* pass(fake, machine([REPO], 3)); @@ -883,12 +963,43 @@ describe("runs that are confirmed and still in triage", () => { }), ); - it.effect("a building run is left alone", () => + it.effect("a building run is worked: time limit, watch, advance, in that order", () => Effect.gen(function* () { const fake = new Fake(); - fake.runIn(40, BUILDING); - yield* pass(fake, machine([REPO], 3)); - assert.deepEqual(starts(fake), []); + fake.runIn(40, BUILDING, REPO, true); + const { lines } = yield* pass(fake, machine([REPO], 3)); + const calls = fake.calls.filter((c) => /^(time_limit|watch|advance|start|resume) /.test(c)); + assert.deepEqual(calls, [`time_limit ${REPO}#40`, `watch ${REPO}#40`, `advance ${REPO}#40`]); + assert.equal( + runLine(lines, `${REPO}#40 building:`), + `${REPO}#40 building: session still running`, + ); + }), + ); + + it.effect("a run that is not confirmed is never time-limited, watched or advanced", () => + Effect.gen(function* () { + const fake = new Fake(); + for (const [issue, state] of [ + [12, "triage"], + [40, BUILDING], + [41, "stuck"], + ] as const) { + fake.runs.push({ + id: rid(issue), + state, + unreadable: false, + confirmed: false, + session: true, + prReported: false, + }); + } + const { lines } = yield* pass(fake, machine([REPO], 3)); + const calls = fake.calls.filter((c) => /^(time_limit|watch|advance|start|resume) /.test(c)); + assert.deepEqual(calls, []); + for (const issue of [12, 40, 41]) { + assert.ok(!lines.some((l) => l.startsWith(`${REPO}#${issue} `)), lines.join("\n")); + } }), ); @@ -953,6 +1064,7 @@ describe("claims that have no run in the ledger", () => { unreadable: false, confirmed: false, session: false, + prReported: false, }); fake.triage[REPO] = [12]; fake.pushStart(`${REPO}#12`, { kind: "stuck", run: rid(12), reason: "could not confirm" }); @@ -971,6 +1083,7 @@ describe("claims that have no run in the ledger", () => { unreadable: false, confirmed: false, session: false, + prReported: false, }); fake.ready[REPO] = [3]; fake.pushStart(`${REPO}#3`, started(3)); @@ -1511,6 +1624,7 @@ describe("what the pass records", () => { unreadable: true, confirmed: false, session: false, + prReported: false, }); const { report, lines } = yield* pass(fake, machine([REPO], 1)); assert.deepEqual(fake.events[0], { event: "pass_started", data: {} }); @@ -1762,3 +1876,277 @@ describe("text that is printed", () => { const _loaded: Loaded = { kind: "broken", name: REPO }; void _loaded; void runIdToString; + +// ---- following a building run ---- + +describe("following a building run", () => { + const stopped = (reason = "the reason") => ({ kind: "stopped" as const, reason }); + const callsOf = (fake: Fake) => + fake.calls.filter((c) => /^(time_limit|watch|advance|start|resume) /.test(c)); + + it.effect("a triage run gets the time limit before resume", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(12, "triage"); + yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#12`, `resume ${REPO}#12`]); + }), + ); + + it.effect("a triage run stopped at its limit is not resumed and its line says stuck", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(12, "triage"); + fake.timeLimits[`${REPO}#12`] = [stopped()]; + const { lines } = yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#12`]); + assert.equal( + runLine(lines, `${REPO}#12 `), + `${REPO}#12 triage: stuck, the reason is in the ledger of run ${runIdToString(rid(12))}`, + ); + }), + ); + + it.effect("a building run whose pull request waits for the gate gets no watch or advance", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runs.push({ + id: rid(40), + state: BUILDING, + unreadable: false, + confirmed: true, + session: true, + prReported: true, + }); + const { lines } = yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#40`]); + assert.equal( + runLine(lines, `${REPO}#40 `), + `${REPO}#40 building: session finished, ready for the gate`, + ); + }), + ); + + it.effect("a building run stopped by the time limit gets no watch or advance", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(40, BUILDING, REPO, true); + fake.timeLimits[`${REPO}#40`] = [stopped()]; + const { lines } = yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#40`]); + assert.include(runLine(lines, `${REPO}#40 `), "stuck, the reason is in the ledger of run"); + }), + ); + + it.effect("a building run stopped by watch gets no advance", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(40, BUILDING, REPO, true); + fake.watches[`${REPO}#40`] = [stopped()]; + const { lines } = yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#40`, `watch ${REPO}#40`]); + assert.include(runLine(lines, `${REPO}#40 `), "stuck, the reason is in the ledger of run"); + }), + ); + + const advanceLines: ReadonlyArray = [ + [{ kind: "running" }, "session still running"], + [{ kind: "reminded" }, "session ended without a result line, sent it a reminder"], + [ + { kind: "paused" }, + "session stopped by hand, waiting for an instructor's comment on the pull request or the issue", + ], + [ + { kind: "ready-for-gate", pr: 3, prUrl: "https://github.com/rawphp/softfac/pull/3" }, + "session finished, ready for the gate", + ], + [ + { kind: "needs-info" }, + "session asked questions, the issue is now needs-info and the run is finished", + ], + [ + { kind: "stuck", reason: "r" }, + `stuck, the reason is in the ledger of run ${runIdToString(rid(40))}`, + ], + ]; + + it.effect("each advance outcome has its line", () => + Effect.gen(function* () { + for (const [outcome, words] of advanceLines) { + const fake = new Fake(); + fake.runIn(40, BUILDING, REPO, true); + fake.advances[`${REPO}#40`] = [outcome]; + const { lines } = yield* pass(fake, machine([REPO], 3)); + assert.equal(runLine(lines, `${REPO}#40 `), `${REPO}#40 building: ${words}`); + } + }), + ); + + const failing = ( + script: "advances" | "watches" | "timeLimits", + error: BuildingError, + step: string, + kind: ErrorKind, + ) => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(40, BUILDING, REPO, true); + fake[script][`${REPO}#40`] = [error]; + const { report, lines } = yield* pass(fake, machine([REPO], 3)); + assert.equal(report.errors, 1); + assert.ok( + lines.some((l) => l.startsWith("error: ")), + lines.join("\n"), + ); + assert.equal(fake.runErrors.length, 1); + assert.equal(fake.runErrors[0]?.step, step); + assert.equal(fake.runErrors[0]?.kind, kind); + return fake; + }); + + it.effect("a failing advance is an error line on the run, its kind by the error", () => + Effect.gen(function* () { + yield* failing( + "advances", + new NotBuildingError({ run: { repo: REPO, issue: "40", attempt: 1 } }), + "advance", + "ledger", + ); + yield* failing( + "advances", + new SessionsError({ operation: "status", cause: new Error("x") }), + "advance", + "t3", + ); + yield* failing( + "advances", + new GitHubFailed({ args: ["pr", "view"], status: 1, stderr: "boom" }), + "advance", + "tracker", + ); + }), + ); + + it.effect("a failing watch is recorded with step watch", () => + Effect.gen(function* () { + const fake = yield* failing( + "watches", + new GitHubFailed({ args: ["issue", "view"], status: 1, stderr: "boom" }), + "watch", + "tracker", + ); + assert.ok(!fake.calls.some((c) => c.startsWith("advance "))); + }), + ); + + it.effect( + "a failing time limit is recorded with step pass and the run is left for the pass", + () => + Effect.gen(function* () { + const fake = yield* failing( + "timeLimits", + new SessionsError({ operation: "status", cause: new Error("x") }), + "pass", + "t3", + ); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#40`]); + }), + ); + + it.effect("a building run that goes stuck in the pass frees its slot for a claim", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(40, BUILDING, REPO, true); + fake.advances[`${REPO}#40`] = [{ kind: "stuck", reason: "r" }]; + fake.ready[REPO] = [7]; + fake.pushStart(`${REPO}#7`, started(7)); + const { report } = yield* pass(fake, machine([REPO], 1)); + assert.deepEqual(starts(fake), [`start ${REPO}#7`]); + assert.equal(report.claimed, 1); + }), + ); + + it.effect("a needs-info run is advanced, with no watch and no resume", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(42, "needs-info", REPO, true); + fake.advances[`${REPO}#42`] = [{ kind: "needs-info" }]; + const { lines } = yield* pass(fake, machine([REPO], 5)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#42`, `advance ${REPO}#42`]); + assert.equal( + runLine(lines, `${REPO}#42 `), + `${REPO}#42 needs-info: session asked questions, the issue is now needs-info and the run is finished`, + ); + }), + ); + + it.effect("a stuck run is advanced, with no watch and no resume", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(43, "stuck", REPO, true); + fake.advances[`${REPO}#43`] = [{ kind: "stuck", reason: "r" }]; + const { lines } = yield* pass(fake, machine([REPO], 5)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#43`, `advance ${REPO}#43`]); + assert.equal( + runLine(lines, `${REPO}#43 `), + `${REPO}#43 stuck: stuck, the reason is in the ledger of run ${runIdToString(rid(43))}`, + ); + }), + ); + + it.effect("a failing advance of a needs-info run is recorded with step advance", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(42, "needs-info", REPO, true); + fake.advances[`${REPO}#42`] = [ + new SessionsError({ operation: "archive", cause: new Error("x") }), + ]; + const { report } = yield* pass(fake, machine([REPO], 5)); + assert.equal(report.errors, 1); + assert.equal(fake.runErrors[0]?.step, "advance"); + }), + ); + + it.effect("a pr-open run gets only the time limit", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(41, "pr-open", REPO, true); + const { lines } = yield* pass(fake, machine([REPO], 5)); + assert.deepEqual(callsOf(fake), [`time_limit ${REPO}#41`]); + assert.ok(!lines.some((l) => l.startsWith(`${REPO}#41`)), lines.join("\n")); + }), + ); + + it.effect("a needs-info run that its advance finishes frees its slot for a claim", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(42, "needs-info", REPO, true); + fake.advances[`${REPO}#42`] = [{ kind: "needs-info" }]; + fake.ready[REPO] = [7]; + fake.pushStart(`${REPO}#7`, started(7)); + const { report } = yield* pass(fake, machine([REPO], 1)); + assert.deepEqual(starts(fake), [`start ${REPO}#7`]); + assert.equal(report.claimed, 1); + }), + ); + + it.effect("a building run whose gate is running is not worked", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(40, BUILDING, REPO, true); + fake.gateLocks = [runIdToString(rid(40))]; + const { lines } = yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), []); + assert.equal(runLine(lines, `${REPO}#40 `), `${REPO}#40 building: gate running`); + }), + ); + + it.effect("a building run of a repo that is not loaded is left alone", () => + Effect.gen(function* () { + const fake = new Fake(); + fake.runIn(40, BUILDING, "acme/other", true); + yield* pass(fake, machine([REPO], 3)); + assert.deepEqual(callsOf(fake), []); + }), + ); +}); diff --git a/apps/server/src/factory/pass.ts b/apps/server/src/factory/pass.ts index d0a952e9ee30..0b3451ff1273 100644 --- a/apps/server/src/factory/pass.ts +++ b/apps/server/src/factory/pass.ts @@ -13,13 +13,19 @@ import * as Effect from "effect/Effect"; import type { HandPause } from "./claims.ts"; import type { MachineConfig, ProjectConfig, WatchedRepo } from "./config.ts"; import { stateLabel, type State } from "./domain/lifecycle.ts"; -import { sanitize } from "./gate.ts"; import type { LedgerError } from "./ledger/errors.ts"; -import type { ErrorKind, MachineEvent } from "./ledger/events.ts"; +import type { CauseStep, ErrorKind, MachineEvent } from "./ledger/events.ts"; import { holdsSlot, type RunInfo } from "./ledger/listRuns.ts"; import { asciiLower, runIdToString, type RunId } from "./ledger/runId.ts"; import { duration, formatTs } from "./ledger/time.ts"; +import type { + AdvanceOutcome, + BuildingError, + LimitOutcome, + WatchOutcome, +} from "./runner/building.ts"; import { mention, type StartError, type StartOutcome } from "./runner/start.ts"; +import { sanitize } from "./text.ts"; import { LIST_LIMIT, type Blocker, type Listing, type UnassignedClaim } from "./tracker/github.ts"; /** An error of the pass's own: what kind it is for the machine log, and what it says. */ @@ -76,12 +82,18 @@ export interface PassWorld { readonly start: (repo: Repo, issue: number) => Effect.Effect; /** The claim gone on with, never made: `resume` in `runner/start.ts`. */ readonly resume: (repo: Repo, issue: number) => Effect.Effect; + /** The time limit: `runner/building.ts` `timeLimit`. */ + readonly timeLimit: (repo: Repo, run: RunId) => Effect.Effect; + /** The text watch: `watch`. */ + readonly watch: (repo: Repo, run: RunId) => Effect.Effect; + /** The session's result: `advance`. */ + readonly advance: (repo: Repo, run: RunId) => Effect.Effect; /** Whether the run's gate is running now. */ readonly gateRunning: (run: RunId) => boolean; /** Appends an `error` line to the run's own ledger. */ readonly recordError: ( run: RunId, - step: "pass" | "start", + step: CauseStep, kind: ErrorKind, message: string, ) => Effect.Effect; @@ -112,7 +124,7 @@ interface Notes { interface NoteError { /** The error line, without its `error: ` prefix. */ readonly line: string; - readonly step: "pass" | "start"; + readonly step: CauseStep; readonly kind: ErrorKind; /** What went wrong, before it is cleaned for the ledger. */ readonly message: string; @@ -130,7 +142,7 @@ const passError = (kind: ErrorKind, what: string, detail: string): NoteError => }); /** Every call `start` makes is a tracker call, whatever host the tracker runs on. */ -const errorKind = (error: StartError): ErrorKind => { +const errorKind = (error: StartError | BuildingError): ErrorKind => { switch (error._tag) { case "GitHubSpawnError": case "GitHubInvalidArgument": @@ -143,7 +155,11 @@ const errorKind = (error: StartError): ErrorKind => { case "GateRunningError": return "gate"; case "FactorySessionsError": + case "FactoryThreadNotFoundError": + case "FactoryMessageRejectedError": return "t3"; + case "WrongRepoError": + return "config"; default: // The run's own record disagrees with what the step expected. return "ledger"; @@ -154,6 +170,8 @@ const errorKind = (error: StartError): ErrorKind => { const sameRepo = (a: string, b: string) => asciiLower(a) === asciiLower(b); const GATE_RUNNING = "gate running"; +/** The line of a building run whose session reported its pull request. */ +const READY_FOR_GATE = "session finished, ready for the gate"; const FROM_TEXT = " (from Depends on, add the GitHub link)"; @@ -252,11 +270,11 @@ class Pass { return { label: plain(label), state: state ?? "starting", parts: [], errors: [] }; } - /** The step `name` failed for this issue. */ - failed(notes: Notes, name: string, error: StartError): void { + /** The step called `label` in the pass line failed for this issue; the ledger records it under `step`. */ + failed(notes: Notes, label: string, step: CauseStep, error: StartError | BuildingError): void { notes.errors.push({ - line: `${notes.label} ${name}: ${plain(error.message)}`, - step: name === "start" ? "start" : "pass", + line: `${notes.label} ${label}: ${plain(error.message)}`, + step, kind: errorKind(error), message: error.message, }); @@ -375,8 +393,8 @@ const runs = (world: PassWorld, pass: Pass): Effect.Effect => Effect.gen(function* () { for (const info of [...known.infos]) { - if (!inTriage(info)) continue; + if (info.unreadable || !info.confirmed) continue; const loaded = repos.find( (l) => l.kind === "ready" && sameRepo(l.repo.watched.repo, info.id.repo), ); @@ -417,35 +440,137 @@ const workRuns = ( const name = repo.watched.repo; const issue = Number(info.id.issue); const notes = pass.notes(`${name}${mention(issue)}`, info.state); - const outcome = yield* Effect.result(world.resume(repo, issue)); - if (outcome._tag === "Failure") { - if (outcome.failure._tag === "GateRunningError") notes.parts.push(GATE_RUNNING); - else pass.failed(notes, "start", outcome.failure); - } else { - switch (outcome.success.kind) { - case "started": - notes.parts.push("session started, the issue is building"); - break; - case "stuck": - pass.stuck(notes, runIdToString(outcome.success.run)); - break; - case "skipped": - case "refused": { - const moved = outcome.success.kind === "skipped" ? outcome.success.moved : undefined; - if (moved === "ready") notes.parts.push(READY_AGAIN); - else { - const run = runIdToString(info.id); - yield* stalled(pass, notes, run, outcome.success.reason, moved === "closed"); - } - break; - } + if (world.gateRunning(info.id)) { + // The gate has the run: nothing else is done for it (softfac's `work_runs`). + notes.parts.push(GATE_RUNNING); + } else if (yield* withinLimit(world, repo, info, notes, pass)) { + if (inTriage(info)) yield* resumeRun(world, repo, info, notes, pass); + // The session's pull request waits for the gate: nothing is left to read or decide. + else if (info.state === "building" && info.prReported) notes.parts.push(READY_FOR_GATE); + else if (info.state === "building") yield* followBuilding(world, repo, info, notes, pass); + // A run left in `needs-info` or `stuck` has its end done by `advance` (softfac's `finishing`). + else if (info.state === "needs-info" || info.state === "stuck") { + yield* advanceRun(world, repo, info, notes, pass); } + // `pr-open` waits for the gate, a later step. } yield* known.refresh(world, pass, name, issue); yield* pass.flush(notes, info.id); } }); +/** The time limit for the run. False when the run was stopped or the call failed: it is done for this pass. */ +const withinLimit = ( + world: PassWorld, + repo: Repo, + info: RunInfo, + notes: Notes, + pass: Pass, +): Effect.Effect => + Effect.gen(function* () { + const outcome = yield* Effect.result(world.timeLimit(repo, info.id)); + if (outcome._tag === "Failure") { + pass.failed(notes, "time limit", "pass", outcome.failure); + return false; + } + if (outcome.success.kind === "stopped") { + pass.stuck(notes, runIdToString(info.id)); + return false; + } + return true; + }); + +/** `resume` for a run in `triage`. */ +const resumeRun = ( + world: PassWorld, + repo: Repo, + info: RunInfo, + notes: Notes, + pass: Pass, +): Effect.Effect => + Effect.gen(function* () { + const issue = Number(info.id.issue); + const outcome = yield* Effect.result(world.resume(repo, issue)); + if (outcome._tag === "Failure") { + if (outcome.failure._tag === "GateRunningError") notes.parts.push(GATE_RUNNING); + else pass.failed(notes, "start", "start", outcome.failure); + return; + } + switch (outcome.success.kind) { + case "started": + notes.parts.push("session started, the issue is building"); + break; + case "stuck": + pass.stuck(notes, runIdToString(outcome.success.run)); + break; + case "skipped": + case "refused": { + const moved = outcome.success.kind === "skipped" ? outcome.success.moved : undefined; + if (moved === "ready") notes.parts.push(READY_AGAIN); + else { + const run = runIdToString(info.id); + yield* stalled(pass, notes, run, outcome.success.reason, moved === "closed"); + } + break; + } + } + }); + +/** The watch, then `advance`, for a building run: what the session did, in a line. */ +const followBuilding = ( + world: PassWorld, + repo: Repo, + info: RunInfo, + notes: Notes, + pass: Pass, +): Effect.Effect => + Effect.gen(function* () { + const watched = yield* Effect.result(world.watch(repo, info.id)); + if (watched._tag === "Failure") return pass.failed(notes, "watch", "watch", watched.failure); + if (watched.success.kind === "stopped") return pass.stuck(notes, runIdToString(info.id)); + yield* advanceRun(world, repo, info, notes, pass); + }); + +/** `advance` for a run, and what it did in a line. */ +const advanceRun = ( + world: PassWorld, + repo: Repo, + info: RunInfo, + notes: Notes, + pass: Pass, +): Effect.Effect => + Effect.gen(function* () { + const run = runIdToString(info.id); + const advanced = yield* Effect.result(world.advance(repo, info.id)); + // A `NotBuildingError` is a run that finished meanwhile: softfac prints it as an error too. + if (advanced._tag === "Failure") + return pass.failed(notes, "advance", "advance", advanced.failure); + switch (advanced.success.kind) { + case "running": + notes.parts.push("session still running"); + break; + case "reminded": + notes.parts.push("session ended without a result line, sent it a reminder"); + break; + case "paused": + notes.parts.push( + "session stopped by hand, waiting for an instructor's comment on the pull request or the issue", + ); + break; + case "ready-for-gate": + notes.parts.push(READY_FOR_GATE); + break; + case "needs-info": + notes.parts.push( + "session asked questions, the issue is now needs-info and the run is finished", + ); + break; + case "stuck": + pass.stuck(notes, run); + break; + } + }); + /** * A ready issue the limit holds back only because of its own triage run whose session may be * running: only the claim would finish that run (`finishOlderRuns`), so the slot never frees. @@ -660,7 +785,7 @@ const noteStart = ( } yield* known.refresh(world, pass, name, issue); if (outcome._tag === "Failure") { - pass.failed(notes, "start", outcome.failure); + pass.failed(notes, "start", "start", outcome.failure); return madeSince(newestRun(known.of(name, issue)), before); } switch (outcome.success.kind) { diff --git a/apps/server/src/factory/runner/building.test.ts b/apps/server/src/factory/runner/building.test.ts new file mode 100644 index 000000000000..f0ffb503b233 --- /dev/null +++ b/apps/server/src/factory/runner/building.test.ts @@ -0,0 +1,2144 @@ +// @effect-diagnostics nodeBuiltinImport:off - the tests read the bytes of real run files. +/** + * `advance` of a building run, ported from softfac's `runner/tests.rs` ("advance" and "advance: + * the result-line reminder"). Left out, for the issues that own them: a merged or closed pull + * request ending the run (`done_with_a_merged_pull_request_ends_the_run_as_merged`, + * `done_with_a_pull_request_closed_without_a_merge_ends_the_run_as_closed`, #20), a closed issue + * abandoning the run (#20), the "a new attempt ..." tests (`start.ts`, covered by its own tests), + * `apply()` internals (`a_stuck_comment_without_a_stop_never_shows_the_lifecycle_reason`), the + * gate and forward interplay, and the crash matrix. For `watch` and `timeLimit` also left out: a + * recorded gate pass (#19), a run whose own merge closed the issue (#20), a relaunch decided by + * an interruption, and the tests over the real `gh` client (`tracker/github.test.ts` owns them). + */ +import * as NodeFS from "node:fs"; + +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import { describe } from "vite-plus/test"; + +import { timeLimitOnlyProjectConfig, type ProjectConfig } from "../config.ts"; +import type { TextCheck } from "../domain/approval.ts"; +import type { State } from "../domain/lifecycle.ts"; +import { makeCause, type RunEvent } from "../ledger/events.ts"; +import { Ledger } from "../ledger/ledger.ts"; +import { readRun } from "../ledger/read.ts"; +import { runPath, type RunId } from "../ledger/runId.ts"; +import { T0, useTempDirs } from "../ledger/testSupport.ts"; +import { + MessageRejectedError, + SessionsError, + ThreadNotFoundError, + type RunStatus, + type Thread, + type ThreadStatus, +} from "../t3/sessions.ts"; +import { + GitHubFailed, + GitHubRateLimited, + type GitHubError, + type IssueDetail, + type PullRequest, +} from "../tracker/github.ts"; +import { + advance, + timeLimit, + watch, + type AdvanceOutcome, + type BuildingContext, +} from "./building.ts"; + +const tempDir = useTempDirs(); + +const REPO = "rawphp/softfac"; +const run10: RunId = { repo: REPO, issue: "10", attempt: 1 }; +const LABELED = "2026-10-04T10:00:00Z"; +const URL12 = "https://github.com/rawphp/softfac/pull/12"; +const DONE_MESSAGE = `Built it.\nFACTORY-DONE: ${URL12}`; +const QUESTIONS = "1. Which database?\n2. Which user?"; +const WORKING_MESSAGE = "Review is running in the background. I'm waiting on it."; +const NO_RESULT_LINE = "the session ended without a FACTORY-DONE or FACTORY-STOP line"; +const SAID_STUCK = "the session stopped and said it was stuck"; +const THREAD_GONE = "the T3 thread no longer exists"; +const REMINDER = + "Your last reply had no result line. If your work is still in progress, finish it. End your reply with one of these lines as the last line: FACTORY-DONE: , FACTORY-STOP: needs-info (after your numbered questions) or FACTORY-STOP: stuck."; +const WRONG_ACCOUNT = + "the session ran as the wrong GitHub account: restart the factory's T3 server so it has GH_CONFIG_DIR, then set factory:ready again"; + +// ---- the ledger of a started run ------------------------------------------------------------ + +type Line = { readonly event: string; readonly data: unknown }; + +const step = (text: string): RunEvent => ({ event: "step_changed", data: { step: text } }); +const moved = (from: State, to: State): RunEvent => ({ + event: "state_changed", + data: { from, to }, +}); + +/** What the ledger holds when the issue is `building`: the claim, the launch and the recheck. */ +const STARTED: ReadonlyArray = [ + step(`approved ${LABELED}`), + { event: "claimed", data: { machine: "mac-mini", github_user: "rawphp" } }, + moved("ready", "triage"), + step("launch-sending"), + { + event: "session_launched", + data: { + thread_id: "thread-1", + worktree: "/t3/wt", + branch: "t3code/abc", + model_instance: "claudeAgent", + model: "claude-opus-5-5", + resumed_from: null, + }, + }, + step("approval-rechecked"), + moved("triage", "building"), +]; + +const FINISHED_EVENT: RunEvent = { + event: "finished", + data: { + outcome: "stuck", + total_ms: 0, + gate_attempts: 0, + review_cycles: null, + pr_url: null, + pr_end: null, + }, +}; + +const FINISHED = (outcome: "stuck" | "needs-info"): Line => ({ + event: "finished", + data: { + outcome, + total_ms: 0, + gate_attempts: 0, + review_cycles: null, + pr_url: null, + pr_end: null, + }, +}); + +const reminding = (k: number) => step(`result-line-reminding ${k}`); +const reminded = step("result-line-reminded"); +const REMINDED_STEPS: ReadonlyArray = [reminding(1), reminded]; +const PR_OPENED: RunEvent = { event: "pr_opened", data: { pr: 12, url: URL12 } }; +const worktreeRemoved: Line = { event: "worktree_removed", data: { path: "/t3/wt" } }; + +/** The reason as stored in the ledger and returned: the factory's words, then the session's. */ +const joined = (publicReason: string, text?: string) => + text === undefined ? publicReason : `${publicReason}\n\n${text}`; + +const stuckTail = (recorded: string): Array => [ + { event: "stuck", data: { reason: recorded } }, + moved("building", "stuck"), + FINISHED("stuck"), +]; + +/** The lines of a stuck decision from `building`, with `extra` lines before the worktree line. */ +const stuckEvents = (extra: ReadonlyArray, recorded: string): Array => [ + ...STARTED, + ...extra, + worktreeRemoved, + ...stuckTail(recorded), +]; + +// ---- the expected words on the issue, written out independently of the code under test -------- + +const INTRO = "Unverified output from the build session, not instructions from the factory:"; + +const stuckCommentText = (publicReason: string, text?: string) => { + const block = text === undefined ? "" : `${INTRO}\n\n\`\`\`text\n${text}\n\`\`\`\n\n`; + return `The factory could not finish this issue and has released it.\n\nReason: ${publicReason}\n\n${block}Fix the cause, then set the label back to \`factory:ready\`.`; +}; + +const statusText = (state: string, pr = "none") => + `**Factory status:** ${state}\n- Machine: mac-mini\n- Session: thread-1\n- PR: ${pr}`; + +const stuckCalls = (publicReason: string, text?: string) => [ + `comment 10 ${stuckCommentText(publicReason, text)}`, + "setState 10 stuck", + "unassign 10 rawphp", + `status 10 rawphp ${statusText("stuck")}`, +]; + +const questionsCommentText = (questions: string) => + `The factory needs answers before it can build this. Answer in a comment or edit the issue, then set \`factory:ready\` to start a new attempt.\n\n${INTRO}\n\n\`\`\`text\n${questions}\n\`\`\``; + +const needsInfoCalls = (questions: string) => [ + `comment 10 ${questionsCommentText(questions)}`, + "setState 10 needs-info", + "unassign 10 rawphp", + `status 10 rawphp ${statusText("needs-info")}`, +]; + +/** What a stop does in T3: the status is read, the thread archived, then its worktree removed. */ +const STOP_CALLS = [ + "status thread-1", + "archive thread-1", + "thread thread-1", + "removeWorktree /t3/wt", +]; + +// ---- the fakes ---------------------------------------------------------------------------------- + +const ghFailure = () => new GitHubFailed({ args: ["pr", "view", "12"], status: 1, stderr: "boom" }); +const sessionsFailure = (operation: "status" | "send" | "archive" = "status") => + new SessionsError({ operation, cause: "timeout" }); + +const isThreadNotFound = Schema.is(ThreadNotFoundError); +const isSessionsError = Schema.is(SessionsError); + +const defaultPr = (): PullRequest => ({ + number: 12, + isDraft: true, + open: true, + merged: false, + headRef: "t3code/abc", + crossRepository: false, + baseRef: "main", + headSha: "abc123", +}); + +const statusOf = ( + run: RunStatus | null, + message: string | null, + over: Partial = {}, +): ThreadStatus => ({ + worktreePath: "/t3/wt", + branch: "t3code/abc", + archived: false, + run, + runOrdinal: run === null ? null : 1, + lastAssistantMessage: message, + ...over, +}); + +/** What the fakes answer. A test changes it before it calls, or between two calls. */ +interface World { + pr: PullRequest | null; + issue: IssueDetail; + textCheck: TextCheck | GitHubError; + status: ThreadStatus | ThreadNotFoundError | SessionsError; + thread: Thread | ThreadNotFoundError | SessionsError; + /** The next call of each method fails with this, once. */ + failNext: { + pullRequest?: GitHubError; + markPrDraft?: GitHubError; + postComment?: GitHubError; + textCheck?: GitHubError; + }; + /** The next send fails; `delivered` says T3 took the message all the same. */ + sendFailure: { + error: SessionsError | MessageRejectedError | ThreadNotFoundError; + delivered: boolean; + } | null; + archiveFails: SessionsError | null; + /** The next read of the thread fails, once. */ + threadOnce: SessionsError | null; + /** The text of each message T3 took, by command id. */ + delivered: Map; + archived: Array; +} + +const issueDetail = (state: State | null, assignees: ReadonlyArray): IssueDetail => ({ + number: 10, + title: "Build step", + body: "Build the runner.", + labels: state === null ? [] : [`factory:${state}`], + assignees, + author: "tomkaczocha", + state, + closed: false, +}); + +const MINUTE = 60_000; + +const projectConfig = (base = "main"): ProjectConfig => { + const config = timeLimitOnlyProjectConfig(); + return { + ...config, + instructors: ["rawphp"], + base, + gate: ["cargo test"], + limits: { ...config.limits, minutes: 90 }, + }; +}; + +/** Seeds a run file with `events`, line `i` written at `ms(i)`. */ +const seed = ( + dataDir: string, + id: RunId, + events: ReadonlyArray, + ms: (index: number) => number = () => T0, +) => { + let now = T0; + const ledger = Ledger.create(dataDir, id, { clock: () => now }); + try { + events.forEach((event, index) => { + now = ms(index); + ledger.append(event, makeCause("start", "seed")); + }); + } finally { + ledger.close(); + } +}; + +interface Rig { + readonly dataDir: string; + readonly run: RunId; + readonly world: World; + /** The calls to the tracker and to T3, in order, and the two interleaved. */ + readonly gh: Array; + readonly t3: Array; + readonly order: Array; + readonly clock: { now: number }; + ctx: BuildingContext; +} + +const rigFor = ( + events: ReadonlyArray = STARTED, + options: { base?: string; ms?: (index: number) => number } = {}, +): Rig => { + const dataDir = tempDir(); + seed(dataDir, run10, events, options.ms); + const world: World = { + pr: defaultPr(), + issue: issueDetail("building", ["rawphp"]), + textCheck: "unchanged", + status: statusOf("completed", DONE_MESSAGE), + thread: { worktreePath: "/t3/wt", branch: "t3code/abc", archived: false }, + failNext: {}, + sendFailure: null, + archiveFails: null, + threadOnce: null, + delivered: new Map(), + archived: [], + }; + const gh: Array = []; + const t3: Array = []; + const order: Array = []; + const clock = { now: T0 }; + const ghCall = (text: string) => { + gh.push(text); + order.push(`gh:${text}`); + }; + const t3Call = (text: string) => { + t3.push(text); + order.push(`t3:${text}`); + }; + const takeFailure = (key: K) => { + const error = world.failNext[key]; + delete world.failNext[key]; + return error; + }; + const tracker: BuildingContext["tracker"] = { + issue: (n) => { + ghCall(`issue ${n}`); + return Effect.succeed(world.issue); + }, + pullRequest: (n) => { + ghCall(`pullRequest ${n}`); + const error = takeFailure("pullRequest"); + return error === undefined ? Effect.succeed(world.pr) : Effect.fail(error); + }, + markPrDraft: (n) => { + ghCall(`markPrDraft ${n}`); + const error = takeFailure("markPrDraft"); + if (error !== undefined) return Effect.fail(error); + if (world.pr !== null) world.pr = { ...world.pr, isDraft: true }; + return Effect.void; + }, + setState: (n, state) => { + ghCall(`setState ${n} ${state}`); + world.issue = { ...world.issue, state, labels: [`factory:${state}`] }; + return Effect.void; + }, + unassign: (n, user) => { + ghCall(`unassign ${n} ${user}`); + world.issue = { ...world.issue, assignees: [] }; + return Effect.void; + }, + postComment: (n, body) => { + ghCall(`comment ${n} ${body}`); + const error = takeFailure("postComment"); + return error === undefined ? Effect.void : Effect.fail(error); + }, + upsertStatusComment: (n, user, body) => { + ghCall(`status ${n} ${user} ${body}`); + return Effect.void; + }, + textCheck: (n, since, instructors) => { + ghCall(`textCheck ${n} ${since} ${instructors.join(",")}`); + const error = takeFailure("textCheck"); + if (error !== undefined) return Effect.fail(error); + return typeof world.textCheck === "string" + ? Effect.succeed(world.textCheck) + : Effect.fail(world.textCheck); + }, + }; + const sessions: BuildingContext["sessions"] = { + status: (threadId) => { + t3Call(`status ${threadId}`); + const answer = world.status; + return isThreadNotFound(answer) || isSessionsError(answer) + ? Effect.fail(answer) + : Effect.succeed(answer); + }, + thread: (threadId) => { + t3Call(`thread ${threadId}`); + const once = world.threadOnce; + world.threadOnce = null; + if (once !== null) return Effect.fail(once); + const answer = world.thread; + return isThreadNotFound(answer) || isSessionsError(answer) + ? Effect.fail(answer) + : Effect.succeed(answer); + }, + send: (threadId, commandId, text) => { + t3Call(`send ${threadId} ${commandId} ${text}`); + const failure = world.sendFailure; + world.sendFailure = null; + if (failure !== null) { + if (failure.delivered) world.delivered.set(commandId, text); + return Effect.fail(failure.error); + } + world.delivered.set(commandId, text); + return Effect.void; + }, + archive: (threadId) => { + t3Call(`archive ${threadId}`); + if (world.archiveFails !== null) return Effect.fail(world.archiveFails); + world.archived.push(threadId); + return Effect.void; + }, + removeWorktree: (path) => { + t3Call(`removeWorktree ${path}`); + return Effect.void; + }, + }; + const ctx: BuildingContext = { + repo: REPO, + trackerUser: "rawphp", + machine: "mac-mini", + project: projectConfig(options.base), + dataDir, + tracker, + sessions, + clock: () => clock.now, + }; + return { dataDir, run: run10, world, gh, t3, order, clock, ctx }; +}; + +const advanceIn = (rig: Rig) => advance(rig.ctx, rig.run); + +const linesOf = (rig: Rig, id: RunId = rig.run): Array => + readRun(rig.dataDir, id).entries.map(({ event, data }) => ({ event, data })); + +const bytesOf = (rig: Rig) => NodeFS.readFileSync(runPath(rig.dataDir, rig.run), "utf8"); + +const lastCause = (rig: Rig) => { + const cause = readRun(rig.dataDir, rig.run).entries.at(-1)?.cause; + return { step: cause?.step, reason: cause?.reason }; +}; + +/** The code, as the causes of the last three lines (`stuck`, `state_changed`, `finished`) carry it. */ +const stuckCauses = (rig: Rig) => + readRun(rig.dataDir, rig.run) + .entries.slice(-3) + .map((entry) => `${entry.cause?.step} ${entry.cause?.reason}`); + +const clearCalls = (rig: Rig) => { + rig.gh.length = 0; + rig.t3.length = 0; + rig.order.length = 0; +}; + +const setStatus = ( + rig: Rig, + run: RunStatus | null, + message: string | null, + over: Partial = {}, +) => { + rig.world.status = statusOf(run, message, over); +}; + +/** Run `ordinal` of the session has ended with `message`. */ +const setEnded = (rig: Rig, ordinal: number, message: string | null) => + setStatus(rig, "completed", message, { runOrdinal: ordinal }); + +const sends = (rig: Rig) => rig.t3.filter((call) => call.startsWith("send ")); +const remindCall = (k = 1) => + `send thread-1 softfac:rawphp__softfac__10__1:remind:${k} ${REMINDER}`; + +/** Asserts `advance` goes stuck: the outcome, the tracker calls, the ledger, and the stop before any write. */ +const assertStuck = ( + rig: Rig, + outcome: AdvanceOutcome, + expected: { + publicReason: string; + text?: string; + before?: ReadonlyArray; + extra?: ReadonlyArray; + recorded?: string; + }, +) => { + const recorded = expected.recorded ?? joined(expected.publicReason, expected.text); + assert.deepStrictEqual(outcome, { kind: "stuck", reason: recorded }); + assert.deepStrictEqual(rig.gh, [ + ...(expected.before ?? []), + ...stuckCalls(expected.publicReason, expected.text), + ]); + assert.deepStrictEqual(linesOf(rig), stuckEvents(expected.extra ?? [], recorded)); + // The session is stopped before anything is written on the issue. + assert.ok( + rig.order.indexOf("t3:archive thread-1") < + rig.order.findIndex((call) => call.startsWith("gh:comment")), + rig.order.join("\n"), + ); +}; + +/** A started run whose run 1 ended with `first` (no result line) and was reminded, and whose run 2 ended with `second`. */ +const remindedRun = (first: string | null, second: string | null) => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, first); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + setEnded(rig, 2, second); + clearCalls(rig); + return rig; + }); + +/** A run whose `state_changed` to `needs-info` is written but that was not ended. */ +const waitingInNeedsInfo = () => { + const rig = rigFor([ + ...STARTED, + { event: "questions_posted", data: { questions: "Which one?" } }, + moved("building", "needs-info"), + ]); + rig.world.issue = issueDetail("needs-info", []); + setStatus(rig, "completed", "Which one?\nFACTORY-STOP: needs-info"); + return rig; +}; + +const WAITING_LINES: ReadonlyArray = [ + { event: "questions_posted", data: { questions: "Which one?" } }, + moved("building", "needs-info"), +]; + +// ---- advance -------------------------------------------------------------------------------------- + +describe("advance: while the session works", () => { + it.effect("does nothing while the run is active, even for a message that looks final", () => + Effect.gen(function* () { + const actives: ReadonlyArray = [ + "preparing", + "queued", + "starting", + "running", + "waiting", + null, + ]; + for (const run of actives) { + const rig = rigFor(); + setStatus(rig, run, DONE_MESSAGE); + const before = bytesOf(rig); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }, String(run)); + assert.deepStrictEqual(rig.gh, [], String(run)); + assert.deepStrictEqual(rig.t3, ["status thread-1"], String(run)); + assert.strictEqual(bytesOf(rig), before, String(run)); + } + }), + ); + + it.effect("leaves a run in triage to start until the text was checked again", () => + Effect.gen(function* () { + const rig = rigFor(STARTED.slice(0, 5)); + const before = bytesOf(rig); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + assert.strictEqual(bytesOf(rig), before); + }), + ); + + it.effect("moves a rechecked run in triage to building, then reads the session", () => + Effect.gen(function* () { + const rig = rigFor(STARTED.slice(0, 6)); + setStatus(rig, "running", "working"); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }); + assert.deepStrictEqual(rig.gh, [ + "setState 10 building", + `status 10 rawphp ${statusText("building")}`, + ]); + assert.deepStrictEqual(linesOf(rig), [...STARTED]); + assert.deepStrictEqual(lastCause(rig), { step: "advance", reason: "session-started" }); + }), + ); +}); + +describe("advance: done", () => { + it.effect("with an open draft pull request on the session's branch is ready for the gate", () => + Effect.gen(function* () { + const rig = rigFor(); + const outcome = yield* advanceIn(rig); + assert.deepStrictEqual(outcome, { kind: "ready-for-gate", pr: 12, prUrl: URL12 }); + // Only a read: the label stays building and the gate moves it on. + assert.deepStrictEqual(rig.gh, ["pullRequest 12"]); + assert.deepStrictEqual(linesOf(rig), [...STARTED, PR_OPENED]); + assert.deepStrictEqual(lastCause(rig), { step: "advance", reason: "pr-opened" }); + }), + ); + + it.effect("again does not record the pull request twice", () => + Effect.gen(function* () { + const rig = rigFor(); + yield* advanceIn(rig); + const afterFirst = bytesOf(rig); + assert.deepStrictEqual(yield* advanceIn(rig), { + kind: "ready-for-gate", + pr: 12, + prUrl: URL12, + }); + assert.strictEqual(bytesOf(rig), afterFirst); + assert.strictEqual(linesOf(rig).length, STARTED.length + 1); + }), + ); + + it.effect("counts with a failed status when the pull request is open", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "failed", DONE_MESSAGE); + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + }), + ); + + it.effect("with a pull request that is not a draft puts it back to draft and goes on", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), isDraft: false }; + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + assert.deepStrictEqual(rig.gh, ["pullRequest 12", "markPrDraft 12"]); + assert.deepStrictEqual(linesOf(rig).slice(STARTED.length), [ + { event: "pr_set_draft", data: { pr: 12 } }, + PR_OPENED, + ]); + assert.deepStrictEqual( + readRun(rig.dataDir, rig.run) + .entries.slice(STARTED.length) + .map((e) => e.cause?.reason), + ["pr-set-draft", "pr-opened"], + ); + }), + ); + + it.effect("with a pull request that is not a draft and cannot be drafted is stuck", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), isDraft: false }; + rig.world.failNext.markPrDraft = ghFailure(); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session reported PR #12 but it is not a draft", + text: DONE_MESSAGE, + before: ["pullRequest 12", "markPrDraft 12"], + }); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("advance pr-not-bound")); + }), + ); + + it.effect("with a rate limit on putting it back to draft is an error, not stuck", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), isDraft: false }; + rig.world.failNext.markPrDraft = new GitHubRateLimited({ args: ["pr"], stderr: "slow down" }); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "GitHubRateLimited"); + // Nothing final was written: the run is still building and the next pass asks again. + assert.deepStrictEqual(linesOf(rig), [...STARTED]); + assert.notStrictEqual(rig.world.issue.state, "stuck"); + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + }), + ); + + it.effect("with someone else's pull request that is not a draft does not draft it", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), isDraft: false, headRef: "someone/else" }; + const outcome = yield* advanceIn(rig); + assert.ok( + outcome.kind === "stuck" && + outcome.reason.startsWith( + "the session reported PR #12 but its branch someone/else is not this session's branch", + ), + JSON.stringify(outcome), + ); + assert.ok(!rig.gh.includes("markPrDraft 12")); + }), + ); + + it.effect("with a merged pull request on someone else's branch is still stuck", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { + ...defaultPr(), + open: false, + merged: true, + isDraft: false, + headRef: "someone/else", + }; + const outcome = yield* advanceIn(rig); + assert.deepStrictEqual(outcome, { + kind: "stuck", + reason: joined( + "the session reported PR #12 but its branch someone/else is not this session's branch", + DONE_MESSAGE, + ), + }); + assert.deepStrictEqual(linesOf(rig).at(-1), FINISHED("stuck")); + }), + ); + + it.effect("with a pull request that does not exist is stuck", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = null; + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session reported PR #12 but it does not exist", + text: DONE_MESSAGE, + before: ["pullRequest 12"], + }); + }), + ); + + it.effect( + "with someone else's pull request is stuck, and the branch stays out of the comment", + () => + Effect.gen(function* () { + for (const head of ["someone/else", "main", "T3CODE/ABC", "t3code/abc2", ""]) { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), headRef: head }; + const recorded = joined( + `the session reported PR #12 but its branch ${head} is not this session's branch`, + DONE_MESSAGE, + ); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session reported PR #12 but its branch is not this session's branch", + text: DONE_MESSAGE, + before: ["pullRequest 12"], + recorded, + }); + assert.ok(!linesOf(rig).some((line) => line.event === "pr_opened"), head); + } + }), + ); + + it.effect("with a pull request from a fork is stuck", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), crossRepository: true }; + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session reported PR #12 but it comes from a fork", + text: DONE_MESSAGE, + before: ["pullRequest 12"], + }); + }), + ); + + it.effect( + "with a pull request into another base is stuck, and the base stays out of the comment", + () => + Effect.gen(function* () { + for (const base of [ + "release/1.0", + "develop", + "MAIN", + "main2", + "release/main", + "@octocat", + ]) { + const rig = rigFor(); + rig.world.pr = { ...defaultPr(), baseRef: base }; + const shown = base.replaceAll("@", "@\u200B"); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: + "the session reported PR #12 but its base is not the configured base branch", + text: DONE_MESSAGE, + before: ["pullRequest 12"], + recorded: joined( + `the session reported PR #12 but its base ${shown} is not the configured base branch`, + DONE_MESSAGE, + ), + }); + } + }), + ); + + it.effect("compares the base with the configured base, not with main", () => + Effect.gen(function* () { + const rig = rigFor(STARTED, { base: "develop" }); + rig.world.pr = { ...defaultPr(), baseRef: "develop" }; + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + }), + ); + + it.effect("reports a fork before a wrong base or branch", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.pr = { + ...defaultPr(), + crossRepository: true, + baseRef: "release", + headRef: "other", + }; + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session reported PR #12 but it comes from a fork", + text: DONE_MESSAGE, + before: ["pullRequest 12"], + }); + }), + ); + + it.effect("when the session has no branch fails closed", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", DONE_MESSAGE, { branch: null }); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session reported PR #12 but the session has no branch", + text: DONE_MESSAGE, + before: ["pullRequest 12"], + }); + }), + ); + + it.effect("is accepted when open, a draft and on the session's branch", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", DONE_MESSAGE, { branch: "t3code/xyz" }); + rig.world.pr = { ...defaultPr(), headRef: "t3code/xyz" }; + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + }), + ); + + it.effect("with an unusable pull request URL is stuck without asking GitHub", () => + Effect.gen(function* () { + const cases = [ + [ + "https://github.com/rawphp/other/pull/12", + "the session reported a pull request that is not on this repo", + ], + [ + "https://github.com/rawphp/softfac/pull/12?x=1", + "the session reported a malformed pull request URL", + ], + [ + "https://github.com/rawphp/softfac/pull/12 and more", + "the session reported a malformed pull request URL", + ], + [ + "https://gitlab.com/rawphp/softfac/pull/12", + "the session reported a malformed pull request URL", + ], + [ + "https://github.com/rawphp/softfac/pull/0", + "the session reported a malformed pull request URL", + ], + [ + "https://github.com/rawphp/softfac/pull/twelve", + "the session reported a malformed pull request URL", + ], + ] as const; + for (const [url, reason] of cases) { + const rig = rigFor(); + const message = `Done.\nFACTORY-DONE: ${url}`; + setStatus(rig, "completed", message); + assertStuck(rig, yield* advanceIn(rig), { publicReason: reason, text: message }); + } + }), + ); + + it.effect("is a runner error that writes nothing when the pull request cannot be read", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.failNext.pullRequest = ghFailure(); + const before = bytesOf(rig); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "GitHubFailed"); + assert.strictEqual(bytesOf(rig), before); + }), + ); +}); + +describe("advance: questions", () => { + it.effect("posts them, releases the issue and ends the run", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", `${QUESTIONS}\nFACTORY-STOP: needs-info`); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + assert.deepStrictEqual(rig.gh, needsInfoCalls(QUESTIONS)); + // The thread is archived and the worktree removed, then the run is finished. + assert.deepStrictEqual(rig.t3, STOP_CALLS); + assert.deepStrictEqual(linesOf(rig), [ + ...STARTED, + { event: "questions_posted", data: { questions: QUESTIONS } }, + moved("building", "needs-info"), + worktreeRemoved, + FINISHED("needs-info"), + ]); + assert.deepStrictEqual(lastCause(rig), { step: "advance", reason: "questions-asked" }); + }), + ); + + it.effect("with no questions is stuck", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", "FACTORY-STOP: needs-info"); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session asked for information but gave no questions", + }); + }), + ); + + it.effect("are not asked by a run that ended badly", () => + Effect.gen(function* () { + const rig = rigFor(); + const message = "Why?\nFACTORY-STOP: needs-info"; + setStatus(rig, "failed", message); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session run ended as failed", + text: message, + }); + }), + ); + + it.effect("fail with the comment as a runner error that a retry can finish", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", "Which one?\nFACTORY-STOP: needs-info"); + rig.world.failNext.postComment = ghFailure(); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "GitHubFailed"); + assert.deepStrictEqual(linesOf(rig), [...STARTED]); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + }), + ); + + it.effect("already posted are not posted again by the next call", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", "Which one?\nFACTORY-STOP: needs-info"); + // The label call fails after the comment was posted and recorded. + rig.world.issue = issueDetail("building", ["rawphp"]); + const original = rig.ctx.tracker.setState; + let failed = false; + rig.ctx = { + ...rig.ctx, + tracker: { + ...rig.ctx.tracker, + setState: (n, state) => { + if (!failed) { + failed = true; + return Effect.fail(ghFailure()); + } + return original(n, state); + }, + }, + }; + assert.strictEqual((yield* Effect.flip(advanceIn(rig)))._tag, "GitHubFailed"); + assert.strictEqual(rig.gh.filter((call) => call.startsWith("comment")).length, 1); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + assert.strictEqual(rig.gh.filter((call) => call.startsWith("comment")).length, 1); + }), + ); +}); + +describe("advance: a run left in needs-info", () => { + it.effect("is ended without posting the questions again", () => + Effect.gen(function* () { + const rig = waitingInNeedsInfo(); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + // Nothing is written on GitHub: the label stays factory:needs-info. + assert.deepStrictEqual(rig.gh, ["issue 10"]); + assert.deepStrictEqual(rig.world.archived, ["thread-1"]); + assert.deepStrictEqual(rig.t3.slice(0, 3), [ + "archive thread-1", + "thread thread-1", + "removeWorktree /t3/wt", + ]); + assert.deepStrictEqual(linesOf(rig), [ + ...STARTED, + ...WAITING_LINES, + worktreeRemoved, + FINISHED("needs-info"), + ]); + assert.deepStrictEqual(lastCause(rig), { step: "advance", reason: "questions-asked" }); + }), + ); + + it.effect("is not carried out as stuck by a decision an older version recorded", () => + Effect.gen(function* () { + // The old timeout path wrote `stuck` before it moved the label, then died. + const rig = rigFor([ + ...STARTED, + { event: "questions_posted", data: { questions: "Which one?" } }, + { event: "stuck", data: { reason: "the questions went unanswered" } }, + moved("building", "needs-info"), + ]); + rig.world.issue = issueDetail("needs-info", []); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + assert.deepStrictEqual(rig.gh, ["issue 10"]); + assert.deepStrictEqual(linesOf(rig).at(-1), FINISHED("needs-info")); + }), + ); + + it.effect("whose issue a person changed is ended and the issue left alone", () => + Effect.gen(function* () { + const cases: ReadonlyArray<[State | null, ReadonlyArray]> = [ + ["building", ["someone"]], + ["building", ["rawphp", "someone"]], + ["building", []], + ["ready", []], + ["stuck", ["rawphp"]], + [null, ["rawphp"]], + ]; + for (const [state, assignees] of cases) { + const rig = waitingInNeedsInfo(); + rig.world.issue = issueDetail(state, assignees); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }, `${state}`); + assert.deepStrictEqual(rig.gh, ["issue 10"], `${state} ${assignees}`); + assert.deepStrictEqual(linesOf(rig).at(-1), FINISHED("needs-info")); + } + }), + ); + + it.effect("that an older version claimed back before its marker is given the issue back", () => + Effect.gen(function* () { + const rig = waitingInNeedsInfo(); + rig.world.issue = issueDetail("building", ["RawPHP"]); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + assert.deepStrictEqual(rig.gh, ["issue 10", "setState 10 needs-info", "unassign 10 rawphp"]); + assert.deepStrictEqual(rig.world.archived, ["thread-1"]); + assert.deepStrictEqual(linesOf(rig).slice(-2), [worktreeRemoved, FINISHED("needs-info")]); + }), + ); + + it.effect("that is ended is not ended again", () => + Effect.gen(function* () { + const rig = waitingInNeedsInfo(); + yield* advanceIn(rig); + const bytes = bytesOf(rig); + clearCalls(rig); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "NotBuildingError"); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); + + it.effect("stays open for the next call when the archive fails", () => + Effect.gen(function* () { + const rig = waitingInNeedsInfo(); + rig.world.archiveFails = sessionsFailure("archive"); + assert.strictEqual((yield* Effect.flip(advanceIn(rig)))._tag, "FactorySessionsError"); + assert.deepStrictEqual(rig.world.archived, []); + assert.notStrictEqual(linesOf(rig).at(-1)?.event, "finished"); + rig.world.archiveFails = null; + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + assert.deepStrictEqual(linesOf(rig).at(-1), FINISHED("needs-info")); + assert.deepStrictEqual(linesOf(rig).at(-2), worktreeRemoved); + }), + ); +}); + +describe("advance: stuck", () => { + it.effect("carries out a decision already in the ledger without asking T3 or posting again", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, { event: "stuck", data: { reason: "gave up" } }]); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "stuck", reason: "gave up" }); + assert.deepStrictEqual(rig.gh, [ + "setState 10 stuck", + "unassign 10 rawphp", + `status 10 rawphp ${statusText("stuck")}`, + ]); + assert.deepStrictEqual(rig.t3, [ + "archive thread-1", + "thread thread-1", + "removeWorktree /t3/wt", + ]); + assert.deepStrictEqual(linesOf(rig).slice(STARTED.length), [ + { event: "stuck", data: { reason: "gave up" } }, + worktreeRemoved, + moved("building", "stuck"), + FINISHED("stuck"), + ]); + assert.deepStrictEqual(lastCause(rig), { step: "advance", reason: "stuck-resumed" }); + }), + ); + + it.effect("only finishes a run whose label a person changed before it was finished", () => + Effect.gen(function* () { + const reason = `${NO_RESULT_LINE}\n\nno marker`; + const rig = rigFor([ + ...STARTED, + { event: "stuck", data: { reason } }, + moved("building", "stuck"), + ]); + rig.world.issue = issueDetail("ready", []); + const world = { ...rig.world.issue }; + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "stuck", reason }); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.world.issue, world); + assert.deepStrictEqual(rig.t3, []); + assert.deepStrictEqual(linesOf(rig).at(-1), FINISHED("stuck")); + assert.strictEqual(linesOf(rig).length, STARTED.length + 3); + }), + ); + + it.effect("puts the session text in the code block, not in the reason line", () => + Effect.gen(function* () { + const rig = rigFor(); + const text = "The gate cannot run here.\nIt needs a database."; + setStatus(rig, "completed", `${text}\nFACTORY-STOP: stuck`); + assertStuck(rig, yield* advanceIn(rig), { publicReason: SAID_STUCK, text }); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("advance session-stuck")); + }), + ); + + it.effect("prepares the session text once for the comment, the ledger and the outcome", () => + Effect.gen(function* () { + const rig = rigFor(); + const message = `ping @octocat \u202e\n\`\`\`\n${"x".repeat(6100)}\nFACTORY-STOP: stuck`; + setStatus(rig, "completed", message); + const outcome = yield* advanceIn(rig); + assert.strictEqual(outcome.kind, "stuck"); + if (outcome.kind !== "stuck") return; + // The mention and the marker are disarmed, the bidi mark is a space, the text is cut. + assert.ok( + outcome.reason.startsWith(`${SAID_STUCK}\n\nping @\u200Boctocat <\u200B!-- softfac --> \n`), + ); + assert.ok(outcome.reason.endsWith("x\n[cut]")); + const comment = rig.gh[0]!; + assert.ok(comment.includes("\n\n````text\nping @\u200Boctocat"), comment); + assert.ok(comment.includes("x\n[cut]\n````\n\nFix the cause"), comment); + const stuckLine = linesOf(rig)[STARTED.length + 1]; + assert.deepStrictEqual(stuckLine, { event: "stuck", data: { reason: outcome.reason } }); + }), + ); + + it.effect("without a reason", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", "FACTORY-STOP: stuck"); + assertStuck(rig, yield* advanceIn(rig), { publicReason: SAID_STUCK }); + }), + ); + + it.effect("ends as session-wrong-account when the first line names another gh account", () => + Effect.gen(function* () { + const rig = rigFor(); + const text = + "the session's gh account is tomkaczocha, not rawphp\nIt cannot see rawphp/softfac."; + setStatus(rig, "completed", `${text}\nFACTORY-STOP: stuck`); + assertStuck(rig, yield* advanceIn(rig), { publicReason: WRONG_ACCOUNT, text }); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("advance session-wrong-account")); + }), + ); + + it.effect("counts the wrong-account line after blank lines, and alone", () => + Effect.gen(function* () { + for (const message of [ + "\n \nthe session's gh account is x, not rawphp\nmore\nFACTORY-STOP: stuck", + "the session's gh account is x, not rawphp\nFACTORY-STOP: stuck", + ]) { + const rig = rigFor(); + setStatus(rig, "completed", message); + const outcome = yield* advanceIn(rig); + assert.ok(outcome.kind === "stuck" && outcome.reason.startsWith(WRONG_ACCOUNT), message); + assert.strictEqual(stuckCauses(rig)[0], "advance session-wrong-account"); + } + }), + ); + + it.effect( + "stays session-stuck when the wrong-account words are anywhere but the first line", + () => + Effect.gen(function* () { + for (const message of [ + "The gate cannot run.\nthe session's gh account is x, not rawphp\nFACTORY-STOP: stuck", + "I think the session's gh account is x, not rawphp\nFACTORY-STOP: stuck", + "FACTORY-STOP: stuck", + ]) { + const rig = rigFor(); + setStatus(rig, "completed", message); + const outcome = yield* advanceIn(rig); + assert.ok(outcome.kind === "stuck" && outcome.reason.startsWith(SAID_STUCK), message); + assert.strictEqual(stuckCauses(rig)[0], "advance session-stuck", message); + } + }), + ); + + it.effect("counts the wrong-account words only with the stuck line", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus( + rig, + "completed", + "the session's gh account is x, not rawphp\nFACTORY-DONE: not a url", + ); + const outcome = yield* advanceIn(rig); + assert.ok( + outcome.kind === "stuck" && + outcome.reason.startsWith("the session reported a malformed pull request URL"), + JSON.stringify(outcome), + ); + assert.strictEqual(stuckCauses(rig)[0], "advance session-stuck"); + }), + ); + + it.effect("when the message has no usable marker, after the one reminder", () => + Effect.gen(function* () { + for (const message of [ + "I changed some files.", + "FACTORY-STOP: stuck\nThanks for waiting.", + "FACTORY-STOP: stuck, sorry", + "FACTORY-STOP: needs-info\n1. Why?", + `FACTORY-DONE: ${URL12}\nbye`, + ]) { + // The first ended run gets the reminder, the second goes stuck. + const rig = yield* remindedRun(message, message); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: NO_RESULT_LINE, + text: message, + extra: REMINDED_STEPS, + }); + } + }), + ); + + it.effect("when there is no final message, after the one reminder", () => + Effect.gen(function* () { + for (const message of [null, "", " \n"]) { + const rig = yield* remindedRun(message, message); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: "the session ended without a final message", + extra: REMINDED_STEPS, + }); + } + }), + ); + + it.effect("when the run ended badly, with its status and no reminder", () => + Effect.gen(function* () { + const ended = ["failed", "cancelled", "rolled_back"] as const; + for (const status of ended) { + const rig = rigFor(); + setStatus(rig, status, "it crashed"); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: `the session run ended as ${status}`, + text: "it crashed", + }); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("advance session-ended-badly")); + } + const rig = rigFor(); + setStatus(rig, "failed", null); + assertStuck(rig, yield* advanceIn(rig), { publicReason: "the session run ended as failed" }); + }), + ); + + it.effect("counts only the known terminal statuses as ended", () => + Effect.gen(function* () { + const statuses = ["completed", "interrupted", "failed", "cancelled", "rolled_back"] as const; + for (const status of statuses) { + const rig = rigFor(); + setStatus(rig, status, "no marker"); + const outcome = yield* advanceIn(rig); + // A run that ended on its own with no result line is reminded; one a person stopped is + // paused; one that ended badly is stuck at once. + if (status === "completed") assert.deepStrictEqual(outcome, { kind: "reminded" }, status); + else if (status === "interrupted") + assert.deepStrictEqual(outcome, { kind: "paused" }, status); + else assert.strictEqual(outcome.kind, "stuck", status); + } + }), + ); + + it.effect("when the thread no longer exists", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.status = new ThreadNotFoundError({ threadId: "thread-1" }); + assertStuck(rig, yield* advanceIn(rig), { publicReason: THREAD_GONE }); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("advance thread-gone")); + }), + ); + + it.effect("when the thread is archived", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", DONE_MESSAGE, { archived: true }); + assertStuck(rig, yield* advanceIn(rig), { publicReason: THREAD_GONE }); + }), + ); + + it.effect("is never decided by a transient T3 error, which writes nothing", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.status = sessionsFailure(); + const before = bytesOf(rig); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "FactorySessionsError"); + assert.deepStrictEqual(rig.gh, []); + assert.strictEqual(bytesOf(rig), before); + }), + ); +}); + +describe("advance: a run a person stopped", () => { + it.effect("is paused, once for each run, with nothing sent and nothing on the issue", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "interrupted", "I was about to"); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "paused" }); + assert.deepStrictEqual(linesOf(rig).slice(STARTED.length), [ + { event: "session_paused", data: { run_ordinal: 1 } }, + ]); + assert.deepStrictEqual(lastCause(rig), { step: "advance", reason: "session-paused" }); + const bytes = bytesOf(rig); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "paused" }); + assert.strictEqual(bytesOf(rig), bytes); + // The person resumes the session and stops its next run. + setStatus(rig, "interrupted", "again", { runOrdinal: 2 }); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "paused" }); + assert.deepStrictEqual(linesOf(rig).slice(STARTED.length), [ + { event: "session_paused", data: { run_ordinal: 1 } }, + { event: "session_paused", data: { run_ordinal: 2 } }, + ]); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(sends(rig), []); + }), + ); + + it.effect("is read as done when its last message says so", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "interrupted", DONE_MESSAGE); + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + }), + ); +}); + +describe("advance: refused before anything is read", () => { + it.effect("on a finished run", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, FINISHED_EVENT]); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "NotBuildingError"); + assert.strictEqual(error.message, "run rawphp__softfac__10__1 is already finished"); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + }), + ); + + it.effect("on a run with no session", () => + Effect.gen(function* () { + const rig = rigFor([ + { event: "claimed", data: { machine: "mac-mini", github_user: "rawphp" } }, + ]); + const error = yield* Effect.flip(advanceIn(rig)); + assert.strictEqual(error._tag, "NoSessionError"); + assert.strictEqual(error.message, "run rawphp__softfac__10__1 has no launched session"); + assert.deepStrictEqual(rig.t3, []); + }), + ); + + it.effect("on a run with no file", () => + Effect.gen(function* () { + const rig = rigFor(); + const error = yield* Effect.flip(advance(rig.ctx, { ...run10, attempt: 2 })); + assert.strictEqual(error._tag, "LedgerNotFoundError"); + }), + ); + + it.effect("on a run from another repo", () => + Effect.gen(function* () { + const rig = rigFor(); + const other: RunId = { repo: "someone/else", issue: "10", attempt: 1 }; + const error = yield* Effect.flip(advance(rig.ctx, other)); + assert.strictEqual(error._tag, "WrongRepoError"); + assert.strictEqual(error.message, "run someone__else__10__1 is not a run of rawphp/softfac"); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + }), + ); + + it.effect("compares the repo case-insensitively", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "running", null); + const outcome = yield* advance({ ...rig.ctx, repo: "RawPHP/SoftFac" }, rig.run); + assert.deepStrictEqual(outcome, { kind: "running" }); + }), + ); +}); + +// ---- advance: the result-line reminder ----------------------------------------------------------- + +describe("advance: the result-line reminder", () => { + it.effect("is sent once, as one fixed message, to a run that ended with no result line", () => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + assert.deepStrictEqual(rig.t3, ["status thread-1", remindCall()]); + // Nothing on GitHub: the issue is still building and still ours. + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(linesOf(rig), [...STARTED, ...REMINDED_STEPS]); + assert.deepStrictEqual( + readRun(rig.dataDir, rig.run) + .entries.slice(STARTED.length) + .map((e) => e.cause?.reason), + ["result-line-reminding", "result-line-reminded"], + ); + }), + ); + + it.effect("writes the wait marker before the send and the step after it", () => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + rig.world.sendFailure = { error: sessionsFailure("send"), delivered: false }; + assert.strictEqual((yield* Effect.flip(advanceIn(rig)))._tag, "FactorySessionsError"); + assert.deepStrictEqual(linesOf(rig), [...STARTED, reminding(1)]); + }), + ); + + it.effect("is not sent again while the session works on it", () => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + const bytes = bytesOf(rig); + // T3 has not started the run yet, then it is running. + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }); + setStatus(rig, "running", "working", { runOrdinal: 2 }); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }); + assert.deepStrictEqual(sends(rig), [remindCall()]); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); + + it.effect("is not sent for a run that is still active", () => + Effect.gen(function* () { + const actives: ReadonlyArray = [ + "preparing", + "queued", + "starting", + "running", + "waiting", + null, + ]; + for (const run of actives) { + const rig = rigFor(); + setStatus(rig, run, WORKING_MESSAGE); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }, String(run)); + assert.deepStrictEqual(sends(rig), [], String(run)); + assert.deepStrictEqual(linesOf(rig), [...STARTED], String(run)); + } + }), + ); + + it.effect("is followed by a newer run that ends with done, which goes to the gate", () => + Effect.gen(function* () { + const rig = yield* remindedRun(WORKING_MESSAGE, `Done.\nFACTORY-DONE: ${URL12}`); + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + assert.deepStrictEqual(sends(rig), []); + assert.strictEqual(rig.world.issue.state, "building"); + assert.deepStrictEqual(linesOf(rig).at(-1), PR_OPENED); + assert.deepStrictEqual(rig.world.archived, []); + }), + ); + + it.effect("is followed by a newer run that ends with questions, which posts them", () => + Effect.gen(function* () { + const rig = yield* remindedRun(WORKING_MESSAGE, `${QUESTIONS}\nFACTORY-STOP: needs-info`); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "needs-info" }); + assert.strictEqual(rig.world.issue.state, "needs-info"); + assert.deepStrictEqual(rig.gh, needsInfoCalls(QUESTIONS)); + assert.deepStrictEqual(sends(rig), []); + }), + ); + + it.effect( + "is followed by a second run with no result line, which goes stuck, with no second reminder", + () => + Effect.gen(function* () { + const rig = yield* remindedRun(WORKING_MESSAGE, "Still working."); + assertStuck(rig, yield* advanceIn(rig), { + publicReason: NO_RESULT_LINE, + text: "Still working.", + extra: REMINDED_STEPS, + }); + assert.deepStrictEqual(sends(rig), []); + assert.strictEqual(rig.world.delivered.size, 1); + }), + ); + + it.effect("waits for the second run in the exact live sequence and releases nothing", () => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + // The session wakes up and finishes: running, then ended with the result line. + setStatus(rig, "running", "working", { runOrdinal: 2 }); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "running" }); + setEnded(rig, 2, `Opened it.\nFACTORY-DONE: ${URL12}`); + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + assert.strictEqual(rig.world.issue.state, "building"); + assert.deepStrictEqual(rig.world.issue.assignees, ["rawphp"]); + assert.deepStrictEqual( + rig.gh.filter((call) => call.startsWith("comment")), + [], + ); + assert.ok(!rig.t3.some((call) => call.startsWith("archive")), rig.t3.join("\n")); + assert.ok(!linesOf(rig).some((line) => line.event === "stuck")); + }), + ); + + it.effect("treats a missing message like one with no result line", () => + Effect.gen(function* () { + for (const message of [null, "", " \n"]) { + const rig = rigFor(); + setEnded(rig, 1, message); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + assert.deepStrictEqual(sends(rig), [remindCall()]); + } + }), + ); + + it.effect("is not sent for a result line that is present but unusable", () => + Effect.gen(function* () { + for (const message of [ + "FACTORY-DONE: nonsense", + "FACTORY-DONE: https://github.com/someone/else/pull/3", + "FACTORY-STOP: needs-info", + ]) { + const rig = rigFor(); + setEnded(rig, 1, message); + assert.strictEqual((yield* advanceIn(rig)).kind, "stuck", message); + assert.deepStrictEqual(sends(rig), [], message); + } + }), + ); + + it.effect("is not sent for a run that ended badly", () => + Effect.gen(function* () { + for (const status of ["failed", "cancelled", "rolled_back"] as const) { + const rig = rigFor(); + setStatus(rig, status, WORKING_MESSAGE); + assert.strictEqual((yield* advanceIn(rig)).kind, "stuck", status); + assert.deepStrictEqual(sends(rig), [], status); + } + }), + ); + + it.effect("cut short before its send is sent on the next call, under one reminding step", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, reminding(1)]); + setEnded(rig, 1, WORKING_MESSAGE); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + assert.deepStrictEqual(sends(rig), [remindCall()]); + assert.deepStrictEqual(linesOf(rig), [...STARTED, ...REMINDED_STEPS]); + }), + ); + + it.effect("whose step is missing is not sent again when the session answered it", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, reminding(1)]); + setEnded(rig, 2, `Done.\nFACTORY-DONE: ${URL12}`); + assert.strictEqual((yield* advanceIn(rig)).kind, "ready-for-gate"); + assert.deepStrictEqual(sends(rig), []); + // The send is taken to have arrived, and the step is made good. + assert.deepStrictEqual(linesOf(rig), [...STARTED, ...REMINDED_STEPS, PR_OPENED]); + }), + ); + + it.effect("sent again after a crash between the send and its step uses the same command id", () => + Effect.gen(function* () { + for (const delivered of [false, true]) { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + rig.world.sendFailure = { error: sessionsFailure("send"), delivered }; + yield* Effect.flip(advanceIn(rig)); + assert.deepStrictEqual(yield* advanceIn(rig), { kind: "reminded" }); + const ids = sends(rig).map((call) => call.split(" ")[2]); + assert.strictEqual(ids.length, 2, `delivered ${delivered}`); + assert.strictEqual(ids[0], ids[1], `delivered ${delivered}`); + assert.strictEqual(rig.world.delivered.size, 1, `delivered ${delivered}`); + assert.deepStrictEqual( + linesOf(rig), + [...STARTED, ...REMINDED_STEPS], + `delivered ${delivered}`, + ); + } + }), + ); + + it.effect("is not owed to a run with no ordinal and no result line, which is stuck at once", () => + Effect.gen(function* () { + const rig = rigFor(); + setStatus(rig, "completed", WORKING_MESSAGE, { runOrdinal: null }); + assert.strictEqual((yield* advanceIn(rig)).kind, "stuck"); + assert.deepStrictEqual(sends(rig), []); + assert.ok( + !linesOf(rig).some( + (line) => + JSON.stringify(line) === + JSON.stringify({ event: "step_changed", data: { step: "result-line-reminded" } }), + ), + ); + }), + ); + + it.effect("with a lost step and a newer run that has no line goes stuck and does not hang", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, reminding(1)]); + setEnded(rig, 2, "Nothing yet."); + assert.strictEqual((yield* advanceIn(rig)).kind, "stuck"); + assert.deepStrictEqual(sends(rig), []); + }), + ); + + const refusal = () => + new MessageRejectedError({ + threadId: "thread-1", + commandId: "softfac:rawphp__softfac__10__1:remind:1", + cause: "no", + }); + + /** A refused reminder is final: T3 keeps its refusal under the command id, so the run goes stuck at once. */ + const assertRefusedReminderIsStuck = (rig: Rig, outcome: AdvanceOutcome) => + Effect.gen(function* () { + const recorded = `${NO_RESULT_LINE}; T3 refused the reminder: ${refusal().message}\n\n${WORKING_MESSAGE}`; + assert.deepStrictEqual(outcome, { kind: "stuck", reason: recorded }); + assert.deepStrictEqual(rig.gh, stuckCalls(NO_RESULT_LINE, WORKING_MESSAGE)); + assert.deepStrictEqual(linesOf(rig), stuckEvents([reminding(1)], recorded)); + assert.deepStrictEqual(rig.world.archived, ["thread-1"]); + assert.strictEqual(stuckCauses(rig)[0], "advance session-stuck"); + // The run is finished: a second call sends nothing. + clearCalls(rig); + assert.strictEqual((yield* Effect.flip(advanceIn(rig)))._tag, "NotBuildingError"); + assert.deepStrictEqual(sends(rig), []); + }); + + it.effect("goes stuck at once, with the marker written, when T3 refuses the message", () => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + rig.world.sendFailure = { error: refusal(), delivered: false }; + const outcome = yield* advanceIn(rig); + yield* assertRefusedReminderIsStuck(rig, outcome); + }), + ); + + it.effect("goes stuck at once when T3 refuses the message of a reminder cut short", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, reminding(1)]); + setEnded(rig, 1, WORKING_MESSAGE); + rig.world.sendFailure = { error: refusal(), delivered: false }; + const outcome = yield* advanceIn(rig); + yield* assertRefusedReminderIsStuck(rig, outcome); + }), + ); + + it.effect("goes stuck as thread-gone when T3 no longer has the thread at the send", () => + Effect.gen(function* () { + const rig = rigFor(); + setEnded(rig, 1, WORKING_MESSAGE); + rig.world.sendFailure = { + error: new ThreadNotFoundError({ threadId: "thread-1" }), + delivered: false, + }; + const outcome = yield* advanceIn(rig); + assert.deepStrictEqual(outcome, { kind: "stuck", reason: THREAD_GONE }); + assert.strictEqual(stuckCauses(rig)[0], "advance thread-gone"); + }), + ); +}); + +// ---- watch ----------------------------------------------------------------------------------------- + +const CHANGED_DURING_BUILD = + "the issue text was changed during the build by someone who is not an instructor"; +const CANNOT_VERIFY_DURING_BUILD = + "the factory could not verify who changed the issue text during the build"; +const CANNOT_CONFIRM = "the factory could not confirm the issue text was approved"; +const TEXT_READ = `textCheck 10 ${LABELED} rawphp`; +/** What a stop that does not read the session does in T3: archive the thread, then remove its worktree. */ +const ARCHIVE_CALLS = ["archive thread-1", "thread thread-1", "removeWorktree /t3/wt"]; + +const approvedLine = (at: string): RunEvent => step(`approved ${at}`); + +describe("watch", () => { + it.effect("leaves a run alone when the text is unchanged since the approval", () => + Effect.gen(function* () { + const rig = rigFor(); + const bytes = bytesOf(rig); + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { kind: "unchanged" }); + // One read, since the moment the run's first line records, with the project's instructors. + assert.deepStrictEqual(rig.gh, [TEXT_READ]); + assert.deepStrictEqual(rig.t3, []); + assert.strictEqual(bytesOf(rig), bytes); + // And again: the same read each pass, never a step that says it passed. + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { kind: "unchanged" }); + assert.deepStrictEqual(rig.gh, [TEXT_READ, TEXT_READ]); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); + + it.effect("checks since the approval the ledger records", () => + Effect.gen(function* () { + const rig = rigFor([approvedLine("2026-10-04T12:34:56Z"), ...STARTED.slice(1)]); + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { kind: "unchanged" }); + assert.deepStrictEqual(rig.gh, ["textCheck 10 2026-10-04T12:34:56Z rawphp"]); + }), + ); + + it.effect("stops a run whose text someone else changed", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.textCheck = "changed"; + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { + kind: "stopped", + reason: CHANGED_DURING_BUILD, + }); + assert.deepStrictEqual(rig.gh, [TEXT_READ, ...stuckCalls(CHANGED_DURING_BUILD)]); + assert.deepStrictEqual(rig.t3, ARCHIVE_CALLS); + assert.deepStrictEqual(linesOf(rig), stuckEvents([], CHANGED_DURING_BUILD)); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("watch text-changed-during-build")); + assert.strictEqual(rig.world.issue.state, "stuck"); + assert.deepStrictEqual(rig.world.issue.assignees, []); + }), + ); + + it.effect("stops a run whose history cannot be judged, without blaming anyone", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.textCheck = "unjudged"; + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { + kind: "stopped", + reason: CANNOT_VERIFY_DURING_BUILD, + }); + assert.deepStrictEqual(rig.gh, [TEXT_READ, ...stuckCalls(CANNOT_VERIFY_DURING_BUILD)]); + assert.deepStrictEqual(rig.t3, ARCHIVE_CALLS); + assert.deepStrictEqual(linesOf(rig), stuckEvents([], CANNOT_VERIFY_DURING_BUILD)); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("watch edits-unverifiable")); + }), + ); + + it.effect("finishes a stopped run, and nothing polls it again", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.textCheck = "changed"; + yield* watch(rig.ctx, rig.run); + clearCalls(rig); + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { kind: "skipped", why: "finished" }); + assert.strictEqual((yield* Effect.flip(advanceIn(rig)))._tag, "NotBuildingError"); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + }), + ); + + it.effect("leaves the draft pull request open and does not mark it ready", () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, PR_OPENED]); + rig.world.textCheck = "changed"; + assert.strictEqual((yield* watch(rig.ctx, rig.run)).kind, "stopped"); + assert.ok( + !rig.gh.some((call) => call.startsWith("pullRequest") || call.startsWith("markPrDraft")), + rig.gh.join("\n"), + ); + // The status comment still names the pull request the run reported. + assert.ok(rig.gh.at(-1)!.endsWith(statusText("stuck", "#12")), rig.gh.at(-1)); + }), + ); + + it.effect("does nothing for a run that is finished, has no session or is not building", () => + Effect.gen(function* () { + const questions: RunEvent = { event: "questions_posted", data: { questions: "1. Which?" } }; + const stuck: RunEvent = { event: "stuck", data: { reason: "gave up" } }; + const cases: ReadonlyArray< + [string, ReadonlyArray, "finished" | "no-session" | "not-building"] + > = [ + ["finished", [...STARTED, FINISHED_EVENT], "finished"], + [ + "no session", + [STARTED[0]!, STARTED[1]!, moved("ready", "triage"), moved("triage", "building")], + "no-session", + ], + ["triage", STARTED.slice(0, 5), "not-building"], + ["needs-info", [...STARTED, questions, moved("building", "needs-info")], "not-building"], + ["stuck", [...STARTED, stuck, moved("building", "stuck")], "not-building"], + // The decision is in the ledger and `state_changed` is not: `advance` finishes it. + ["half stuck", [...STARTED, stuck], "not-building"], + ["half needs-info", [...STARTED, questions], "not-building"], + ]; + for (const [name, events, why] of cases) { + const rig = rigFor(events); + rig.world.textCheck = "changed"; + const bytes = bytesOf(rig); + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { kind: "skipped", why }, name); + assert.deepStrictEqual(rig.gh, [], name); + assert.deepStrictEqual(rig.t3, [], name); + assert.strictEqual(bytesOf(rig), bytes, name); + } + }), + ); + + it.effect("refuses a run of another repo before touching anything", () => + Effect.gen(function* () { + const rig = rigFor(); + const other: RunId = { repo: "rawphp/other", issue: "10", attempt: 1 }; + const error = yield* Effect.flip(watch(rig.ctx, other)); + assert.strictEqual(error._tag, "WrongRepoError"); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + }), + ); + + it.effect("is an error, never a stop or a pass, when the text cannot be read", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.textCheck = "changed"; + rig.world.failNext.textCheck = ghFailure(); + const bytes = bytesOf(rig); + assert.strictEqual((yield* Effect.flip(watch(rig.ctx, rig.run)))._tag, "GitHubFailed"); + assert.deepStrictEqual(rig.gh, [TEXT_READ]); + assert.deepStrictEqual(rig.t3, [], "nothing archived"); + assert.strictEqual(bytesOf(rig), bytes, "nothing written"); + assert.strictEqual(rig.world.issue.state, "building"); + // The next pass reads again and, the text being changed, stops the run. + assert.strictEqual((yield* watch(rig.ctx, rig.run)).kind, "stopped"); + }), + ); + + it.effect( + "is an error with nothing written when the archive fails, and the next pass retries", + () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.textCheck = "changed"; + rig.world.archiveFails = sessionsFailure("archive"); + const bytes = bytesOf(rig); + for (const pass of [1, 2]) { + const error = yield* Effect.flip(watch(rig.ctx, rig.run)); + assert.strictEqual(error._tag, "FactorySessionsError"); + assert.strictEqual(bytesOf(rig), bytes, `pass ${pass}`); + assert.strictEqual(rig.world.issue.state, "building"); + // Only the read: no comment, no label, no release. + assert.strictEqual(rig.gh.length, pass, rig.gh.join("\n")); + } + rig.world.archiveFails = null; + assert.strictEqual((yield* watch(rig.ctx, rig.run)).kind, "stopped"); + assert.deepStrictEqual(rig.world.archived, ["thread-1"]); + assert.deepStrictEqual(linesOf(rig), stuckEvents([], CHANGED_DURING_BUILD)); + }), + ); + + it.effect("does not let a thread T3 no longer knows stop the stop", () => + Effect.gen(function* () { + const rig = rigFor(); + rig.world.textCheck = "changed"; + rig.world.thread = new ThreadNotFoundError({ threadId: "thread-1" }); + assert.strictEqual((yield* watch(rig.ctx, rig.run)).kind, "stopped"); + assert.strictEqual(rig.world.issue.state, "stuck"); + // The worktree is removed from the path the launch recorded. + assert.deepStrictEqual(linesOf(rig), stuckEvents([], CHANGED_DURING_BUILD)); + }), + ); + + it.effect("fails closed for a run with no approval step", () => + Effect.gen(function* () { + const rig = rigFor(STARTED.slice(1)); + assert.deepStrictEqual(yield* watch(rig.ctx, rig.run), { + kind: "stopped", + reason: CANNOT_CONFIRM, + }); + // No read: there is no moment to read since. The archive, then the stuck path. + assert.deepStrictEqual(rig.gh, stuckCalls(CANNOT_CONFIRM)); + assert.deepStrictEqual(rig.t3, ARCHIVE_CALLS); + assert.deepStrictEqual(linesOf(rig), [ + ...STARTED.slice(1), + worktreeRemoved, + ...stuckTail(CANNOT_CONFIRM), + ]); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("watch approval-unconfirmed")); + }), + ); + + it.effect( + "is an error with nothing written for a run with no approval step and a failing archive", + () => + Effect.gen(function* () { + const rig = rigFor(STARTED.slice(1)); + rig.world.archiveFails = sessionsFailure("archive"); + const bytes = bytesOf(rig); + assert.strictEqual( + (yield* Effect.flip(watch(rig.ctx, rig.run)))._tag, + "FactorySessionsError", + ); + assert.deepStrictEqual(rig.gh, []); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); +}); + +// ---- the time limit ----------------------------------------------------------------------------------- + +const TIME_LIMIT = "the run passed its time limit"; +const LIMIT_MS = 90 * MINUTE; + +/** The started run, claimed at `T0` and building one minute later. */ +const startedAtT0 = (index: number) => (index === STARTED.length - 1 ? T0 + MINUTE : T0); + +/** The comment a time-limit stop posts, with the line that says where the work is. */ +const limitComment = (workLine?: string) => { + const work = workLine === undefined ? "" : `${workLine}\n\n`; + return `The factory could not finish this issue and has released it.\n\nReason: ${TIME_LIMIT}\n\n${work}Fix the cause, then set the label back to \`factory:ready\`.`; +}; + +/** The lines of a time-limit stop `elapsed` ms after the run began, with the branch the stop recorded. */ +const limitEvents = (elapsed: number, branch?: string): Array => [ + ...STARTED, + worktreeRemoved, + { event: "stuck", data: { reason: TIME_LIMIT, ...(branch !== undefined && { branch }) } }, + moved("building", "stuck"), + { event: "finished", data: { ...(FINISHED("stuck").data as object), total_ms: elapsed } }, +]; + +/** A started run that is past its limit and stopped by `timeLimit`; returns the elapsed time. */ +const stopAtTheLimit = (rig: Rig) => + Effect.gen(function* () { + const elapsed = LIMIT_MS + 1; + rig.clock.now = T0 + elapsed; + const outcome = yield* timeLimit(rig.ctx, rig.run); + assert.deepStrictEqual(outcome, { kind: "stopped", reason: TIME_LIMIT }); + return elapsed; + }); + +const limitedRig = () => rigFor(STARTED, { ms: startedAtT0 }); + +describe("timeLimit", () => { + it.effect("is within the limit up to and including the last millisecond of it", () => + Effect.gen(function* () { + const rig = limitedRig(); + const bytes = bytesOf(rig); + rig.clock.now = T0 + 1; + assert.deepStrictEqual(yield* timeLimit(rig.ctx, rig.run), { kind: "within" }); + rig.clock.now = T0 + LIMIT_MS; + assert.deepStrictEqual(yield* timeLimit(rig.ctx, rig.run), { kind: "within" }); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); + + it.effect("stops a run past its limit and names the branch without a count of commits", () => + Effect.gen(function* () { + const rig = limitedRig(); + const elapsed = yield* stopAtTheLimit(rig); + assert.deepStrictEqual(linesOf(rig), limitEvents(elapsed, "t3code/abc")); + assert.deepStrictEqual(rig.gh, [ + `comment 10 ${limitComment("The work so far is on branch t3code/abc.")}`, + "setState 10 stuck", + "unassign 10 rawphp", + `status 10 rawphp ${statusText("stuck")}`, + ]); + // The branch is read before the archive, then the archive and the worktree as for any stop. + assert.deepStrictEqual(rig.t3, ["thread thread-1", ...ARCHIVE_CALLS]); + assert.deepStrictEqual(stuckCauses(rig), Array(3).fill("pass time-limit")); + }), + ); + + it.effect( + "stops a run past its limit whose thread has no branch as it would without the work", + () => + Effect.gen(function* () { + const rig = limitedRig(); + rig.world.thread = { worktreePath: "/t3/wt", branch: null, archived: false }; + const elapsed = yield* stopAtTheLimit(rig); + assert.deepStrictEqual(linesOf(rig), limitEvents(elapsed)); + assert.deepStrictEqual(rig.gh[0], `comment 10 ${limitComment()}`); + assert.strictEqual(rig.world.issue.state, "stuck"); + }), + ); + + it.effect( + "stops a run past its limit whose thread cannot be read as it would without the work", + () => + Effect.gen(function* () { + const rig = limitedRig(); + // The first read of the thread is the branch; the stop path's own read goes through. + rig.world.threadOnce = sessionsFailure(); + const elapsed = yield* stopAtTheLimit(rig); + assert.deepStrictEqual(linesOf(rig), limitEvents(elapsed)); + assert.deepStrictEqual(rig.gh[0], `comment 10 ${limitComment()}`); + assert.deepStrictEqual(rig.t3, ["thread thread-1", ...ARCHIVE_CALLS]); + }), + ); + + it.effect( + "stops a run past its limit whose thread T3 no longer knows as it would without the work", + () => + Effect.gen(function* () { + const rig = limitedRig(); + rig.world.thread = new ThreadNotFoundError({ threadId: "thread-1" }); + const elapsed = yield* stopAtTheLimit(rig); + assert.deepStrictEqual(linesOf(rig), limitEvents(elapsed)); + }), + ); + + it.effect( + "is an error with nothing written when the archive fails, and the next call decides again", + () => + Effect.gen(function* () { + const rig = limitedRig(); + rig.clock.now = T0 + LIMIT_MS + 1; + rig.world.archiveFails = sessionsFailure("archive"); + const bytes = bytesOf(rig); + assert.strictEqual( + (yield* Effect.flip(timeLimit(rig.ctx, rig.run)))._tag, + "FactorySessionsError", + ); + assert.strictEqual(bytesOf(rig), bytes); + assert.deepStrictEqual(rig.gh, []); + rig.world.archiveFails = null; + assert.strictEqual((yield* timeLimit(rig.ctx, rig.run)).kind, "stopped"); + }), + ); + + it.effect( + "skips a finished run, and a run that is waiting, stuck or in the middle of either", + () => + Effect.gen(function* () { + const questions: RunEvent = { event: "questions_posted", data: { questions: "1. Which?" } }; + const stuck: RunEvent = { event: "stuck", data: { reason: "gave up" } }; + const cases: ReadonlyArray<[string, ReadonlyArray, "finished" | "not-running"]> = + [ + ["finished", [...STARTED, FINISHED_EVENT], "finished"], + ["needs-info", [...STARTED, questions, moved("building", "needs-info")], "not-running"], + ["stuck", [...STARTED, stuck, moved("building", "stuck")], "not-running"], + ["pr-open", [...STARTED, moved("building", "pr-open")], "not-running"], + ["half stuck", [...STARTED, stuck], "not-running"], + ["half needs-info", [...STARTED, questions], "not-running"], + ]; + for (const [name, events, why] of cases) { + const rig = rigFor(events); + rig.clock.now = T0 + 10 * LIMIT_MS; + const bytes = bytesOf(rig); + assert.deepStrictEqual( + yield* timeLimit(rig.ctx, rig.run), + { kind: "skipped", why }, + name, + ); + assert.deepStrictEqual(rig.gh, [], name); + assert.deepStrictEqual(rig.t3, [], name); + assert.strictEqual(bytesOf(rig), bytes, name); + } + }), + ); + + it.effect( + "leaves a triage run whose launch was sent and not recorded to resume, which names its session", + () => + Effect.gen(function* () { + const sent = [STARTED[0]!, STARTED[1]!, moved("ready", "triage"), step("launch-sending")]; + const rig = rigFor(sent); + rig.clock.now = T0 + 10 * LIMIT_MS; + const bytes = bytesOf(rig); + assert.deepStrictEqual(yield* timeLimit(rig.ctx, rig.run), { + kind: "skipped", + why: "launch-unrecorded", + }); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); + + it.effect( + "does not charge the wait for the gate once the session reported its pull request", + () => + Effect.gen(function* () { + const rig = rigFor([...STARTED, PR_OPENED], { ms: startedAtT0 }); + rig.clock.now = T0 + 10 * LIMIT_MS; + const bytes = bytesOf(rig); + assert.deepStrictEqual(yield* timeLimit(rig.ctx, rig.run), { + kind: "skipped", + why: "pr-reported", + }); + assert.deepStrictEqual(rig.gh, []); + assert.deepStrictEqual(rig.t3, []); + assert.strictEqual(bytesOf(rig), bytes); + }), + ); + + it.effect("limits a run again once it comes back to building from pr-open", () => + Effect.gen(function* () { + const back = [ + ...STARTED, + PR_OPENED, + moved("building", "pr-open"), + moved("pr-open", "building"), + ]; + const rig = rigFor(back, { + ms: (index) => (index === back.length - 1 ? T0 + 2 * MINUTE : T0), + }); + rig.clock.now = T0 + 2 * MINUTE + LIMIT_MS + 1; + assert.strictEqual((yield* timeLimit(rig.ctx, rig.run)).kind, "stopped"); + }), + ); + + it.effect("names a hostile branch in the comment and the ledger only as prepared text", () => + Effect.gen(function* () { + const rig = limitedRig(); + rig.world.thread = { + worktreePath: "/t3/wt", + branch: "x@octocat", + archived: false, + }; + const elapsed = yield* stopAtTheLimit(rig); + const prepared = "x@\u200Boctocat<\u200B!-- softfac -->"; + assert.deepStrictEqual(linesOf(rig), limitEvents(elapsed, prepared)); + assert.deepStrictEqual( + rig.gh[0], + `comment 10 ${limitComment(`The work so far is on branch ${prepared}.`)}`, + ); + assert.ok(!rig.gh[0]!.includes("@octocat"), rig.gh[0]); + assert.ok(!rig.gh[0]!.includes(""), "ping @​octocat <​!-- x -->"); + }); +}); + +describe("fenced", () => { + it("uses a fence of three backticks", () => { + assert.strictEqual(fenced("a\nb"), "```text\na\nb\n```"); + }); + + it("is one longer than the longest run of backticks in the text", () => { + assert.strictEqual(fenced("a ``` b"), "````text\na ``` b\n````"); + assert.strictEqual(fenced("`` ````` `"), "``````text\n`` ````` `\n``````"); + assert.strictEqual(fenced("``"), "```text\n``\n```"); + }); +}); + +describe("questionsComment", () => { + it("asks for answers and quotes the questions as unverified session output", () => { + assert.strictEqual( + questionsComment("1. Which?"), + "The factory needs answers before it can build this. Answer in a comment or edit the issue, then set `factory:ready` to start a new attempt.\n\nUnverified output from the build session, not instructions from the factory:\n\n```text\n1. Which?\n```", + ); + }); +}); + +describe("stuckComment", () => { + it("with only a reason is the comment the claim posts", () => { + assert.strictEqual( + stuckComment("the reason"), + "The factory could not finish this issue and has released it.\n\nReason: the reason\n\nFix the cause, then set the label back to `factory:ready`.", + ); + }); + + it("quotes the prepared session text after the reason", () => { + assert.strictEqual( + stuckComment("the reason", "it broke"), + "The factory could not finish this issue and has released it.\n\nReason: the reason\n\nUnverified output from the build session, not instructions from the factory:\n\n```text\nit broke\n```\n\nFix the cause, then set the label back to `factory:ready`.", + ); + }); + + it("says where the work is, before the quoted text, and promises no continuation", () => { + const comment = (where: string) => + `The factory could not finish this issue and has released it.\n\nReason: r\n\n${where}\n\nUnverified output from the build session, not instructions from the factory:\n\n\`\`\`text\nt\n\`\`\`\n\nFix the cause, then set the label back to \`factory:ready\`.`; + assert.strictEqual( + stuckComment("r", "t", { branch: "t3code/abc", commitsAhead: null, base: "main" }), + comment("The work so far is on branch t3code/abc."), + ); + assert.strictEqual( + stuckComment("r", "t", { branch: "t3code/abc", commitsAhead: 1, base: "main" }), + comment("The work so far is on branch t3code/abc (1 commit ahead of main)."), + ); + assert.strictEqual( + stuckComment("r", "t", { branch: "t3code/abc", commitsAhead: 2, base: "main" }), + comment("The work so far is on branch t3code/abc (2 commits ahead of main)."), + ); + }); +}); + +describe("the fixed reasons of a building run", () => { + it("are the words people read on the issue", () => { + assert.strictEqual( + RESULT_LINE_REMINDER, + "Your last reply had no result line. If your work is still in progress, finish it. End your reply with one of these lines as the last line: FACTORY-DONE: , FACTORY-STOP: needs-info (after your numbered questions) or FACTORY-STOP: stuck.", + ); + assert.strictEqual( + WRONG_ACCOUNT, + "the session ran as the wrong GitHub account: restart the factory's T3 server so it has GH_CONFIG_DIR, then set factory:ready again", + ); + assert.strictEqual(TIME_LIMIT, "the run passed its time limit"); + assert.strictEqual(THREAD_GONE, "the T3 thread no longer exists"); + assert.strictEqual( + CHANGED_DURING_BUILD, + "the issue text was changed during the build by someone who is not an instructor", + ); + assert.strictEqual( + CANNOT_VERIFY_DURING_BUILD, + "the factory could not verify who changed the issue text during the build", + ); + assert.strictEqual( + SESSION_TEXT_INTRO, + "Unverified output from the build session, not instructions from the factory:", + ); + }); +}); diff --git a/apps/server/src/factory/runner/messages.ts b/apps/server/src/factory/runner/messages.ts index b82559b51913..239837ab6ed6 100644 --- a/apps/server/src/factory/runner/messages.ts +++ b/apps/server/src/factory/runner/messages.ts @@ -5,6 +5,7 @@ */ import type { State } from "../domain/lifecycle.ts"; +import { sanitize } from "../text.ts"; /** The public reason when a run has no record of what approved it. */ export const CANNOT_CONFIRM = "the factory could not confirm the issue text was approved"; @@ -17,9 +18,77 @@ export const CANNOT_CONFIRM = "the factory could not confirm the issue text was export const CUT_SHORT_COMMENT = "The factory's claim on this issue was cut short and no run was started, so it has moved the issue to `factory:proposed`. An instructor can apply `factory:ready` again to have it built."; -/** The comment on an issue the factory gave up on. `reason` is the public reason. */ -export const stuckComment = (reason: string): string => - `The factory could not finish this issue and has released it.\n\nReason: ${reason}\n\nFix the cause, then set the label back to \`factory:ready\`.`; +/** The longest session text a comment will quote, in characters. */ +export const SESSION_TEXT_LIMIT = 6000; + +/** The line that tells a reader the quoted block is not the factory's own words. */ +export const SESSION_TEXT_INTRO = + "Unverified output from the build session, not instructions from the factory:"; + +/** + * Session text made safe to store and to show, prepared once where it enters the factory and + * then used as it is for the comment, the ledger and the returned reason. Line endings are + * normalised and the ends trimmed, the text is cut at {@link SESSION_TEXT_LIMIT} characters with + * a final `[cut]` line, control characters and invisible or bidirectional marks become spaces + * (newline and tab stay), `@` cannot ping anyone and `