From 520aaab0b0d6ce62bb1a3938a42105ab892b2419 Mon Sep 17 00:00:00 2001 From: Tom Kaczocha Date: Fri, 16 Oct 2026 16:07:24 +1000 Subject: [PATCH 1/5] test: install and rollback wait for the server to come up Fails until the scripts check health: the unhealthy install, closed port, old log line, no-previous, and both rollback-check cases. The healthy and truncated-log cases already pass. Issue: #59 --- .../server/src/factory/installScripts.test.ts | 153 +++++++++++++++++- 1 file changed, 150 insertions(+), 3 deletions(-) diff --git a/apps/server/src/factory/installScripts.test.ts b/apps/server/src/factory/installScripts.test.ts index a2b818dc057f..b62fc8e61059 100644 --- a/apps/server/src/factory/installScripts.test.ts +++ b/apps/server/src/factory/installScripts.test.ts @@ -1,10 +1,11 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeNet from "node:net"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; -import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vite-plus/test"; const scripts = NodePath.resolve(import.meta.dirname, "../../../../factory/scripts"); const label = "io.github.rawphp.t3fac"; @@ -12,6 +13,7 @@ const uid = process.getuid?.() ?? 0; const v1 = "0.0.1-test+fac.aaaaaaa"; const v2 = "0.0.2-test+fac.bbbbbbb"; +const v3 = "0.0.3-test+fac.ccccccc"; let tmp: string; let home: string; @@ -21,8 +23,23 @@ let launchctlLoaded: string; let launchctlDrain: string; const archives: Record = {}; +// One listener shared by every test: the scripts check this port instead of 3876. +let listener: NodeNet.Server; +let healthPort = 0; + +beforeAll(async () => { + listener = NodeNet.createServer((socket) => socket.destroy()); + await new Promise((resolve) => listener.listen(0, "127.0.0.1", resolve)); + healthPort = (listener.address() as NodeNet.AddressInfo).port; +}); + +afterAll(async () => { + await new Promise((resolve) => listener.close(() => resolve())); +}); + const plistPath = () => NodePath.join(home, "Library/LaunchAgents", `${label}.plist`); const root = () => NodePath.join(home, ".local/share/t3fac"); +const logPath = () => NodePath.join(home, "Library/Logs/t3fac/t3fac.log"); const run = (script: string, ...args: Array) => runWith({}, script, ...args); @@ -35,6 +52,9 @@ const runWith = (extraEnv: Record, script: string, ...args: Arra LAUNCHCTL_LOG: launchctlLog, LAUNCHCTL_LOADED: launchctlLoaded, LAUNCHCTL_DRAIN: launchctlDrain, + T3FAC_PORT: String(healthPort), + T3FAC_HEALTH_WAIT: "2", + LAUNCHCTL_HEALTHY: `versions/${v1} versions/${v2} versions/${v3}`, ...extraEnv, }, encoding: "utf8", @@ -71,7 +91,9 @@ const makeArchive = (version: string, options: { binary?: boolean; label?: strin // A little launchd-like: bootout leaves the job loaded for two more `print` polls (or // LAUNCHCTL_BOOTOUT_DRAIN), and bootstrap fails with EIO while it is still loaded (or when -// LAUNCHCTL_BOOTSTRAP_FAIL is set). +// LAUNCHCTL_BOOTSTRAP_FAIL is set). A bootstrap writes "factory started" to the log when +// LAUNCHCTL_HEALTHY (space-separated `current` targets) lists the version `current` points at; +// LAUNCHCTL_TRUNCATE_LOG empties the log first, as a rotation would. const setupSandbox = () => { tmp = NodeFS.realpathSync(NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3fac-install-"))); home = NodePath.join(tmp, "home"); @@ -101,7 +123,17 @@ const setupSandbox = () => { ' echo "Bootstrap failed: 5: Input/output error" >&2', " exit 5", " fi", - ' touch "$LAUNCHCTL_LOADED" ;;', + ' touch "$LAUNCHCTL_LOADED"', + ' if [ -n "${LAUNCHCTL_HEALTHY+x}" ]; then', + ' target=$(readlink "$HOME/.local/share/t3fac/current")', + " for entry in $LAUNCHCTL_HEALTHY; do", + ' if [ "$entry" = "$target" ]; then', + ' mkdir -p "$HOME/Library/Logs/t3fac"', + ' if [ -n "$LAUNCHCTL_TRUNCATE_LOG" ]; then : > "$HOME/Library/Logs/t3fac/t3fac.log"; fi', + ' echo "factory started" >> "$HOME/Library/Logs/t3fac/t3fac.log"', + " fi", + " done", + " fi ;;", ' bootout) echo "${LAUNCHCTL_BOOTOUT_DRAIN:-2}" > "$LAUNCHCTL_DRAIN" ;;', "esac", "exit 0", @@ -320,6 +352,121 @@ describe("install.sh and rollback.sh when the job never unloads", () => { }); }); +describe("install.sh and rollback.sh health check", () => { + const installV1 = () => { + setupSandbox(); + for (const version of [v1, v2, v3]) archives[version] = makeArchive(version); + expect(run("install.sh", archives[v1]!).status).toBe(0); + }; + + const installV1ThenV2 = () => { + installV1(); + expect(run("install.sh", archives[v2]!).status).toBe(0); + }; + + const seedLog = (content: string) => { + NodeFS.mkdirSync(NodePath.dirname(logPath()), { recursive: true }); + NodeFS.writeFileSync(logPath(), content); + }; + + const link = (name: string) => NodeFS.readlinkSync(NodePath.join(root(), name)); + const bootstraps = (calls: Array) => calls.filter((c) => c.startsWith("bootstrap")); + + afterEach(() => { + NodeFS.rmSync(tmp, { recursive: true, force: true }); + }); + + it("keeps a healthy install", () => { + installV1(); + const result = run("install.sh", archives[v2]!); + expect(result.status).toBe(0); + expect(result.output).toContain(`installed ${v2}`); + expect(link("current")).toBe(`versions/${v2}`); + expect(link("previous")).toBe(`versions/${v1}`); + expect(bootstraps(result.calls)).toHaveLength(1); + }); + + it("rolls back an install whose server never logs that it started", () => { + installV1(); + seedLog("boot line one\nboot line two\n"); + const result = runWith({ LAUNCHCTL_HEALTHY: `versions/${v1}` }, "install.sh", archives[v2]!); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + `install of ${v2} failed its health check; rolled back to ${v1}`, + ); + expect(result.stderr).toContain("boot line two"); + expect(link("current")).toBe(`versions/${v1}`); + expect(NodeFS.existsSync(NodePath.join(root(), "previous"))).toBe(false); + expect(bootstraps(result.calls)).toHaveLength(2); + }); + + it("rolls back an install whose port does not accept connections", () => { + installV1(); + const closed = NodeNet.createServer(); + closed.listen(0, "127.0.0.1"); + const port = (closed.address() as NodeNet.AddressInfo).port; + closed.close(); + const result = runWith({ T3FAC_PORT: String(port) }, "install.sh", archives[v2]!); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + `install of ${v2} failed its health check; rolled back to ${v1}`, + ); + expect(link("current")).toBe(`versions/${v1}`); + }); + + it("does not count a factory started line that was there before the restart", () => { + installV1(); + seedLog("factory started\n"); + const result = runWith({ LAUNCHCTL_HEALTHY: `versions/${v1}` }, "install.sh", archives[v2]!); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + `install of ${v2} failed its health check; rolled back to ${v1}`, + ); + expect(link("current")).toBe(`versions/${v1}`); + }); + + it("counts a line written after the log was truncated to a size below the noted one", () => { + installV1(); + seedLog("older log content\n".repeat(10)); + const result = runWith({ LAUNCHCTL_TRUNCATE_LOG: "1" }, "install.sh", archives[v2]!); + expect(result.status).toBe(0); + expect(link("current")).toBe(`versions/${v2}`); + }); + + it("exits 1 and says so when an unhealthy first install has nothing to roll back to", () => { + setupSandbox(); + archives[v1] = makeArchive(v1); + const result = runWith({ LAUNCHCTL_HEALTHY: "" }, "install.sh", archives[v1]!); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + `install of ${v1} failed its health check; no earlier version to roll back to`, + ); + expect(link("current")).toBe(`versions/${v1}`); + expect(NodeFS.existsSync(NodePath.join(root(), "previous"))).toBe(false); + expect(bootstraps(result.calls)).toHaveLength(1); + }); + + it("rollback.sh exits 1 naming both versions when the restored version is unhealthy too", () => { + installV1ThenV2(); + const result = runWith({ LAUNCHCTL_HEALTHY: "" }, "rollback.sh"); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`rollback to ${v1} from ${v2} failed its health check`); + expect(link("current")).toBe(`versions/${v1}`); + expect(NodeFS.existsSync(NodePath.join(root(), "previous"))).toBe(false); + }); + + it("install.sh reports when the rollback fails its health check as well", () => { + installV1(); + const result = runWith({ LAUNCHCTL_HEALTHY: "" }, "install.sh", archives[v3]!); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`rollback to ${v1} from ${v3} failed its health check`); + expect(result.stderr).toContain( + `install of ${v3} failed its health check; rollback to ${v1} failed too`, + ); + expect(link("current")).toBe(`versions/${v1}`); + }); +}); + describe("build.sh --print-version", () => { let repo: string; From b33d8dfcd74407975b4f935ce810d5af07bf113a Mon Sep 17 00:00:00 2001 From: Tom Kaczocha Date: Fri, 16 Oct 2026 16:17:48 +1000 Subject: [PATCH 2/5] feat(factory): roll back an install that fails its health check After the restart, install.sh and rollback.sh wait up to T3FAC_HEALTH_WAIT seconds (90) for the port and a fresh factory started or no factory config log line. A failed install runs rollback.sh; a failed rollback exits 1 naming both versions. Issue: #59 --- .../server/src/factory/installScripts.test.ts | 13 +++--- factory/scripts/install.sh | 18 +++++++- factory/scripts/lib/t3fac.sh | 44 +++++++++++++++++-- factory/scripts/rollback.sh | 10 ++++- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/apps/server/src/factory/installScripts.test.ts b/apps/server/src/factory/installScripts.test.ts index b62fc8e61059..c2001901317e 100644 --- a/apps/server/src/factory/installScripts.test.ts +++ b/apps/server/src/factory/installScripts.test.ts @@ -400,18 +400,19 @@ describe("install.sh and rollback.sh health check", () => { expect(bootstraps(result.calls)).toHaveLength(2); }); - it("rolls back an install whose port does not accept connections", () => { + it("rolls back an install whose port does not accept connections", async () => { installV1(); const closed = NodeNet.createServer(); - closed.listen(0, "127.0.0.1"); + await new Promise((resolve) => closed.listen(0, "127.0.0.1", resolve)); const port = (closed.address() as NodeNet.AddressInfo).port; - closed.close(); + await new Promise((resolve) => closed.close(() => resolve())); + // The port stays closed for the rollback's own check too, so that one fails as well. const result = runWith({ T3FAC_PORT: String(port) }, "install.sh", archives[v2]!); expect(result.status).toBe(1); - expect(result.stderr).toContain( - `install of ${v2} failed its health check; rolled back to ${v1}`, - ); + expect(result.stderr).toContain(`port ${port} does not accept connections`); + expect(result.stderr).toContain(`install of ${v2} failed its health check; rollback`); expect(link("current")).toBe(`versions/${v1}`); + expect(bootstraps(result.calls)).toHaveLength(2); }); it("does not count a factory started line that was there before the restart", () => { diff --git a/factory/scripts/install.sh b/factory/scripts/install.sh index 367a05788246..371f911711ee 100755 --- a/factory/scripts/install.sh +++ b/factory/scripts/install.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Installs a t3fac archive under ~/.local/share/t3fac/versions, points `current` at it, -# writes the LaunchAgent plist and restarts the job. Usage: install.sh +# writes the LaunchAgent plist and restarts the job, then waits for it to come up and runs +# rollback.sh when it does not. Usage: install.sh set -euo pipefail source "$(dirname "$0")/lib/t3fac.sh" @@ -50,6 +51,21 @@ ln -sfn "versions/$version" "$root/current" restart_job +if ! wait_healthy "$version"; then + if [[ -L $root/previous && -e $root/previous ]]; then + prev=$(readlink "$root/previous") + prev=${prev#versions/} + if bash "$(dirname "$0")/rollback.sh"; then + echo "install of $version failed its health check; rolled back to $prev" >&2 + else + echo "install of $version failed its health check; rollback to $prev failed too" >&2 + fi + else + echo "install of $version failed its health check; no earlier version to roll back to" >&2 + fi + exit 1 +fi + echo "installed $version" echo "current -> versions/$version" echo "job $label restarted" diff --git a/factory/scripts/lib/t3fac.sh b/factory/scripts/lib/t3fac.sh index ddd465e67418..c15f6eebc019 100644 --- a/factory/scripts/lib/t3fac.sh +++ b/factory/scripts/lib/t3fac.sh @@ -1,10 +1,14 @@ -# Sourced by install.sh and rollback.sh: where versions live, the LaunchAgent job, and -# how to restart it. +# Sourced by install.sh and rollback.sh: where versions live, the LaunchAgent job, how to +# restart it and how to tell it came up. root="$HOME/.local/share/t3fac" label=io.github.rawphp.t3fac template="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../launchd" && pwd)/$label.plist" plist="$HOME/Library/LaunchAgents/$label.plist" +log="$HOME/Library/Logs/t3fac/t3fac.log" +# The port the plist serves on. T3FAC_PORT exists only so tests can point the health check at +# a stub listener; the plist still says 3876. +port=${T3FAC_PORT:-3876} # Waits for launchd to report the job unloaded, up to T3FAC_UNLOAD_WAIT seconds (10). wait_unloaded() { @@ -18,10 +22,12 @@ wait_unloaded() { } # Writes the plist from the template, boots the job out when it is loaded, waits for it to -# unload and bootstraps it. +# unload and bootstraps it. Notes the log size first (log_before) so wait_healthy reads only +# what the restarted job writes. restart_job() { local uid uid=$(id -u) + log_before=$(($(wc -c 2> /dev/null < "$log" || echo 0))) mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs/t3fac" "$HOME/.config/t3fac" sed "s|@HOME@|$HOME|g" "$template" > "$plist" if launchctl print "gui/$uid/$label" > /dev/null 2>&1; then @@ -30,3 +36,35 @@ restart_job() { fi launchctl bootstrap "gui/$uid" "$plist" } + +# Waits up to T3FAC_HEALTH_WAIT seconds (90) after restart_job for the server to accept +# connections on $port and for a "factory started" or "no factory config" line to appear in +# the bytes appended to the log since the restart. A log smaller than it was before the +# restart was truncated, so it is read from the start. On timeout prints what is missing and +# the last 50 log lines to stderr and returns 1. +wait_healthy() { + local version=$1 wait=${T3FAC_HEALTH_WAIT:-90} offset=${log_before:-0} + local i size port_ok log_ok missing + for ((i = 0; i <= wait * 2; i++)); do + port_ok=0 + log_ok=0 + if (exec 3<> "/dev/tcp/127.0.0.1/$port") 2> /dev/null; then port_ok=1; fi + size=$(($(wc -c 2> /dev/null < "$log" || echo 0))) + if ((size < offset)); then offset=0; fi + # No -q: grep must read all of tail's output, or pipefail sees tail die of SIGPIPE. + if tail -c +$((offset + 1)) "$log" 2> /dev/null | grep -e 'factory started' -e 'no factory config' > /dev/null; then + log_ok=1 + fi + if ((port_ok && log_ok)); then return 0; fi + if ((i < wait * 2)); then sleep 0.5; fi + done + missing="" + if ((!port_ok)); then missing="port $port does not accept connections"; fi + if ((!log_ok)); then + missing="${missing:+$missing, }no 'factory started' or 'no factory config' line in the log since the restart" + fi + echo "version $version failed its health check within ${wait}s: $missing" >&2 + echo "last 50 lines of $log:" >&2 + tail -n 50 "$log" >&2 2> /dev/null || true + return 1 +} diff --git a/factory/scripts/rollback.sh b/factory/scripts/rollback.sh index 247e5df45848..8b54839db836 100755 --- a/factory/scripts/rollback.sh +++ b/factory/scripts/rollback.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Points `current` at the version installed before it and restarts the LaunchAgent job. -# Exits 1 when there is no earlier version. +# Points `current` at the version installed before it, restarts the LaunchAgent job and +# checks it comes up. Exits 1 when there is no earlier version or the restored one is unhealthy. set -euo pipefail source "$(dirname "$0")/lib/t3fac.sh" @@ -11,6 +11,7 @@ if [[ ! -L $root/previous || ! -e $root/previous ]]; then fi target=$(readlink "$root/previous") +from=$(readlink "$root/current") ln -sfn "$target" "$root/current" restart_job @@ -18,4 +19,9 @@ restart_job # Only now is the rollback done; a failed restart above leaves `previous` for a rerun. rm -f "$root/previous" +if ! wait_healthy "${target#versions/}"; then + echo "rollback to ${target#versions/} from ${from#versions/} failed its health check" >&2 + exit 1 +fi + echo "rolled back to ${target#versions/}" From 626a4971b3782f62d45c740f38a4ff41d0433a11 Mon Sep 17 00:00:00 2001 From: Tom Kaczocha Date: Fri, 16 Oct 2026 16:18:22 +1000 Subject: [PATCH 3/5] docs(factory): recovery section Issue: #59 --- factory/README.md | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/factory/README.md b/factory/README.md index 02f0e5878a62..ff8ff743f872 100644 --- a/factory/README.md +++ b/factory/README.md @@ -30,10 +30,46 @@ Build, install and roll back the darwin-arm64 binary (LaunchAgent `io.github.raw ```sh factory/scripts/build.sh # clean tree only: build, smoke test, then -> release/t3fac--darwin-arm64.tar.gz -factory/scripts/install.sh # unpack, point `current` at it, restart the job -factory/scripts/rollback.sh # point `current` at the previous version, restart the job +factory/scripts/install.sh # unpack, point `current` at it, restart the job, wait for it to come up, roll back if it does not +factory/scripts/rollback.sh # point `current` at the previous version, restart the job and check it comes up ``` +## Recovery + +`install.sh` and `rollback.sh` check that the job came up. After the restart they wait up to `T3FAC_HEALTH_WAIT` seconds (default 90) for the server to accept connections on port 3876 and for a `factory started` or `no factory config` line to be written to `~/Library/Logs/t3fac/t3fac.log` after the restart. Lines from before the restart do not count. If either is missing, the script prints the last 50 log lines. A failed `install.sh` then runs `rollback.sh` and exits 1 with `install of failed its health check; rolled back to `. With no earlier version it exits 1 and says so. + +### Telling the fork is down + +```sh +launchctl print gui/$(id -u)/io.github.rawphp.t3fac +tail -n 50 ~/Library/Logs/t3fac/t3fac.log +``` + +A non-zero exit from `launchctl print`, or output with no `pid` line, means the job is not running. A `last exit code` other than 0 together with a changing `runs` count is a crash loop. In the log, look for `factory started` or `no factory config` after the last start. Neither means the server never got as far as the factory. + +### Rolling back by hand + +```sh +factory/scripts/rollback.sh +``` + +It points `current` at `previous`, restarts the job, removes `previous` and runs the same health check on the restored version. It exits 1 with `rollback to from failed its health check` when that check fails too. When there is no `previous`, install an older archive from `release/` with `factory/scripts/install.sh `. + +### Handing this repo back to the Rust factory + +Do this when every installed version has a bug that stops the fork from building its own fix. It is the reverse of the #21 switch. + +1. Check that no run of either factory holds an issue for this repo: `softfac runs list` for the Rust factory, `softfac runs list --data-dir ~/.config/t3fac/factory` for the fork, and `gh issue list --repo original-solutions/t3code-factory --label factory:building` and `--label factory:triage` both come back empty. +2. Remove `original-solutions/t3code-factory` from `repos` in `~/.config/t3fac/factory/config.yml`. +3. Add it to `repos` in `~/.config/softfac/config.yml` with `repo`, `path` and `user`. +4. Restart each factory so it rereads its config. + +The Rust factory then builds the fix. + +### Moving it back + +Once `install.sh` has installed a fixed archive and it passed its health check, repeat the steps above in reverse: check that no run holds an issue, remove the repo from `~/.config/softfac/config.yml`, add it to `repos` in `~/.config/t3fac/factory/config.yml`, and restart each factory. + ## Syncing with upstream `factory/scripts/sync.sh` merges upstream's newest `v*-nightly.*` tag into `main` through a pull request. `.github/workflows/factory-sync.yml` runs it daily at 03:17 UTC and on `workflow_dispatch`, as two jobs: From 51fe118b362ec0e31f54c3ee355d774e45e5acc7 Mon Sep 17 00:00:00 2001 From: Tom Kaczocha Date: Fri, 16 Oct 2026 16:35:30 +1000 Subject: [PATCH 4/5] test: a server that logs no factory config is healthy, and the log tail is 50 lines Review findings: nothing exercised the "no factory config" line, and the tail assertion could not tell 50 lines from the whole log. Both already pass. Issue: #59 --- .../server/src/factory/installScripts.test.ts | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/server/src/factory/installScripts.test.ts b/apps/server/src/factory/installScripts.test.ts index c2001901317e..660dea0922eb 100644 --- a/apps/server/src/factory/installScripts.test.ts +++ b/apps/server/src/factory/installScripts.test.ts @@ -92,8 +92,9 @@ const makeArchive = (version: string, options: { binary?: boolean; label?: strin // A little launchd-like: bootout leaves the job loaded for two more `print` polls (or // LAUNCHCTL_BOOTOUT_DRAIN), and bootstrap fails with EIO while it is still loaded (or when // LAUNCHCTL_BOOTSTRAP_FAIL is set). A bootstrap writes "factory started" to the log when -// LAUNCHCTL_HEALTHY (space-separated `current` targets) lists the version `current` points at; -// LAUNCHCTL_TRUNCATE_LOG empties the log first, as a rotation would. +// LAUNCHCTL_HEALTHY (space-separated `current` targets) lists the version `current` points at +// (LAUNCHCTL_HEALTHY_LINE writes another line instead); LAUNCHCTL_TRUNCATE_LOG empties the log +// first, as a rotation would. const setupSandbox = () => { tmp = NodeFS.realpathSync(NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3fac-install-"))); home = NodePath.join(tmp, "home"); @@ -130,7 +131,7 @@ const setupSandbox = () => { ' if [ "$entry" = "$target" ]; then', ' mkdir -p "$HOME/Library/Logs/t3fac"', ' if [ -n "$LAUNCHCTL_TRUNCATE_LOG" ]; then : > "$HOME/Library/Logs/t3fac/t3fac.log"; fi', - ' echo "factory started" >> "$HOME/Library/Logs/t3fac/t3fac.log"', + ' echo "${LAUNCHCTL_HEALTHY_LINE:-factory started}" >> "$HOME/Library/Logs/t3fac/t3fac.log"', " fi", " done", " fi ;;", @@ -388,18 +389,34 @@ describe("install.sh and rollback.sh health check", () => { it("rolls back an install whose server never logs that it started", () => { installV1(); - seedLog("boot line one\nboot line two\n"); + const lines = Array.from({ length: 60 }, (_, i) => `boot line ${i + 1}`); + seedLog(lines.join("\n") + "\n"); const result = runWith({ LAUNCHCTL_HEALTHY: `versions/${v1}` }, "install.sh", archives[v2]!); expect(result.status).toBe(1); expect(result.stderr).toContain( `install of ${v2} failed its health check; rolled back to ${v1}`, ); - expect(result.stderr).toContain("boot line two"); + // The last 50 lines: line 11 is the oldest shown, line 10 is cut. + expect(result.stderr).toContain("boot line 60"); + expect(result.stderr).toContain("boot line 11\n"); + expect(result.stderr).not.toContain("boot line 10\n"); expect(link("current")).toBe(`versions/${v1}`); expect(NodeFS.existsSync(NodePath.join(root(), "previous"))).toBe(false); expect(bootstraps(result.calls)).toHaveLength(2); }); + it("keeps an install whose server logs that it has no factory config", () => { + installV1(); + const result = runWith( + { LAUNCHCTL_HEALTHY_LINE: "no factory config" }, + "install.sh", + archives[v2]!, + ); + expect(result.status).toBe(0); + expect(link("current")).toBe(`versions/${v2}`); + expect(bootstraps(result.calls)).toHaveLength(1); + }); + it("rolls back an install whose port does not accept connections", async () => { installV1(); const closed = NodeNet.createServer(); From 9ed810d227cd7c758e8e8b5d76371ad17a98c551 Mon Sep 17 00:00:00 2001 From: Tom Kaczocha Date: Fri, 16 Oct 2026 17:42:35 +1000 Subject: [PATCH 5/5] fix(factory): rollback without current, late-server tests, runbook config notes - rollback.sh restores previous when current is missing, naming only the restored version on failure - tests: a server whose log line or port comes up after the first check is kept - README: a factory config error fails the health check; hand-back moves config.yml aside when the repo is the only one - plist and lib/t3fac.sh point at each other for the port and log path --- .../server/src/factory/installScripts.test.ts | 108 +++++++++++++++--- factory/README.md | 6 +- factory/launchd/io.github.rawphp.t3fac.plist | 1 + factory/scripts/lib/t3fac.sh | 4 +- factory/scripts/rollback.sh | 5 +- 5 files changed, 102 insertions(+), 22 deletions(-) diff --git a/apps/server/src/factory/installScripts.test.ts b/apps/server/src/factory/installScripts.test.ts index 660dea0922eb..198e205ff355 100644 --- a/apps/server/src/factory/installScripts.test.ts +++ b/apps/server/src/factory/installScripts.test.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeNet from "node:net"; @@ -43,20 +43,22 @@ const logPath = () => NodePath.join(home, "Library/Logs/t3fac/t3fac.log"); const run = (script: string, ...args: Array) => runWith({}, script, ...args); +const scriptEnv = (extraEnv: Record) => ({ + HOME: home, + PATH: `${bin}:/usr/bin:/bin`, + LAUNCHCTL_LOG: launchctlLog, + LAUNCHCTL_LOADED: launchctlLoaded, + LAUNCHCTL_DRAIN: launchctlDrain, + T3FAC_PORT: String(healthPort), + T3FAC_HEALTH_WAIT: "2", + LAUNCHCTL_HEALTHY: `versions/${v1} versions/${v2} versions/${v3}`, + ...extraEnv, +}); + const runWith = (extraEnv: Record, script: string, ...args: Array) => { NodeFS.writeFileSync(launchctlLog, ""); const result = NodeChildProcess.spawnSync("bash", [NodePath.join(scripts, script), ...args], { - env: { - HOME: home, - PATH: `${bin}:/usr/bin:/bin`, - LAUNCHCTL_LOG: launchctlLog, - LAUNCHCTL_LOADED: launchctlLoaded, - LAUNCHCTL_DRAIN: launchctlDrain, - T3FAC_PORT: String(healthPort), - T3FAC_HEALTH_WAIT: "2", - LAUNCHCTL_HEALTHY: `versions/${v1} versions/${v2} versions/${v3}`, - ...extraEnv, - }, + env: scriptEnv(extraEnv), encoding: "utf8", }); const calls = NodeFS.readFileSync(launchctlLog, "utf8").split("\n").filter(Boolean); @@ -93,8 +95,9 @@ const makeArchive = (version: string, options: { binary?: boolean; label?: strin // LAUNCHCTL_BOOTOUT_DRAIN), and bootstrap fails with EIO while it is still loaded (or when // LAUNCHCTL_BOOTSTRAP_FAIL is set). A bootstrap writes "factory started" to the log when // LAUNCHCTL_HEALTHY (space-separated `current` targets) lists the version `current` points at -// (LAUNCHCTL_HEALTHY_LINE writes another line instead); LAUNCHCTL_TRUNCATE_LOG empties the log -// first, as a rotation would. +// (LAUNCHCTL_HEALTHY_LINE writes another line instead, LAUNCHCTL_HEALTHY_DELAY writes it that many +// seconds later, as a server still starting would); LAUNCHCTL_TRUNCATE_LOG empties the log first, +// as a rotation would. const setupSandbox = () => { tmp = NodeFS.realpathSync(NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3fac-install-"))); home = NodePath.join(tmp, "home"); @@ -131,7 +134,12 @@ const setupSandbox = () => { ' if [ "$entry" = "$target" ]; then', ' mkdir -p "$HOME/Library/Logs/t3fac"', ' if [ -n "$LAUNCHCTL_TRUNCATE_LOG" ]; then : > "$HOME/Library/Logs/t3fac/t3fac.log"; fi', - ' echo "${LAUNCHCTL_HEALTHY_LINE:-factory started}" >> "$HOME/Library/Logs/t3fac/t3fac.log"', + ' line="${LAUNCHCTL_HEALTHY_LINE:-factory started}"', + ' if [ -n "$LAUNCHCTL_HEALTHY_DELAY" ]; then', + ' (sleep "$LAUNCHCTL_HEALTHY_DELAY"; echo "$line" >> "$HOME/Library/Logs/t3fac/t3fac.log") > /dev/null 2>&1 &', + " else", + ' echo "$line" >> "$HOME/Library/Logs/t3fac/t3fac.log"', + " fi", " fi", " done", " fi ;;", @@ -328,6 +336,33 @@ describe("rollback.sh when the previous version is gone from disk", () => { }); }); +describe("rollback.sh when current is missing", () => { + beforeAll(() => { + setupInstalledSandbox(); + }); + + afterAll(() => { + NodeFS.rmSync(tmp, { recursive: true, force: true }); + }); + + it("still restores previous", () => { + NodeFS.rmSync(NodePath.join(root(), "current")); + const result = run("rollback.sh"); + expect(result.status).toBe(0); + expect(result.output).toContain(`rolled back to ${v1}`); + expect(NodeFS.readlinkSync(NodePath.join(root(), "current"))).toBe(`versions/${v1}`); + expect(NodeFS.existsSync(NodePath.join(root(), "previous"))).toBe(false); + }); + + it("names only the restored version when its health check fails", () => { + setupInstalledSandbox(); + NodeFS.rmSync(NodePath.join(root(), "current")); + const result = runWith({ LAUNCHCTL_HEALTHY: "" }, "rollback.sh"); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`rollback to ${v1} failed its health check`); + }); +}); + describe("install.sh and rollback.sh when the job never unloads", () => { const stuck = { LAUNCHCTL_BOOTOUT_DRAIN: "1000", T3FAC_UNLOAD_WAIT: "1" }; @@ -405,6 +440,49 @@ describe("install.sh and rollback.sh health check", () => { expect(bootstraps(result.calls)).toHaveLength(2); }); + it("keeps an install whose server logs that it started only after the first check", () => { + installV1(); + const result = runWith( + { LAUNCHCTL_HEALTHY_DELAY: "1", T3FAC_HEALTH_WAIT: "3" }, + "install.sh", + archives[v2]!, + ); + expect(result.status).toBe(0); + expect(link("current")).toBe(`versions/${v2}`); + expect(bootstraps(result.calls)).toHaveLength(1); + }); + + it("keeps an install whose port accepts connections only after the first check", async () => { + installV1(); + const late = NodeNet.createServer((socket) => socket.destroy()); + await new Promise((resolve) => late.listen(0, "127.0.0.1", resolve)); + const port = (late.address() as NodeNet.AddressInfo).port; + await new Promise((resolve) => late.close(() => resolve())); + NodeFS.writeFileSync(launchctlLog, ""); + const child = NodeChildProcess.spawn( + "bash", + [NodePath.join(scripts, "install.sh"), archives[v2]!], + { + env: scriptEnv({ T3FAC_PORT: String(port), T3FAC_HEALTH_WAIT: "5" }), + stdio: "ignore", + }, + ); + const exited = new Promise((resolve) => child.on("exit", resolve)); + // Open the port a poll after the bootstrap, so the first check finds it closed and only a + // later one can pass. + while (!NodeFS.readFileSync(launchctlLog, "utf8").includes("bootstrap")) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + await new Promise((resolve) => setTimeout(resolve, 600)); + await new Promise((resolve) => late.listen(port, "127.0.0.1", resolve)); + try { + expect(await exited).toBe(0); + } finally { + await new Promise((resolve) => late.close(() => resolve())); + } + expect(link("current")).toBe(`versions/${v2}`); + }); + it("keeps an install whose server logs that it has no factory config", () => { installV1(); const result = runWith( diff --git a/factory/README.md b/factory/README.md index ff8ff743f872..734c9fd8b4f6 100644 --- a/factory/README.md +++ b/factory/README.md @@ -45,7 +45,7 @@ launchctl print gui/$(id -u)/io.github.rawphp.t3fac tail -n 50 ~/Library/Logs/t3fac/t3fac.log ``` -A non-zero exit from `launchctl print`, or output with no `pid` line, means the job is not running. A `last exit code` other than 0 together with a changing `runs` count is a crash loop. In the log, look for `factory started` or `no factory config` after the last start. Neither means the server never got as far as the factory. +A non-zero exit from `launchctl print`, or output with no `pid` line, means the job is not running. A `last exit code` other than 0 together with a changing `runs` count is a crash loop. In the log, look for `factory started` or `no factory config` after the last start. Neither means the server never got as far as the factory, or the factory found an error in `~/.config/t3fac/factory/config.yml`: it logs each error as `: : ` and stays idle. A config error fails the health check too, so `install.sh` rolls back a good archive until the config is fixed. ### Rolling back by hand @@ -60,7 +60,7 @@ It points `current` at `previous`, restarts the job, removes `previous` and runs Do this when every installed version has a bug that stops the fork from building its own fix. It is the reverse of the #21 switch. 1. Check that no run of either factory holds an issue for this repo: `softfac runs list` for the Rust factory, `softfac runs list --data-dir ~/.config/t3fac/factory` for the fork, and `gh issue list --repo original-solutions/t3code-factory --label factory:building` and `--label factory:triage` both come back empty. -2. Remove `original-solutions/t3code-factory` from `repos` in `~/.config/t3fac/factory/config.yml`. +2. Remove `original-solutions/t3code-factory` from `repos` in `~/.config/t3fac/factory/config.yml`. When it is the only repo there, move `config.yml` aside instead (`mv ~/.config/t3fac/factory/config.yml ~/.config/t3fac/factory/config.yml.off`): an empty `repos` is a config error, which fails every later health check, while a missing config logs `no factory config` and passes. 3. Add it to `repos` in `~/.config/softfac/config.yml` with `repo`, `path` and `user`. 4. Restart each factory so it rereads its config. @@ -68,7 +68,7 @@ The Rust factory then builds the fix. ### Moving it back -Once `install.sh` has installed a fixed archive and it passed its health check, repeat the steps above in reverse: check that no run holds an issue, remove the repo from `~/.config/softfac/config.yml`, add it to `repos` in `~/.config/t3fac/factory/config.yml`, and restart each factory. +Once `install.sh` has installed a fixed archive and it passed its health check, repeat the steps above in reverse: check that no run holds an issue, remove the repo from `~/.config/softfac/config.yml`, add it to `repos` in `~/.config/t3fac/factory/config.yml` (or move `config.yml.off` back to `config.yml`), and restart each factory. ## Syncing with upstream diff --git a/factory/launchd/io.github.rawphp.t3fac.plist b/factory/launchd/io.github.rawphp.t3fac.plist index 9d13924487e7..c72adf3c2cae 100644 --- a/factory/launchd/io.github.rawphp.t3fac.plist +++ b/factory/launchd/io.github.rawphp.t3fac.plist @@ -1,6 +1,7 @@ + Label io.github.rawphp.t3fac diff --git a/factory/scripts/lib/t3fac.sh b/factory/scripts/lib/t3fac.sh index c15f6eebc019..77445f9bac8c 100644 --- a/factory/scripts/lib/t3fac.sh +++ b/factory/scripts/lib/t3fac.sh @@ -6,8 +6,8 @@ label=io.github.rawphp.t3fac template="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../launchd" && pwd)/$label.plist" plist="$HOME/Library/LaunchAgents/$label.plist" log="$HOME/Library/Logs/t3fac/t3fac.log" -# The port the plist serves on. T3FAC_PORT exists only so tests can point the health check at -# a stub listener; the plist still says 3876. +# The log and port the plist template writes to and serves on; change them together. T3FAC_PORT +# exists only so tests can point the health check at a stub listener; the plist still says 3876. port=${T3FAC_PORT:-3876} # Waits for launchd to report the job unloaded, up to T3FAC_UNLOAD_WAIT seconds (10). diff --git a/factory/scripts/rollback.sh b/factory/scripts/rollback.sh index 8b54839db836..a03eabdcc7e7 100755 --- a/factory/scripts/rollback.sh +++ b/factory/scripts/rollback.sh @@ -11,7 +11,8 @@ if [[ ! -L $root/previous || ! -e $root/previous ]]; then fi target=$(readlink "$root/previous") -from=$(readlink "$root/current") +# `current` can be missing after a hand repair; the rollback still restores `previous`. +from=$(readlink "$root/current" || true) ln -sfn "$target" "$root/current" restart_job @@ -20,7 +21,7 @@ restart_job rm -f "$root/previous" if ! wait_healthy "${target#versions/}"; then - echo "rollback to ${target#versions/} from ${from#versions/} failed its health check" >&2 + echo "rollback to ${target#versions/}${from:+ from ${from#versions/}} failed its health check" >&2 exit 1 fi