Explicit step completion tracking and unified stage status

apps/api/src/app.ts

@@ -5,26 +5,20 @@ import { logger } from "hono/logger"
 import { errorHandler } from "./middleware/error-handler.js"
 import { healthRoutes } from "./routes/health.js"
 import { createBookRoutes } from "./routes/books.js"
-import { createPipelineRoutes } from "./routes/pipeline.js"
 import { createPageRoutes } from "./routes/pages.js"
 import { createDebugRoutes } from "./routes/debug.js"
 import { createGlossaryRoutes } from "./routes/glossary.js"
 import { createQuizRoutes } from "./routes/quizzes.js"
-import { createPipelineService } from "./services/pipeline-service.js"
-import { createPipelineRunner } from "./services/pipeline-runner.js"
 import { createProofService } from "./services/proof-service.js"
 import { createProofRunner } from "./services/proof-runner.js"
 import { createProofRoutes } from "./routes/proof.js"
-import { createMasterService } from "./services/master-service.js"
-import { createMasterRunner } from "./services/master-runner.js"
-import { createMasterRoutes } from "./routes/master.js"
 import { createPackageRoutes } from "./routes/package.js"
 import { createPromptRoutes } from "./routes/prompts.js"
 import { createTextCatalogRoutes } from "./routes/text-catalog.js"
 import { createTTSRoutes } from "./routes/tts.js"
-import { createStepRoutes } from "./routes/steps.js"
-import { createStepService } from "./services/step-service.js"
-import { createStepRunner } from "./services/step-runner.js"
+import { createStageRoutes } from "./routes/stages.js"
+import { createStageService } from "./services/stage-service.js"
+import { createStageRunner } from "./services/stage-runner.js"
 import { createPresetRoutes } from "./routes/presets.js"
 import { createAdtPreviewRoutes } from "./routes/adt-preview.js"
 import { createSpeechConfigRoutes } from "./routes/speech-config.js"
@@ -42,14 +36,10 @@ const webAssetsDir = path.resolve(
   process.env.WEB_ASSETS_DIR ?? path.join(projectRoot, "assets", "adt")
 )
 
-const pipelineRunner = createPipelineRunner()
-const pipelineService = createPipelineService(pipelineRunner)
 const proofRunner = createProofRunner()
 const proofService = createProofService(proofRunner)
-const masterRunner = createMasterRunner()
-const masterService = createMasterService(masterRunner)
-const stepRunner = createStepRunner()
-const stepService = createStepService(stepRunner, pipelineService)
+const stageRunner = createStageRunner()
+const stageService = createStageService(stageRunner)
 
 const app = new Hono()
 
@@ -71,18 +61,16 @@ app.onError(errorHandler)
 
 app.route("/api", healthRoutes)
 app.route("/api", createBookRoutes(booksDir, webAssetsDir, configPath))
-app.route("/api", createPipelineRoutes(pipelineService, booksDir, promptsDir, configPath))
 app.route("/api", createPageRoutes(booksDir, promptsDir, configPath))
 app.route("/api", createProofRoutes(proofService, booksDir, promptsDir, configPath))
-app.route("/api", createMasterRoutes(masterService, booksDir, promptsDir, configPath))
 app.route("/api", createGlossaryRoutes(booksDir))
-app.route("/api", createDebugRoutes(pipelineService, booksDir, promptsDir, configPath))
+app.route("/api", createDebugRoutes(booksDir, promptsDir, configPath))
 app.route("/api", createQuizRoutes(booksDir))
 app.route("/api", createPackageRoutes(booksDir, webAssetsDir, configPath))
 app.route("/api", createPromptRoutes(promptsDir, booksDir))
 app.route("/api", createTextCatalogRoutes(booksDir))
 app.route("/api", createTTSRoutes(booksDir))
-app.route("/api", createStepRoutes(stepService, pipelineService, booksDir, promptsDir, configPath))
+app.route("/api", createStageRoutes(stageService, booksDir, promptsDir, configPath))
 app.route("/api", createPresetRoutes(configPath))
 app.route("/api", createAdtPreviewRoutes(booksDir, webAssetsDir, configPath))
 app.route("/api", createSpeechConfigRoutes(configPath))

apps/api/src/routes/books.test.ts

@@ -4,7 +4,9 @@ import os from "node:os"
 import path from "node:path"
 import { openBookDb, createBookStorage } from "@adt/storage"
 import { SCHEMA_VERSION } from "@adt/types"
+import type { StageService } from "../services/stage-service.js"
 import { createBookRoutes } from "./books.js"
+import { createStageRoutes } from "./stages.js"
 
 let tmpDir: string
 
@@ -513,33 +515,59 @@ describe("POST /books/:label/accept-storyboard", () => {
 })
 
 describe("GET /books/:label/step-status", () => {
-  it("does not mark extract complete when only pages exist", async () => {
+  /** Minimal mock StageService — no active runs, no queue */
+  function mockStageService(): StageService {
+    return {
+      getStatus: () => ({ active: null, queue: [] }),
+      getStageStates: () => ({}),
+      addListener: () => () => {},
+      startStageRun: () => ({ status: "started" as const, id: "mock" }),
+    }
+  }
+
+  it("does not mark extract complete when only some steps are done", async () => {
     createTestBook("extract-incomplete")
-    addExtractPages("extract-incomplete", 2)
-    const app = createBookRoutes(tmpDir)
+    const storage = createBookStorage("extract-incomplete", tmpDir)
+    try {
+      storage.markStepComplete("extract")
+      storage.markStepComplete("metadata")
+    } finally {
+      storage.close()
+    }
+    const app = createStageRoutes(mockStageService(), tmpDir, "")
 
     const res = await app.request("/books/extract-incomplete/step-status")
     expect(res.status).toBe(200)
     const body = await res.json()
-    expect(body.steps.extract).not.toBe(true)
+    // Extract stage should not be done — only 2 of its steps are complete
+    expect(body.stages.extract).not.toBe("done")
+    // Individual steps should be marked done
+    expect(body.steps.extract).toBe("done")
+    expect(body.steps.metadata).toBe("done")
   })
 
-  it("marks extract complete only when summary and per-page classifications exist", async () => {
+  it("marks extract complete when all extract steps are done", async () => {
     createTestBook("extract-complete")
-    addExtractPages("extract-complete", 2)
-    addExtractNodes("extract-complete", 2, false)
-    const app = createBookRoutes(tmpDir)
-
-    const beforeSummaryRes = await app.request("/books/extract-complete/step-status")
-    expect(beforeSummaryRes.status).toBe(200)
-    const beforeSummaryBody = await beforeSummaryRes.json()
-    expect(beforeSummaryBody.steps.extract).not.toBe(true)
+    const storage = createBookStorage("extract-complete", tmpDir)
+    try {
+      // Mark all extract stage steps as complete
+      for (const step of [
+        "extract", "metadata", "image-filtering", "image-segmentation",
+        "image-cropping", "image-meaningfulness", "text-classification",
+        "book-summary", "translation",
+      ]) {
+        storage.markStepComplete(step)
+      }
+    } finally {
+      storage.close()
+    }
+    const app = createStageRoutes(mockStageService(), tmpDir, "")
 
-    addExtractNodes("extract-complete", 2, true)
-    const afterSummaryRes = await app.request("/books/extract-complete/step-status")
-    expect(afterSummaryRes.status).toBe(200)
-    const afterSummaryBody = await afterSummaryRes.json()
-    expect(afterSummaryBody.steps.extract).toBe(true)
+    const res = await app.request("/books/extract-complete/step-status")
+    expect(res.status).toBe(200)
+    const body = await res.json()
+    // All extract steps done → stage should be done
+    expect(body.stages.extract).toBe("done")
   })
 })
 

apps/api/src/routes/books.ts

@@ -2,7 +2,7 @@ import fs from "node:fs"
 import path from "node:path"
 import { Hono } from "hono"
 import { HTTPException } from "hono/http-exception"
-import { parseBookLabel } from "@adt/types"
+import { parseBookLabel, PIPELINE } from "@adt/types"
 import { openBookDb } from "@adt/storage"
 import {
   listBooks,
@@ -156,82 +156,7 @@ export function createBookRoutes(
     }
   })
 
-  // GET /books/:label/step-status — Which pipeline steps have data
-  app.get("/books/:label/step-status", (c) => {
-    const { label } = c.req.param()
-    let safeLabel: string
-    try {
-      safeLabel = parseBookLabel(label)
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err)
-      throw new HTTPException(400, { message })
-    }
-    const resolvedDir = path.resolve(booksDir)
-    const dbPath = path.join(resolvedDir, safeLabel, `${safeLabel}.db`)
-
-    if (!fs.existsSync(dbPath)) {
-      return c.json({ steps: {} })
-    }
-
-    const db = openBookDb(dbPath)
-    try {
-      // Extract is only complete when all extract artifacts exist.
-      const pageRows = db.all("SELECT COUNT(*) as count FROM pages") as Array<{ count: number }>
-      const pageCount = pageRows[0]?.count ?? 0
-      const hasPages = pageCount > 0
-
-      // Get all distinct nodes that have data
-      const nodeRows = db.all("SELECT DISTINCT node FROM node_data") as Array<{ node: string }>
-      const nodes = new Set(nodeRows.map((r) => r.node))
-
-      const textClassificationRows = db.all(
-        "SELECT COUNT(DISTINCT item_id) as count FROM node_data WHERE node = 'text-classification'"
-      ) as Array<{ count: number }>
-      const textClassificationCount = textClassificationRows[0]?.count ?? 0
-
-      const imageClassificationRows = db.all(
-        "SELECT COUNT(DISTINCT item_id) as count FROM node_data WHERE node = 'image-filtering'"
-      ) as Array<{ count: number }>
-      const imageClassificationCount = imageClassificationRows[0]?.count ?? 0
-
-      const hasMetadata = nodes.has("metadata")
-      const hasBookSummary = nodes.has("book-summary")
-      const hasAllPageClassifications =
-        hasPages &&
-        textClassificationCount >= pageCount &&
-        imageClassificationCount >= pageCount
-
-      // Map nodes → step slugs
-      const NODE_TO_STEP: Record<string, string> = {
-        "page-sectioning": "storyboard",
-        "web-rendering": "storyboard",
-        "storyboard-acceptance": "storyboard",
-        "quiz-generation": "quizzes",
-        "image-captioning": "captions",
-        glossary: "glossary",
-        "text-catalog": "text-and-speech",
-        "text-catalog-translation": "text-and-speech",
-        tts: "text-and-speech",
-      }
-
-      const steps: Record<string, boolean> = {}
-      if (hasMetadata && hasBookSummary && hasAllPageClassifications) {
-        steps.extract = true
-      }
-      for (const node of nodes) {
-        const step = NODE_TO_STEP[node]
-        if (step) steps[step] = true
-      }
-
-      // Check if ADT is packaged (preview step)
-      const adtDir = path.join(resolvedDir, safeLabel, "adt")
-      if (fs.existsSync(adtDir)) steps.preview = true
-
-      return c.json({ steps })
-    } finally {
-      db.close()
-    }
-  })
+  // NOTE: step-status endpoint is now in stages.ts (needs StageService for run state)
 
   // GET /books/:label/export — Download book as ZIP or EPUB
   app.get("/books/:label/export", async (c) => {

apps/api/src/routes/debug.test.ts

@@ -7,18 +7,6 @@ import { createBookStorage } from "@adt/storage"
 import { openBookDb } from "@adt/storage"
 import { errorHandler } from "../middleware/error-handler.js"
 import { createDebugRoutes } from "./debug.js"
-import type { PipelineService } from "../services/pipeline-service.js"
-
-function makeMockPipelineService(
-  overrides?: Partial<PipelineService>
-): PipelineService {
-  return {
-    getStatus: () => null,
-    addListener: () => () => {},
-    startPipeline: async () => {},
-    ...overrides,
-  }
-}
 
 describe("Debug routes", () => {
   let tmpDir: string
@@ -136,8 +124,7 @@ describe("Debug routes", () => {
     const dbPath = path.join(tmpDir, label, `${label}.db`)
     seedLlmLogs(dbPath)
 
-    const pipelineService = makeMockPipelineService()
-    const routes = createDebugRoutes(pipelineService, tmpDir, tmpDir)
+    const routes = createDebugRoutes(tmpDir, tmpDir)
     app = new Hono()
     app.onError(errorHandler)
     app.route("/api", routes)
@@ -228,26 +215,11 @@ describe("Debug routes", () => {
       expect(textStep.cacheHits).toBe(1)
     })
 
-    it("includes pipeline run timing when available", async () => {
-      const pipelineService = makeMockPipelineService({
-        getStatus: () => ({
-          label,
-          status: "completed",
-          startedAt: 1000,
-          completedAt: 5000,
-        }),
-      })
-      const routes = createDebugRoutes(pipelineService, tmpDir, tmpDir)
-      const appWithTiming = new Hono()
-      appWithTiming.onError(errorHandler)
-      appWithTiming.route("/api", routes)
-
-      const res = await appWithTiming.request(`/api/books/${label}/debug/stats`)
+    it("returns null pipeline run timing", async () => {
+      const res = await app.request(`/api/books/${label}/debug/stats`)
       expect(res.status).toBe(200)
       const body = await res.json()
-      expect(body.pipelineRun).toBeDefined()
-      expect(body.pipelineRun.status).toBe("completed")
-      expect(body.pipelineRun.wallClockMs).toBe(4000)
+      expect(body.pipelineRun).toBeNull()
     })
 
     it("returns 404 for nonexistent book", async () => {

apps/api/src/routes/debug.ts

@@ -5,7 +5,6 @@ import { Hono } from "hono"
 import { HTTPException } from "hono/http-exception"
 import { parseBookLabel } from "@adt/types"
 import { openBookDb } from "@adt/storage"
-import type { PipelineService } from "../services/pipeline-service.js"
 
 function getDbPath(label: string, booksDir: string): string {
   const safeLabel = parseBookLabel(label)
@@ -34,7 +33,6 @@ function parseLogsQuery(query: Record<string, string>) {
 }
 
 export function createDebugRoutes(
-  pipelineService: PipelineService,
   booksDir: string,
   promptsDir: string,
   configPath?: string
@@ -94,7 +92,7 @@ export function createDebugRoutes(
   // GET /books/:label/debug/stats — aggregate pipeline metrics
   app.get("/books/:label/debug/stats", (c) => {
     const { label } = c.req.param()
-    const { safeLabel, dbPath } = requireDb(label, booksDir)
+    const { dbPath } = requireDb(label, booksDir)
 
     const db = openBookDb(dbPath)
     try {
@@ -141,19 +139,8 @@ export function createDebugRoutes(
         totals.errorCount += row.errorCount
       }
 
-      // Pipeline run timing
-      const job = pipelineService.getStatus(safeLabel)
-      const pipelineRun = job
-        ? {
-            status: job.status,
-            startedAt: job.startedAt,
-            completedAt: job.completedAt,
-            wallClockMs:
-              job.startedAt && job.completedAt
-                ? job.completedAt - job.startedAt
-                : undefined,
-          }
-        : null
+      // Full-pipeline job tracking was removed; keep nullable field for compatibility.
+      const pipelineRun = null
 
       return c.json({ steps: stepRows, totals, pipelineRun })
     } finally {