From a31ef75b83c676e3c38283f7f6a950a06eb47552 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 19:13:54 -0300 Subject: [PATCH 01/10] refactor: moved job to core --- packages/core/src/index.ts | 1 + packages/core/src/job/index.ts | 1 + packages/core/src/job/job.test.ts | 116 +++++++++ packages/{engine => core}/src/job/job.ts | 19 +- packages/core/tsconfig.json | 3 +- packages/engine/src/engine.ts | 3 +- packages/engine/src/job/index.ts | 1 - packages/engine/src/job/job-builder.test.ts | 133 +++++++++- packages/engine/src/job/job-builder.ts | 2 +- packages/engine/src/job/job.test.ts | 237 ------------------ .../engine/src/shared-runner/runner-pool.ts | 1 + packages/engine/src/shared-runner/runner.ts | 3 +- .../sidequest/src/operations/sidequest.ts | 4 +- 13 files changed, 262 insertions(+), 262 deletions(-) create mode 100644 packages/core/src/job/index.ts create mode 100644 packages/core/src/job/job.test.ts rename packages/{engine => core}/src/job/job.ts (97%) delete mode 100644 packages/engine/src/job/job.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e540e0e..12531da 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,4 +1,5 @@ export * from "./backends"; +export * from "./job"; export * from "./logger"; export * from "./schema"; export * from "./tools"; diff --git a/packages/core/src/job/index.ts b/packages/core/src/job/index.ts new file mode 100644 index 0000000..9fc5009 --- /dev/null +++ b/packages/core/src/job/index.ts @@ -0,0 +1 @@ +export * from "./job"; diff --git a/packages/core/src/job/job.test.ts b/packages/core/src/job/job.test.ts new file mode 100644 index 0000000..280a62f --- /dev/null +++ b/packages/core/src/job/job.test.ts @@ -0,0 +1,116 @@ +import { sidequestTest } from "@/tests/fixture"; +import { CompletedResult, RetryResult, SnoozeResult } from "@sidequest/core"; +import { Job } from "./job"; + +export class DummyJob extends Job { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + constructor(..._optional) { + super(); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + run(..._optional) { + return "dummy job"; + } +} + +describe("job.ts", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + sidequestTest("should expose script and className correctly", async () => { + const job = new DummyJob(); + await job.ready(); + expect(typeof job.script).toBe("string"); + expect(job.className).toBe("DummyJob"); + }); + + sidequestTest("creates a complete transition", () => { + const job = new DummyJob(); + const transition = job.complete("foo bar"); + expect(transition.result).toBe("foo bar"); + }); + + sidequestTest("creates a fail transition", () => { + const job = new DummyJob(); + const transition = job.fail("error"); + expect(transition.error).toEqual({ message: "error" }); + }); + + sidequestTest("creates a retry transition", () => { + const job = new DummyJob(); + const transition = job.retry("reason", 1000); + expect(transition.error).toEqual({ message: "reason" }); + expect(transition.delay).toEqual(1000); + }); + + sidequestTest("creates a snooze transition", () => { + const job = new DummyJob(); + const transition = job.snooze(1000); + expect(transition.delay).toBe(1000); + }); + + sidequestTest("fail/retry should accept an Error object", () => { + const job = new DummyJob(); + const error = new Error("fail"); + expect(job.fail(error).error.message).toEqual("fail"); + expect(job.retry(error).error.message).toEqual("fail"); + }); + + describe("perform", () => { + sidequestTest("should return CompleteResult if run returns a value", async () => { + class ValueJob extends Job { + run() { + return "abc"; + } + } + const job = new ValueJob(); + const result = (await job.perform()) as CompletedResult; + expect(result.type).toBe("completed"); + expect(result.result).toBe("abc"); + }); + + sidequestTest("should return the JobResult return by run", async () => { + class TransitionJob extends Job { + run() { + return { __is_job_transition__: true, type: "snooze" } as SnoozeResult; + } + } + const job = new TransitionJob(); + const result = (await job.perform()) as SnoozeResult; + expect(result.type).toBe("snooze"); + }); + + sidequestTest("should return RetryResult if run throws", async () => { + class ErrorJob extends Job { + run() { + throw new Error("fail!"); + } + } + const job = new ErrorJob(); + const result = (await job.perform()) as RetryResult; + expect(result.type).toBe("retry"); + expect(result.error.message).toEqual("fail!"); + }); + + sidequestTest("should return RetryResult if run unhandled promise", async () => { + class DummyUnhandled extends Job { + run() { + return new Promise(() => { + throw new Error("unhandled error"); + }); + } + } + + const job = new DummyUnhandled(); + const result = (await job.perform()) as RetryResult; + expect(result.type).toBe("retry"); + expect(result.error.message).toEqual("unhandled error"); + }); + }); +}); diff --git a/packages/engine/src/job/job.ts b/packages/core/src/job/job.ts similarity index 97% rename from packages/engine/src/job/job.ts rename to packages/core/src/job/job.ts index 3ab8a36..5190d1e 100644 --- a/packages/engine/src/job/job.ts +++ b/packages/core/src/job/job.ts @@ -1,19 +1,10 @@ -import { - CompletedResult, - ErrorData, - FailedResult, - isJobResult, - JobData, - JobResult, - JobState, - logger, - RetryResult, - SnoozeResult, - toErrorData, - UniquenessConfig, -} from "@sidequest/core"; import { access } from "fs/promises"; import { pathToFileURL } from "url"; +import { logger } from "../logger"; +import { ErrorData, JobData, JobState } from "../schema"; +import { toErrorData } from "../tools"; +import { CompletedResult, FailedResult, isJobResult, JobResult, RetryResult, SnoozeResult } from "../transitions"; +import { UniquenessConfig } from "../uniquiness"; /** * Type for a job class constructor. diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index df59da5..87b647b 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -1,8 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "rootDir": "src", "outDir": "dist" }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "../../tests/**/*"] } diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index d3c4479..f7326f8 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -1,10 +1,9 @@ import { Backend, BackendConfig, LazyBackend, MISC_FALLBACK, NewQueueData, QUEUE_FALLBACK } from "@sidequest/backend"; -import { configureLogger, logger, LoggerOptions } from "@sidequest/core"; +import { configureLogger, JobClassType, logger, LoggerOptions } from "@sidequest/core"; import { ChildProcess, fork } from "child_process"; import { cpus } from "os"; import path from "path"; import { JOB_BUILDER_FALLBACK } from "./job/constants"; -import { JobClassType } from "./job/job"; import { JobBuilder, JobBuilderDefaults } from "./job/job-builder"; import { grantQueueConfig, QueueDefaults } from "./queue/grant-queue-config"; import { clearGracefulShutdown, gracefulShutdown } from "./utils/shutdown"; diff --git a/packages/engine/src/job/index.ts b/packages/engine/src/job/index.ts index 7e550b4..eb5e4cd 100644 --- a/packages/engine/src/job/index.ts +++ b/packages/engine/src/job/index.ts @@ -1,4 +1,3 @@ export * from "./constants"; -export * from "./job"; export * from "./job-builder"; export * from "./job-transitioner"; diff --git a/packages/engine/src/job/job-builder.test.ts b/packages/engine/src/job/job-builder.test.ts index b80bdd3..af7546a 100644 --- a/packages/engine/src/job/job-builder.test.ts +++ b/packages/engine/src/job/job-builder.test.ts @@ -1,6 +1,6 @@ import { sidequestTest } from "@/tests/fixture"; import { Backend } from "@sidequest/backend"; -import { JobData } from "@sidequest/core"; +import { JobData, JobState, UniquenessFactory } from "@sidequest/core"; import nodeCron from "node-cron"; import { DummyJob } from "../test-jobs/dummy-job"; import { JobBuilder } from "./job-builder"; @@ -72,6 +72,137 @@ describe("JobBuilder", () => { expect(new Date(jobData.available_at as unknown as string).getTime()).toBeCloseTo(futureDate.getTime(), -2); }); + sidequestTest("should enqueue job", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).enqueue(); + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(1); + }); + + sidequestTest("should enqueue job in different queue", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).queue("test-queue").enqueue(); + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + queue: "test-queue", + }); + + expect(jobData.length).toBe(1); + }); + + sidequestTest("should enqueue job with timeout", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).timeout(100).enqueue(); + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(1); + expect(jobData[0].timeout).toBe(100); + }); + + sidequestTest("should be able to enqueue duplicated jobs", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).enqueue(); + await new JobBuilder(backend, DummyJob).enqueue(); + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(2); + }); + + sidequestTest("should not be able to enqueue duplicated jobs", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).unique(true).enqueue(); + await expect(new JobBuilder(backend, DummyJob).unique(true).enqueue()).rejects.toThrow(); + + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(1); + }); + + sidequestTest("should not be able to enqueue duplicated jobs in the same period", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue(); + await expect(new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue()).rejects.toThrow(); + vi.advanceTimersByTime(1100); + await new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue(); + + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(2); + }); + + sidequestTest( + "should not be able to enqueue duplicated jobs with different args withargs=false", + async ({ backend }) => { + await new JobBuilder(backend, DummyJob).unique({ withArgs: false }).enqueue(); + await expect(new JobBuilder(backend, DummyJob).unique({ withArgs: false }).enqueue("arg1")).rejects.toThrow(); + + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(1); + }, + ); + + sidequestTest("should be able to enqueue duplicated jobs with different args", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue(); + await new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue("arg1"); + + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(2); + }); + + sidequestTest("should not be able to enqueue duplicated jobs with same args withargs=true", async ({ backend }) => { + await new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue("arg1"); + await expect(new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue("arg1")).rejects.toThrow(); + + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(1); + }); + + sidequestTest.for([ + { expected: 1, state: "waiting" }, + { expected: 1, state: "running" }, + { expected: 1, state: "claimed" }, + { expected: 2, state: "canceled" }, + { expected: 2, state: "failed" }, + { expected: 2, state: "completed" }, + ] as { expected: number; state: JobState }[])( + "should have %i jobs if first job is %s", + async ({ expected, state }, { backend }) => { + const job1 = await new JobBuilder(backend, DummyJob).unique(true).enqueue(); + + const newData = { ...job1, state }; + + const uniqueness = UniquenessFactory.create(newData.uniqueness_config!); + newData.unique_digest = uniqueness.digest(newData); + await backend.updateJob(newData); + + try { + await new JobBuilder(backend, DummyJob).unique(true).enqueue(); + } catch { + // noop + } + + const jobData = await backend.listJobs({ + jobClass: DummyJob.name, + }); + + expect(jobData.length).toBe(expected); + }, + ); + describe("constructor defaults", () => { sidequestTest("uses default queue when no defaults provided", async ({ backend }) => { const jobData = await new JobBuilder(backend, DummyJob).enqueue(); diff --git a/packages/engine/src/job/job-builder.ts b/packages/engine/src/job/job-builder.ts index 596cde1..3728ef5 100644 --- a/packages/engine/src/job/job-builder.ts +++ b/packages/engine/src/job/job-builder.ts @@ -6,6 +6,7 @@ import { FixedWindowConfig, // eslint-disable-next-line @typescript-eslint/no-unused-vars type FixedWindowUniqueness, + JobClassType, JobData, logger, TimePeriod, @@ -14,7 +15,6 @@ import { } from "@sidequest/core"; import nodeCron, { ScheduledTask } from "node-cron"; import { JOB_BUILDER_FALLBACK } from "./constants"; -import { JobClassType } from "./job"; /** * Configuration for job uniqueness constraints. diff --git a/packages/engine/src/job/job.test.ts b/packages/engine/src/job/job.test.ts deleted file mode 100644 index 2e03a02..0000000 --- a/packages/engine/src/job/job.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { sidequestTest } from "@/tests/fixture"; -import { CompletedResult, JobState, RetryResult, SnoozeResult, UniquenessFactory } from "@sidequest/core"; -import { DummyJob } from "../test-jobs/dummy-job"; -import { Job } from "./job"; -import { JobBuilder } from "./job-builder"; - -describe("job.ts", () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - sidequestTest("should expose script and className correctly", async () => { - const job = new DummyJob(); - await job.ready(); - expect(typeof job.script).toBe("string"); - expect(job.className).toBe("DummyJob"); - }); - - sidequestTest("should enqueue job", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).enqueue(); - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(1); - }); - - sidequestTest("should enqueue job in different queue", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).queue("test-queue").enqueue(); - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - queue: "test-queue", - }); - - expect(jobData.length).toBe(1); - }); - - sidequestTest("should enqueue job with timeout", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).timeout(100).enqueue(); - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(1); - expect(jobData[0].timeout).toBe(100); - }); - - sidequestTest("should be able to enqueue duplicated jobs", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).enqueue(); - await new JobBuilder(backend, DummyJob).enqueue(); - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(2); - }); - - sidequestTest("should not be able to enqueue duplicated jobs", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).unique(true).enqueue(); - await expect(new JobBuilder(backend, DummyJob).unique(true).enqueue()).rejects.toThrow(); - - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(1); - }); - - sidequestTest("should not be able to enqueue duplicated jobs in the same period", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue(); - await expect(new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue()).rejects.toThrow(); - vi.advanceTimersByTime(1100); - await new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue(); - - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(2); - }); - - sidequestTest( - "should not be able to enqueue duplicated jobs with different args withargs=false", - async ({ backend }) => { - await new JobBuilder(backend, DummyJob).unique({ withArgs: false }).enqueue(); - await expect(new JobBuilder(backend, DummyJob).unique({ withArgs: false }).enqueue("arg1")).rejects.toThrow(); - - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(1); - }, - ); - - sidequestTest("should be able to enqueue duplicated jobs with different args", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue(); - await new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue("arg1"); - - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(2); - }); - - sidequestTest("should not be able to enqueue duplicated jobs with same args withargs=true", async ({ backend }) => { - await new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue("arg1"); - await expect(new JobBuilder(backend, DummyJob).unique({ withArgs: true }).enqueue("arg1")).rejects.toThrow(); - - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(1); - }); - - sidequestTest.for([ - { expected: 1, state: "waiting" }, - { expected: 1, state: "running" }, - { expected: 1, state: "claimed" }, - { expected: 2, state: "canceled" }, - { expected: 2, state: "failed" }, - { expected: 2, state: "completed" }, - ] as { expected: number; state: JobState }[])( - "should have %i jobs if first job is %s", - async ({ expected, state }, { backend }) => { - const job1 = await new JobBuilder(backend, DummyJob).unique(true).enqueue(); - - const newData = { ...job1, state }; - - const uniqueness = UniquenessFactory.create(newData.uniqueness_config!); - newData.unique_digest = uniqueness.digest(newData); - await backend.updateJob(newData); - - try { - await new JobBuilder(backend, DummyJob).unique(true).enqueue(); - } catch { - // noop - } - - const jobData = await backend.listJobs({ - jobClass: DummyJob.name, - }); - - expect(jobData.length).toBe(expected); - }, - ); - - sidequestTest("creates a complete transition", () => { - const job = new DummyJob(); - const transition = job.complete("foo bar"); - expect(transition.result).toBe("foo bar"); - }); - - sidequestTest("creates a fail transition", () => { - const job = new DummyJob(); - const transition = job.fail("error"); - expect(transition.error).toEqual({ message: "error" }); - }); - - sidequestTest("creates a retry transition", () => { - const job = new DummyJob(); - const transition = job.retry("reason", 1000); - expect(transition.error).toEqual({ message: "reason" }); - expect(transition.delay).toEqual(1000); - }); - - sidequestTest("creates a snooze transition", () => { - const job = new DummyJob(); - const transition = job.snooze(1000); - expect(transition.delay).toBe(1000); - }); - - sidequestTest("fail/retry should accept an Error object", () => { - const job = new DummyJob(); - const error = new Error("fail"); - expect(job.fail(error).error.message).toEqual("fail"); - expect(job.retry(error).error.message).toEqual("fail"); - }); - - describe("perform", () => { - sidequestTest("should return CompleteResult if run returns a value", async () => { - class ValueJob extends Job { - run() { - return "abc"; - } - } - const job = new ValueJob(); - const result = (await job.perform()) as CompletedResult; - expect(result.type).toBe("completed"); - expect(result.result).toBe("abc"); - }); - - sidequestTest("should return the JobResult return by run", async () => { - class TransitionJob extends Job { - run() { - return { __is_job_transition__: true, type: "snooze" } as SnoozeResult; - } - } - const job = new TransitionJob(); - const result = (await job.perform()) as SnoozeResult; - expect(result.type).toBe("snooze"); - }); - - sidequestTest("should return RetryResult if run throws", async () => { - class ErrorJob extends Job { - run() { - throw new Error("fail!"); - } - } - const job = new ErrorJob(); - const result = (await job.perform()) as RetryResult; - expect(result.type).toBe("retry"); - expect(result.error.message).toEqual("fail!"); - }); - - sidequestTest("should return RetryResult if run unhandled promise", async () => { - class DummyUnhandled extends Job { - run() { - return new Promise(() => { - throw new Error("unhandled error"); - }); - } - } - - const job = new DummyUnhandled(); - const result = (await job.perform()) as RetryResult; - expect(result.type).toBe("retry"); - expect(result.error.message).toEqual("unhandled error"); - }); - }); -}); diff --git a/packages/engine/src/shared-runner/runner-pool.ts b/packages/engine/src/shared-runner/runner-pool.ts index 65e9a51..80a9dff 100644 --- a/packages/engine/src/shared-runner/runner-pool.ts +++ b/packages/engine/src/shared-runner/runner-pool.ts @@ -22,6 +22,7 @@ export class RunnerPool { filename: runnerPath, minThreads: this.nonNullConfig.minThreads, maxThreads: this.nonNullConfig.maxThreads, + idleTimeout: 10_000, }); logger("RunnerPool").debug( `Created worker pool with min ${this.nonNullConfig.minThreads} threads and max ${this.nonNullConfig.maxThreads} threads`, diff --git a/packages/engine/src/shared-runner/runner.ts b/packages/engine/src/shared-runner/runner.ts index ca7bdd5..1f6969b 100644 --- a/packages/engine/src/shared-runner/runner.ts +++ b/packages/engine/src/shared-runner/runner.ts @@ -1,6 +1,5 @@ -import { JobData, JobResult, logger, toErrorData } from "@sidequest/core"; +import { Job, JobClassType, JobData, JobResult, logger, toErrorData } from "@sidequest/core"; import { EngineConfig } from "../engine"; -import { Job, JobClassType } from "../job/job"; import { importSidequest } from "../utils"; /** diff --git a/packages/sidequest/src/operations/sidequest.ts b/packages/sidequest/src/operations/sidequest.ts index fd6e5b3..c8189ef 100644 --- a/packages/sidequest/src/operations/sidequest.ts +++ b/packages/sidequest/src/operations/sidequest.ts @@ -1,6 +1,6 @@ -import { logger } from "@sidequest/core"; +import { JobClassType, logger } from "@sidequest/core"; import { DashboardConfig, SidequestDashboard } from "@sidequest/dashboard"; -import { Engine, EngineConfig, JobClassType } from "@sidequest/engine"; +import { Engine, EngineConfig } from "@sidequest/engine"; import { JobOperations } from "./job"; import { QueueOperations } from "./queue"; From 9b4184f70629ce3254dc10c313b7d6659be5a686 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 19:16:47 -0300 Subject: [PATCH 02/10] refactor: update import paths to use @sidequest/core --- packages/engine/src/test-jobs/dummy-failed-job.js | 2 +- packages/engine/src/test-jobs/dummy-job.js | 2 +- packages/engine/src/test-jobs/dynamic-dummy-job.js | 2 +- packages/sidequest/src/operations/job.test.ts | 3 +-- packages/sidequest/src/operations/sidequest.test.ts | 3 ++- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/test-jobs/dummy-failed-job.js b/packages/engine/src/test-jobs/dummy-failed-job.js index 488f233..55ab2ef 100644 --- a/packages/engine/src/test-jobs/dummy-failed-job.js +++ b/packages/engine/src/test-jobs/dummy-failed-job.js @@ -1,4 +1,4 @@ -import { Job } from "../job"; +import { Job } from "@sidequest/core"; export class DummyJob extends Job { run() { diff --git a/packages/engine/src/test-jobs/dummy-job.js b/packages/engine/src/test-jobs/dummy-job.js index 60d94c2..b159105 100644 --- a/packages/engine/src/test-jobs/dummy-job.js +++ b/packages/engine/src/test-jobs/dummy-job.js @@ -1,4 +1,4 @@ -import { Job } from "../job"; +import { Job } from "@sidequest/core"; export class DummyJob extends Job { // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/packages/engine/src/test-jobs/dynamic-dummy-job.js b/packages/engine/src/test-jobs/dynamic-dummy-job.js index 4d1cce4..e039a8c 100644 --- a/packages/engine/src/test-jobs/dynamic-dummy-job.js +++ b/packages/engine/src/test-jobs/dynamic-dummy-job.js @@ -1,4 +1,4 @@ -import { Job } from "../job"; +import { Job } from "@sidequest/core"; export class DynamicDummyJob extends Job { async run() { diff --git a/packages/sidequest/src/operations/job.test.ts b/packages/sidequest/src/operations/job.test.ts index e9a8f33..9c92061 100644 --- a/packages/sidequest/src/operations/job.test.ts +++ b/packages/sidequest/src/operations/job.test.ts @@ -1,7 +1,6 @@ import { sidequestTest, SidequestTestFixture } from "@/tests/fixture"; import { NewJobData, UpdateJobData } from "@sidequest/backend"; -import { CancelTransition, JobData, JobState, RerunTransition, SnoozeTransition } from "@sidequest/core"; -import { Job } from "@sidequest/engine"; +import { CancelTransition, Job, JobData, JobState, RerunTransition, SnoozeTransition } from "@sidequest/core"; import { JobOperations } from "./job"; // Mock JobTransitioner to control its behavior diff --git a/packages/sidequest/src/operations/sidequest.test.ts b/packages/sidequest/src/operations/sidequest.test.ts index 112e509..3a8c7f3 100644 --- a/packages/sidequest/src/operations/sidequest.test.ts +++ b/packages/sidequest/src/operations/sidequest.test.ts @@ -1,4 +1,5 @@ -import { Job, NonNullableEngineConfig } from "@sidequest/engine"; +import { Job } from "@sidequest/core"; +import { NonNullableEngineConfig } from "@sidequest/engine"; import { JobOperations } from "./job"; import { QueueOperations } from "./queue"; import { Sidequest, SidequestConfig } from "./sidequest"; From 3533bc7eeb1d1e7a9418deb0e3df14d9007024e7 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 19:33:58 -0300 Subject: [PATCH 03/10] refactor: add @sidequest/engine dependency and update job transition logic --- packages/dashboard/package.json | 1 + packages/dashboard/src/resources/jobs.ts | 14 +++++++++----- yarn.lock | 3 ++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/dashboard/package.json b/packages/dashboard/package.json index e124bff..eda006d 100644 --- a/packages/dashboard/package.json +++ b/packages/dashboard/package.json @@ -21,6 +21,7 @@ "dependencies": { "@sidequest/backend": "workspace:*", "@sidequest/core": "workspace:*", + "@sidequest/engine": "workspace:*", "ejs": "^3.1.10", "express": "^5.1.0", "express-basic-auth": "^1.2.1", diff --git a/packages/dashboard/src/resources/jobs.ts b/packages/dashboard/src/resources/jobs.ts index 6c492b4..798061c 100644 --- a/packages/dashboard/src/resources/jobs.ts +++ b/packages/dashboard/src/resources/jobs.ts @@ -1,5 +1,6 @@ import { Backend } from "@sidequest/backend"; -import { JobState } from "@sidequest/core"; +import { CancelTransition, JobState, RerunTransition, SnoozeTransition } from "@sidequest/core"; +import { JobTransitioner } from "@sidequest/engine"; import { Router } from "express"; export function createJobsRouter(backend: Backend) { @@ -111,7 +112,11 @@ export function createJobsRouter(backend: Backend) { const job = await backend?.getJob(jobId); if (job) { - await backend.updateJob({ id: job.id, available_at: new Date() }); + if (job.state === "canceled") { + await JobTransitioner.apply(backend, job, new RerunTransition()); + } else { + await JobTransitioner.apply(backend, job, new SnoozeTransition(0)); + } res.header("HX-Trigger", "jobChanged").status(200).end(); } else { res.status(404).end(); @@ -123,7 +128,7 @@ export function createJobsRouter(backend: Backend) { const job = await backend?.getJob(jobId); if (job) { - await backend.updateJob({ ...job, state: "canceled" }); + await JobTransitioner.apply(backend, job, new CancelTransition()); res.header("HX-Trigger", "jobChanged").status(200).end(); } else { res.status(404).end(); @@ -135,8 +140,7 @@ export function createJobsRouter(backend: Backend) { const job = await backend?.getJob(jobId); if (job) { - const maxAttempts = job.max_attempts === job.attempt ? job.max_attempts + 1 : job.max_attempts; - await backend.updateJob({ ...job, state: "waiting", max_attempts: maxAttempts }); + await JobTransitioner.apply(backend, job, new RerunTransition()); res.header("HX-Trigger", "jobChanged").status(200).end(); } else { res.status(404).end(); diff --git a/yarn.lock b/yarn.lock index f3c944b..5023ddf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2624,6 +2624,7 @@ __metadata: "@highlightjs/cdn-assets": "npm:^11.11.1" "@sidequest/backend": "workspace:*" "@sidequest/core": "workspace:*" + "@sidequest/engine": "workspace:^" "@types/feather-icons": "npm:^4" "@types/morgan": "npm:^1" chart.js: "npm:^4.5.0" @@ -2644,7 +2645,7 @@ __metadata: languageName: unknown linkType: soft -"@sidequest/engine@workspace:*, @sidequest/engine@workspace:packages/engine": +"@sidequest/engine@workspace:*, @sidequest/engine@workspace:^, @sidequest/engine@workspace:packages/engine": version: 0.0.0-use.local resolution: "@sidequest/engine@workspace:packages/engine" dependencies: From ab1cf99c824df91d93ea74b5f51de5d869da61ae Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 19:38:54 -0300 Subject: [PATCH 04/10] test: enhance test setup and teardown for backend tests --- packages/backends/backend-test/src/index.ts | 5 +++++ packages/engine/src/job/job-builder.test.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/packages/backends/backend-test/src/index.ts b/packages/backends/backend-test/src/index.ts index ecddb95..37e2cce 100644 --- a/packages/backends/backend-test/src/index.ts +++ b/packages/backends/backend-test/src/index.ts @@ -28,6 +28,11 @@ import defineUpdateJobTestSuite from "./updateJob"; * @param backendFactory - A factory function that creates a backend instance. */ export function testBackend(backendFactory: () => Backend) { + beforeAll(async () => { + setTestBackend(backendFactory()); + await backend.truncate(); + }); + beforeEach(async () => { setTestBackend(backendFactory()); await backend.migrate(); diff --git a/packages/engine/src/job/job-builder.test.ts b/packages/engine/src/job/job-builder.test.ts index af7546a..bb58630 100644 --- a/packages/engine/src/job/job-builder.test.ts +++ b/packages/engine/src/job/job-builder.test.ts @@ -16,6 +16,10 @@ const scheduleMock = vi.mocked(nodeCron.schedule); const validateMock = vi.mocked(nodeCron.validate); describe("JobBuilder", () => { + afterEach(() => { + vi.useRealTimers(); + }); + sidequestTest("enqueues a job at default queue", async ({ backend }) => { const jobData = await new JobBuilder(backend, DummyJob).enqueue(); expect(jobData).toEqual( @@ -123,6 +127,7 @@ describe("JobBuilder", () => { }); sidequestTest("should not be able to enqueue duplicated jobs in the same period", async ({ backend }) => { + vi.useFakeTimers(); await new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue(); await expect(new JobBuilder(backend, DummyJob).unique({ period: "second" }).enqueue()).rejects.toThrow(); vi.advanceTimersByTime(1100); From 26433157cf1507e21a99c0ef36994a861757d75d Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 19:48:29 -0300 Subject: [PATCH 05/10] refactor: enhance run method logic for job re-running and state handling --- packages/sidequest/src/operations/job.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/sidequest/src/operations/job.ts b/packages/sidequest/src/operations/job.ts index 6a347da..21179d7 100644 --- a/packages/sidequest/src/operations/job.ts +++ b/packages/sidequest/src/operations/job.ts @@ -167,11 +167,14 @@ export class JobOperations { * * If `force` is `false`, it will simply update the available_at time for waiting jobs. * It is effectively a snooze with 0 delay. - * This can only be applied to "waiting" and "running" jobs. + * This can only be applied to `waiting` and `running` jobs. + * Calling this on a running job will do nothing other than update the available_at time. * * If `force` is `true`, it will use RerunTransition to completely reset and re-run the job, - * similar to the dashboard's re-run functionality. - * This can only be applied to jobs that are in "completed", "canceled", or "failed" states. + * similar to the dashboard's re-run functionality. It will completely ignore the number of attempts + * and the current state of the job, allowing it to be re-run regardless of its current state. + * This can only be applied to jobs that are in `waiting`, `running`, `completed`, `canceled`, or `failed` states. + * Calling this on a running job will do nothing other than update the available_at time. * * @param jobId - The ID of the job to run * @param force - Whether to force re-run the job regardless of state and attempts @@ -180,19 +183,20 @@ export class JobOperations { */ async run(jobId: number, force = false): Promise { const backend = this.getBackend(); - const job = await backend.getJob(jobId); + let job = await backend.getJob(jobId); if (!job) { throw new Error(`Job with ID ${jobId} not found`); } + // Simple run - just update available_at to make it available immediately + job = await JobTransitioner.apply(backend, job, new SnoozeTransition(0)); + // If force, we apply RerunTransition to disregard the current state and attempts if (force) { // Use RerunTransition to force a new run, regardless of current state and attempts - return await JobTransitioner.apply(backend, job, new RerunTransition()); - } else { - // Simple run - just update available_at to make it available immediately - return await JobTransitioner.apply(backend, job, new SnoozeTransition(0)); + job = await JobTransitioner.apply(backend, job, new RerunTransition()); } + return job; } /** From 08bbcccd7d9387e69a014865eaec51e39e3faa16 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 20:02:48 -0300 Subject: [PATCH 06/10] refactor: add idleWorkerTimeout to EngineConfig and update RunnerPool to use it --- packages/docs/engine/configuration.md | 3 +++ packages/engine/src/engine.ts | 3 +++ packages/engine/src/shared-runner/runner-pool.ts | 2 +- packages/sidequest/src/operations/job.ts | 6 +++--- yarn.lock | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/docs/engine/configuration.md b/packages/docs/engine/configuration.md index f4e7023..647ab95 100644 --- a/packages/docs/engine/configuration.md +++ b/packages/docs/engine/configuration.md @@ -97,6 +97,7 @@ await Sidequest.start({ maxConcurrentJobs: 50, minThreads: 4, maxThreads: 8, + idleWorkerTimeout: 10000, // 10 seconds // 4. Migration and startup skipMigration: false, @@ -155,6 +156,7 @@ await Sidequest.start({ | `maxConcurrentJobs` | Maximum number of jobs processed simultaneously across all queues | `10` | | `minThreads` | Minimum number of worker threads to use | Number of CPU cores | | `maxThreads` | Maximum number of worker threads to use | `minThreads * 2` | +| `idleWorkerTimeout` | Timeout (milliseconds) for idle workers before they are terminated | `10000` (10 seconds) | | `skipMigration` | Whether to skip database migration on startup | `false` | | `releaseStaleJobsIntervalMin` | Frequency (minutes) for releasing stale jobs. Set to `false` to disable | `60` | | `releaseStaleJobsMaxStaleMs` | Maximum age (milliseconds) for a running job to be considered stale | `600000` (10 minutes) | @@ -272,6 +274,7 @@ await Sidequest.start({ maxConcurrentJobs: 100, minThreads: 8, maxThreads: 16, + idleWorkerTimeout: 30000, // 30 seconds for high throughput releaseStaleJobsIntervalMin: 30, // More frequent stale job cleanup cleanupFinishedJobsIntervalMin: 30, // More frequent cleanup queueDefaults: { diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index f7326f8..cdba19f 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -40,6 +40,8 @@ export interface EngineConfig { minThreads?: number; /** Maximum number of worker threads to use. Defaults to `minThreads * 2` */ maxThreads?: number; + /** Timeout in milliseconds for idle workers before they are terminated. Defaults to 10 seconds */ + idleWorkerTimeout?: number; /** * Default job builder configuration. @@ -127,6 +129,7 @@ export class Engine { gracefulShutdown: config?.gracefulShutdown ?? true, minThreads: config?.minThreads ?? cpus().length, maxThreads: config?.maxThreads ?? cpus().length * 2, + idleWorkerTimeout: config?.idleWorkerTimeout ?? 10_000, releaseStaleJobsMaxStaleMs: config?.releaseStaleJobsMaxStaleMs ?? MISC_FALLBACK.maxStaleMs, // 10 minutes releaseStaleJobsMaxClaimedMs: config?.releaseStaleJobsMaxClaimedMs ?? MISC_FALLBACK.maxClaimedMs, // 1 minute jobDefaults: { diff --git a/packages/engine/src/shared-runner/runner-pool.ts b/packages/engine/src/shared-runner/runner-pool.ts index 80a9dff..474387e 100644 --- a/packages/engine/src/shared-runner/runner-pool.ts +++ b/packages/engine/src/shared-runner/runner-pool.ts @@ -22,7 +22,7 @@ export class RunnerPool { filename: runnerPath, minThreads: this.nonNullConfig.minThreads, maxThreads: this.nonNullConfig.maxThreads, - idleTimeout: 10_000, + idleTimeout: this.nonNullConfig.idleWorkerTimeout, }); logger("RunnerPool").debug( `Created worker pool with min ${this.nonNullConfig.minThreads} threads and max ${this.nonNullConfig.maxThreads} threads`, diff --git a/packages/sidequest/src/operations/job.ts b/packages/sidequest/src/operations/job.ts index 21179d7..18d7c77 100644 --- a/packages/sidequest/src/operations/job.ts +++ b/packages/sidequest/src/operations/job.ts @@ -168,13 +168,13 @@ export class JobOperations { * If `force` is `false`, it will simply update the available_at time for waiting jobs. * It is effectively a snooze with 0 delay. * This can only be applied to `waiting` and `running` jobs. - * Calling this on a running job will do nothing other than update the available_at time. * * If `force` is `true`, it will use RerunTransition to completely reset and re-run the job, * similar to the dashboard's re-run functionality. It will completely ignore the number of attempts * and the current state of the job, allowing it to be re-run regardless of its current state. * This can only be applied to jobs that are in `waiting`, `running`, `completed`, `canceled`, or `failed` states. - * Calling this on a running job will do nothing other than update the available_at time. + * + * Calling this method on a running job will do nothing other than update the available_at time. * * @param jobId - The ID of the job to run * @param force - Whether to force re-run the job regardless of state and attempts @@ -189,7 +189,7 @@ export class JobOperations { throw new Error(`Job with ID ${jobId} not found`); } - // Simple run - just update available_at to make it available immediately + // First update available_at to make it available immediately job = await JobTransitioner.apply(backend, job, new SnoozeTransition(0)); // If force, we apply RerunTransition to disregard the current state and attempts if (force) { diff --git a/yarn.lock b/yarn.lock index 5023ddf..ab1ab5b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2624,7 +2624,7 @@ __metadata: "@highlightjs/cdn-assets": "npm:^11.11.1" "@sidequest/backend": "workspace:*" "@sidequest/core": "workspace:*" - "@sidequest/engine": "workspace:^" + "@sidequest/engine": "workspace:*" "@types/feather-icons": "npm:^4" "@types/morgan": "npm:^1" chart.js: "npm:^4.5.0" @@ -2645,7 +2645,7 @@ __metadata: languageName: unknown linkType: soft -"@sidequest/engine@workspace:*, @sidequest/engine@workspace:^, @sidequest/engine@workspace:packages/engine": +"@sidequest/engine@workspace:*, @sidequest/engine@workspace:packages/engine": version: 0.0.0-use.local resolution: "@sidequest/engine@workspace:packages/engine" dependencies: From 83699545aa27e2cacf772750dd8f96497bcd0b41 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 20:21:02 -0300 Subject: [PATCH 07/10] refactor: move backend truncation to beforeEach for consistent test setup --- packages/backends/backend-test/src/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/backends/backend-test/src/index.ts b/packages/backends/backend-test/src/index.ts index 37e2cce..394e9a1 100644 --- a/packages/backends/backend-test/src/index.ts +++ b/packages/backends/backend-test/src/index.ts @@ -28,14 +28,10 @@ import defineUpdateJobTestSuite from "./updateJob"; * @param backendFactory - A factory function that creates a backend instance. */ export function testBackend(backendFactory: () => Backend) { - beforeAll(async () => { - setTestBackend(backendFactory()); - await backend.truncate(); - }); - beforeEach(async () => { setTestBackend(backendFactory()); await backend.migrate(); + await backend.truncate(); }); afterEach(async () => { From 4bcfe934a4d1dc0f89992676005d630ab1f79bd8 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 20:28:12 -0300 Subject: [PATCH 08/10] refactor: update vitest configuration to include test setup and path resolution --- packages/core/vitest.config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/vitest.config.js b/packages/core/vitest.config.js index 8c3948e..fac8a71 100644 --- a/packages/core/vitest.config.js +++ b/packages/core/vitest.config.js @@ -1,4 +1,5 @@ +import path from "path"; import { defineConfig } from "vitest/config"; import { createVitestConfig } from "../../vitest.base.config"; -export default defineConfig(createVitestConfig()); +export default defineConfig(createVitestConfig(["../../tests/vitest.setup.ts"], path.resolve("..", "..", "./tests"))); From 0e0a6f329e44a74e1845484b547e0f813e4aa3b2 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 20:30:40 -0300 Subject: [PATCH 09/10] refactor: replace sidequestTest with it in job tests for consistency refactor: update tsconfig to remove test directory from include refactor: simplify vitest configuration by removing unnecessary parameters --- packages/core/src/job/job.test.ts | 21 ++++++++++----------- packages/core/tsconfig.json | 3 ++- packages/core/vitest.config.js | 3 +-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/packages/core/src/job/job.test.ts b/packages/core/src/job/job.test.ts index 280a62f..c0ec02b 100644 --- a/packages/core/src/job/job.test.ts +++ b/packages/core/src/job/job.test.ts @@ -1,4 +1,3 @@ -import { sidequestTest } from "@/tests/fixture"; import { CompletedResult, RetryResult, SnoozeResult } from "@sidequest/core"; import { Job } from "./job"; @@ -23,39 +22,39 @@ describe("job.ts", () => { vi.restoreAllMocks(); }); - sidequestTest("should expose script and className correctly", async () => { + it("should expose script and className correctly", async () => { const job = new DummyJob(); await job.ready(); expect(typeof job.script).toBe("string"); expect(job.className).toBe("DummyJob"); }); - sidequestTest("creates a complete transition", () => { + it("creates a complete transition", () => { const job = new DummyJob(); const transition = job.complete("foo bar"); expect(transition.result).toBe("foo bar"); }); - sidequestTest("creates a fail transition", () => { + it("creates a fail transition", () => { const job = new DummyJob(); const transition = job.fail("error"); expect(transition.error).toEqual({ message: "error" }); }); - sidequestTest("creates a retry transition", () => { + it("creates a retry transition", () => { const job = new DummyJob(); const transition = job.retry("reason", 1000); expect(transition.error).toEqual({ message: "reason" }); expect(transition.delay).toEqual(1000); }); - sidequestTest("creates a snooze transition", () => { + it("creates a snooze transition", () => { const job = new DummyJob(); const transition = job.snooze(1000); expect(transition.delay).toBe(1000); }); - sidequestTest("fail/retry should accept an Error object", () => { + it("fail/retry should accept an Error object", () => { const job = new DummyJob(); const error = new Error("fail"); expect(job.fail(error).error.message).toEqual("fail"); @@ -63,7 +62,7 @@ describe("job.ts", () => { }); describe("perform", () => { - sidequestTest("should return CompleteResult if run returns a value", async () => { + it("should return CompleteResult if run returns a value", async () => { class ValueJob extends Job { run() { return "abc"; @@ -75,7 +74,7 @@ describe("job.ts", () => { expect(result.result).toBe("abc"); }); - sidequestTest("should return the JobResult return by run", async () => { + it("should return the JobResult return by run", async () => { class TransitionJob extends Job { run() { return { __is_job_transition__: true, type: "snooze" } as SnoozeResult; @@ -86,7 +85,7 @@ describe("job.ts", () => { expect(result.type).toBe("snooze"); }); - sidequestTest("should return RetryResult if run throws", async () => { + it("should return RetryResult if run throws", async () => { class ErrorJob extends Job { run() { throw new Error("fail!"); @@ -98,7 +97,7 @@ describe("job.ts", () => { expect(result.error.message).toEqual("fail!"); }); - sidequestTest("should return RetryResult if run unhandled promise", async () => { + it("should return RetryResult if run unhandled promise", async () => { class DummyUnhandled extends Job { run() { return new Promise(() => { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 87b647b..df59da5 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + "rootDir": "src", "outDir": "dist" }, - "include": ["src/**/*.ts", "../../tests/**/*"] + "include": ["src/**/*.ts"] } diff --git a/packages/core/vitest.config.js b/packages/core/vitest.config.js index fac8a71..8c3948e 100644 --- a/packages/core/vitest.config.js +++ b/packages/core/vitest.config.js @@ -1,5 +1,4 @@ -import path from "path"; import { defineConfig } from "vitest/config"; import { createVitestConfig } from "../../vitest.base.config"; -export default defineConfig(createVitestConfig(["../../tests/vitest.setup.ts"], path.resolve("..", "..", "./tests"))); +export default defineConfig(createVitestConfig()); From 33f4d6bc2be94d48ac3b645722ebda03fe8a1d5c Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 30 Jul 2025 20:34:35 -0300 Subject: [PATCH 10/10] refactor: update package list in development guide for clarity and consistency --- packages/docs/development.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/docs/development.md b/packages/docs/development.md index 848f977..f884733 100644 --- a/packages/docs/development.md +++ b/packages/docs/development.md @@ -16,15 +16,17 @@ description: Development guide for Sidequest.js Sidequest is built as a monorepo with the following packages: - **`sidequest`** - Main package combining all components +- **`@sidequest/docs`** - Documentation site using Vitepress - **`@sidequest/core`** - Core functionality, logging, and schema definitions - **`@sidequest/engine`** - Job processing engine with worker thread management - **`@sidequest/backend`** - Abstract backend interface +- **`@sidequest/backend-test`** - Test suite for backend implementations - **`@sidequest/sqlite-backend`** - SQLite backend implementation - **`@sidequest/postgres-backend`** - PostgreSQL backend implementation - **`@sidequest/mysql-backend`** - MySQL backend implementation +- **`@sidequest/mongo-backend`** - MongoDB backend implementation - **`@sidequest/dashboard`** - Web dashboard with Express.js, EJS, and HTMX - **`@sidequest/cli`** - Command-line interface tools -- **`@sidequest/backend-test`** - Test suite for backend implementations ## Setup