feat: Add exponential backoff retry with DLQ support (Vibe Kanban) (#34)

* All tasks are complete. Here's a summary of the changes made:

## Summary of Changes

### New Features
1. **Exponential Backoff Retry Strategy**
   - Added `RetryStrategy` type: `(retryCount: number) => number`
   - Default strategy: exponential backoff with 1s base, 60s max
   - Customizable via `retryStrategy` option per event

2. **Retry Counter & DLQ**
   - Added `retryCount` field to track retry attempts
   - Added `status` field (`'pending' | 'dlq'`)
   - Events move to DLQ after `maxRetries` (default: 5) failures

### Naming Improvements
- `expiresAtTTL` → `retentionPeriod`
- `retryEveryMilliseconds` → `pollingInterval`
- `maxOutboxTransportEventPerRetry` → `maxEventsPerPoll`
- `maxExecutionTimeTTL` → `maxExecutionTime`
- `readyToRetryAfterTTL` → replaced by `retryStrategy`

### Files Modified
- **Core**: `outbox-transport-event.interface.ts`, `outbox.module-definition.ts`, `retryable-outbox-event.poller.ts`, `transactional-event-emitter.ts`
- **MikroORM**: Model, driver, and new migration `MigrationOutbox1734900000.ts`
- **TypeORM**: Model, driver, and new migration `MigrationOutbox1734900000.ts`
- **Tests**: All unit and integration tests updated
- **Documentation**: All README files updated

### Build & Tests
- Build passes for all packages
- 58 unit tests pass

* Good question. Let me think through the options:

| Status | Pros | Cons |
|--------|------|------|
| `failed` | Clear meaning, standard term | Might imply permanent failure (could be retried manually?) |
| `dlq` | Industry standard term (Dead Letter Queue) | Abbreviation, less readable |
| `dead` | Short, clear | Might sound too dramatic |
| `exhausted` | Describes what happened (retries exhausted) | Longer, less common |
| `abandoned` | Clear meaning | Implies we gave up, not that it failed |

My recommendation: **`dead`** or keep **`failed`**

- `dead` is concise and matches "Dead Letter Queue" concept
- `failed` is clearer for developers who don't know DLQ terminology

`cancelled` doesn't fit well because the event wasn't cancelled - it failed after max retries.

What's your preference? I can quickly rename if you'd like to use `dead` instead.

* fix(test): convert bigint to number in attemptAt assertion

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

* feat: increase default maxRetries from 5 to 10

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: jtomaszewski

packages/core/README.md

@@ -143,14 +143,16 @@ import {
           {
             name: OrderCreatedEvent.name,
             listeners: {
-              expiresAtTTL: 1000 * 60 * 60 * 24,   // 24 hours
-              maxExecutionTimeTTL: 1000 * 15,       // 15 seconds
-              readyToRetryAfterTTL: 10000,          // 10 seconds
+              retentionPeriod: 1000 * 60 * 60 * 24, // 24 hours - how long events are retained
+              maxExecutionTime: 1000 * 15,          // 15 seconds - max listener execution time
+              maxRetries: 5,                        // max retry attempts before DLQ
+              retryStrategy: (retryCount) =>        // optional: custom exponential backoff
+                Math.min(1000 * Math.pow(2, retryCount), 60_000),
             },
           },
         ],
-        retryEveryMilliseconds: 30_000,
-        maxOutboxTransportEventPerRetry: 10,
+        pollingInterval: 30_000,                    // 30 seconds - polling frequency
+        maxEventsPerPoll: 10,                       // batch size per poll
       }),
       inject: [DataSource],
     }),
@@ -166,18 +168,19 @@ export class AppModule {}
 | Option | Description |
 |--------|-------------|
 | `name` | Event class name |
-| `listeners.expiresAtTTL` | How long events are retained and retried (ms) |
-| `listeners.maxExecutionTimeTTL` | Max listener execution time before retry (ms) |
-| `listeners.readyToRetryAfterTTL` | Delay before retrying failed events (ms) |
+| `listeners.retentionPeriod` | How long events are retained before expiring (ms) |
+| `listeners.maxExecutionTime` | Max listener execution time before timeout (ms) |
+| `listeners.maxRetries` | Max retry attempts before moving to DLQ (default: 10) |
+| `listeners.retryStrategy` | Optional function `(retryCount) => delayMs` for custom backoff. Default: exponential backoff with 1s base, 60s max |
 
 ### Module Options
 
 | Option | Description |
 |--------|-------------|
 | `driverFactory` | Database driver factory instance |
 | `events` | Array of event configurations |
-| `retryEveryMilliseconds` | Polling interval for retry mechanism |
-| `maxOutboxTransportEventPerRetry` | Batch size per polling cycle |
+| `pollingInterval` | How often to poll for pending events (ms) |
+| `maxEventsPerPoll` | Batch size per polling cycle |
 | `isGlobal` | Register module globally (optional) |
 | `enableDefaultMiddlewares` | Enable default middlewares like LoggerMiddleware (default: `true`) |
 | `middlewares` | Array of custom middleware classes |
@@ -214,8 +217,8 @@ import { MikroORMDatabaseDriverFactory } from '@fullstackhouse/nestjs-outbox-mik
       useFactory: (orm: MikroORM) => ({
         driverFactory: new MikroORMDatabaseDriverFactory(orm),
         events: [/* ... */],
-        retryEveryMilliseconds: 30_000,
-        maxOutboxTransportEventPerRetry: 10,
+        pollingInterval: 30_000,
+        maxEventsPerPoll: 10,
       }),
       inject: [MikroORM],
     }),
@@ -333,8 +336,8 @@ import { MikroORMDatabaseDriverFactory } from '@fullstackhouse/nestjs-outbox-mik
       useFactory: (orm: MikroORM) => ({
         driverFactory: new MikroORMDatabaseDriverFactory(orm),
         events: [/* ... */],
-        retryEveryMilliseconds: 30_000,
-        maxOutboxTransportEventPerRetry: 10,
+        pollingInterval: 30_000,
+        maxEventsPerPoll: 10,
       }),
       inject: [MikroORM],
       middlewares: [LoggingMiddleware, SentryMiddleware],

packages/core/src/driver/database.driver.ts

@@ -2,7 +2,7 @@ import { OutboxTransportEvent } from '../model/outbox-transport-event.interface'
 import { DatabaseDriverPersister } from './database.driver-persister';
 
 export interface DatabaseDriver extends DatabaseDriverPersister {
-  createOutboxTransportEvent(eventName: string, eventPayload: any, expireAt: number, readyToRetryAfter: number | null): OutboxTransportEvent;
+  createOutboxTransportEvent(eventName: string, eventPayload: any, expireAt: number, attemptAt: number | null): OutboxTransportEvent;
   findAndExtendReadyToRetryEvents(limit: number): Promise<OutboxTransportEvent[]>;
   findPendingEvents(limit: number): Promise<OutboxTransportEvent[]>;
 }

packages/core/src/emitter/transactional-event-emitter.ts

@@ -55,7 +55,7 @@ export class TransactionalEventEmitter {
     const outboxTransportEvent = databaseDriver.createOutboxTransportEvent(
       processedEvent.name,
       processedEvent,
-      currentTimestamp + eventOptions.listeners.expiresAtTTL,
+      currentTimestamp + eventOptions.listeners.retentionPeriod,
       currentTimestamp,
     );
     const persister = customDatabaseDriverPersister ?? databaseDriver;

packages/core/src/model/outbox-transport-event.interface.ts

@@ -1,9 +1,13 @@
+export type OutboxEventStatus = 'pending' | 'failed';
+
 export interface OutboxTransportEvent {
   id: number;
   eventName: string;
   eventPayload: any;
   deliveredToListeners: string[];
-  readyToRetryAfter: number | null;
+  attemptAt: number | null;
+  retryCount: number;
+  status: OutboxEventStatus;
   expireAt: number;
   insertedAt: number;
 }

packages/core/src/outbox.module-definition.ts

@@ -2,19 +2,29 @@ import { ConfigurableModuleBuilder, Type } from '@nestjs/common';
 import { DatabaseDriverFactory } from './driver/database-driver.factory';
 import { OutboxMiddleware } from './middleware/outbox-middleware.interface';
 
+export type RetryStrategy = (retryCount: number) => number;
+
+export const defaultRetryStrategy: RetryStrategy = (retryCount: number) => {
+  const baseDelayMs = 1000;
+  const maxDelayMs = 60_000;
+  const delay = Math.min(baseDelayMs * Math.pow(2, retryCount), maxDelayMs);
+  return delay;
+};
+
 export interface OutboxModuleEventOptions {
   name: string;
   listeners: {
-    expiresAtTTL: number;
-    readyToRetryAfterTTL: number;
-    maxExecutionTimeTTL: number;
+    retentionPeriod: number;
+    retryStrategy?: RetryStrategy;
+    maxRetries?: number;
+    maxExecutionTime: number;
   };
 }
 
 export interface OutboxModuleOptions {
   events: OutboxModuleEventOptions[];
-  retryEveryMilliseconds: number;
-  maxOutboxTransportEventPerRetry: number;
+  pollingInterval: number;
+  maxEventsPerPoll: number;
   driverFactory: DatabaseDriverFactory;
 }
 

packages/core/src/poller/retryable-outbox-event.poller.ts

@@ -25,7 +25,7 @@ export class RetryableOutboxEventPoller implements OnModuleInit, OnModuleDestroy
   ) {}
 
   async onModuleInit() {
-    this.logger.log(`Poller options: retryEveryMilliseconds: ${this.options.retryEveryMilliseconds}, maxOutboxTransportEventPerRetry: ${this.options.maxOutboxTransportEventPerRetry}, events: ${JSON.stringify(this.options.events)}, driver: ${this.options.driverFactory.constructor.name}`);
+    this.logger.log(`Poller options: pollingInterval: ${this.options.pollingInterval}, maxEventsPerPoll: ${this.options.maxEventsPerPoll}, events: ${JSON.stringify(this.options.events)}, driver: ${this.options.driverFactory.constructor.name}`);
 
     if (this.eventListener) {
       try {
@@ -36,7 +36,7 @@ export class RetryableOutboxEventPoller implements OnModuleInit, OnModuleDestroy
       }
     }
 
-    const pollingSource$ = interval(this.options.retryEveryMilliseconds);
+    const pollingSource$ = interval(this.options.pollingInterval);
     const eventSource$ = this.eventListener?.events$ ?? EMPTY;
 
     this.subscription = merge(pollingSource$, eventSource$)
@@ -84,10 +84,10 @@ export class RetryableOutboxEventPoller implements OnModuleInit, OnModuleDestroy
 
   async poolRetryableEvents() {
     try {
-      const maxOutboxTransportEventPerRetry = this.options.maxOutboxTransportEventPerRetry;
+      const maxEventsPerPoll = this.options.maxEventsPerPoll;
       const databaseDriver = this.databaseDriverFactory.create(this.eventConfigurationResolver);
 
-      const readyToRetryEvents = await databaseDriver.findAndExtendReadyToRetryEvents(maxOutboxTransportEventPerRetry);
+      const readyToRetryEvents = await databaseDriver.findAndExtendReadyToRetryEvents(maxEventsPerPoll);
 
       if (readyToRetryEvents.length === 0) {
         return;

packages/core/src/processor/outbox-event.processor.ts

@@ -86,7 +86,7 @@ export class OutboxEventProcessor implements OutboxEventProcessorContract {
             listenerName: listener.getName(),
             hasFailed: true,
           });
-        }, eventOptions.listeners.maxExecutionTimeTTL);
+        }, eventOptions.listeners.maxExecutionTime);
 
         await this.wrapExecution(context, async () => {
           await listener.handle(outboxTransportEvent.eventPayload, outboxTransportEvent.eventName);

packages/core/src/test/unit/event-validator.spec.ts

@@ -4,24 +4,24 @@ import { OutboxModuleOptions } from '../../outbox.module-definition';
 describe('EventValidator', () => {
   it('should throw an exception when event names are not unique', () => {
     const mockOptions: OutboxModuleOptions = {
-      driverFactory: null,
-      maxOutboxTransportEventPerRetry: 10,
-      retryEveryMilliseconds: 1000,
+      driverFactory: null as any,
+      maxEventsPerPoll: 10,
+      pollingInterval: 1000,
       events: [
         {
           name: 'event1',
           listeners: {
-            expiresAtTTL: 1000,
-            readyToRetryAfterTTL: 1000,
-            maxExecutionTimeTTL: 1000,
+            retentionPeriod: 1000,
+            maxRetries: 5,
+            maxExecutionTime: 1000,
           },
         },
         {
           name: 'event1',
           listeners: {
-            expiresAtTTL: 1000,
-            readyToRetryAfterTTL: 1000,
-            maxExecutionTimeTTL: 1000,
+            retentionPeriod: 1000,
+            maxRetries: 5,
+            maxExecutionTime: 1000,
           },
         },
       ],
@@ -34,24 +34,24 @@ describe('EventValidator', () => {
 
   it('should not throw an exception when event names are unique', () => {
     const mockOptions: OutboxModuleOptions = {
-      driverFactory: null,
-      maxOutboxTransportEventPerRetry: 10,
-      retryEveryMilliseconds: 1000,
+      driverFactory: null as any,
+      maxEventsPerPoll: 10,
+      pollingInterval: 1000,
       events: [
         {
           name: 'event1',
           listeners: {
-            expiresAtTTL: 1000,
-            readyToRetryAfterTTL: 1000,
-            maxExecutionTimeTTL: 1000,
+            retentionPeriod: 1000,
+            maxRetries: 5,
+            maxExecutionTime: 1000,
           },
         },
         {
           name: 'event2',
           listeners: {
-            expiresAtTTL: 1000,
-            readyToRetryAfterTTL: 1000,
-            maxExecutionTimeTTL: 1000,
+            retentionPeriod: 1000,
+            maxRetries: 5,
+            maxExecutionTime: 1000,
           },
         },
       ],

packages/core/src/test/unit/mock/outbox-options.mock.ts

@@ -1,15 +1,17 @@
 import { DatabaseDriverFactory } from "../../../driver/database-driver.factory";
+import { RetryStrategy } from "../../../outbox.module-definition";
 
 export const createMockedOutboxOptionsFactory = (mockedDriverFactory: DatabaseDriverFactory, events: {
     name: string,
     listeners: {
-        expiresAtTTL: number,
-        readyToRetryAfterTTL: number,
-        maxExecutionTimeTTL: number
+        retentionPeriod: number,
+        retryStrategy?: RetryStrategy,
+        maxRetries?: number,
+        maxExecutionTime: number
     }
 }[]) => ({
     driverFactory: mockedDriverFactory,
-    retryEveryMilliseconds: 1000,
-    maxOutboxTransportEventPerRetry: 1000,
+    pollingInterval: 1000,
+    maxEventsPerPoll: 1000,
     events
 });