Add X-Max-Output-Tokens header for output truncation

src/api/crawler.ts

@@ -1018,6 +1018,9 @@ export class CrawlerHost extends RPCHost {
         this.threadLocal.set('retainImages', opts.retainImages);
         this.threadLocal.set('noGfm', opts.noGfm);
         this.threadLocal.set('DNT', Boolean(opts.doNotTrack));
+        if (opts.maxOutputTokens) {
+            this.threadLocal.set('maxOutputTokens', opts.maxOutputTokens);
+        }
         if (opts.markdown) {
             this.threadLocal.set('turndownOpts', opts.markdown);
         }

src/dto/crawler-options.ts

@@ -221,6 +221,11 @@ class Viewport extends AutoCastable {
                     in: 'header',
                     schema: { type: 'string' }
                 },
+                'X-Max-Output-Tokens': {
+                    description: 'Limit the output content to approximately this many tokens.\n\nThe content will be truncated to fit within the specified token limit. Useful for workflows that only need a partial extraction.',
+                    in: 'header',
+                    schema: { type: 'string' }
+                },
                 'X-Respond-Timing': {
                     description: `Explicitly specify the respond timing. One of the following:\n\n` +
                         `- html: directly return unrendered HTML\n` +
@@ -406,6 +411,13 @@ export class CrawlerOptions extends AutoCastable {
     @Prop()
     tokenBudget?: number;
 
+    @Prop({
+        validate: (v: number) => v > 0,
+        type: Number,
+        nullable: true,
+    })
+    maxOutputTokens?: number;
+
     @Prop()
     viewport?: Viewport;
 
@@ -576,6 +588,14 @@ export class CrawlerOptions extends AutoCastable {
         const tokenBudget = ctx?.get('x-token-budget');
         instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined;
 
+        const maxOutputTokens = ctx?.get('x-max-output-tokens');
+        if (maxOutputTokens) {
+            const parsed = parseInt(maxOutputTokens);
+            if (!isNaN(parsed) && parsed > 0) {
+                instance.maxOutputTokens ??= parsed;
+            }
+        }
+
         const baseMode = ctx?.get('x-base');
         if (baseMode) {
             instance.base = baseMode as any;

src/services/snapshot-formatter.ts

@@ -97,6 +97,36 @@ export class SnapshotFormatter extends AsyncService {
     }
 
 
+    truncateToTokenLimit(text: string, maxTokens: number): string {
+        const totalTokens = countGPTToken(text);
+        if (totalTokens <= maxTokens) {
+            return text;
+        }
+
+        // Estimate character-to-token ratio and cut proportionally
+        const ratio = text.length / totalTokens;
+        let cutPoint = Math.floor(maxTokens * ratio);
+
+        // Avoid cutting in the middle of a word
+        while (cutPoint < text.length && text[cutPoint] !== ' ' && text[cutPoint] !== '\n') {
+            cutPoint++;
+        }
+
+        let truncated = text.slice(0, cutPoint);
+
+        // Verify and adjust if we overshot
+        let truncatedTokens = countGPTToken(truncated);
+        while (truncatedTokens > maxTokens && truncated.length > 0) {
+            // Trim by ~10% of remaining overshoot
+            const overshoot = truncatedTokens - maxTokens;
+            const charsToRemove = Math.max(1, Math.floor(overshoot * ratio * 0.5));
+            truncated = truncated.slice(0, truncated.length - charsToRemove);
+            truncatedTokens = countGPTToken(truncated);
+        }
+
+        return truncated.trimEnd();
+    }
+
     @Threaded()
     async formatSnapshot(mode: string | 'markdown' | 'html' | 'text' | 'screenshot' | 'pageshot', snapshot: PageSnapshot & {
         screenshotUrl?: string;
@@ -190,6 +220,19 @@ export class SnapshotFormatter extends AsyncService {
         if (modeOK && (mode.includes('lm') ||
             (!mode.includes('markdown') && !mode.includes('content')))
         ) {
+            const maxOutputTokens = this.threadLocal.get('maxOutputTokens') as number | undefined;
+            if (maxOutputTokens) {
+                if (f.content) {
+                    f.content = this.truncateToTokenLimit(f.content, maxOutputTokens);
+                }
+                if (f.text) {
+                    f.text = this.truncateToTokenLimit(f.text, maxOutputTokens);
+                }
+                if (f.html) {
+                    f.html = this.truncateToTokenLimit(f.html, maxOutputTokens);
+                }
+            }
+
             const dt = Date.now() - t0;
             this.logger.debug(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });
 
@@ -387,6 +430,11 @@ export class SnapshotFormatter extends AsyncService {
             }
         } while (false);
 
+        const maxOutputTokens = this.threadLocal.get('maxOutputTokens') as number | undefined;
+        if (maxOutputTokens && contentText) {
+            contentText = this.truncateToTokenLimit(contentText, maxOutputTokens);
+        }
+
         const formatted: FormattedPage = {
             title: (snapshot.parsed?.title || snapshot.title || '').trim(),
             description: (snapshot.description || '').trim(),