diff --git a/src/api/crawler.ts b/src/api/crawler.ts index c5679bcd..fcff1d76 100644 --- a/src/api/crawler.ts +++ b/src/api/crawler.ts @@ -1018,6 +1018,9 @@ export class CrawlerHost extends RPCHost { this.threadLocal.set('retainImages', opts.retainImages); this.threadLocal.set('noGfm', opts.noGfm); this.threadLocal.set('DNT', Boolean(opts.doNotTrack)); + if (opts.maxOutputTokens) { + this.threadLocal.set('maxOutputTokens', opts.maxOutputTokens); + } if (opts.markdown) { this.threadLocal.set('turndownOpts', opts.markdown); } diff --git a/src/dto/crawler-options.ts b/src/dto/crawler-options.ts index 01dea593..ee846e9a 100644 --- a/src/dto/crawler-options.ts +++ b/src/dto/crawler-options.ts @@ -221,6 +221,11 @@ class Viewport extends AutoCastable { in: 'header', schema: { type: 'string' } }, + 'X-Max-Output-Tokens': { + description: 'Limit the output content to approximately this many tokens.\n\nThe content will be truncated to fit within the specified token limit. Useful for workflows that only need a partial extraction.', + in: 'header', + schema: { type: 'string' } + }, 'X-Respond-Timing': { description: `Explicitly specify the respond timing. One of the following:\n\n` + `- html: directly return unrendered HTML\n` + @@ -406,6 +411,13 @@ export class CrawlerOptions extends AutoCastable { @Prop() tokenBudget?: number; + @Prop({ + validate: (v: number) => v > 0, + type: Number, + nullable: true, + }) + maxOutputTokens?: number; + @Prop() viewport?: Viewport; @@ -576,6 +588,14 @@ export class CrawlerOptions extends AutoCastable { const tokenBudget = ctx?.get('x-token-budget'); instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined; + const maxOutputTokens = ctx?.get('x-max-output-tokens'); + if (maxOutputTokens) { + const parsed = parseInt(maxOutputTokens); + if (!isNaN(parsed) && parsed > 0) { + instance.maxOutputTokens ??= parsed; + } + } + const baseMode = ctx?.get('x-base'); if (baseMode) { instance.base = baseMode as any; diff --git a/src/services/snapshot-formatter.ts b/src/services/snapshot-formatter.ts index 3064a88b..20aa8b8e 100644 --- a/src/services/snapshot-formatter.ts +++ b/src/services/snapshot-formatter.ts @@ -97,6 +97,36 @@ export class SnapshotFormatter extends AsyncService { } + truncateToTokenLimit(text: string, maxTokens: number): string { + const totalTokens = countGPTToken(text); + if (totalTokens <= maxTokens) { + return text; + } + + // Estimate character-to-token ratio and cut proportionally + const ratio = text.length / totalTokens; + let cutPoint = Math.floor(maxTokens * ratio); + + // Avoid cutting in the middle of a word + while (cutPoint < text.length && text[cutPoint] !== ' ' && text[cutPoint] !== '\n') { + cutPoint++; + } + + let truncated = text.slice(0, cutPoint); + + // Verify and adjust if we overshot + let truncatedTokens = countGPTToken(truncated); + while (truncatedTokens > maxTokens && truncated.length > 0) { + // Trim by ~10% of remaining overshoot + const overshoot = truncatedTokens - maxTokens; + const charsToRemove = Math.max(1, Math.floor(overshoot * ratio * 0.5)); + truncated = truncated.slice(0, truncated.length - charsToRemove); + truncatedTokens = countGPTToken(truncated); + } + + return truncated.trimEnd(); + } + @Threaded() async formatSnapshot(mode: string | 'markdown' | 'html' | 'text' | 'screenshot' | 'pageshot', snapshot: PageSnapshot & { screenshotUrl?: string; @@ -190,6 +220,19 @@ export class SnapshotFormatter extends AsyncService { if (modeOK && (mode.includes('lm') || (!mode.includes('markdown') && !mode.includes('content'))) ) { + const maxOutputTokens = this.threadLocal.get('maxOutputTokens') as number | undefined; + if (maxOutputTokens) { + if (f.content) { + f.content = this.truncateToTokenLimit(f.content, maxOutputTokens); + } + if (f.text) { + f.text = this.truncateToTokenLimit(f.text, maxOutputTokens); + } + if (f.html) { + f.html = this.truncateToTokenLimit(f.html, maxOutputTokens); + } + } + const dt = Date.now() - t0; this.logger.debug(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt }); @@ -387,6 +430,11 @@ export class SnapshotFormatter extends AsyncService { } } while (false); + const maxOutputTokens = this.threadLocal.get('maxOutputTokens') as number | undefined; + if (maxOutputTokens && contentText) { + contentText = this.truncateToTokenLimit(contentText, maxOutputTokens); + } + const formatted: FormattedPage = { title: (snapshot.parsed?.title || snapshot.title || '').trim(), description: (snapshot.description || '').trim(),