diff --git a/src/api/crawler.ts b/src/api/crawler.ts index c5679bcd..c52e5af3 100644 --- a/src/api/crawler.ts +++ b/src/api/crawler.ts @@ -1021,6 +1021,9 @@ export class CrawlerHost extends RPCHost { if (opts.markdown) { this.threadLocal.set('turndownOpts', opts.markdown); } + if (opts.freshness) { + this.threadLocal.set('x-brave-freshness', opts.freshness); + } const crawlOpts: ExtraScrappingOptions = { proxyUrl: opts.proxyUrl, diff --git a/src/dto/crawler-options.ts b/src/dto/crawler-options.ts index 01dea593..487aa95c 100644 --- a/src/dto/crawler-options.ts +++ b/src/dto/crawler-options.ts @@ -429,6 +429,9 @@ export class CrawlerOptions extends AutoCastable { }) respondTiming?: RESPOND_TIMING; + @Prop() + freshness?: string; + _hintIps?: string[]; static override from(input: any) { @@ -589,6 +592,11 @@ export class CrawlerOptions extends AutoCastable { instance.respondTiming ??= respondTiming as RESPOND_TIMING; } + const freshness = ctx?.get('x-brave-freshness'); + if (freshness && /^(pd|pw|pm|py|\d{4}-\d{2}-\d{2}to\d{4}-\d{2}-\d{2})$/.test(freshness)) { + instance.freshness ??= freshness; + } + if (instance.cacheTolerance) { instance.cacheTolerance = instance.cacheTolerance * 1000; } diff --git a/src/services/brave-search.ts b/src/services/brave-search.ts index 8dcfe2cd..b6f01548 100644 --- a/src/services/brave-search.ts +++ b/src/services/brave-search.ts @@ -33,7 +33,7 @@ export class BraveSearchService extends AsyncService { this.braveSearchHTTP = new BraveSearchHTTP(this.secretExposer.BRAVE_SEARCH_API_KEY); } - async webSearch(query: WebSearchQueryParams) { + async webSearch(query: WebSearchQueryParams & { freshness?: string }) { const ip = this.threadLocal.get('ip'); const extraHeaders: WebSearchOptionalHeaderOptions = {}; if (ip) { @@ -61,11 +61,16 @@ export class BraveSearchService extends AsyncService { extraHeaders['User-Agent'] = this.threadLocal.get('userAgent'); } - const encoded = { ...query }; + const encoded: Record = { ...query }; if (encoded.q) { encoded.q = (Buffer.from(encoded.q).toString('ascii') === encoded.q) ? encoded.q : encodeURIComponent(encoded.q); } + const freshness = this.threadLocal.get('x-brave-freshness') || query.freshness; + if (freshness) { + encoded.freshness = freshness; + } + let maxTries = 11; while (maxTries--) {