Skip to content

Commit e75ac4b

Browse files
authored
fix: call error_handler for SessionError (#557)
Closes: #546
1 parent 734c797 commit e75ac4b

File tree

2 files changed

+24
-1
lines changed

2 files changed

+24
-1
lines changed

src/crawlee/basic_crawler/_basic_crawler.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -851,6 +851,9 @@ async def __run_task_function(self) -> None:
851851
if not crawling_context.session:
852852
raise RuntimeError('SessionError raised in a crawling context without a session') from session_error
853853

854+
if self._error_handler:
855+
await self._error_handler(crawling_context, session_error)
856+
854857
if self._should_retry_request(crawling_context, session_error):
855858
self._logger.warning('Encountered a session error, rotating session and retrying')
856859

tests/unit/basic_crawler/test_basic_crawler.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from datetime import timedelta
1010
from pathlib import Path
1111
from typing import TYPE_CHECKING, Any
12-
from unittest.mock import Mock
12+
from unittest.mock import AsyncMock, Mock
1313

1414
import httpx
1515
import pytest
@@ -189,6 +189,26 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ
189189
assert calls[1][2] == 1
190190

191191

192+
async def test_calls_error_handler_for_sesion_errors() -> None:
193+
crawler = BasicCrawler(
194+
max_session_rotations=1,
195+
)
196+
197+
@crawler.router.default_handler
198+
async def handler(context: BasicCrawlingContext) -> None:
199+
raise SessionError('Arbitrary session error for testing purposes')
200+
201+
error_handler_mock = AsyncMock()
202+
203+
@crawler.error_handler
204+
async def error_handler(context: BasicCrawlingContext, error: Exception) -> None:
205+
await error_handler_mock(context, error)
206+
207+
await crawler.run(['https://crawlee.dev'])
208+
209+
assert error_handler_mock.call_count == 1
210+
211+
192212
async def test_handles_error_in_error_handler() -> None:
193213
crawler = BasicCrawler(
194214
request_provider=RequestList(['http://a.com/', 'http://b.com/', 'http://c.com/']),

0 commit comments

Comments
 (0)