feat!: Implement RequestManagerTandem, remove add_request from RequestList, accept any iterable in RequestList constructor (#777)

> Tandem, or in tandem, is an arrangement in which two or more animals,
machines, or people are lined up one behind another, all facing in the
same
direction.[[1]](https://en.wikipedia.org/wiki/Tandem#cite_note-OED-1)
Tandem can also be used more generally to refer to any group of persons
or objects working together, not necessarily in
line.[[1]](https://en.wikipedia.org/wiki/Tandem#cite_note-OED-1)
(https://en.wikipedia.org/wiki/Tandem)

- Inspired by
https://github.com/apify/crawlee/blob/4c95847d5cedd6514620ccab31d5b242ba76de80/packages/basic-crawler/src/internals/basic-crawler.ts#L1154-L1177
and related code in the same class
- In my opinion, it implements the feature more cleanly and without
polluting `BasicCrawler` (...any further)
- The motivation for the feature is twofold:
1. Apify Actor development - it is common that an Actor receives a
`requestListSources` input from the user, which may be pretty complex
(regexp-based extraction from remote URL lists), and which is usually
parsed using `apify.RequestList.open`. At the same time, the Actor wants
to use the built in `RequestQueue`.
2. Sitemap parsing (#248) - similar to 1, but not coupled to the Apify
platform - we want to read URLs from a sitemap in the background, but
the URLs should go through the standard request queue

## Breaking changes

- `RequestList` does not support `.drop()`, `.reclaim_request()`,
`.add_request()` and `add_requests_batched()` anymore
- `RequestManagerTandem` with a `RequestQueue` should be used for this
use case, `await list.to_tandem()` can be used as a shortcut
- The `RequestProvider` interface has been renamed to `RequestManager`
and moved to the `crawlee.request_loaders` package
- `RequestList` has been moved to the `crawlee.request_loaders` package
- The `BasicCrawler.get_request_provider` method has been renamed to
`BasicCrawler.get_request_manager` and it does not accept the `id` and
`name` arguments anymore
- The `request_provider` parameter of `BasicCrawler.__init__` has been
renamed to `request_manager`
 
## TODO
- [x] new tests
- [x] fix existing tests

---------

Co-authored-by: Vlada Dusek <[email protected]>

Author: janbuchar

docs/guides/code/request_storage/rl_basic_example.py

@@ -1,6 +1,6 @@
 import asyncio
 
-from crawlee.storages import RequestList
+from crawlee.request_loaders import RequestList
 
 
 async def main() -> None:
@@ -11,24 +11,13 @@ async def main() -> None:
         requests=['https://apify.com/', 'https://crawlee.dev/', 'https://crawlee.dev/python/'],
     )
 
-    # You can interact with the request list in the same way as with the request queue.
-    await request_list.add_requests_batched(
-        [
-            'https://crawlee.dev/python/docs/quick-start',
-            'https://crawlee.dev/python/api',
-        ]
-    )
-
     # Fetch and process requests from the queue.
     while request := await request_list.fetch_next_request():
         # Do something with it...
 
         # And mark it as handled.
         await request_list.mark_request_as_handled(request)
 
-    # Remove the request queue.
-    await request_list.drop()
-
 
 if __name__ == '__main__':
     asyncio.run(main())

docs/guides/code/request_storage/rl_with_crawler_example.py

@@ -1,7 +1,7 @@
 import asyncio
 
 from crawlee.http_crawler import HttpCrawler, HttpCrawlingContext
-from crawlee.storages import RequestList
+from crawlee.request_loaders import RequestList
 
 
 async def main() -> None:
@@ -12,9 +12,11 @@ async def main() -> None:
         requests=['https://apify.com/', 'https://crawlee.dev/'],
     )
 
-    # Create a new crawler (it can be any subclass of BasicCrawler) and pass the request
-    # list as request provider to it. It will be managed by the crawler.
-    crawler = HttpCrawler(request_provider=request_list)
+    # Join the request list into a tandem with the default request queue
+    request_manager = await request_list.to_tandem()
+
+    # Create a new crawler (it can be any subclass of BasicCrawler) and pass the request manager tandem
+    crawler = HttpCrawler(request_manager=request_manager)
 
     # Define the default request handler, which will be called for every request.
     @crawler.router.default_handler

docs/guides/code/request_storage/rq_with_crawler_explicit_example.py

@@ -14,7 +14,7 @@ async def main() -> None:
 
     # Create a new crawler (it can be any subclass of BasicCrawler) and pass the request
     # list as request provider to it. It will be managed by the crawler.
-    crawler = HttpCrawler(request_provider=request_queue)
+    crawler = HttpCrawler(request_manager=request_queue)
 
     # Define the default request handler, which will be called for every request.
     @crawler.router.default_handler

docs/guides/code/request_storage/tandem_example.py

@@ -0,0 +1,23 @@
+import asyncio
+
+from crawlee.parsel_crawler import ParselCrawler, ParselCrawlingContext
+from crawlee.request_loaders import RequestList
+
+
+async def main() -> None:
+    # Create a static request list
+    request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
+
+    crawler = ParselCrawler(
+        # Requests from the list will be processed first, but they will be enqueued in the default request queue first
+        request_manager=await request_list.to_tandem(),
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        await context.enqueue_links()  # New links will be enqueued directly to the queue
+
+    await crawler.run()
+
+
+asyncio.run(main())

docs/guides/code/request_storage/tandem_example_explicit.py

@@ -0,0 +1,27 @@
+import asyncio
+
+from crawlee.parsel_crawler import ParselCrawler, ParselCrawlingContext
+from crawlee.request_loaders import RequestList, RequestManagerTandem
+from crawlee.storages import RequestQueue
+
+
+async def main() -> None:
+    # Create a static request list
+    request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
+
+    # Open the default request queue
+    request_queue = await RequestQueue.open()
+
+    crawler = ParselCrawler(
+        # Requests from the list will be processed first, but they will be enqueued in the default request queue first
+        request_manager=RequestManagerTandem(request_list, request_queue),
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        await context.enqueue_links()  # New links will be enqueued directly to the queue
+
+    await crawler.run()
+
+
+asyncio.run(main())

docs/guides/request_storage.mdx

@@ -18,12 +18,14 @@ import RsHelperAddRequestsExample from '!!raw-loader!./code/request_storage/help
 import RsHelperEnqueueLinksExample from '!!raw-loader!./code/request_storage/helper_enqueue_links_example.py';
 import RsDoNotPurgeExample from '!!raw-loader!./code/request_storage/do_not_purge_example.py';
 import RsPurgeExplicitlyExample from '!!raw-loader!./code/request_storage/purge_explicitly_example.py';
+import TandemExample from '!!raw-loader!./code/request_storage/tandem_example.py';
+import ExplicitTandemExample from '!!raw-loader!./code/request_storage/tandem_example_explicit.py';
 
 This guide explains the different types of request storage available in Crawlee, how to store the requests that your crawler will process, and which storage type to choose based on your needs.
 
 ## Introduction
 
-All request storage types in Crawlee implement the same interface - <ApiLink to="class/RequestProvider">`RequestProvider`</ApiLink>. This unified interface allows them to be used in a consistent manner, regardless of the storage backend. The request providers are managed by storage clients - subclasses of <ApiLink to="class/BaseStorageClient">`BaseStorageClient`</ApiLink>. For instance, <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink> stores data in memory while it can also offload them to the local directory. Data are stored in the following directory structure:
+All request storage types in Crawlee implement the same interface - <ApiLink to="class/RequestManager">`RequestManager`</ApiLink>. This unified interface allows them to be used in a consistent manner, regardless of the storage backend. The request providers are managed by storage clients - subclasses of <ApiLink to="class/BaseStorageClient">`BaseStorageClient`</ApiLink>. For instance, <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink> stores data in memory while it can also offload them to the local directory. Data are stored in the following directory structure:
 
 ```text
 {CRAWLEE_STORAGE_DIR}/{request_provider}/{QUEUE_ID}/
@@ -95,6 +97,28 @@ TODO: write this section, once https://github.com/apify/crawlee-python/issues/99
 
 */}
 
+## Processing requests from multiple sources
+
+In some cases, you might need to combine requests from multiple sources, most frequently from a static list of URLs (such as <ApiLink to="class/RequestList">`RequestList`</ApiLink>) and a <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>, where the queue takes care of persistence and retrying failed requests.
+
+This use case is supported via the <ApiLink to="class/RequestManagerTandem">`RequestManagerTandem`</ApiLink> class. You may also use the <ApiLink to="class/RequestLoader#to_tandem">`RequestLoader.to_tandem`</ApiLink> method as a shortcut.
+
+<CodeBlock className="language-python">
+    {TandemExample}
+</CodeBlock>
+<Tabs groupId="request_manager_tandem">
+    <TabItem value="request_manager_tandem_helper" label="Using to_tandem helper" default>
+        <CodeBlock className="language-python">
+            {TandemExample}
+        </CodeBlock>
+    </TabItem>
+    <TabItem value="request_manager_tandem_explicit" label="Explicitly using RequestManagerTandem">
+        <CodeBlock className="language-python">
+            {ExplicitTandemExample}
+        </CodeBlock>
+    </TabItem>
+</Tabs>
+
 ## Request-related helpers
 
 We offer several helper functions to simplify interactions with request storages:

docs/introduction/code/02_bs.py

@@ -12,7 +12,7 @@ async def main() -> None:
     # And then you add one or more requests to it.
     await rq.add_request('https://crawlee.dev')
 
-    crawler = BeautifulSoupCrawler(request_provider=rq)
+    crawler = BeautifulSoupCrawler(request_manager=rq)
 
     # Define a request handler and attach it to the crawler using the decorator.
     @crawler.router.default_handler

docs/upgrading/upgrading_to_v0x.md

@@ -26,6 +26,17 @@ This section summarizes the breaking changes between v0.4.x and v0.5.0.
 
 - Removed properties `json_` and `order_no`.
 
+### Request storages and loaders
+
+- The `request_provider` parameter of `BasicCrawler.__init__` has been renamed to `request_manager`
+- The `BasicCrawler.get_request_provider` method has been renamed to `BasicCrawler.get_request_manager` and it does not accept the `id` and `name` arguments anymore
+    - If using a specific request queue is desired, pass it as the `request_manager` on `BasicCrawler` creation
+- The `RequestProvider` interface has been renamed to `RequestManager` and moved to the `crawlee.request_loaders` package
+- `RequestList` has been moved to the `crawlee.request_loaders` package
+- `RequestList` does not support `.drop()`, `.reclaim_request()`, `.add_request()` and `add_requests_batched()` anymore
+    - It implements the new `RequestLoader` interface instead of `RequestManager`
+    - `RequestManagerTandem` with a `RequestQueue` should be used to enable passing a `RequestList` (or any other `RequestLoader` implementation) as a `request_manager`, `await list.to_tandem()` can be used as a shortcut
+
 ### PlaywrightCrawler
 
 - The `PlaywrightPreNavigationContext` was renamed to `PlaywrightPreNavCrawlingContext`.