diff --git a/crewai_tools/tools/firecrawl_crawl_website_tool/README.md b/crewai_tools/tools/firecrawl_crawl_website_tool/README.md index 46d01160..d8e8f140 100644 --- a/crewai_tools/tools/firecrawl_crawl_website_tool/README.md +++ b/crewai_tools/tools/firecrawl_crawl_website_tool/README.md @@ -4,6 +4,10 @@ [Firecrawl](https://firecrawl.dev) is a platform for crawling and convert any website into clean markdown or structured data. +## Version Compatibility + +This implementation is compatible with FireCrawl API v1 + ## Installation - Get an API key from [firecrawl.dev](https://firecrawl.dev) and set it in environment variables (`FIRECRAWL_API_KEY`). @@ -27,16 +31,27 @@ tool = FirecrawlCrawlWebsiteTool(url='firecrawl.dev') - `api_key`: Optional. Specifies Firecrawl API key. Defaults is the `FIRECRAWL_API_KEY` environment variable. - `url`: The base URL to start crawling from. -- `page_options`: Optional. - - `onlyMainContent`: Optional. Only return the main content of the page excluding headers, navs, footers, etc. - - `includeHtml`: Optional. Include the raw HTML content of the page. Will output a html key in the response. -- `crawler_options`: Optional. Options for controlling the crawling behavior. - - `includes`: Optional. URL patterns to include in the crawl. - - `exclude`: Optional. URL patterns to exclude from the crawl. - - `generateImgAltText`: Optional. Generate alt text for images using LLMs (requires a paid plan). - - `returnOnlyUrls`: Optional. If true, returns only the URLs as a list in the crawl status. Note: the response will be a list of URLs inside the data, not a list of documents. - - `maxDepth`: Optional. Maximum depth to crawl. Depth 1 is the base URL, depth 2 includes the base URL and its direct children, and so on. - - `mode`: Optional. The crawling mode to use. Fast mode crawls 4x faster on websites without a sitemap but may not be as accurate and shouldn't be used on heavily JavaScript-rendered websites. - - `limit`: Optional. Maximum number of pages to crawl. - - `timeout`: Optional. Timeout in milliseconds for the crawling operation. +- `maxDepth`: Optional. Maximum depth to crawl. Depth 1 is the base URL, depth 2 includes the base URL and its direct children and so on. +- `limit`: Optional. Maximum number of pages to crawl. +- `allowExternalLinks`: Allows the crawler to follow links that point to external domains. +- `formats`: Optional. Formats for the page's content to be returned (eg. markdown, html, screenshot, links). +- `timeout`: Optional. Timeout in milliseconds for the crawling operation. + +## Configurations Example +This is the default configuration + +```python + DEFAULT_CRAWLING_OPTIONS = { + "maxDepth": 2, + "ignoreSitemap": True, + "limit": 100, + "allowBackwardLinks": False, + "allowExternalLinks": False, + "scrapeOptions": { + "formats": ["markdown", "screenshot", "links"], + "onlyMainContent": True, + "timeout": 30000 + } + } +``` diff --git a/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py b/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py index b95199c8..f91ad318 100644 --- a/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py +++ b/crewai_tools/tools/firecrawl_crawl_website_tool/firecrawl_crawl_website_tool.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Type +from typing import Any, Optional, Type from crewai.tools import BaseTool from pydantic import BaseModel, ConfigDict, Field, PrivateAttr @@ -12,8 +12,20 @@ class FirecrawlCrawlWebsiteToolSchema(BaseModel): url: str = Field(description="Website URL") - crawler_options: Optional[Dict[str, Any]] = Field( - default=None, description="Options for crawling" + maxDepth: Optional[int] = Field( + default=2, + description="Maximum depth to crawl. Depth 1 is the base URL, depth 2 includes the base URL and its direct children and so on.", + ) + limit: Optional[int] = Field( + default=100, description="Maximum number of pages to crawl." + ) + allowExternalLinks: Optional[bool] = Field( + default=False, + description="Allows the crawler to follow links that point to external domains.", + ) + formats: Optional[list[str]] = Field( + default=["markdown", "screenshot", "links"], + description="Formats for the page's content to be returned (eg. markdown, html, screenshot, links).", ) timeout: Optional[int] = Field( default=30000, @@ -64,17 +76,38 @@ def _initialize_firecrawl(self) -> None: def _run( self, url: str, - crawler_options: Optional[Dict[str, Any]] = None, + maxDepth: Optional[int] = 2, + limit: Optional[int] = 100, + allowExternalLinks: Optional[bool] = False, + formats: Optional[list[str]] = ["markdown", "screenshot", "links"], timeout: Optional[int] = 30000, ): - if crawler_options is None: - crawler_options = {} - - options = { - "crawlerOptions": crawler_options, - "timeout": timeout, + # Default options for timeout and crawling + DEFAULT_TIMEOUT = 30000 + DEFAULT_CRAWLING_OPTIONS = { + "maxDepth": 2, + "ignoreSitemap": True, + "limit": 100, + "allowBackwardLinks": False, + "allowExternalLinks": False, + "scrapeOptions": { + "formats": ["markdown", "screenshot", "links"], + "onlyMainContent": True, + "timeout": DEFAULT_TIMEOUT, + }, } - return self._firecrawl.crawl_url(url, options) + + # Add default options not present as parameters + crawling_options = DEFAULT_CRAWLING_OPTIONS + + # Update the values of parameters present + crawling_options["maxDepth"] = maxDepth + crawling_options["limit"] = limit + crawling_options["allowExternalLinks"] = allowExternalLinks + crawling_options["scrapeOptions"]["formats"] = formats + crawling_options["scrapeOptions"]["timeout"] = timeout + + return self._firecrawl.crawl_url(url, crawling_options) try: