feat: add header generator and integrate it into HTTPX client (#530)

### Description

- This is the first version of the header generator, providing common
HTTP headers including user agent.
- User-agent is picked randomly from a random pool of 1000 user agents
from the Apify fingerprint dataset.
- This is integrated into the HTTPX client and will be further used in
the Playwright fingerprint injector (#401).

### Issues

- Closes: #402 

### Testing

- New unit tests implemented.

### Checklist

- [x] CI passed

Author: vdusek

src/crawlee/fingerprint_suite/__init__.py

@@ -0,0 +1 @@
+from ._header_generator import HeaderGenerator

src/crawlee/fingerprint_suite/_consts.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING
+
+from crawlee.fingerprint_suite._consts import COMMON_ACCEPT, COMMON_ACCEPT_LANGUAGE, USER_AGENT_POOL
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+
+class HeaderGenerator:
+    """Generates common headers for HTTP requests."""
+
+    def get_common_headers(self) -> Mapping[str, str]:
+        """Get common headers for HTTP requests.
+
+        We do not modify the 'Accept-Encoding', 'Connection' and other headers. They should be included and handled
+        by the HTTP client.
+
+        Returns:
+            Dictionary containing common headers.
+        """
+        return {
+            'Accept': COMMON_ACCEPT,
+            'Accept-Language': COMMON_ACCEPT_LANGUAGE,
+            'User-Agent': random.choice(USER_AGENT_POOL),
+        }

src/crawlee/fingerprint_suite/_header_generator.py

@@ -6,15 +6,17 @@
 import httpx
 from typing_extensions import override
 
+from crawlee._types import HttpHeaders
 from crawlee._utils.blocked import ROTATE_PROXY_ERRORS
 from crawlee.errors import ProxyError
+from crawlee.fingerprint_suite import HeaderGenerator
 from crawlee.http_clients import BaseHttpClient, HttpCrawlingResult, HttpResponse
 from crawlee.sessions import Session
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from crawlee._types import HttpHeaders, HttpMethod
+    from crawlee._types import HttpMethod
     from crawlee.base_storage_client._models import Request
     from crawlee.proxy_configuration import ProxyInfo
     from crawlee.statistics import Statistics
@@ -77,6 +79,8 @@ class HttpxHttpClient(BaseHttpClient):
     See the `BaseHttpClient` class for more common information about HTTP clients.
     """
 
+    _DEFAULT_HEADER_GENERATOR = HeaderGenerator()
+
     def __init__(
         self,
         *,
@@ -85,6 +89,7 @@ def __init__(
         ignore_http_error_status_codes: Iterable[int] = (),
         http1: bool = True,
         http2: bool = True,
+        header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR,
         **async_client_kwargs: Any,
     ) -> None:
         """Create a new instance.
@@ -95,6 +100,7 @@ def __init__(
             ignore_http_error_status_codes: HTTP status codes to ignore as errors.
             http1: Whether to enable HTTP/1.1 support.
             http2: Whether to enable HTTP/2 support.
+            header_generator: Header generator instance to use for generating common headers.
             async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`.
         """
         super().__init__(
@@ -105,6 +111,7 @@ def __init__(
         self._http1 = http1
         self._http2 = http2
         self._async_client_kwargs = async_client_kwargs
+        self._header_generator = header_generator
 
         self._client_by_proxy_url = dict[Optional[str], httpx.AsyncClient]()
 
@@ -118,11 +125,12 @@ async def crawl(
         statistics: Statistics | None = None,
     ) -> HttpCrawlingResult:
         client = self._get_client(proxy_info.url if proxy_info else None)
+        headers = self._combine_headers(HttpHeaders(request.headers))
 
         http_request = client.build_request(
             url=request.url,
             method=request.method,
-            headers=request.headers,
+            headers=headers,
             params=request.query_params,
             data=request.data,
             cookies=session.cookies if session else None,
@@ -164,6 +172,7 @@ async def send_request(
         proxy_info: ProxyInfo | None = None,
     ) -> HttpResponse:
         client = self._get_client(proxy_info.url if proxy_info else None)
+        headers = self._combine_headers(headers)
 
         http_request = client.build_request(
             url=url,
@@ -215,6 +224,16 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
 
         return self._client_by_proxy_url[proxy_url]
 
+    def _combine_headers(self, explicit_headers: HttpHeaders | None) -> HttpHeaders | None:
+        """Helper to get the headers for a HTTP request."""
+        common_headers = self._header_generator.get_common_headers() if self._header_generator else {}
+        headers = HttpHeaders(common_headers)
+
+        if explicit_headers:
+            headers = HttpHeaders({**headers, **explicit_headers})
+
+        return headers if headers else None
+
     @staticmethod
     def _is_proxy_error(error: httpx.TransportError) -> bool:
         """Helper to check whether the given error is a proxy-related error."""

src/crawlee/fingerprint_suite/py.typed

@@ -0,0 +1,10 @@
+from crawlee.fingerprint_suite import HeaderGenerator
+
+
+def test_get_common_headers() -> None:
+    header_generator = HeaderGenerator()
+    headers = header_generator.get_common_headers()
+
+    assert 'Accept' in headers
+    assert 'Accept-Language' in headers
+    assert 'User-Agent' in headers

src/crawlee/http_clients/_httpx.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+import json
 import os
 from typing import TYPE_CHECKING
 
 import pytest
 
 from crawlee import Request
 from crawlee.errors import ProxyError
+from crawlee.fingerprint_suite._consts import COMMON_ACCEPT, COMMON_ACCEPT_LANGUAGE, USER_AGENT_POOL
 from crawlee.http_clients import HttpxHttpClient
 from crawlee.statistics import Statistics
 
@@ -82,3 +84,22 @@ async def test_send_request_with_proxy_disabled(
 
     with pytest.raises(ProxyError):
         await http_client.send_request(url, proxy_info=disabled_proxy)
+
+
+async def test_common_headers() -> None:
+    client = HttpxHttpClient()
+
+    response = await client.send_request('https://httpbin.org/get')
+    response_dict = json.loads(response.read().decode())
+    response_headers = response_dict.get('headers', {})
+
+    assert 'Accept' in response_headers
+    assert response_headers['Accept'] == COMMON_ACCEPT
+
+    assert 'Accept-Language' in response_headers
+    assert response_headers['Accept-Language'] == COMMON_ACCEPT_LANGUAGE
+
+    # By default, HTTPX uses its own User-Agent, which should be replaced by the one from the header generator.
+    assert 'User-Agent' in response_headers
+    assert 'python-httpx' not in response_headers['User-Agent']
+    assert response_headers['User-Agent'] in USER_AGENT_POOL