Firefox‐based HTTP fetching bridge proof-of-concept

gallery_dl/downloader/http.py

@@ -13,6 +13,7 @@
 from requests.exceptions import RequestException, ConnectionError, Timeout
 from .common import DownloaderBase
 from .. import text, util, output
+from ..ff_fetch import FFFetchHandler
 from ssl import SSLError
 
 
@@ -89,6 +90,8 @@ def __init__(self, job):
         else:
             self.interval_429 = util.build_duration_func(interval_429)
 
+        self._ff_fetch_handler = FFFetchHandler(self.log)
+
     def download(self, url, pathfmt):
         try:
             return self._download_impl(url, pathfmt)
@@ -106,6 +109,8 @@ def _download_impl(self, url, pathfmt):
         response = None
         tries = code = 0
         msg = ""
+        url = self._ff_fetch_handler.adjust_request(
+            self.proxies, self.session, url)
 
         metadata = self.metadata
         kwdict = pathfmt.kwdict

gallery_dl/extractor/common.py

@@ -24,6 +24,7 @@
 from requests.adapters import HTTPAdapter
 from .message import Message
 from .. import config, output, text, util, cache, exception
+from ..ff_fetch import FFFetchHandler
 urllib3 = requests.packages.urllib3
 
 
@@ -60,6 +61,7 @@ def __init__(self, match):
             self.category = CATEGORY_MAP[self.category]
         self._cfgpath = ("extractor", self.category, self.subcategory)
         self._parentdir = ""
+        self._ff_fetch_handler = FFFetchHandler(self.log)
 
     @classmethod
     def from_url(cls, url):
@@ -164,6 +166,8 @@ def request(self, url, method="GET", session=None,
                 else:
                     kwargs["headers"] = {"Content-Type": "application/json"}
 
+        url = self._ff_fetch_handler.adjust_request(
+            kwargs["proxies"], session, url)
         response = None
         tries = 1
 

gallery_dl/ff_fetch.py

@@ -0,0 +1,47 @@
+
+import requests
+
+class FFFetchHandler:
+    def __init__(self, log):
+        self.log = log
+        self._is_ff_fetch_proxy = dict()
+
+    def adjust_request(self, proxies, session, url):
+        self.log.debug('FFFetchHandler.adjust_request %s proxies=%s', url, proxies)
+        single_proxy = proxies
+        # Special handling for the same copied proxy
+        if isinstance(proxies, list) and len(set(proxies)) == 1:
+            single_proxy = proxies[0]
+        elif isinstance(proxies, dict):
+            copied = set(v for k, v in proxies.items()
+                         if k.lower() in ['http', 'https'])
+            if len(copied) == 1:
+                single_proxy = list(copied)[0]
+
+        # Only single proxy specification is supported
+        if not isinstance(single_proxy, str):
+            return url
+
+        # Detecting Firefox Fetcher
+        if single_proxy not in self._is_ff_fetch_proxy:
+            # caching the detection result
+            r = requests.get(
+                'http://name.nonexistent?X-FFFetch-SelfIdent',
+                proxies=proxies
+            )
+            self._is_ff_fetch_proxy[single_proxy] = (
+                r.status_code == 200
+                and r.text == 'X-FFFetch-SelfIdent: OK'
+            )
+            if self._is_ff_fetch_proxy[single_proxy]:
+                self.log.info("Using Firefox Fetcher: '%s'", single_proxy)
+
+        if self._is_ff_fetch_proxy[single_proxy]:
+            # When using Firefox Fetcher, we have to resort to HTTP since
+            # HTTPS bypasses proxying and nullifies our imitation effort.
+            # Do not worry, Firefox Fetcher always uses HTTPS while
+            # fetching.
+            if url.lower().startswith('https://'):
+                url = 'http://' + url[8:]
+
+        return url