add prefetching of index in legacy repositories

Author: tgolsson

docs/repositories.md

@@ -170,3 +170,18 @@ default = true
 ```
 
 A default source will also be the fallback source if you add other sources.
+
+### Enabling indexing for PEP503 repositories
+
+Repositories following the [PEP503](https://peps.python.org/pep-0503/) specification should expose a
+root page with individual links for each package it serves. This isn't reliably implemented
+everywhere, which leads to increased network traffic and slower resolve times. If you're using a
+repository which has a valid listing, you can add the `indexed` property to let Poetry prefetch and
+cache this package list.
+
+```toml
+[[tool.poetry.source]]
+name = "foo"
+url = "https://foo.bar/simple/"
+indexed = true
+```

src/poetry/factory.py

@@ -168,13 +168,15 @@ def create_legacy_repository(
             raise RuntimeError("Missing [name] in source.")
         name = source["name"]
         url = source["url"]
+        indexed = bool(source.get("indexed", False))
 
         return LegacyRepository(
             name,
             url,
             config=auth_config,
             cert=get_cert(auth_config, name),
             client_cert=get_client_cert(auth_config, name),
+            indexed=indexed,
         )
 
     @classmethod

src/poetry/repositories/legacy_repository.py

@@ -8,6 +8,7 @@
 from poetry.inspection.info import PackageInfo
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http import HTTPRepository
+from poetry.repositories.link_sources.html import SimpleIndexPage
 from poetry.repositories.link_sources.html import SimpleRepositoryPage
 from poetry.utils.helpers import canonicalize_name
 
@@ -30,6 +31,7 @@ def __init__(
         disable_cache: bool = False,
         cert: Path | None = None,
         client_cert: Path | None = None,
+        indexed: bool = False,
     ) -> None:
         if name == "pypi":
             raise ValueError("The name [pypi] is reserved for repositories")
@@ -38,6 +40,10 @@ def __init__(
             name, url.rstrip("/"), config, disable_cache, cert, client_cert
         )
 
+        self._index_page = None
+        if indexed:
+            self._index_page = self._get_index_page()
+
     def find_packages(self, dependency: Dependency) -> list[Package]:
         packages = []
         constraint, allow_prereleases = self._get_constraints_from_dependency(
@@ -53,6 +59,11 @@ def find_packages(self, dependency: Dependency) -> list[Package]:
         if self._cache.store("matches").has(key):
             versions = self._cache.store("matches").get(key)
         else:
+            if self._index_page is not None and not self._index_page.serves_package(
+                dependency.name
+            ):
+                return []
+
             page = self._get_page(f"/{dependency.name.replace('.', '-')}/")
             if page is None:
                 return []
@@ -152,3 +163,9 @@ def _get_page(self, endpoint: str) -> SimpleRepositoryPage | None:
         if not response:
             return None
         return SimpleRepositoryPage(response.url, response.text)
+
+    def _get_index_page(self) -> SimpleIndexPage | None:
+        response = self._get_response("")
+        if not response:
+            return None
+        return SimpleIndexPage(response.url, response.text)

src/poetry/repositories/link_sources/html.py

@@ -43,3 +43,30 @@ def __init__(self, url: str, content: str) -> None:
         if not url.endswith("/"):
             url += "/"
         super().__init__(url=url, content=content)
+
+
+class SimpleIndexPage:
+    """Describes the root page of a PEP503 compliant repository.
+
+    This contains a list of links, each one corresponding to a served project.
+    """
+
+    def __init__(self, url: str, content: str) -> None:
+        if not url.endswith("/"):
+            url += "/"
+
+        self._url = url
+        self._content = content
+        self._parsed = html5lib.parse(content, namespaceHTMLElements=False)
+        self._cached_packages = set(self.links)
+
+    @property
+    def links(self) -> Iterator[Link]:
+        # Note: PEP426 specifies that comparisons should be
+        # case-insensitive. For simplicity, we'll do lookups using
+        # lowercase-naming, and treating - and _ equivalently.
+        for anchor in self._parsed.findall(".//a"):
+            yield anchor.text.lower().replace("-", "_")
+
+    def serves_package(self, name: str) -> bool:
+        return name.lower().replace("-", "_") in self._cached_packages

tests/repositories/fixtures/legacy/index.html

@@ -0,0 +1,3 @@
+<a href="pyyaml/">pyyaml</a>
+<a href="missing-version/">missing-version</a>
+<a href="black/">black</a>

tests/repositories/test_legacy_repository.py

@@ -14,6 +14,7 @@
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.exceptions import RepositoryError
 from poetry.repositories.legacy_repository import LegacyRepository
+from poetry.repositories.link_sources.html import SimpleIndexPage
 from poetry.repositories.link_sources.html import SimpleRepositoryPage
 
 
@@ -32,8 +33,10 @@ class MockRepository(LegacyRepository):
 
     FIXTURES = Path(__file__).parent / "fixtures" / "legacy"
 
-    def __init__(self) -> None:
-        super().__init__("legacy", url="http://legacy.foo.bar", disable_cache=True)
+    def __init__(self, indexed: bool = False) -> None:
+        super().__init__(
+            "legacy", url="http://legacy.foo.bar", disable_cache=True, indexed=indexed
+        )
 
     def _get_page(self, endpoint: str) -> SimpleRepositoryPage | None:
         parts = endpoint.split("/")
@@ -338,6 +341,64 @@ def test_get_package_retrieves_packages_with_no_hashes():
     ] == package.files
 
 
+def test_unindexed_has_no_root_page():
+    repo = MockRepository()
+    assert not repo._index_page
+
+
+class MockIndexedRepository(MockRepository):
+    def __init__(self) -> None:
+        super().__init__(True)
+
+    def _get_index_page(self) -> SimpleIndexPage | None:
+        fixture = self.FIXTURES / "index.html"
+        if not fixture.exists():
+            return
+
+        with fixture.open(encoding="utf-8") as f:
+            return SimpleIndexPage(self._url + "/", f.read())
+
+
+def test_indexed_has_root_page():
+    repo = MockIndexedRepository()
+    assert repo._index_page
+
+
+def test_indexed_root_page_has_valid_content():
+    repo = MockIndexedRepository()
+    assert repo._index_page.serves_package("pyyaml")
+
+
+def test_indexed_fails_on_missing():
+    repo = MockIndexedRepository()
+
+    packages = repo.find_packages(Factory.create_dependency("this-doesnt-exist", "*"))
+
+    assert packages == []
+
+
+def test_indexed_succeeds_on_existing():
+    repo = MockIndexedRepository()
+
+    packages = repo.find_packages(Factory.create_dependency("pyyaml", "*"))
+
+    assert len(packages) == 1
+
+
+def test_indexed_pep426_underscore_hyphen():
+    repo = MockIndexedRepository()
+
+    # 'missing-version' in the index
+    assert repo._index_page.serves_package("missing_version")
+
+
+def test_indexed_pep426_case_insensitive():
+    repo = MockIndexedRepository()
+
+    # 'black' in the index
+    assert repo._index_page.serves_package("Black")
+
+
 class MockHttpRepository(LegacyRepository):
     def __init__(self, endpoint_responses: dict, http: type[httpretty.httpretty]):
         base_url = "http://legacy.foo.bar"