diff --git a/docs/notes/2.31.x.md b/docs/notes/2.31.x.md index fde66b6ea0d..3c82d0bd4a6 100644 --- a/docs/notes/2.31.x.md +++ b/docs/notes/2.31.x.md @@ -27,6 +27,12 @@ pantsd now preserves web proxy environment variables (`HTTP_PROXY`, `HTTPS_PROXY ### Goals +#### `generate-lockfiles` + +Previously if any Python resolve set `find-links`, then *all* Python resolves used `find-links` during lockfile generation. Notably this included the `find-links` automatically injected by `pants.backend.plugin_development`. In Pants 2.31, `find-links` are correctly used per resolve. + +Having extraneous un-scoped `find-links` can materially affect dependency resolution time. In some real world user report >30% improvements in `generate-lockfiles` time. + ### Backends #### Helm diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 30750325f8a..d00fdbf4704 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -334,13 +334,13 @@ async def setup_user_lockfile_requests( return UserGenerateLockfiles() resolve_to_requirements_fields = defaultdict(set) - find_links: set[str] = set() + resolve_to_find_links: dict[str, set[str]] = defaultdict(set) for tgt in all_targets: if not tgt.has_fields((PythonRequirementResolveField, PythonRequirementsField)): continue resolve = tgt[PythonRequirementResolveField].normalized_value(python_setup) resolve_to_requirements_fields[resolve].add(tgt[PythonRequirementsField]) - find_links.update(tgt[PythonRequirementFindLinksField].value or ()) + resolve_to_find_links[resolve].update(tgt[PythonRequirementFindLinksField].value or ()) tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase) @@ -352,7 +352,7 @@ async def setup_user_lockfile_requests( requirements=PexRequirements.req_strings_from_requirement_fields( resolve_to_requirements_fields[resolve] ), - find_links=FrozenOrderedSet(find_links), + find_links=FrozenOrderedSet(resolve_to_find_links[resolve]), interpreter_constraints=InterpreterConstraints( python_setup.resolves_to_interpreter_constraints.get( resolve, python_setup.interpreter_constraints @@ -381,7 +381,7 @@ async def setup_user_lockfile_requests( out.add( GeneratePythonLockfile( requirements=FrozenOrderedSet(sorted(tool.requirements)), - find_links=FrozenOrderedSet(find_links), + find_links=FrozenOrderedSet(), interpreter_constraints=ic, resolve_name=resolve, lockfile_dest=DEFAULT_TOOL_LOCKFILE, diff --git a/src/python/pants/backend/python/goals/lockfile_test.py b/src/python/pants/backend/python/goals/lockfile_test.py index 33291ea1099..9709c9edf8e 100644 --- a/src/python/pants/backend/python/goals/lockfile_test.py +++ b/src/python/pants/backend/python/goals/lockfile_test.py @@ -341,6 +341,58 @@ def test_multiple_resolves() -> None: } +def test_find_links_scoped_to_resolve() -> None: + rule_runner = PythonRuleRunner( + rules=[ + setup_user_lockfile_requests, + *PythonSetup.rules(), + QueryRule(UserGenerateLockfiles, [RequestedPythonUserResolveNames]), + ], + target_types=[PythonRequirementTarget], + ) + rule_runner.write_files( + { + "BUILD": dedent( + """\ + # Using the underscore field directly instead of pulling all of + # pants.backend.plugin_development into the test + python_requirement( + name='a', + requirements=['a'], + resolve='a', + _find_links=['https://example.com/wheels'], + ) + python_requirement( + name='b', + requirements=['b'], + resolve='b', + ) + """ + ), + } + ) + rule_runner.set_options( + [ + "--python-resolves={'a': 'a.lock', 'b': 'b.lock'}", + "--python-enable-resolves", + ], + env_inherit=PYTHON_BOOTSTRAP_ENV, + ) + result = rule_runner.request( + UserGenerateLockfiles, [RequestedPythonUserResolveNames(["a", "b"])] + ) + assert all(isinstance(r, GeneratePythonLockfile) for r in result) + result_by_resolve = {r.resolve_name: r for r in result} + assert isinstance(result_by_resolve["a"], GeneratePythonLockfile) + assert isinstance(result_by_resolve["b"], GeneratePythonLockfile) + + assert result_by_resolve["a"].requirements == FrozenOrderedSet(["a"]) + assert result_by_resolve["b"].requirements == FrozenOrderedSet(["b"]) + + assert result_by_resolve["a"].find_links == FrozenOrderedSet(["https://example.com/wheels"]) + assert result_by_resolve["b"].find_links == FrozenOrderedSet([]) + + def test_empty_requirements(rule_runner: PythonRuleRunner) -> None: with pytest.raises(ExecutionError) as excinfo: json.loads(