use shutil.which() to detect python executables

Author: dimbleby

src/poetry/utils/env.py

@@ -9,6 +9,7 @@
 import platform
 import plistlib
 import re
+import shutil
 import subprocess
 import sys
 import sysconfig
@@ -472,6 +473,11 @@ def __init__(self, e: CalledProcessError, input: str | None = None) -> None:
         super().__init__("\n\n".join(message_parts))
 
 
+class PythonVersionNotFound(EnvError):
+    def __init__(self, expected: str) -> None:
+        super().__init__(f"Could not find the python executable {expected}")
+
+
 class NoCompatiblePythonVersionFound(EnvError):
     def __init__(self, expected: str, given: str | None = None) -> None:
         if given:
@@ -517,41 +523,47 @@ def __init__(self, poetry: Poetry, io: None | IO = None) -> None:
         self._io = io or NullIO()
 
     @staticmethod
-    def _full_python_path(python: str) -> Path:
+    def _full_python_path(python: str) -> Path | None:
+        # eg first find pythonXY.bat on windows.
+        path_python = shutil.which(python)
+        if path_python is None:
+            return None
+
         try:
             executable = decode(
                 subprocess.check_output(
-                    [python, "-c", "import sys; print(sys.executable)"],
+                    [path_python, "-c", "import sys; print(sys.executable)"],
                 ).strip()
             )
-        except CalledProcessError as e:
-            raise EnvCommandError(e)
+            return Path(executable)
 
-        return Path(executable)
+        except CalledProcessError:
+            return None
 
     @staticmethod
     def _detect_active_python(io: None | IO = None) -> Path | None:
         io = io or NullIO()
-        executable = None
+        io.write_error_line(
+            (
+                "Trying to detect current active python executable as specified in"
+                " the config."
+            ),
+            verbosity=Verbosity.VERBOSE,
+        )
 
-        try:
-            io.write_error_line(
-                (
-                    "Trying to detect current active python executable as specified in"
-                    " the config."
-                ),
-                verbosity=Verbosity.VERBOSE,
-            )
-            executable = EnvManager._full_python_path("python")
+        executable = EnvManager._full_python_path("python")
+
+        if executable is not None:
             io.write_error_line(f"Found: {executable}", verbosity=Verbosity.VERBOSE)
-        except EnvCommandError:
+        else:
             io.write_error_line(
                 (
                     "Unable to detect the current active python executable. Falling"
                     " back to default."
                 ),
                 verbosity=Verbosity.VERBOSE,
             )
+
         return executable
 
     @staticmethod
@@ -592,6 +604,8 @@ def activate(self, python: str) -> Env:
             pass
 
         python_path = self._full_python_path(python)
+        if python_path is None:
+            raise PythonVersionNotFound(python)
 
         try:
             python_version_string = decode(
@@ -949,25 +963,26 @@ def create_venv(
                 "Trying to find and use a compatible version.</warning> "
             )
 
-            for python_to_try in sorted(
+            for suffix in sorted(
                 self._poetry.package.AVAILABLE_PYTHONS,
                 key=lambda v: (v.startswith("3"), -len(v), v),
                 reverse=True,
             ):
-                if len(python_to_try) == 1:
-                    if not parse_constraint(f"^{python_to_try}.0").allows_any(
+                if len(suffix) == 1:
+                    if not parse_constraint(f"^{suffix}.0").allows_any(
                         supported_python
                     ):
                         continue
-                elif not supported_python.allows_any(
-                    parse_constraint(python_to_try + ".*")
-                ):
+                elif not supported_python.allows_any(parse_constraint(suffix + ".*")):
                     continue
 
-                python = "python" + python_to_try
-
+                python_name = f"python{suffix}"
                 if self._io.is_debug():
-                    self._io.write_error_line(f"<debug>Trying {python}</debug>")
+                    self._io.write_error_line(f"<debug>Trying {python_name}</debug>")
+
+                python = self._full_python_path(python_name)
+                if python is None:
+                    continue
 
                 try:
                     python_patch = decode(
@@ -979,14 +994,11 @@ def create_venv(
                 except CalledProcessError:
                     continue
 
-                if not python_patch:
-                    continue
-
                 if supported_python.allows(Version.parse(python_patch)):
                     self._io.write_error_line(
-                        f"Using <c1>{python}</c1> ({python_patch})"
+                        f"Using <c1>{python_name}</c1> ({python_patch})"
                     )
-                    executable = self._full_python_path(python)
+                    executable = python
                     python_minor = ".".join(python_patch.split(".")[:2])
                     break
 

tests/console/commands/env/helpers.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import os
+
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
@@ -28,9 +30,11 @@ def check_output(cmd: list[str], *args: Any, **kwargs: Any) -> str:
         elif "sys.version_info[:2]" in python_cmd:
             return f"{version.major}.{version.minor}"
         elif "import sys; print(sys.executable)" in python_cmd:
-            return f"/usr/bin/{cmd[0]}"
+            executable = cmd[0]
+            basename = os.path.basename(executable)
+            return f"/usr/bin/{basename}"
         else:
             assert "import sys; print(sys.prefix)" in python_cmd
-            return str(Path("/prefix"))
+            return "/prefix"
 
     return check_output

tests/console/commands/env/test_use.py

@@ -56,6 +56,7 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file(
     venv_name: str,
     venvs_in_cache_config: None,
 ) -> None:
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(),

tests/utils/test_env.py

@@ -27,6 +27,7 @@
 from poetry.utils.env import InvalidCurrentPythonVersionError
 from poetry.utils.env import MockEnv
 from poetry.utils.env import NoCompatiblePythonVersionFound
+from poetry.utils.env import PythonVersionNotFound
 from poetry.utils.env import SystemEnv
 from poetry.utils.env import VirtualEnv
 from poetry.utils.env import build_environment
@@ -197,10 +198,12 @@ def check_output(cmd: list[str], *args: Any, **kwargs: Any) -> str:
         elif "sys.version_info[:2]" in python_cmd:
             return f"{version.major}.{version.minor}"
         elif "import sys; print(sys.executable)" in python_cmd:
-            return f"/usr/bin/{cmd[0]}"
+            executable = cmd[0]
+            basename = os.path.basename(executable)
+            return f"/usr/bin/{basename}"
         else:
             assert "import sys; print(sys.prefix)" in python_cmd
-            return str(Path("/prefix"))
+            return "/prefix"
 
     return check_output
 
@@ -218,6 +221,7 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file(
 
     config.merge({"virtualenvs": {"path": str(tmp_dir)}})
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(),
@@ -252,6 +256,30 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file(
     assert env.base == Path("/prefix")
 
 
+def test_activate_fails_when_python_cannot_be_found(
+    tmp_dir: str,
+    manager: EnvManager,
+    poetry: Poetry,
+    config: Config,
+    mocker: MockerFixture,
+    venv_name: str,
+) -> None:
+    if "VIRTUAL_ENV" in os.environ:
+        del os.environ["VIRTUAL_ENV"]
+
+    os.mkdir(os.path.join(tmp_dir, f"{venv_name}-py3.7"))
+
+    config.merge({"virtualenvs": {"path": str(tmp_dir)}})
+
+    mocker.patch("shutil.which", return_value=None)
+
+    with pytest.raises(PythonVersionNotFound) as e:
+        manager.activate("python3.7")
+
+    expected_message = "Could not find the python executable python3.7"
+    assert str(e.value) == expected_message
+
+
 def test_activate_activates_existing_virtualenv_no_envs_file(
     tmp_dir: str,
     manager: EnvManager,
@@ -267,6 +295,7 @@ def test_activate_activates_existing_virtualenv_no_envs_file(
 
     config.merge({"virtualenvs": {"path": str(tmp_dir)}})
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(),
@@ -311,6 +340,7 @@ def test_activate_activates_same_virtualenv_with_envs_file(
 
     config.merge({"virtualenvs": {"path": str(tmp_dir)}})
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(),
@@ -354,6 +384,7 @@ def test_activate_activates_different_virtualenv_with_envs_file(
 
     config.merge({"virtualenvs": {"path": str(tmp_dir)}})
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(Version.parse("3.6.6")),
@@ -407,6 +438,7 @@ def test_activate_activates_recreates_for_different_patch(
 
     config.merge({"virtualenvs": {"path": str(tmp_dir)}})
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(),
@@ -474,6 +506,7 @@ def test_activate_does_not_recreate_when_switching_minor(
 
     config.merge({"virtualenvs": {"path": str(tmp_dir)}})
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(Version.parse("3.6.6")),
@@ -1070,6 +1103,7 @@ def test_create_venv_tries_to_find_a_compatible_python_executable_using_generic_
     poetry.package.python_versions = "^3.6"
 
     mocker.patch("sys.version_info", (2, 7, 16))
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(Version.parse("3.7.5")),
@@ -1093,6 +1127,34 @@ def test_create_venv_tries_to_find_a_compatible_python_executable_using_generic_
     )
 
 
+def test_create_venv_finds_no_python_executable(
+    manager: EnvManager,
+    poetry: Poetry,
+    config: Config,
+    mocker: MockerFixture,
+    config_virtualenvs_path: Path,
+    venv_name: str,
+) -> None:
+    if "VIRTUAL_ENV" in os.environ:
+        del os.environ["VIRTUAL_ENV"]
+
+    poetry.package.python_versions = "^3.6"
+
+    mocker.patch("sys.version_info", (2, 7, 16))
+    mocker.patch("shutil.which", return_value=None)
+
+    with pytest.raises(NoCompatiblePythonVersionFound) as e:
+        manager.create_venv()
+
+    expected_message = (
+        "Poetry was unable to find a compatible version. "
+        "If you have one, you can explicitly use it "
+        'via the "env use" command.'
+    )
+
+    assert str(e.value) == expected_message
+
+
 def test_create_venv_tries_to_find_a_compatible_python_executable_using_specific_ones(
     manager: EnvManager,
     poetry: Poetry,
@@ -1107,8 +1169,10 @@ def test_create_venv_tries_to_find_a_compatible_python_executable_using_specific
     poetry.package.python_versions = "^3.6"
 
     mocker.patch("sys.version_info", (2, 7, 16))
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
-        "subprocess.check_output", side_effect=["3.5.3", "3.9.0", "/usr/bin/python3.9"]
+        "subprocess.check_output",
+        side_effect=["/usr/bin/python3", "3.5.3", "/usr/bin/python3.9", "3.9.0"],
     )
     m = mocker.patch(
         "poetry.utils.env.EnvManager.build_venv", side_effect=lambda *args, **kwargs: ""
@@ -1309,6 +1373,7 @@ def test_activate_with_in_project_setting_does_not_fail_if_no_venvs_dir(
         }
     )
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(),
@@ -1546,13 +1611,15 @@ def test_create_venv_accepts_fallback_version_w_nonzero_patchlevel(
 
     def mock_check_output(cmd: str, *args: Any, **kwargs: Any) -> str:
         if GET_PYTHON_VERSION_ONELINER in cmd:
-            if "python3.5" in cmd:
+            executable = cmd[0]
+            if "python3.5" in str(executable):
                 return "3.5.12"
             else:
                 return "3.7.1"
         else:
             return "/usr/bin/python3.5"
 
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     check_output = mocker.patch(
         "subprocess.check_output",
         side_effect=mock_check_output,
@@ -1662,6 +1729,7 @@ def test_create_venv_project_name_empty_sets_correct_prompt(
     venv_name = manager.generate_env_name("", str(poetry.file.parent))
 
     mocker.patch("sys.version_info", (2, 7, 16))
+    mocker.patch("shutil.which", side_effect=lambda py: f"/usr/bin/{py}")
     mocker.patch(
         "subprocess.check_output",
         side_effect=check_output_wrapper(Version.parse("3.7.5")),