Merge pull request #209 from robotpy/fix-cross-compilation

Fix installation of build dependencies for cross compilation

Author: virtuald

.github/workflows/dist.yml

@@ -258,7 +258,7 @@ jobs:
     - name: Install deps
       shell: bash
       run: |
-        /build/venv/bin/cross-pip --disable-pip-version-check install -r rdev_requirements.txt
+        /build/venv/bin/build-pip --disable-pip-version-check install -r rdev_requirements.txt
 
     - name: Build + test wheels
       shell: bash
@@ -320,7 +320,7 @@ jobs:
     - name: Install deps
       shell: bash
       run: |
-        /build/venv/bin/cross-pip --disable-pip-version-check install -r rdev_requirements.txt
+        /build/venv/bin/build-pip --disable-pip-version-check install -r rdev_requirements.txt
 
     - name: Build wheels
       shell: bash

devtools/__main__.py

@@ -7,7 +7,6 @@
 from .ctx import Context
 from . import ci
 from . import update_pyproject
-from . import util
 
 
 #
@@ -83,7 +82,7 @@ def install_prereqs(ctx: Context):
                 if req.name not in repo_deps:
                     reqs.add(req)
 
-    util.run_pip("install", *map(str, reqs))
+    ctx.run_pip("install", *map(str, reqs))
 
 
 @main.command()

devtools/ctx.py

@@ -3,13 +3,13 @@
 import subprocess
 import sys
 import sysconfig
-import typing
+import typing as T
 
 import toposort
 
 from . import config
 from .subproject import Subproject
-from .util import run_pip
+from .util import run_cmd
 
 
 class Context:
@@ -27,13 +27,13 @@ def __init__(self, verbose: bool) -> None:
         self.wheel_path = self.root_path / "dist"
         self.other_wheel_path = self.root_path / "dist-other"
 
-        subprojects: typing.List[Subproject] = []
+        subprojects: T.List[Subproject] = []
         for project, cfg in self.cfg.subprojects.items():
             # Skip projects that aren't compatible with the robot
             if self.is_robot and not cfg.robot:
                 continue
 
-            subprojects.append(Subproject(cfg, self.subprojects_path / project))
+            subprojects.append(Subproject(self, cfg, self.subprojects_path / project))
 
         # Create a sorted dictionary of subprojects ordered by build order
         si = {p.pyproject_name: i for i, p in enumerate(subprojects)}
@@ -47,6 +47,29 @@ def __init__(self, verbose: bool) -> None:
             for i in toposort.toposort_flatten(ti, sort=False)
         }
 
+        # build_python is for build dependencies, python is for the target environment
+        # - if crossenv is specified, then we use that instead
+        self._build_python = None
+        self.python = sys.executable
+
+    @property
+    def build_python(self):
+        if self._build_python is None:
+            self._build_python = self.python
+
+            # try to detect if we're running in crossenv's cross python and
+            # use the build python instead
+            if getattr(sys, "cross_compiling", False) == True:
+                pth = pathlib.Path(self._build_python).resolve()
+                if pth.parts[-3:-1] == ("cross", "bin"):
+                    self._build_python = str(
+                        pathlib.Path(
+                            *(pth.parts[:-3] + ("build", "bin", pth.parts[-1]))
+                        )
+                    )
+
+        return self._build_python
+
     def git_commit(self, msg: str, *relpath: str):
         subprocess.run(
             ["git", "commit", "-F", "-", "--"] + list(relpath),
@@ -102,13 +125,14 @@ def install_build_deps(
                 external.append(req)
 
         if external:
-            run_pip(
+            self.run_pip(
                 "install",
                 *[str(req) for req in external],
+                installing_build_deps=True,
             )
 
         if internal:
-            run_pip(
+            self.run_pip(
                 "install",
                 "--no-index",
                 "--find-links",
@@ -117,3 +141,11 @@ def install_build_deps(
                 str(self.other_wheel_path),
                 *[str(req) for req in internal],
             )
+
+    def run_pip(self, *args: str, cwd=None, installing_build_deps: bool = False):
+        if installing_build_deps:
+            python = self.build_python
+        else:
+            python = self.python
+
+        run_cmd(python, "-m", "pip", "--disable-pip-version-check", *args, cwd=cwd)

devtools/subproject.py

@@ -8,11 +8,17 @@
 import tomli
 
 from .config import SubprojectConfig
-from .util import run_cmd, run_pip
+from .util import run_cmd
+
+if T.TYPE_CHECKING:
+    from .ctx import Context
 
 
 class Subproject:
-    def __init__(self, cfg: SubprojectConfig, path: pathlib.Path) -> None:
+    def __init__(
+        self, ctx: "Context", cfg: SubprojectConfig, path: pathlib.Path
+    ) -> None:
+        self.ctx = ctx
         self.cfg = cfg
         self.path = path
         self.pyproject_path = self.path / "pyproject.toml"
@@ -43,15 +49,17 @@ def is_meson_project(self) -> bool:
     #
 
     def develop(self):
-        run_pip("install", "-v", "-e", ".", "--no-build-isolation", cwd=self.path)
+        self.ctx.run_pip(
+            "install", "-v", "-e", ".", "--no-build-isolation", cwd=self.path
+        )
 
     def uninstall(self):
-        run_pip("uninstall", "-y", self.pyproject_name)
+        self.ctx.run_pip("uninstall", "-y", self.pyproject_name)
 
     def scan_headers(self):
         """Returns True if no headers found or False if missing headers were found"""
         result = run_cmd(
-            sys.executable,
+            self.ctx.python,
             "-m",
             "semiwrap",
             "scan-headers",
@@ -64,7 +72,7 @@ def scan_headers(self):
     def update_yaml(self):
         """Resyncs the yaml files with their header files"""
         result = run_cmd(
-            sys.executable,
+            self.ctx.python,
             "-m",
             "semiwrap",
             "update-yaml",
@@ -76,7 +84,7 @@ def update_yaml(self):
 
     def update_init(self):
         run_cmd(
-            sys.executable,
+            self.ctx.python,
             "-m",
             "semiwrap",
             "update-init",
@@ -91,14 +99,14 @@ def test(self, *, install_requirements=False):
         if install_requirements:
             requirements = tests_path / "requirements.txt"
             if requirements.exists():
-                run_pip(
+                self.ctx.run_pip(
                     "install",
                     "-r",
                     str(requirements),
                 )
 
         run_cmd(
-            sys.executable,
+            self.ctx.python,
             "run_tests.py",
             cwd=tests_path,
         )
@@ -120,7 +128,7 @@ def build_wheel(
         with tempfile.TemporaryDirectory() as td:
             # I wonder if we should use hatch build instead?
             run_cmd(
-                sys.executable,
+                self.ctx.python,
                 "-m",
                 "build",
                 "--no-isolation",
@@ -138,7 +146,7 @@ def build_wheel(
 
         if install:
             # Install the wheel
-            run_pip(
+            self.ctx.run_pip(
                 "install",
                 "--find-links",
                 str(wheel_path),

devtools/util.py

@@ -47,7 +47,3 @@ def parse_input(value: typing.Any, spec: typing.Type[T], fname) -> T:
 def run_cmd(*args: str, cwd=None, check=True):
     print("+", shlex.join(args))
     return subprocess.run(args, cwd=cwd, check=check)
-
-
-def run_pip(*args: str, cwd=None):
-    run_cmd(sys.executable, "-m", "pip", "--disable-pip-version-check", *args, cwd=cwd)