Patch sysconfig

Author: charliermarsh

Cargo.lock

@@ -72,6 +72,7 @@ windows-result = { workspace = true }
 anyhow = { version = "1.0.89" }
 assert_fs = { version = "1.1.2" }
 indoc = { workspace = true }
+insta = { version = "1.40.0" }
 itertools = { version = "0.13.0" }
 temp-env = { version = "0.3.6" }
 tempfile = { workspace = true }

crates/uv-python/Cargo.toml

@@ -163,6 +163,7 @@ impl PythonInstallation {
 
         let installed = ManagedPythonInstallation::new(path)?;
         installed.ensure_externally_managed()?;
+        installed.ensure_sysconfig_patched()?;
         installed.ensure_canonical_executables()?;
 
         Ok(Self {

crates/uv-python/src/installation.rs

@@ -8,7 +8,7 @@ pub use crate::discovery::{
     find_python_installations, EnvironmentPreference, Error as DiscoveryError, PythonDownloads,
     PythonNotFound, PythonPreference, PythonRequest, PythonSource, PythonVariant, VersionRequest,
 };
-pub use crate::environment::{InvalidEnvironment, InvalidEnvironmentKind, PythonEnvironment};
+pub use crate::environment::{InvalidEnvironmentKind, PythonEnvironment};
 pub use crate::implementation::ImplementationName;
 pub use crate::installation::{PythonInstallation, PythonInstallationKey};
 pub use crate::interpreter::{Error as InterpreterError, Interpreter};
@@ -39,6 +39,7 @@ mod prefix;
 #[cfg(windows)]
 mod py_launcher;
 mod python_version;
+mod sysconfig;
 mod target;
 mod version_files;
 mod virtualenv;

crates/uv-python/src/lib.rs

@@ -25,7 +25,7 @@ use crate::libc::LibcDetectionError;
 use crate::platform::Error as PlatformError;
 use crate::platform::{Arch, Libc, Os};
 use crate::python_version::PythonVersion;
-use crate::{PythonRequest, PythonVariant};
+use crate::{sysconfig, PythonRequest, PythonVariant};
 #[derive(Error, Debug)]
 pub enum Error {
     #[error(transparent)]
@@ -40,6 +40,8 @@ pub enum Error {
     InvalidPythonVersion(String),
     #[error(transparent)]
     ExtractError(#[from] uv_extract::Error),
+    #[error(transparent)]
+    SysconfigError(#[from] sysconfig::Error),
     #[error("Failed to copy to: {0}", to.user_display())]
     CopyError {
         to: PathBuf,
@@ -491,6 +493,18 @@ impl ManagedPythonInstallation {
         Ok(())
     }
 
+    /// Ensure that the `sysconfig` data is patched to match the installation path.
+    pub fn ensure_sysconfig_patched(&self) -> Result<(), Error> {
+        if cfg!(unix) {
+            sysconfig::update_sysconfig(
+                self.path(),
+                self.version().major(),
+                self.version().minor(),
+            )?;
+        }
+        Ok(())
+    }
+
     /// Create a link to the managed Python executable.
     ///
     /// If the file already exists at the target path, an error will be returned.

crates/uv-python/src/managed.rs

@@ -0,0 +1,148 @@
+#![allow(dead_code)]
+
+use std::str::Chars;
+
+pub(super) const EOF_CHAR: char = '\0';
+
+/// A cursor represents a pointer in the source code.
+///
+/// Based on [`rustc`'s `Cursor`](https://github.com/rust-lang/rust/blob/d1b7355d3d7b4ead564dbecb1d240fcc74fff21b/compiler/rustc_lexer/src/cursor.rs)
+#[derive(Clone, Debug)]
+pub(super) struct Cursor<'src> {
+    /// An iterator over the [`char`]'s of the source code.
+    chars: Chars<'src>,
+
+    /// Stores the previous character for debug assertions.
+    #[cfg(debug_assertions)]
+    prev_char: char,
+}
+
+impl<'src> Cursor<'src> {
+    pub(super) fn new(source: &'src str) -> Self {
+        Self {
+            chars: source.chars(),
+            #[cfg(debug_assertions)]
+            prev_char: EOF_CHAR,
+        }
+    }
+
+    /// Returns the previous character. Useful for debug assertions.
+    #[cfg(debug_assertions)]
+    pub(super) const fn previous(&self) -> char {
+        self.prev_char
+    }
+
+    /// Peeks the next character from the input stream without consuming it.
+    /// Returns [`EOF_CHAR`] if the position is past the end of the file.
+    pub(super) fn first(&self) -> char {
+        self.chars.clone().next().unwrap_or(EOF_CHAR)
+    }
+
+    /// Peeks the second character from the input stream without consuming it.
+    /// Returns [`EOF_CHAR`] if the position is past the end of the file.
+    pub(super) fn second(&self) -> char {
+        let mut chars = self.chars.clone();
+        chars.next();
+        chars.next().unwrap_or(EOF_CHAR)
+    }
+
+    /// Returns the remaining text to lex.
+    ///
+    /// Use [`Cursor::text_len`] to get the length of the remaining text.
+    pub(super) fn rest(&self) -> &'src str {
+        self.chars.as_str()
+    }
+
+    /// Returns `true` if the cursor is at the end of file.
+    pub(super) fn is_eof(&self) -> bool {
+        self.chars.as_str().is_empty()
+    }
+
+    /// Moves the cursor to the next character, returning the previous character.
+    /// Returns [`None`] if there is no next character.
+    pub(super) fn bump(&mut self) -> Option<char> {
+        let prev = self.chars.next()?;
+
+        #[cfg(debug_assertions)]
+        {
+            self.prev_char = prev;
+        }
+
+        Some(prev)
+    }
+
+    pub(super) fn eat_char(&mut self, c: char) -> bool {
+        if self.first() == c {
+            self.bump();
+            true
+        } else {
+            false
+        }
+    }
+
+    pub(super) fn eat_char2(&mut self, c1: char, c2: char) -> bool {
+        let mut chars = self.chars.clone();
+        if chars.next() == Some(c1) && chars.next() == Some(c2) {
+            self.bump();
+            self.bump();
+            true
+        } else {
+            false
+        }
+    }
+
+    pub(super) fn eat_char3(&mut self, c1: char, c2: char, c3: char) -> bool {
+        let mut chars = self.chars.clone();
+        if chars.next() == Some(c1) && chars.next() == Some(c2) && chars.next() == Some(c3) {
+            self.bump();
+            self.bump();
+            self.bump();
+            true
+        } else {
+            false
+        }
+    }
+
+    pub(super) fn eat_if<F>(&mut self, mut predicate: F) -> Option<char>
+    where
+        F: FnMut(char) -> bool,
+    {
+        if predicate(self.first()) && !self.is_eof() {
+            self.bump()
+        } else {
+            None
+        }
+    }
+
+    /// Eats symbols while predicate returns true or until the end of file is reached.
+    #[inline]
+    pub(super) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
+        // It was tried making optimized version of this for eg. line comments, but
+        // LLVM can inline all of this and compile it down to fast iteration over bytes.
+        while predicate(self.first()) && !self.is_eof() {
+            self.bump();
+        }
+    }
+
+    /// Skips the next `count` bytes.
+    ///
+    /// ## Panics
+    ///  - If `count` is larger than the remaining bytes in the input stream.
+    ///  - If `count` indexes into a multi-byte character.
+    pub(super) fn skip_bytes(&mut self, count: usize) {
+        #[cfg(debug_assertions)]
+        {
+            self.prev_char = self.chars.as_str()[..count]
+                .chars()
+                .next_back()
+                .unwrap_or('\0');
+        }
+
+        self.chars = self.chars.as_str()[count..].chars();
+    }
+
+    /// Skips to the end of the input stream.
+    pub(super) fn skip_to_end(&mut self) {
+        self.chars = "".chars();
+    }
+}

crates/uv-python/src/sysconfig/cursor.rs

@@ -0,0 +1,164 @@
+//! Patch `sysconfig` data in a Python installation.
+//!
+//! Inspired by: <https://github.com/bluss/sysconfigpatcher/blob/c1ebf8ab9274dcde255484d93ce0f1fd1f76a248/src/sysconfigpatcher.py#L137C1-L140C100>,
+//! available under the MIT license:
+//!
+//! ```text
+//! Copyright 2024 Ulrik Sverdrup "bluss"
+//!
+//! Permission is hereby granted, free of charge, to any person obtaining a copy of
+//! this software and associated documentation files (the "Software"), to deal in
+//! the Software without restriction, including without limitation the rights to
+//! use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+//! the Software, and to permit persons to whom the Software is furnished to do so,
+//! subject to the following conditions:
+//!
+//! The above copyright notice and this permission notice shall be included in all
+//! copies or substantial portions of the Software.
+//!
+//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+//! FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+//! COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+//! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+//! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//! ```
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use tracing::trace;
+
+use crate::sysconfig::parser::{Error as ParseError, SysconfigData, Value};
+
+mod cursor;
+mod parser;
+
+/// Update the `sysconfig` data in a Python installation.
+pub(crate) fn update_sysconfig(install_root: &Path, major: u8, minor: u8) -> Result<(), Error> {
+    // Find the `_sysconfigdata_` file in the Python installation.
+    let real_prefix = std::path::absolute(install_root)?;
+    let sysconfigdata = find_sysconfigdata(&real_prefix, major, minor)?;
+    trace!(
+        "Discovered `sysconfig` data at: {}",
+        sysconfigdata.display()
+    );
+
+    // Update the `_sysconfigdata_` file in-memory.
+    let contents = std::fs::read_to_string(&sysconfigdata)?;
+    let data = patch_sysconfigdata(&contents, &real_prefix)?;
+    let contents = data.to_string_pretty()?;
+
+    // Write the updated `_sysconfigdata_` file.
+    let mut file = std::fs::OpenOptions::new()
+        .write(true)
+        .truncate(true)
+        .create(true)
+        .open(&sysconfigdata)?;
+    file.write_all(contents.as_bytes())?;
+    file.sync_data()?;
+
+    Ok(())
+}
+
+/// Find the `_sysconfigdata_` file in a Python installation.
+///
+/// For example, on macOS, returns `{real_prefix}/lib/python3.12/_sysconfigdata__darwin_darwin.py"`.
+fn find_sysconfigdata(real_prefix: &Path, major: u8, minor: u8) -> Result<PathBuf, Error> {
+    // Find the `lib` directory in the Python installation.
+    let lib = real_prefix
+        .join("lib")
+        .join(format!("python{major}.{minor}"));
+    if !lib.exists() {
+        return Err(Error::MissingLib);
+    }
+
+    // Probe the `lib` directory for `_sysconfigdata_`.
+    for entry in lib.read_dir()? {
+        let entry = entry?;
+
+        if entry.path().extension().is_none_or(|ext| ext != "py") {
+            continue;
+        }
+
+        if !entry
+            .path()
+            .file_stem()
+            .and_then(|stem| stem.to_str())
+            .is_some_and(|stem| stem.starts_with("_sysconfigdata_"))
+        {
+            continue;
+        }
+
+        let metadata = entry.metadata()?;
+        if metadata.is_symlink() {
+            continue;
+        };
+
+        if metadata.is_file() {
+            return Ok(entry.path());
+        }
+    }
+
+    Err(Error::MissingSysconfigdata)
+}
+
+/// Patch the given `_sysconfigdata_` contents.
+fn patch_sysconfigdata(contents: &str, real_prefix: &Path) -> Result<SysconfigData, Error> {
+    /// Update the `/install` prefix in a whitespace-separated string.
+    fn update_prefix(s: &str, real_prefix: &Path) -> String {
+        s.split_whitespace()
+            .map(|part| {
+                if let Some(rest) = part.strip_prefix("/install") {
+                    if rest.is_empty() {
+                        real_prefix.display().to_string()
+                    } else {
+                        real_prefix.join(&rest[1..]).display().to_string()
+                    }
+                } else {
+                    part.to_string()
+                }
+            })
+            .collect::<Vec<_>>()
+            .join(" ")
+    }
+
+    // Parse the `_sysconfigdata_` file.
+    let mut data = SysconfigData::from_str(contents)?;
+
+    // Patch each value, as needed.
+    let mut count = 0;
+    for (key, value) in data.iter_mut() {
+        let Value::String(value) = value else {
+            continue;
+        };
+        let patched = update_prefix(value, real_prefix);
+        if *value != patched {
+            trace!("Updated `{key}` from `{value}` to `{patched}`");
+            count += 1;
+            *value = patched;
+        }
+    }
+
+    match count {
+        0 => trace!("No updates required"),
+        1 => trace!("Updated 1 value"),
+        n => trace!("Updated {n} values"),
+    }
+
+    Ok(data)
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+    #[error("Python installation is missing a `lib` directory")]
+    MissingLib,
+    #[error("Python installation is missing a `_sysconfigdata_` file")]
+    MissingSysconfigdata,
+    #[error(transparent)]
+    Parse(#[from] ParseError),
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+}