feat(cli, python): programmatic interface to cli with python bindings (#2677)

Closes #2671

## Goals
1. Have `hugr` cli available in environment when installed via uv (`uvx
hugr --help`)
2. Allow use of cli functionality from python package without spawning
subprocess

## Implications
- Either ship binary with wheel or bind the function directly - chose
the latter as it is more flexible and avoids a bunch of build faff
- CLI uses argc + stdin for input and stdout for output (by default), we
need a way to do this via binding i/o to avoid subprocess

## Primary outcomes
- Cli invocation bound via pyo3 to python "package script" with same
name as binary, meeting goal 1
- hugr-cli refactored to read inputs from a reader and write outputs to
a writer when appropriate, satisfying goal 2. Uses an "override" pattern
- when i/o is specified it overrides the corresponding cli-arg

## Secondary outcomes
- CLI main.rs contents moved to lib.rs, allowing use by library
dependants
- Make tracing a default feature for hugr-cli (not used by hugr-py)
- `cli` module in python to wrap core sub-commands and parameters, with
simply typed interfaces
- Python test cli use replaced with bound function calls - meaning no
need to build cli before running python tests
- Model loading in python via conversion to json uses cli convert
function rather than dedicated binding
- Dedicated cargo profile for python wheel for smaller wheels (very open
to critique on this)
- Pydantic models to parse `describe` json output
- Remove parallel pytest since it is now faster without it

## Possible issues
- Versioning: the cli version is pinned to the rust crates, the python
package is not. Meaning `uvx hugr --version` will report a different
value to `uv --with hugr run python -c "import hugr;
print(hugr.__version__)"`. Are we ok with this?

---------

Co-authored-by: Alec Edgington <[email protected]>

Author: ss2165

.github/workflows/ci-py.yml

@@ -13,8 +13,6 @@ on:
 
 env:
   SCCACHE_GHA_ENABLED: "true"
-  HUGR_BIN_DIR: ${{ github.workspace }}/target/debug
-  HUGR_BIN: ${{ github.workspace }}/target/debug/hugr
   # Pinned version for the uv package manager
   UV_VERSION: "0.9.5"
   UV_FROZEN: 1
@@ -68,30 +66,8 @@ jobs:
       - name: Lint with ruff
         run: uv run ruff check
 
-  build_binary:
-    needs: changes
-    if: ${{ needs.changes.outputs.python == 'true' }}
-
-    name: Build HUGR binary
-    runs-on: ubuntu-latest
-    env:
-      SCCACHE_GHA_ENABLED: "true"
-      RUSTC_WRAPPER: "sccache"
-
-    steps:
-      - uses: actions/checkout@v5
-      - uses: mozilla-actions/[email protected]
-      - name: Install stable toolchain
-        uses: dtolnay/rust-toolchain@stable
-      - name: Build HUGR binary
-        run: cargo build -p hugr-cli
-      - name: Upload the binary to the artifacts
-        uses: actions/upload-artifact@v5
-        with:
-          name: hugr_binary
-          path: target/debug/hugr
   test:
-    needs: [changes, build_binary]
+    needs: [changes]
     if: ${{ needs.changes.outputs.python == 'true' }}
     name: test python ${{ matrix.python-version.py }}
     runs-on: ubuntu-latest
@@ -110,12 +86,6 @@ jobs:
           version: ${{ env.UV_VERSION }}
           enable-cache: true
 
-      - name: Download the hugr binary
-        uses: actions/download-artifact@v6
-        with:
-          name: hugr_binary
-          path: ${{env.HUGR_BIN_DIR}}
-
       - name: Setup dependencies
         run: uv sync --python ${{ matrix.python-version.py }}
 
@@ -125,13 +95,11 @@ jobs:
       - name: Run tests
         if: github.event_name == 'merge_group' || !matrix.python-version.coverage
         run: |
-          chmod +x $HUGR_BIN
           HUGR_RENDER_DOT=1 uv run pytest
 
       - name: Run python tests with coverage instrumentation
         if: github.event_name != 'merge_group' && matrix.python-version.coverage
         run: |
-          chmod +x $HUGR_BIN
           HUGR_RENDER_DOT=1 uv run pytest --cov=./ --cov-report=xml
 
       - name: Upload python coverage to codecov.io

Cargo.lock

@@ -129,3 +129,10 @@ jsonschema.debug = 1
 [profile.dist]
 inherits = "release"
 lto = "thin"
+
+# The profile that 'hugr-py' will build with
+[profile.release-py]
+inherits = "release"
+lto = "fat"
+strip = "symbols"
+panic = "abort"

Cargo.toml

@@ -15,6 +15,10 @@ categories = ["compilers"]
 [lib]
 bench = false
 
+[features]
+default = ["tracing"]
+tracing = ["dep:tracing", "dep:tracing-subscriber"]
+
 [dependencies]
 clap = { workspace = true, features = ["derive", "cargo"] }
 clap-verbosity-flag.workspace = true
@@ -25,8 +29,8 @@ serde = { workspace = true, features = ["derive"] }
 clio = { workspace = true, features = ["clap-parse"] }
 anyhow.workspace = true
 thiserror.workspace = true
-tracing = "0.1.41"
-tracing-subscriber = { version = "0.3.20", features = ["fmt"] }
+tracing = { version = "0.1.41", optional = true }
+tracing-subscriber = { version = "0.3.20", features = ["fmt"], optional = true }
 tabled = "0.20.0"
 schemars = { workspace = true, features = ["derive"] }
 

hugr-cli/Cargo.toml

@@ -3,6 +3,7 @@ use anyhow::Result;
 use clap::Parser;
 use clio::Output;
 use hugr::envelope::{EnvelopeConfig, EnvelopeFormat, ZstdConfig};
+use std::io::{Read, Write};
 
 use crate::CliError;
 use crate::hugr_io::HugrInputArgs;
@@ -47,9 +48,20 @@ pub struct ConvertArgs {
 }
 
 impl ConvertArgs {
-    /// Convert a HUGR between different envelope formats
-    pub fn run_convert(&mut self) -> Result<()> {
-        let (env_config, package) = self.input_args.get_described_package()?;
+    /// Convert a HUGR between different envelope formats with optional input/output overrides.
+    ///
+    /// # Arguments
+    ///
+    /// * `input_override` - Optional reader to use instead of the CLI input argument.
+    /// * `output_override` - Optional writer to use instead of the CLI output argument.
+    pub fn run_convert_with_io<R: Read, W: Write>(
+        &mut self,
+        input_override: Option<R>,
+        mut output_override: Option<W>,
+    ) -> Result<()> {
+        let (env_config, package) = self
+            .input_args
+            .get_described_package_with_reader(input_override)?;
 
         // Handle text and binary format flags, which override the format option
         let mut config = if self.text {
@@ -78,8 +90,17 @@ impl ConvertArgs {
         }
 
         // Write the package with the requested format
-        hugr::envelope::write_envelope(&mut self.output, &package, config)?;
+        if let Some(ref mut writer) = output_override {
+            hugr::envelope::write_envelope(writer, &package, config)?;
+        } else {
+            hugr::envelope::write_envelope(&mut self.output, &package, config)?;
+        }
 
         Ok(())
     }
+
+    /// Convert a HUGR between different envelope formats
+    pub fn run_convert(&mut self) -> Result<()> {
+        self.run_convert_with_io(None::<&[u8]>, None::<Vec<u8>>)
+    }
 }

hugr-cli/src/convert.rs

@@ -8,7 +8,7 @@ use hugr::envelope::ReadError;
 use hugr::envelope::description::{ExtensionDesc, ModuleDesc, PackageDesc};
 use hugr::extension::Version;
 use hugr::package::Package;
-use std::io::Write;
+use std::io::{Read, Write};
 use tabled::Tabled;
 use tabled::derive::display;
 
@@ -73,20 +73,37 @@ impl ModuleArgs {
     }
 }
 impl DescribeArgs {
-    /// Load and describe the HUGR package.
-    pub fn run_describe(&mut self) -> Result<()> {
+    /// Load and describe the HUGR package with optional input/output overrides.
+    ///
+    /// # Arguments
+    ///
+    /// * `input_override` - Optional reader to use instead of the CLI input argument.
+    /// * `output_override` - Optional writer to use instead of the CLI output argument.
+    pub fn run_describe_with_io<R: Read, W: Write>(
+        &mut self,
+        input_override: Option<R>,
+        mut output_override: Option<W>,
+    ) -> Result<()> {
         if self.json_schema {
             let schema = schemars::schema_for!(PackageDescriptionJson);
             let schema_json = serde_json::to_string_pretty(&schema)?;
-            writeln!(self.output, "{schema_json}")?;
+            if let Some(ref mut writer) = output_override {
+                writeln!(writer, "{schema_json}")?;
+            } else {
+                writeln!(self.output, "{schema_json}")?;
+            }
             return Ok(());
         }
-        let (mut desc, res) = match self.input_args.get_described_package() {
+
+        let (mut desc, res) = match self
+            .input_args
+            .get_described_package_with_reader(input_override)
+        {
             Ok((desc, pkg)) => (desc, Ok(pkg)),
             Err(crate::CliError::ReadEnvelope(ReadError::Payload {
                 source,
                 partial_description,
-            })) => (partial_description, Err(source)), // keep error for later
+            })) => (partial_description, Err(source)),
             Err(e) => return Err(e.into()),
         };
 
@@ -96,91 +113,116 @@ impl DescribeArgs {
         }
 
         let res = res.map_err(anyhow::Error::from);
+
+        let writer: &mut dyn Write = if let Some(ref mut w) = output_override {
+            w
+        } else {
+            &mut self.output
+        };
+
         if self.json {
             if !self.packaged_extensions {
                 desc.packaged_extensions.clear();
             }
-            self.output_json(desc, &res)?;
+            output_json(desc, &res, writer)?;
         } else {
-            self.print_description(desc)?;
+            print_description(desc, self.packaged_extensions, writer)?;
         }
 
         // bubble up any errors
         res.map(|_| ())
     }
 
-    fn print_description(&mut self, desc: PackageDesc) -> Result<()> {
-        let header = desc.header();
-        let n_modules = desc.n_modules();
-        let n_extensions = desc.n_packaged_extensions();
-        let module_str = if n_modules == 1 { "module" } else { "modules" };
-        let extension_str = if n_extensions == 1 {
-            "extension"
-        } else {
-            "extensions"
-        };
-        writeln!(
-            self.output,
-            "{header}\nPackage contains {n_modules} {module_str} and {n_extensions} {extension_str}",
-        )?;
-        let summaries: Vec<ModuleSummary> = desc
-            .modules
-            .iter()
-            .map(|m| m.as_ref().map(Into::into).unwrap_or_default())
-            .collect();
-        let summary_table = tabled::Table::builder(summaries).index().build();
-        writeln!(self.output, "{summary_table}")?;
+    /// Load and describe the HUGR package.
+    pub fn run_describe(&mut self) -> Result<()> {
+        self.run_describe_with_io(None::<&[u8]>, None::<Vec<u8>>)
+    }
+}
 
-        for (i, module) in desc.modules.into_iter().enumerate() {
-            writeln!(self.output, "\nModule {i}:")?;
-            if let Some(module) = module {
-                self.display_module(module)?;
-            }
-        }
-        if self.packaged_extensions {
-            writeln!(self.output, "Packaged extensions:")?;
-            let ext_rows: Vec<ExtensionRow> = desc
-                .packaged_extensions
-                .into_iter()
-                .flatten()
-                .map(Into::into)
-                .collect();
-            let ext_table = tabled::Table::new(ext_rows);
-            writeln!(self.output, "{ext_table}")?;
+/// Print a human-readable description of a package.
+fn print_description<W: Write + ?Sized>(
+    desc: PackageDesc,
+    show_packaged_extensions: bool,
+    writer: &mut W,
+) -> Result<()> {
+    let header = desc.header();
+    let n_modules = desc.n_modules();
+    let n_extensions = desc.n_packaged_extensions();
+    let module_str = if n_modules == 1 { "module" } else { "modules" };
+    let extension_str = if n_extensions == 1 {
+        "extension"
+    } else {
+        "extensions"
+    };
+
+    writeln!(
+        writer,
+        "{header}\nPackage contains {n_modules} {module_str} and {n_extensions} {extension_str}",
+    )?;
+
+    let summaries: Vec<ModuleSummary> = desc
+        .modules
+        .iter()
+        .map(|m| m.as_ref().map(Into::into).unwrap_or_default())
+        .collect();
+    let summary_table = tabled::Table::builder(summaries).index().build();
+    writeln!(writer, "{summary_table}")?;
+
+    for (i, module) in desc.modules.into_iter().enumerate() {
+        writeln!(writer, "\nModule {i}:")?;
+        if let Some(module) = module {
+            display_module(module, writer)?;
         }
-        Ok(())
     }
-
-    fn output_json(&mut self, package_desc: PackageDesc, res: &Result<Package>) -> Result<()> {
-        let err_str = res.as_ref().err().map(|e| format!("{e:?}"));
-        let json_desc = PackageDescriptionJson {
-            package_desc,
-            error: err_str,
-        };
-        serde_json::to_writer_pretty(&mut self.output, &json_desc)?;
-        Ok(())
+    if show_packaged_extensions {
+        writeln!(writer, "Packaged extensions:")?;
+        let ext_rows: Vec<ExtensionRow> = desc
+            .packaged_extensions
+            .into_iter()
+            .flatten()
+            .map(Into::into)
+            .collect();
+        let ext_table = tabled::Table::new(ext_rows);
+        writeln!(writer, "{ext_table}")?;
     }
+    Ok(())
+}
 
-    fn display_module(&mut self, desc: ModuleDesc) -> Result<()> {
-        if let Some(exts) = desc.used_extensions_resolved {
-            let ext_rows: Vec<ExtensionRow> = exts.into_iter().map(Into::into).collect();
-            let ext_table = tabled::Table::new(ext_rows);
-            writeln!(self.output, "Resolved extensions:\n{ext_table}")?;
-        }
+/// Output a package description as JSON.
+fn output_json<W: Write + ?Sized>(
+    package_desc: PackageDesc,
+    res: &Result<Package>,
+    writer: &mut W,
+) -> Result<()> {
+    let err_str = res.as_ref().err().map(|e| format!("{e:?}"));
+    let json_desc = PackageDescriptionJson {
+        package_desc,
+        error: err_str,
+    };
+    serde_json::to_writer_pretty(writer, &json_desc)?;
+    Ok(())
+}
 
-        if let Some(syms) = desc.public_symbols {
-            let sym_table = tabled::Table::new(syms.into_iter().map(|s| SymbolRow { symbol: s }));
-            writeln!(self.output, "Public symbols:\n{sym_table}")?;
-        }
+/// Display information about a single module.
+fn display_module<W: Write + ?Sized>(desc: ModuleDesc, writer: &mut W) -> Result<()> {
+    if let Some(exts) = desc.used_extensions_resolved {
+        let ext_rows: Vec<ExtensionRow> = exts.into_iter().map(Into::into).collect();
+        let ext_table = tabled::Table::new(ext_rows);
+        writeln!(writer, "Resolved extensions:\n{ext_table}")?;
+    }
 
-        if let Some(exts) = desc.used_extensions_generator {
-            let ext_rows: Vec<ExtensionRow> = exts.into_iter().map(Into::into).collect();
-            let ext_table = tabled::Table::new(ext_rows);
-            writeln!(self.output, "Generator claimed extensions:\n{ext_table}")?;
-        }
+    if let Some(syms) = desc.public_symbols {
+        let sym_table = tabled::Table::new(syms.into_iter().map(|s| SymbolRow { symbol: s }));
+        writeln!(writer, "Public symbols:\n{sym_table}")?;
+    }
 
-        Ok(())
+    if let Some(exts) = desc.used_extensions_generator {
+        let ext_rows: Vec<ExtensionRow> = exts.into_iter().map(Into::into).collect();
+        let ext_table = tabled::Table::new(ext_rows);
+        writeln!(writer, "Generator claimed extensions:\n{ext_table}")?;
     }
+
+    Ok(())
 }
 
 #[derive(serde::Serialize, schemars::JsonSchema)]