From 6d38fe3a10573144033ac879b071143f164faaa8 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 24 Oct 2023 14:42:05 -0400 Subject: [PATCH 1/7] Remove libprotobuf as a build time dependency of Conda --- conda/recipes/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index 113e7a441..2f957a253 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -41,7 +41,6 @@ requirements: - {{ compiler('rust') }} - cargo-bundle-licenses - maturin >=0.15,<0.16 - - libprotobuf =3 host: - python - maturin >=0.15,<0.16 From 90410bbd2276bb09d3f8515a610eff8d07892fa2 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 25 Oct 2023 13:55:57 -0400 Subject: [PATCH 2/7] Add substrait feature which enables building with substrait features --- Cargo.toml | 3 ++- src/lib.rs | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7c1a57bd3..06834b92f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ include = ["/src", "/datafusion", "/LICENSE.txt", "pyproject.toml", "Cargo.toml" [features] default = ["mimalloc"] protoc = [ "datafusion-substrait/protoc" ] +substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } @@ -41,7 +42,7 @@ datafusion-common = { version = "32.0.0", features = ["pyarrow"] } datafusion-expr = { version = "32.0.0" } datafusion-optimizer = { version = "32.0.0" } datafusion-sql = { version = "32.0.0" } -datafusion-substrait = { version = "32.0.0" } +datafusion-substrait = { version = "32.0.0", optional = true } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } diff --git a/src/lib.rs b/src/lib.rs index 2512aefa4..49e673cd5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,8 @@ pub use datafusion_common; pub use datafusion_expr; pub use datafusion_optimizer; pub use datafusion_sql; + +#[cfg(feature = "substrait")] pub use datafusion_substrait; #[allow(clippy::borrow_deref_ref)] @@ -48,6 +50,8 @@ mod pyarrow_filter_expression; mod record_batch; pub mod sql; pub mod store; + +#[cfg(feature = "substrait")] pub mod substrait; #[allow(clippy::borrow_deref_ref)] mod udaf; @@ -108,8 +112,11 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { m.add_submodule(store)?; // Register substrait as a submodule + #[cfg(feature = "substrait")] let substrait = PyModule::new(py, "substrait")?; + #[cfg(feature = "substrait")] substrait::init_module(substrait)?; + #[cfg(feature = "substrait")] m.add_submodule(substrait)?; Ok(()) From f380345ed63325246e1340ad155d8467add5826d Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 26 Oct 2023 09:43:42 -0400 Subject: [PATCH 3/7] Add datafusion._internal/substrait to the pyproject.toml file so substrait will be built for pytests when using pip --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4fdc4586f..231a01280 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,4 @@ include = [ exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true +features = ["datafusion._internal/substrait"] From 45e177282d17566754959032605f410ea65fed1f Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 26 Oct 2023 09:59:12 -0400 Subject: [PATCH 4/7] Rename feature to just substrait --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 231a01280..d35360519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,4 +62,4 @@ include = [ exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true -features = ["datafusion._internal/substrait"] +features = ["substrait"] From 0c4838de1219c965b03fac2adc368f28641d8f74 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 26 Oct 2023 10:17:02 -0400 Subject: [PATCH 5/7] Add protoc since libprotobuf was removed from conda requirements file --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d35360519..d6b7c57a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,4 +62,4 @@ include = [ exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true -features = ["substrait"] +features = ["substrait", "protoc"] From 03a8bd90f72381b046240d2b1b1167f2f3355bd7 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 26 Oct 2023 10:34:56 -0400 Subject: [PATCH 6/7] Refactor substrait module being initialized as a function --- src/lib.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 49e673cd5..413b2a429 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,11 +113,15 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { // Register substrait as a submodule #[cfg(feature = "substrait")] + setup_substrait_module(py, m)?; + + Ok(()) +} + +#[cfg(feature = "substrait")] +fn setup_substrait_module(py: Python, m: &PyModule) -> PyResult<()> { let substrait = PyModule::new(py, "substrait")?; - #[cfg(feature = "substrait")] substrait::init_module(substrait)?; - #[cfg(feature = "substrait")] m.add_submodule(substrait)?; - Ok(()) } From dd5d099f5aef973aa6f8c09071130560732dac4b Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 26 Oct 2023 12:04:27 -0400 Subject: [PATCH 7/7] Reintroduce libprotobuf as build dependency for conda build --- conda/recipes/meta.yaml | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index 2f957a253..113e7a441 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -41,6 +41,7 @@ requirements: - {{ compiler('rust') }} - cargo-bundle-licenses - maturin >=0.15,<0.16 + - libprotobuf =3 host: - python - maturin >=0.15,<0.16 diff --git a/pyproject.toml b/pyproject.toml index d6b7c57a9..d35360519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,4 +62,4 @@ include = [ exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true -features = ["substrait", "protoc"] +features = ["substrait"]