From 4b92c4f50837dbd756ba9b7c0cde0473a88e8d04 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 4 Jun 2025 07:59:37 -0600 Subject: [PATCH 1/3] Upgrade to DataFusion 48.0.0-rc1 --- native/Cargo.lock | 140 +++++++++++----------- native/Cargo.toml | 4 +- native/core/Cargo.toml | 2 +- native/core/src/parquet/parquet_exec.rs | 14 ++- native/core/src/parquet/schema_adapter.rs | 1 + 5 files changed, 82 insertions(+), 79 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 375e1125ae..fe3cf07ce2 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "assertables" -version = "9.5.2" +version = "9.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "218459a1ca1459c9fbfedc0dc14febf22d0b809708d496b2368e407da91ff5a8" +checksum = "46109705783fa5733709a155971ad89cdd188d45b7e20fba7906f0d6b4f864e3" [[package]] name = "async-trait" @@ -586,9 +586,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e44697a9bded898dcd0b1cb997430d949b87f4f8940d91023ae9062bf218250" +checksum = "073d330f94bdf1f47bb3e0f5d45dda1e372a54a553c39ab6e9646902c8c81594" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -917,9 +917,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.24" +version = "1.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" +checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" dependencies = [ "jobserver", "libc", @@ -1264,8 +1264,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "arrow-ipc", @@ -1313,8 +1313,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1338,8 +1338,8 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1449,8 +1449,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "ahash", "arrow", @@ -1471,8 +1471,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "futures", "log", @@ -1481,8 +1481,8 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1510,8 +1510,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1534,8 +1534,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1558,8 +1558,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1588,13 +1588,13 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" [[package]] name = "datafusion-execution" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "dashmap", @@ -1611,8 +1611,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "chrono", @@ -1630,8 +1630,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "datafusion-common", @@ -1642,8 +1642,8 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "arrow-buffer", @@ -1670,8 +1670,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "ahash", "arrow", @@ -1690,8 +1690,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "ahash", "arrow", @@ -1702,8 +1702,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "arrow-ord", @@ -1722,8 +1722,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1737,8 +1737,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "datafusion-common", @@ -1754,8 +1754,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1763,8 +1763,8 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "datafusion-expr", "quote", @@ -1773,8 +1773,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "chrono", @@ -1790,8 +1790,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "ahash", "arrow", @@ -1811,8 +1811,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "ahash", "arrow", @@ -1824,8 +1824,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "datafusion-common", @@ -1841,8 +1841,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "ahash", "arrow", @@ -1870,8 +1870,8 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "async-trait", @@ -1893,8 +1893,8 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "datafusion-catalog", @@ -1908,8 +1908,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "47.0.0" -source = "git+https://github.com/apache/datafusion?rev=2c2f225#2c2f225926958b6abf06b01fcfb594017531043c" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" dependencies = [ "arrow", "bigdecimal", @@ -3474,9 +3474,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.32" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" dependencies = [ "proc-macro2", "syn 2.0.101", @@ -3811,9 +3811,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.18" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" dependencies = [ "base64", "bytes", @@ -4573,9 +4573,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags 2.9.1", "bytes", diff --git a/native/Cargo.toml b/native/Cargo.toml index 970572d81b..2eafa357c1 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,8 +38,8 @@ arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.10.0" } parquet = { version = "55.1.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/apache/datafusion", rev = "2c2f225", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "2c2f225" } +datafusion = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc1", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc1" } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index cc39b174c2..4c8dcaaec5 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -84,7 +84,7 @@ jni = { version = "0.21", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "2c2f225" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc1" } [features] default = [] diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs index 13961ebf90..4b587b7ba6 100644 --- a/native/core/src/parquet/parquet_exec.rs +++ b/native/core/src/parquet/parquet_exec.rs @@ -69,10 +69,8 @@ pub(crate) fn init_datasource_exec( ) -> Result, ExecutionError> { let (table_parquet_options, spark_parquet_options) = get_options(session_timezone, case_sensitive); - let mut parquet_source = - ParquetSource::new(table_parquet_options).with_schema_adapter_factory(Arc::new( - SparkSchemaAdapterFactory::new(spark_parquet_options, default_values), - )); + let mut parquet_source = ParquetSource::new(table_parquet_options); + // Create a conjunctive form of the vector because ParquetExecBuilder takes // a single expression if let Some(data_filters) = data_filters { @@ -89,6 +87,10 @@ pub(crate) fn init_datasource_exec( } } + let file_source = parquet_source.with_schema_adapter_factory(Arc::new( + SparkSchemaAdapterFactory::new(spark_parquet_options, default_values), + ))?; + let file_groups = file_groups .iter() .map(|files| FileGroup::new(files.clone())) @@ -101,7 +103,7 @@ pub(crate) fn init_datasource_exec( partition_schema, file_groups, object_store_url, - Arc::new(parquet_source), + file_source, ) .with_projection(Some(projection_vector)) .with_table_partition_cols(partition_fields) @@ -112,7 +114,7 @@ pub(crate) fn init_datasource_exec( partition_schema, file_groups, object_store_url, - Arc::new(parquet_source), + file_source, ) .build(), }; diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index 1d210a8688..e27676bc29 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -278,6 +278,7 @@ mod test { use datafusion::common::config::TableParquetOptions; use datafusion::common::DataFusionError; use datafusion::datasource::listing::PartitionedFile; + use datafusion::datasource::physical_plan::FileSource; use datafusion::datasource::physical_plan::{FileGroup, FileScanConfigBuilder, ParquetSource}; use datafusion::datasource::source::DataSourceExec; use datafusion::execution::object_store::ObjectStoreUrl; From 4ec549014c3082f30cdc25dd881190198bc5a45a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 4 Jun 2025 08:22:21 -0600 Subject: [PATCH 2/3] update tests --- native/core/src/execution/planner.rs | 32 +++++++++++------------ native/core/src/parquet/schema_adapter.rs | 9 +++---- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 578c24e341..a587192ae5 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -2564,7 +2564,9 @@ mod tests { use datafusion::catalog::memory::DataSourceExec; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; - use datafusion::datasource::physical_plan::{FileGroup, FileScanConfigBuilder, ParquetSource}; + use datafusion::datasource::physical_plan::{ + FileGroup, FileScanConfigBuilder, FileSource, ParquetSource, + }; use datafusion::error::DataFusionError; use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ExecutionPlan; @@ -3201,14 +3203,12 @@ mod tests { } } - let source = Arc::new( - ParquetSource::default().with_schema_adapter_factory(Arc::new( - SparkSchemaAdapterFactory::new( - SparkParquetOptions::new(EvalMode::Ansi, "", false), - None, - ), - )), - ); + let source = ParquetSource::default().with_schema_adapter_factory(Arc::new( + SparkSchemaAdapterFactory::new( + SparkParquetOptions::new(EvalMode::Ansi, "", false), + None, + ), + ))?; let object_store_url = ObjectStoreUrl::local_filesystem(); let file_scan_config = @@ -3276,14 +3276,12 @@ mod tests { } } - let source = Arc::new( - ParquetSource::default().with_schema_adapter_factory(Arc::new( - SparkSchemaAdapterFactory::new( - SparkParquetOptions::new(EvalMode::Ansi, "", false), - None, - ), - )), - ); + let source = ParquetSource::default().with_schema_adapter_factory(Arc::new( + SparkSchemaAdapterFactory::new( + SparkParquetOptions::new(EvalMode::Ansi, "", false), + None, + ), + ))?; // Define schema Comet reads with let required_schema = Schema::new(Fields::from(vec![Field::new( diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index e27676bc29..b321d902a9 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -345,11 +345,10 @@ mod test { let mut spark_parquet_options = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false); spark_parquet_options.allow_cast_unsigned_ints = true; - let parquet_source = Arc::new( - ParquetSource::new(TableParquetOptions::new()).with_schema_adapter_factory(Arc::new( - SparkSchemaAdapterFactory::new(spark_parquet_options, None), - )), - ); + let parquet_source = + ParquetSource::new(TableParquetOptions::new()).with_schema_adapter_factory( + Arc::new(SparkSchemaAdapterFactory::new(spark_parquet_options, None)), + )?; let files = FileGroup::new(vec![PartitionedFile::from_path(filename.to_string())?]); let file_scan_config = From 70d98cb1af526bb343c6bc8fd61ae44c876a2a77 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 6 Jun 2025 07:36:41 -0600 Subject: [PATCH 3/3] Upgrade to DataFusion 48.0.0-rc2 --- native/Cargo.lock | 106 ++++++++++++++++++++--------------------- native/Cargo.toml | 4 +- native/core/Cargo.toml | 2 +- 3 files changed, 56 insertions(+), 56 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index fe3cf07ce2..0709e50945 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -93,9 +93,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anyhow" @@ -466,9 +466,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.71.0" +version = "1.72.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a4fd09d6e863655d99cd2260f271c6d1030dc6bfad68e19e126d2e4c8ceb18" +checksum = "13118ad30741222f67b1a18e5071385863914da05124652b38e172d6d3d9ce31" dependencies = [ "aws-credential-types", "aws-runtime", @@ -488,9 +488,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.72.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3224ab02ebb3074467a33d57caf6fcb487ca36f3697fdd381b0428dc72380696" +checksum = "f879a8572b4683a8f84f781695bebf2f25cf11a81a2693c31fc0e0215c2c1726" dependencies = [ "aws-credential-types", "aws-runtime", @@ -510,9 +510,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.72.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6933f189ed1255e78175fbd73fb200c0aae7240d220ed3346f567b0ddca3083" +checksum = "f1e9c3c24e36183e2f698235ed38dcfbbdff1d09b9232dc866c4be3011e0b47e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -877,9 +877,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" [[package]] name = "bytemuck" @@ -917,9 +917,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.25" +version = "1.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" +checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac" dependencies = [ "jobserver", "libc", @@ -1265,7 +1265,7 @@ dependencies = [ [[package]] name = "datafusion" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "arrow-ipc", @@ -1314,7 +1314,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1339,7 +1339,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1450,7 +1450,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1472,7 +1472,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "futures", "log", @@ -1482,7 +1482,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1511,7 +1511,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1535,7 +1535,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1559,7 +1559,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1589,12 +1589,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" [[package]] name = "datafusion-execution" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "dashmap", @@ -1612,7 +1612,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "chrono", @@ -1631,7 +1631,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "datafusion-common", @@ -1643,7 +1643,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "arrow-buffer", @@ -1671,7 +1671,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1691,7 +1691,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1703,7 +1703,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "arrow-ord", @@ -1723,7 +1723,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1738,7 +1738,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "datafusion-common", @@ -1755,7 +1755,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1764,7 +1764,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "datafusion-expr", "quote", @@ -1774,7 +1774,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "chrono", @@ -1791,7 +1791,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1812,7 +1812,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1825,7 +1825,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "datafusion-common", @@ -1842,7 +1842,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "ahash", "arrow", @@ -1871,7 +1871,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "async-trait", @@ -1894,7 +1894,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "datafusion-catalog", @@ -1909,7 +1909,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc1#992d156c46f6ad4f0096c4a62b293cabef63718d" +source = "git+https://github.com/apache/datafusion?rev=48.0.0-rc2#85f6621a6b1680b40d483a56b10ff3495861ece3" dependencies = [ "arrow", "bigdecimal", @@ -2441,9 +2441,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.6" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", "hyper", @@ -2458,9 +2458,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ "base64", "bytes", @@ -2878,9 +2878,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -4614,9 +4614,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" dependencies = [ "proc-macro2", "quote", @@ -4625,9 +4625,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", ] @@ -5336,9 +5336,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" diff --git a/native/Cargo.toml b/native/Cargo.toml index 2eafa357c1..1d61257407 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,8 +38,8 @@ arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.10.0" } parquet = { version = "55.1.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc1", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc1" } +datafusion = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc2", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc2" } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 4c8dcaaec5..e4c7d73bfb 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -84,7 +84,7 @@ jni = { version = "0.21", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc1" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "48.0.0-rc2" } [features] default = []