Skip to content

Add support for Arrow Duration type in Substrait #16503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alamb merged 3 commits into from
Jun 25, 2025
Merged

Add support for Arrow Duration type in Substrait #16503

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8047a3e
Add support for Arrow Duration type in Substrait
jkosh44 Jun 12, 2025
ca19d52
Respond to comments
jkosh44 Jun 24, 2025
f1d862c
Fix docs
jkosh44 Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions datafusion/substrait/src/logical_plan/consumer/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ use super::SubstraitConsumer;
use crate::variation_const::{
DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,
DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION,
DEFAULT_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION,
INTERVAL_DAY_TIME_TYPE_REF, INTERVAL_MONTH_DAY_NANO_TYPE_NAME,
INTERVAL_MONTH_DAY_NANO_TYPE_REF, INTERVAL_YEAR_MONTH_TYPE_REF,
LARGE_CONTAINER_TYPE_VARIATION_REF, TIMESTAMP_MICRO_TYPE_VARIATION_REF,
Expand Down Expand Up @@ -213,7 +214,28 @@ pub fn from_substrait_type(
r#type::Kind::IntervalYear(_) => {
Ok(DataType::Interval(IntervalUnit::YearMonth))
}
r#type::Kind::IntervalDay(_) => Ok(DataType::Interval(IntervalUnit::DayTime)),
r#type::Kind::IntervalDay(i) => match i.type_variation_reference {
DEFAULT_INTERVAL_DAY_TYPE_VARIATION => {
Ok(DataType::Interval(IntervalUnit::DayTime))
}
DURATION_INTERVAL_DAY_TYPE_VARIATION => {
let duration_unit = match i.precision {
Some(0) => Ok(TimeUnit::Second),
Some(3) => Ok(TimeUnit::Millisecond),
Some(6) => Ok(TimeUnit::Microsecond),
Some(9) => Ok(TimeUnit::Nanosecond),
p => {
not_impl_err!(
"Unsupported Substrait precision {p:?} for Duration"
)
}
}?;
Ok(DataType::Duration(duration_unit))
}
v => not_impl_err!(
"Unsupported Substrait type variation {v} of type {s_kind:?}"
),
},
r#type::Kind::IntervalCompound(_) => {
Ok(DataType::Interval(IntervalUnit::MonthDayNano))
}
Expand Down
25 changes: 23 additions & 2 deletions datafusion/substrait/src/logical_plan/producer/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ use crate::logical_plan::producer::utils::flatten_names;
use crate::variation_const::{
DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,
DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION,
DEFAULT_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION,
LARGE_CONTAINER_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,
VIEW_CONTAINER_TYPE_VARIATION_REF,
};
Expand Down Expand Up @@ -153,7 +154,7 @@ pub(crate) fn to_substrait_type(
}),
IntervalUnit::DayTime => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::IntervalDay(r#type::IntervalDay {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
type_variation_reference: DEFAULT_INTERVAL_DAY_TYPE_VARIATION,
nullability,
precision: Some(3), // DayTime precision is always milliseconds
})),
Expand All @@ -171,6 +172,21 @@ pub(crate) fn to_substrait_type(
}
}
}
DataType::Duration(duration_unit) => {
let precision = match duration_unit {
TimeUnit::Second => 0,
TimeUnit::Millisecond => 3,
TimeUnit::Microsecond => 6,
TimeUnit::Nanosecond => 9,
};
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::IntervalDay(r#type::IntervalDay {
type_variation_reference: DURATION_INTERVAL_DAY_TYPE_VARIATION,
nullability,
precision: Some(precision),
})),
})
}
DataType::Binary => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Binary(r#type::Binary {
type_variation_reference: DEFAULT_CONTAINER_TYPE_VARIATION_REF,
Expand Down Expand Up @@ -388,6 +404,11 @@ mod tests {
round_trip_type(DataType::Interval(IntervalUnit::MonthDayNano))?;
round_trip_type(DataType::Interval(IntervalUnit::DayTime))?;

round_trip_type(DataType::Duration(TimeUnit::Second))?;
round_trip_type(DataType::Duration(TimeUnit::Millisecond))?;
round_trip_type(DataType::Duration(TimeUnit::Microsecond))?;
round_trip_type(DataType::Duration(TimeUnit::Nanosecond))?;

Ok(())
}

Expand Down
2 changes: 2 additions & 0 deletions datafusion/substrait/src/variation_const.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ pub const LARGE_CONTAINER_TYPE_VARIATION_REF: u32 = 1;
pub const VIEW_CONTAINER_TYPE_VARIATION_REF: u32 = 2;
pub const DECIMAL_128_TYPE_VARIATION_REF: u32 = 0;
pub const DECIMAL_256_TYPE_VARIATION_REF: u32 = 1;
pub const DEFAULT_INTERVAL_DAY_TYPE_VARIATION: u32 = 0;
pub const DURATION_INTERVAL_DAY_TYPE_VARIATION: u32 = 1;
Copy link
Contributor

@gabotechs gabotechs Jun 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice idea using the type variation capabilities, this looks good to me 👍.

Nit 1: maybe prefixing this constants with _REF as all the others?
Nit 2: maybe adding a comment explaining that the default variation maps to DataType::Interval(DayTime) and the other maps to DataType::Duration?


// For [user-defined types](https://substrait.io/types/type_classes/#user-defined-types).
/// For [`DataType::Interval`] with [`IntervalUnit::YearMonth`].
Expand Down