Skip to content

Commit 76f50b0

Browse files
waynexiawaynexia
authored
docs: add documents to substrait type variation consts (#10719)
* docs: add documents to substrait type variation consts Signed-off-by: Ruihang Xia <[email protected]> * rename and add todo Signed-off-by: Ruihang Xia <[email protected]> * fix link style Signed-off-by: Ruihang Xia <[email protected]> --------- Signed-off-by: Ruihang Xia <[email protected]>
1 parent 100b30e commit 76f50b0

3 files changed

Lines changed: 229 additions & 180 deletions

File tree

‎datafusion/substrait/src/logical_plan/consumer.rs‎

Lines changed: 88 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,14 @@ use std::str::FromStr;
7373
use std::sync::Arc;
7474

7575
use crate::variation_const::{
76-
DATE_32_TYPE_REF, DATE_64_TYPE_REF, DECIMAL_128_TYPE_REF, DECIMAL_256_TYPE_REF,
77-
DEFAULT_CONTAINER_TYPE_REF, DEFAULT_TYPE_REF, INTERVAL_DAY_TIME_TYPE_REF,
78-
INTERVAL_MONTH_DAY_NANO_TYPE_REF, INTERVAL_YEAR_MONTH_TYPE_REF,
79-
LARGE_CONTAINER_TYPE_REF, TIMESTAMP_MICRO_TYPE_REF, TIMESTAMP_MILLI_TYPE_REF,
80-
TIMESTAMP_NANO_TYPE_REF, TIMESTAMP_SECOND_TYPE_REF, UNSIGNED_INTEGER_TYPE_REF,
76+
DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,
77+
DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,
78+
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
79+
INTERVAL_DAY_TIME_TYPE_REF, INTERVAL_MONTH_DAY_NANO_TYPE_REF,
80+
INTERVAL_YEAR_MONTH_TYPE_REF, LARGE_CONTAINER_TYPE_VARIATION_REF,
81+
TIMESTAMP_MICRO_TYPE_VARIATION_REF, TIMESTAMP_MILLI_TYPE_VARIATION_REF,
82+
TIMESTAMP_NANO_TYPE_VARIATION_REF, TIMESTAMP_SECOND_TYPE_VARIATION_REF,
83+
UNSIGNED_INTEGER_TYPE_VARIATION_REF,
8184
};
8285

8386
enum ScalarFunctionType {
@@ -1130,62 +1133,62 @@ fn from_substrait_type(
11301133
Some(s_kind) => match s_kind {
11311134
r#type::Kind::Bool(_) => Ok(DataType::Boolean),
11321135
r#type::Kind::I8(integer) => match integer.type_variation_reference {
1133-
DEFAULT_TYPE_REF => Ok(DataType::Int8),
1134-
UNSIGNED_INTEGER_TYPE_REF => Ok(DataType::UInt8),
1136+
DEFAULT_TYPE_VARIATION_REF => Ok(DataType::Int8),
1137+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(DataType::UInt8),
11351138
v => not_impl_err!(
11361139
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11371140
),
11381141
},
11391142
r#type::Kind::I16(integer) => match integer.type_variation_reference {
1140-
DEFAULT_TYPE_REF => Ok(DataType::Int16),
1141-
UNSIGNED_INTEGER_TYPE_REF => Ok(DataType::UInt16),
1143+
DEFAULT_TYPE_VARIATION_REF => Ok(DataType::Int16),
1144+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(DataType::UInt16),
11421145
v => not_impl_err!(
11431146
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11441147
),
11451148
},
11461149
r#type::Kind::I32(integer) => match integer.type_variation_reference {
1147-
DEFAULT_TYPE_REF => Ok(DataType::Int32),
1148-
UNSIGNED_INTEGER_TYPE_REF => Ok(DataType::UInt32),
1150+
DEFAULT_TYPE_VARIATION_REF => Ok(DataType::Int32),
1151+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(DataType::UInt32),
11491152
v => not_impl_err!(
11501153
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11511154
),
11521155
},
11531156
r#type::Kind::I64(integer) => match integer.type_variation_reference {
1154-
DEFAULT_TYPE_REF => Ok(DataType::Int64),
1155-
UNSIGNED_INTEGER_TYPE_REF => Ok(DataType::UInt64),
1157+
DEFAULT_TYPE_VARIATION_REF => Ok(DataType::Int64),
1158+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(DataType::UInt64),
11561159
v => not_impl_err!(
11571160
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11581161
),
11591162
},
11601163
r#type::Kind::Fp32(_) => Ok(DataType::Float32),
11611164
r#type::Kind::Fp64(_) => Ok(DataType::Float64),
11621165
r#type::Kind::Timestamp(ts) => match ts.type_variation_reference {
1163-
TIMESTAMP_SECOND_TYPE_REF => {
1166+
TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
11641167
Ok(DataType::Timestamp(TimeUnit::Second, None))
11651168
}
1166-
TIMESTAMP_MILLI_TYPE_REF => {
1169+
TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
11671170
Ok(DataType::Timestamp(TimeUnit::Millisecond, None))
11681171
}
1169-
TIMESTAMP_MICRO_TYPE_REF => {
1172+
TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
11701173
Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
11711174
}
1172-
TIMESTAMP_NANO_TYPE_REF => {
1175+
TIMESTAMP_NANO_TYPE_VARIATION_REF => {
11731176
Ok(DataType::Timestamp(TimeUnit::Nanosecond, None))
11741177
}
11751178
v => not_impl_err!(
11761179
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11771180
),
11781181
},
11791182
r#type::Kind::Date(date) => match date.type_variation_reference {
1180-
DATE_32_TYPE_REF => Ok(DataType::Date32),
1181-
DATE_64_TYPE_REF => Ok(DataType::Date64),
1183+
DATE_32_TYPE_VARIATION_REF => Ok(DataType::Date32),
1184+
DATE_64_TYPE_VARIATION_REF => Ok(DataType::Date64),
11821185
v => not_impl_err!(
11831186
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11841187
),
11851188
},
11861189
r#type::Kind::Binary(binary) => match binary.type_variation_reference {
1187-
DEFAULT_CONTAINER_TYPE_REF => Ok(DataType::Binary),
1188-
LARGE_CONTAINER_TYPE_REF => Ok(DataType::LargeBinary),
1190+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::Binary),
1191+
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::LargeBinary),
11891192
v => not_impl_err!(
11901193
"Unsupported Substrait type variation {v} of type {s_kind:?}"
11911194
),
@@ -1194,8 +1197,8 @@ fn from_substrait_type(
11941197
Ok(DataType::FixedSizeBinary(fixed.length))
11951198
}
11961199
r#type::Kind::String(string) => match string.type_variation_reference {
1197-
DEFAULT_CONTAINER_TYPE_REF => Ok(DataType::Utf8),
1198-
LARGE_CONTAINER_TYPE_REF => Ok(DataType::LargeUtf8),
1200+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::Utf8),
1201+
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::LargeUtf8),
11991202
v => not_impl_err!(
12001203
"Unsupported Substrait type variation {v} of type {s_kind:?}"
12011204
),
@@ -1209,18 +1212,18 @@ fn from_substrait_type(
12091212
is_substrait_type_nullable(inner_type)?,
12101213
));
12111214
match list.type_variation_reference {
1212-
DEFAULT_CONTAINER_TYPE_REF => Ok(DataType::List(field)),
1213-
LARGE_CONTAINER_TYPE_REF => Ok(DataType::LargeList(field)),
1215+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::List(field)),
1216+
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::LargeList(field)),
12141217
v => not_impl_err!(
12151218
"Unsupported Substrait type variation {v} of type {s_kind:?}"
12161219
)?,
12171220
}
12181221
}
12191222
r#type::Kind::Decimal(d) => match d.type_variation_reference {
1220-
DECIMAL_128_TYPE_REF => {
1223+
DECIMAL_128_TYPE_VARIATION_REF => {
12211224
Ok(DataType::Decimal128(d.precision as u8, d.scale as i8))
12221225
}
1223-
DECIMAL_256_TYPE_REF => {
1226+
DECIMAL_256_TYPE_VARIATION_REF => {
12241227
Ok(DataType::Decimal256(d.precision as u8, d.scale as i8))
12251228
}
12261229
v => not_impl_err!(
@@ -1397,55 +1400,65 @@ fn from_substrait_literal(
13971400
let scalar_value = match &lit.literal_type {
13981401
Some(LiteralType::Boolean(b)) => ScalarValue::Boolean(Some(*b)),
13991402
Some(LiteralType::I8(n)) => match lit.type_variation_reference {
1400-
DEFAULT_TYPE_REF => ScalarValue::Int8(Some(*n as i8)),
1401-
UNSIGNED_INTEGER_TYPE_REF => ScalarValue::UInt8(Some(*n as u8)),
1403+
DEFAULT_TYPE_VARIATION_REF => ScalarValue::Int8(Some(*n as i8)),
1404+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => ScalarValue::UInt8(Some(*n as u8)),
14021405
others => {
14031406
return substrait_err!("Unknown type variation reference {others}");
14041407
}
14051408
},
14061409
Some(LiteralType::I16(n)) => match lit.type_variation_reference {
1407-
DEFAULT_TYPE_REF => ScalarValue::Int16(Some(*n as i16)),
1408-
UNSIGNED_INTEGER_TYPE_REF => ScalarValue::UInt16(Some(*n as u16)),
1410+
DEFAULT_TYPE_VARIATION_REF => ScalarValue::Int16(Some(*n as i16)),
1411+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => ScalarValue::UInt16(Some(*n as u16)),
14091412
others => {
14101413
return substrait_err!("Unknown type variation reference {others}");
14111414
}
14121415
},
14131416
Some(LiteralType::I32(n)) => match lit.type_variation_reference {
1414-
DEFAULT_TYPE_REF => ScalarValue::Int32(Some(*n)),
1415-
UNSIGNED_INTEGER_TYPE_REF => ScalarValue::UInt32(Some(*n as u32)),
1417+
DEFAULT_TYPE_VARIATION_REF => ScalarValue::Int32(Some(*n)),
1418+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => ScalarValue::UInt32(Some(*n as u32)),
14161419
others => {
14171420
return substrait_err!("Unknown type variation reference {others}");
14181421
}
14191422
},
14201423
Some(LiteralType::I64(n)) => match lit.type_variation_reference {
1421-
DEFAULT_TYPE_REF => ScalarValue::Int64(Some(*n)),
1422-
UNSIGNED_INTEGER_TYPE_REF => ScalarValue::UInt64(Some(*n as u64)),
1424+
DEFAULT_TYPE_VARIATION_REF => ScalarValue::Int64(Some(*n)),
1425+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => ScalarValue::UInt64(Some(*n as u64)),
14231426
others => {
14241427
return substrait_err!("Unknown type variation reference {others}");
14251428
}
14261429
},
14271430
Some(LiteralType::Fp32(f)) => ScalarValue::Float32(Some(*f)),
14281431
Some(LiteralType::Fp64(f)) => ScalarValue::Float64(Some(*f)),
14291432
Some(LiteralType::Timestamp(t)) => match lit.type_variation_reference {
1430-
TIMESTAMP_SECOND_TYPE_REF => ScalarValue::TimestampSecond(Some(*t), None),
1431-
TIMESTAMP_MILLI_TYPE_REF => ScalarValue::TimestampMillisecond(Some(*t), None),
1432-
TIMESTAMP_MICRO_TYPE_REF => ScalarValue::TimestampMicrosecond(Some(*t), None),
1433-
TIMESTAMP_NANO_TYPE_REF => ScalarValue::TimestampNanosecond(Some(*t), None),
1433+
TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
1434+
ScalarValue::TimestampSecond(Some(*t), None)
1435+
}
1436+
TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
1437+
ScalarValue::TimestampMillisecond(Some(*t), None)
1438+
}
1439+
TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
1440+
ScalarValue::TimestampMicrosecond(Some(*t), None)
1441+
}
1442+
TIMESTAMP_NANO_TYPE_VARIATION_REF => {
1443+
ScalarValue::TimestampNanosecond(Some(*t), None)
1444+
}
14341445
others => {
14351446
return substrait_err!("Unknown type variation reference {others}");
14361447
}
14371448
},
14381449
Some(LiteralType::Date(d)) => ScalarValue::Date32(Some(*d)),
14391450
Some(LiteralType::String(s)) => match lit.type_variation_reference {
1440-
DEFAULT_CONTAINER_TYPE_REF => ScalarValue::Utf8(Some(s.clone())),
1441-
LARGE_CONTAINER_TYPE_REF => ScalarValue::LargeUtf8(Some(s.clone())),
1451+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => ScalarValue::Utf8(Some(s.clone())),
1452+
LARGE_CONTAINER_TYPE_VARIATION_REF => ScalarValue::LargeUtf8(Some(s.clone())),
14421453
others => {
14431454
return substrait_err!("Unknown type variation reference {others}");
14441455
}
14451456
},
14461457
Some(LiteralType::Binary(b)) => match lit.type_variation_reference {
1447-
DEFAULT_CONTAINER_TYPE_REF => ScalarValue::Binary(Some(b.clone())),
1448-
LARGE_CONTAINER_TYPE_REF => ScalarValue::LargeBinary(Some(b.clone())),
1458+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => ScalarValue::Binary(Some(b.clone())),
1459+
LARGE_CONTAINER_TYPE_VARIATION_REF => {
1460+
ScalarValue::LargeBinary(Some(b.clone()))
1461+
}
14491462
others => {
14501463
return substrait_err!("Unknown type variation reference {others}");
14511464
}
@@ -1484,11 +1497,10 @@ fn from_substrait_literal(
14841497
}
14851498
let element_type = elements[0].data_type();
14861499
match lit.type_variation_reference {
1487-
DEFAULT_CONTAINER_TYPE_REF => ScalarValue::List(ScalarValue::new_list(
1488-
elements.as_slice(),
1489-
&element_type,
1490-
)),
1491-
LARGE_CONTAINER_TYPE_REF => ScalarValue::LargeList(
1500+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => ScalarValue::List(
1501+
ScalarValue::new_list(elements.as_slice(), &element_type),
1502+
),
1503+
LARGE_CONTAINER_TYPE_VARIATION_REF => ScalarValue::LargeList(
14921504
ScalarValue::new_large_list(elements.as_slice(), &element_type),
14931505
),
14941506
others => {
@@ -1503,10 +1515,10 @@ fn from_substrait_literal(
15031515
name_idx,
15041516
)?;
15051517
match lit.type_variation_reference {
1506-
DEFAULT_CONTAINER_TYPE_REF => {
1518+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => {
15071519
ScalarValue::List(ScalarValue::new_list(&[], &element_type))
15081520
}
1509-
LARGE_CONTAINER_TYPE_REF => ScalarValue::LargeList(
1521+
LARGE_CONTAINER_TYPE_VARIATION_REF => ScalarValue::LargeList(
15101522
ScalarValue::new_large_list(&[], &element_type),
15111523
),
15121524
others => {
@@ -1590,68 +1602,70 @@ fn from_substrait_null(
15901602
match kind {
15911603
r#type::Kind::Bool(_) => Ok(ScalarValue::Boolean(None)),
15921604
r#type::Kind::I8(integer) => match integer.type_variation_reference {
1593-
DEFAULT_TYPE_REF => Ok(ScalarValue::Int8(None)),
1594-
UNSIGNED_INTEGER_TYPE_REF => Ok(ScalarValue::UInt8(None)),
1605+
DEFAULT_TYPE_VARIATION_REF => Ok(ScalarValue::Int8(None)),
1606+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(ScalarValue::UInt8(None)),
15951607
v => not_impl_err!(
15961608
"Unsupported Substrait type variation {v} of type {kind:?}"
15971609
),
15981610
},
15991611
r#type::Kind::I16(integer) => match integer.type_variation_reference {
1600-
DEFAULT_TYPE_REF => Ok(ScalarValue::Int16(None)),
1601-
UNSIGNED_INTEGER_TYPE_REF => Ok(ScalarValue::UInt16(None)),
1612+
DEFAULT_TYPE_VARIATION_REF => Ok(ScalarValue::Int16(None)),
1613+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(ScalarValue::UInt16(None)),
16021614
v => not_impl_err!(
16031615
"Unsupported Substrait type variation {v} of type {kind:?}"
16041616
),
16051617
},
16061618
r#type::Kind::I32(integer) => match integer.type_variation_reference {
1607-
DEFAULT_TYPE_REF => Ok(ScalarValue::Int32(None)),
1608-
UNSIGNED_INTEGER_TYPE_REF => Ok(ScalarValue::UInt32(None)),
1619+
DEFAULT_TYPE_VARIATION_REF => Ok(ScalarValue::Int32(None)),
1620+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(ScalarValue::UInt32(None)),
16091621
v => not_impl_err!(
16101622
"Unsupported Substrait type variation {v} of type {kind:?}"
16111623
),
16121624
},
16131625
r#type::Kind::I64(integer) => match integer.type_variation_reference {
1614-
DEFAULT_TYPE_REF => Ok(ScalarValue::Int64(None)),
1615-
UNSIGNED_INTEGER_TYPE_REF => Ok(ScalarValue::UInt64(None)),
1626+
DEFAULT_TYPE_VARIATION_REF => Ok(ScalarValue::Int64(None)),
1627+
UNSIGNED_INTEGER_TYPE_VARIATION_REF => Ok(ScalarValue::UInt64(None)),
16161628
v => not_impl_err!(
16171629
"Unsupported Substrait type variation {v} of type {kind:?}"
16181630
),
16191631
},
16201632
r#type::Kind::Fp32(_) => Ok(ScalarValue::Float32(None)),
16211633
r#type::Kind::Fp64(_) => Ok(ScalarValue::Float64(None)),
16221634
r#type::Kind::Timestamp(ts) => match ts.type_variation_reference {
1623-
TIMESTAMP_SECOND_TYPE_REF => Ok(ScalarValue::TimestampSecond(None, None)),
1624-
TIMESTAMP_MILLI_TYPE_REF => {
1635+
TIMESTAMP_SECOND_TYPE_VARIATION_REF => {
1636+
Ok(ScalarValue::TimestampSecond(None, None))
1637+
}
1638+
TIMESTAMP_MILLI_TYPE_VARIATION_REF => {
16251639
Ok(ScalarValue::TimestampMillisecond(None, None))
16261640
}
1627-
TIMESTAMP_MICRO_TYPE_REF => {
1641+
TIMESTAMP_MICRO_TYPE_VARIATION_REF => {
16281642
Ok(ScalarValue::TimestampMicrosecond(None, None))
16291643
}
1630-
TIMESTAMP_NANO_TYPE_REF => {
1644+
TIMESTAMP_NANO_TYPE_VARIATION_REF => {
16311645
Ok(ScalarValue::TimestampNanosecond(None, None))
16321646
}
16331647
v => not_impl_err!(
16341648
"Unsupported Substrait type variation {v} of type {kind:?}"
16351649
),
16361650
},
16371651
r#type::Kind::Date(date) => match date.type_variation_reference {
1638-
DATE_32_TYPE_REF => Ok(ScalarValue::Date32(None)),
1639-
DATE_64_TYPE_REF => Ok(ScalarValue::Date64(None)),
1652+
DATE_32_TYPE_VARIATION_REF => Ok(ScalarValue::Date32(None)),
1653+
DATE_64_TYPE_VARIATION_REF => Ok(ScalarValue::Date64(None)),
16401654
v => not_impl_err!(
16411655
"Unsupported Substrait type variation {v} of type {kind:?}"
16421656
),
16431657
},
16441658
r#type::Kind::Binary(binary) => match binary.type_variation_reference {
1645-
DEFAULT_CONTAINER_TYPE_REF => Ok(ScalarValue::Binary(None)),
1646-
LARGE_CONTAINER_TYPE_REF => Ok(ScalarValue::LargeBinary(None)),
1659+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(ScalarValue::Binary(None)),
1660+
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(ScalarValue::LargeBinary(None)),
16471661
v => not_impl_err!(
16481662
"Unsupported Substrait type variation {v} of type {kind:?}"
16491663
),
16501664
},
16511665
// FixedBinary is not supported because `None` doesn't have length
16521666
r#type::Kind::String(string) => match string.type_variation_reference {
1653-
DEFAULT_CONTAINER_TYPE_REF => Ok(ScalarValue::Utf8(None)),
1654-
LARGE_CONTAINER_TYPE_REF => Ok(ScalarValue::LargeUtf8(None)),
1667+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(ScalarValue::Utf8(None)),
1668+
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(ScalarValue::LargeUtf8(None)),
16551669
v => not_impl_err!(
16561670
"Unsupported Substrait type variation {v} of type {kind:?}"
16571671
),
@@ -1671,12 +1685,12 @@ fn from_substrait_null(
16711685
true,
16721686
);
16731687
match l.type_variation_reference {
1674-
DEFAULT_CONTAINER_TYPE_REF => Ok(ScalarValue::List(Arc::new(
1675-
GenericListArray::new_null(field.into(), 1),
1676-
))),
1677-
LARGE_CONTAINER_TYPE_REF => Ok(ScalarValue::LargeList(Arc::new(
1678-
GenericListArray::new_null(field.into(), 1),
1679-
))),
1688+
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(ScalarValue::List(
1689+
Arc::new(GenericListArray::new_null(field.into(), 1)),
1690+
)),
1691+
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(ScalarValue::LargeList(
1692+
Arc::new(GenericListArray::new_null(field.into(), 1)),
1693+
)),
16801694
v => not_impl_err!(
16811695
"Unsupported Substrait type variation {v} of type {kind:?}"
16821696
),

0 commit comments

Comments
 (0)