Skip to content

Commit a6b089f

Browse files
wangyumdongjoon-hyun
authored andcommitted
[SPARK-45909][SQL] Remove NumericType cast if it can safely up-cast in IsNotNull
### What changes were proposed in this pull request? Similar to SPARK-37922. We can remove the cast if it can safely up-cast in `IsNotNull`. ### Why are the changes needed? Improve the query performance. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unit test. ### Was this patch authored or co-authored using generative AI tooling? No. Closes apache#43785 from wangyum/SPARK-45909. Authored-by: Yuming Wang <yumwang@ebay.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent 2b046d2 commit a6b089f

2 files changed

Lines changed: 23 additions & 0 deletions

File tree

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1063,6 +1063,8 @@ object SimplifyCasts extends Rule[LogicalPlan] {
10631063
if fromKey == toKey && fromValue == toValue => e
10641064
case _ => c
10651065
}
1066+
case IsNotNull(Cast(e, dataType: NumericType, _, _)) if isWiderCast(e.dataType, dataType) =>
1067+
IsNotNull(e)
10661068
}
10671069

10681070
// Returns whether the from DataType can be safely casted to the to DataType without losing

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,4 +117,25 @@ class SimplifyCastsSuite extends PlanTest {
117117
input.select($"d".cast(LongType).cast(StringType).as("casted")).analyze),
118118
input.select($"d".cast(LongType).cast(StringType).as("casted")).analyze)
119119
}
120+
121+
test("SPARK-45909: Remove the cast if it can safely up-cast in IsNotNull") {
122+
val input = LocalRelation($"a".int, $"b".decimal(18, 0))
123+
// Remove cast
124+
comparePlans(
125+
Optimize.execute(
126+
input.select($"a".cast(DecimalType(18, 1)).isNotNull.as("v")).analyze),
127+
input.select($"a".isNotNull.as("v")).analyze)
128+
comparePlans(
129+
Optimize.execute(input.select($"a".cast(LongType).isNotNull.as("v")).analyze),
130+
input.select($"a".isNotNull.as("v")).analyze)
131+
comparePlans(
132+
Optimize.execute(input.select($"b".cast(LongType).isNotNull.as("v")).analyze),
133+
input.select($"b".isNotNull.as("v")).analyze)
134+
135+
// Can not remove cast
136+
comparePlans(
137+
Optimize.execute(
138+
input.select($"a".cast(DecimalType(2, 1)).as("v")).analyze),
139+
input.select($"a".cast(DecimalType(2, 1)).as("v")).analyze)
140+
}
120141
}

0 commit comments

Comments
 (0)