Skip to content

Commit 5272097

Browse files
committed
Address some comments
1 parent 5555731 commit 5272097

2 files changed

Lines changed: 4 additions & 9 deletions

File tree

sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -173,11 +173,7 @@ case class FilterExec(condition: Expression, child: SparkPlan)
173173
// TODO: revisit this. We can consider reordering predicates as well.
174174
val generatedIsNotNullChecks = new Array[Boolean](notNullPreds.length)
175175

176-
val extraIsNotNullReferences = mutable.Set[Attribute]()
177-
178-
def outputContainsNotNull(ref: Attribute): Boolean = {
179-
output.exists { attr => attr.exprId == ref.exprId }
180-
}
176+
val extraIsNotNullAttrs = mutable.Set[Attribute]()
181177

182178
val generated = otherPreds.map { c =>
183179
val nullChecks = c.references.map { r =>
@@ -186,8 +182,8 @@ case class FilterExec(condition: Expression, child: SparkPlan)
186182
generatedIsNotNullChecks(idx) = true
187183
// Use the child's output. The nullability is what the child produced.
188184
genPredicate(notNullPreds(idx), input, child.output)
189-
} else if (outputContainsNotNull(r) && !extraIsNotNullReferences.contains(r)) {
190-
extraIsNotNullReferences += r
185+
} else if (notNullAttributes.contains(r.exprId) && !extraIsNotNullAttrs.contains(r)) {
186+
extraIsNotNullAttrs += r
191187
genPredicate(IsNotNull(r), input, child.output)
192188
} else {
193189
""

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3193,8 +3193,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession {
31933193
}
31943194
}
31953195

3196-
test("SPARK-29213 Make it consistent when get notnull output and generate null " +
3197-
"checks in FilterExec") {
3196+
test("SPARK-29213: FilterExec should not throw NPE") {
31983197
withView("t1", "t2", "t3") {
31993198
sql("select ''").as[String].map(identity).toDF("x").createOrReplaceTempView("t1")
32003199
sql("select * from values 0, cast(null as bigint)")

0 commit comments

Comments
 (0)