Skip to content

Commit a0decfc

Browse files
sadikovicloud-fan
authored andcommitted
[SPARK-35378][SQL][FOLLOW-UP] Fix incorrect return type in CommandResultExec.executeCollect()
### What changes were proposed in this pull request? This PR is a follow-up for #32513 and fixes an issue introduced by that patch. CommandResultExec is supposed to return `UnsafeRow` records in all of the `executeXYZ` methods but `executeCollect` was left out which causes issues like this one: ``` Error in SQL statement: ClassCastException: org.apache.spark.sql.catalyst.expressions.GenericInternalRow cannot be cast to org.apache.spark.sql.catalyst.expressions.UnsafeRow ``` We need to return `unsafeRows` instead of `rows` in `executeCollect` similar to other methods in the class. ### Why are the changes needed? Fixes a bug in CommandResultExec. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? I added a unit test to check the return type of all commands. Closes #36632 from sadikovi/fix-command-exec. Authored-by: Ivan Sadikov <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent 82f2a98 commit a0decfc

File tree

2 files changed

+11
-2
lines changed

2 files changed

+11
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/CommandResultExec.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ case class CommandResultExec(
7676
}
7777

7878
override def executeCollect(): Array[InternalRow] = {
79-
longMetric("numOutputRows").add(rows.size)
80-
rows.toArray
79+
longMetric("numOutputRows").add(unsafeRows.size)
80+
unsafeRows
8181
}
8282

8383
override def executeTake(limit: Int): Array[InternalRow] = {

sql/core/src/test/scala/org/apache/spark/sql/execution/QueryExecutionSuite.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import scala.io.Source
2020

2121
import org.apache.spark.sql.{AnalysisException, FastOperator}
2222
import org.apache.spark.sql.catalyst.analysis.UnresolvedNamespace
23+
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
2324
import org.apache.spark.sql.catalyst.plans.QueryPlan
2425
import org.apache.spark.sql.catalyst.plans.logical.{CommandResult, LogicalPlan, OneRowRelation, Project, ShowTables, SubqueryAlias}
2526
import org.apache.spark.sql.catalyst.trees.TreeNodeTag
@@ -262,6 +263,14 @@ class QueryExecutionSuite extends SharedSparkSession {
262263
assert(cmdResultExec.commandPhysicalPlan.isInstanceOf[ShowTablesExec])
263264
}
264265

266+
test("SPARK-35378: Return UnsafeRow in CommandResultExecCheck execute methods") {
267+
val plan = spark.sql("SHOW FUNCTIONS").queryExecution.executedPlan
268+
assert(plan.isInstanceOf[CommandResultExec])
269+
plan.executeCollect().foreach { row => assert(row.isInstanceOf[UnsafeRow]) }
270+
plan.executeTake(10).foreach { row => assert(row.isInstanceOf[UnsafeRow]) }
271+
plan.executeTail(10).foreach { row => assert(row.isInstanceOf[UnsafeRow]) }
272+
}
273+
265274
test("SPARK-38198: check specify maxFields when call toFile method") {
266275
withTempDir { dir =>
267276
val path = dir.getCanonicalPath + "/plans.txt"

0 commit comments

Comments
 (0)