Skip to content

Commit ad1d83f

Browse files
author
Davies Liu
committed
Fix a bug around split expression in whole-stage codegen
1 parent 2ffe46f commit ad1d83f

File tree

3 files changed

+2
-4
lines changed

3 files changed

+2
-4
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -584,7 +584,7 @@ class CodegenContext {
584584
* @param expressions the codes to evaluate expressions.
585585
*/
586586
def splitExpressions(row: String, expressions: Seq[String]): String = {
587-
if (row == null) {
587+
if (row == null || currentVars != null) {
588588
// Cannot split these expressions because they are not created from a row object.
589589
return expressions.mkString("\n")
590590
}

sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -603,8 +603,6 @@ case class HashAggregateExec(
603603

604604
// create grouping key
605605
ctx.currentVars = input
606-
// make sure that the generated code will not be splitted as multiple functions
607-
ctx.INPUT_ROW = null
608606
val unsafeRowKeyCode = GenerateUnsafeProjection.createCode(
609607
ctx, groupingExpressions.map(e => BindReferences.bindReference[Expression](e, child.output)))
610608
val vectorizedRowKeys = ctx.generateExpressions(

sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BenchmarkWideTable.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import org.apache.spark.util.Benchmark
2929
*/
3030
class BenchmarkWideTable extends BenchmarkBase {
3131

32-
test("project on wide table") {
32+
ignore("project on wide table") {
3333
val N = 1 << 20
3434
val df = sparkSession.range(N)
3535
val columns = (0 until 400).map{ i => s"id as id$i"}

0 commit comments

Comments
 (0)