Skip to content

Commit 4bcb894

Browse files
arayyhuai
authored andcommitted
[SPARK-12205][SQL] Pivot fails Analysis when aggregate is UnresolvedFunction
Delays application of ResolvePivot until all aggregates are resolved to prevent problems with UnresolvedFunction and adds unit test Author: Andrew Ray <ray.andrew@gmail.com> Closes apache#10202 from aray/sql-pivot-unresolved-function.
1 parent 872a2ee commit 4bcb894

2 files changed

Lines changed: 9 additions & 1 deletion

File tree

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ class Analyzer(
259259

260260
object ResolvePivot extends Rule[LogicalPlan] {
261261
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
262-
case p: Pivot if !p.childrenResolved => p
262+
case p: Pivot if !p.childrenResolved | !p.aggregates.forall(_.resolved) => p
263263
case Pivot(groupByExprs, pivotColumn, pivotValues, aggregates, child) =>
264264
val singleAgg = aggregates.size == 1
265265
val pivotAggregates: Seq[NamedExpression] = pivotValues.flatMap { value =>

sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,4 +85,12 @@ class DataFramePivotSuite extends QueryTest with SharedSQLContext{
8585
sqlContext.conf.setConf(SQLConf.DATAFRAME_PIVOT_MAX_VALUES,
8686
SQLConf.DATAFRAME_PIVOT_MAX_VALUES.defaultValue.get)
8787
}
88+
89+
test("pivot with UnresolvedFunction") {
90+
checkAnswer(
91+
courseSales.groupBy("year").pivot("course", Seq("dotNET", "Java"))
92+
.agg("earnings" -> "sum"),
93+
Row(2012, 15000.0, 20000.0) :: Row(2013, 48000.0, 30000.0) :: Nil
94+
)
95+
}
8896
}

0 commit comments

Comments
 (0)