Skip to content

Commit f0b55f9

Browse files
committed
Fix cube-related data quality problem
1 parent 92df232 commit f0b55f9

2 files changed

Lines changed: 38 additions & 1 deletion

File tree

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2923,7 +2923,22 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
29232923
val extraAggExprs = new LinkedHashMap[Expression, NamedExpression]
29242924
val transformed = exprs.map { e =>
29252925
if (!e.resolved) {
2926-
e
2926+
val aggregatedCondition =
2927+
Aggregate(
2928+
agg.groupingExpressions,
2929+
Alias(e, "havingCondition")() :: Nil,
2930+
agg.child)
2931+
val resolvedOperator = executeSameContext(aggregatedCondition)
2932+
def resolvedAggregateFilter =
2933+
resolvedOperator
2934+
.asInstanceOf[Aggregate]
2935+
.aggregateExpressions.head
2936+
2937+
if (resolvedOperator.resolved) {
2938+
buildAggExprList(resolvedAggregateFilter, agg, extraAggExprs)
2939+
} else {
2940+
e
2941+
}
29272942
} else {
29282943
buildAggExprList(e, agg, extraAggExprs)
29292944
}

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5059,6 +5059,28 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
50595059
}
50605060
}
50615061
}
5062+
5063+
test("SPARK-53094: Fix cube-related data quality problem") {
5064+
withTable("table1") {
5065+
withSQLConf() {
5066+
sql(
5067+
"""CREATE TABLE table1(product string, amount bigint,
5068+
|region string) using csv""".stripMargin)
5069+
5070+
sql("INSERT INTO table1 " + "VALUES('a', 100, 'east')")
5071+
sql("INSERT INTO table1 " + "VALUES('b', 200, 'east')")
5072+
sql("INSERT INTO table1 " + "VALUES('a', 150, 'west')")
5073+
sql("INSERT INTO table1 " + "VALUES('b', 250, 'west')")
5074+
sql("INSERT INTO table1 " + "VALUES('a', 120, 'east')")
5075+
5076+
checkAnswer(
5077+
sql("select product, region, sum(amount) as s " +
5078+
"from table1 group by product, region with cube having count(product) > 2 " +
5079+
"order by s desc"),
5080+
Seq(Row(null, null, 820), Row(null, "east", 420), Row("a", null, 370)))
5081+
}
5082+
}
5083+
}
50625084
}
50635085

50645086
case class Foo(bar: Option[String])

0 commit comments

Comments
 (0)