-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-18207][SQL] Fix a compilation error due to HashExpression.doGenCode #15745
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
71c3284
aa11c59
24a6320
3fdc813
049e477
6a57ba5
e96742e
2c405a1
a2544df
798fd85
61504c3
fa31521
a2b2408
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1728,4 +1728,23 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { | |
| val df = spark.createDataFrame(spark.sparkContext.makeRDD(rows), schema) | ||
| assert(df.filter($"array1" === $"array2").count() == 1) | ||
| } | ||
|
|
||
|
||
| test("SPARK-18207: Compute hash for wider table") { | ||
| import org.apache.spark.sql.types.{StructType, StringType} | ||
|
||
|
|
||
| val COLMAX = 1000 | ||
| val schema: StructType = (1 to COLMAX) | ||
| .foldLeft(new StructType())((s, i) => s.add(s"g$i", StringType, nullable = true)) | ||
| val rdds = spark.sparkContext.parallelize(Seq(Row.fromSeq((1 to COLMAX).map(_.toString)))) | ||
| val wideDF = spark.createDataFrame(rdds, schema) | ||
|
|
||
| val widePlus = wideDF.withColumn("d_rank", lit(1)) | ||
| widePlus.createOrReplaceTempView("wide_plus") | ||
| val widePlus2 = widePlus.withColumn("d_rank", lit(0)) | ||
| widePlus2.createOrReplaceTempView("wide_plus2") | ||
|
|
||
| // union operation in this SQL involves computation of hash for a row | ||
|
||
| val df = spark.sqlContext.sql("select * from wide_plus union select * from wide_plus2") | ||
| df.count | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think you need to revert this change.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sorry for my typo. It is time for me to have to sleep.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No problem. Happens to us all :)