-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-22500][SQL] Fix 64KB JVM bytecode limit problem with cast #19730
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -827,4 +827,49 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { | |
|
|
||
| checkEvaluation(cast(Literal.create(input, from), to), input) | ||
| } | ||
|
|
||
| test("SPARK-22500: cast for struct should not generate codes beyond 64KB") { | ||
| val N = 1000 | ||
| val M = 250 | ||
|
|
||
| val from1 = new StructType( | ||
| (1 to N).map(i => StructField(s"s$i", StringType)).toArray) | ||
| val to1 = new StructType( | ||
| (1 to N).map(i => StructField(s"i$i", IntegerType)).toArray) | ||
| val input1 = Row.fromSeq((1 to N).map(i => i.toString)) | ||
| val output1 = Row.fromSeq((1 to N)) | ||
| checkEvaluation(cast(Literal.create(input1, from1), to1), output1) | ||
|
|
||
| val from2 = new StructType( | ||
| (1 to N).map(i => StructField(s"a$i", ArrayType(StringType, containsNull = false))).toArray) | ||
|
||
| val to2 = new StructType( | ||
| (1 to N).map(i => StructField(s"i$i", ArrayType(IntegerType, containsNull = true))).toArray) | ||
| val input2 = Row.fromSeq((1 to N).map(_ => Seq("456", "true", "78.9"))) | ||
| val output2 = Row.fromSeq((1 to N).map(_ => Seq(456, null, 78))) | ||
| checkEvaluation(cast(Literal.create(input2, from2), to2), output2) | ||
|
|
||
| val from3 = new StructType( | ||
| (1 to N).map(i => StructField(s"s$i", | ||
| StructType(Seq(StructField("l$i", IntegerType, nullable = true))))).toArray) | ||
| val to3 = new StructType( | ||
| (1 to N).map(i => StructField(s"s$i", | ||
| StructType(Seq(StructField("l$i", LongType, nullable = true))))).toArray) | ||
| val input3 = Row.fromSeq((1 to N).map(i => Row(i))) | ||
| val output3 = Row.fromSeq((1 to N).map(i => Row(i.toLong))) | ||
| checkEvaluation(cast(Literal.create(input3, from3), to3), output3) | ||
|
|
||
| val fromInner = new StructType( | ||
| (1 to M).map(i => StructField(s"s$i", DoubleType)).toArray) | ||
| val toInner = new StructType( | ||
| (1 to M).map(i => StructField(s"i$i", IntegerType)).toArray) | ||
| val inputInner = Row.fromSeq((1 to M).map(i => i + 0.5)) | ||
| val outputInner = Row.fromSeq((1 to M)) | ||
| val fromOuter = new StructType( | ||
| (1 to M).map(i => StructField(s"s$i", fromInner)).toArray) | ||
| val toOuter = new StructType( | ||
| (1 to M).map(i => StructField(s"s$i", toInner)).toArray) | ||
| val inputOuter = Row.fromSeq((1 to M).map(_ => inputInner)) | ||
| val outputOuter = Row.fromSeq((1 to M).map(_ => outputInner)) | ||
| checkEvaluation(cast(Literal.create(inputOuter, fromOuter), toOuter), outputOuter) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this case is good enough to cover all the above cases? |
||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
shouldn't be
ctx.currentVars != null?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If
ctx.currentVars != null, we need to usemkString("\n").