Skip to content
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1039,13 +1039,19 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
}
}
"""
}.mkString("\n")
}
val fieldsEvalCodes = if (ctx.INPUT_ROW != null && ctx.currentVars == null) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't be ctx.currentVars != null?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If ctx.currentVars != null, we need to use mkString("\n").

ctx.splitExpressions(fieldsEvalCode, "castStruct",
("InternalRow", tmpRow) :: (rowClass, result) :: Nil)
} else {
fieldsEvalCode.mkString("\n")
}

(c, evPrim, evNull) =>
s"""
final $rowClass $result = new $rowClass(${fieldsCasts.length});
final InternalRow $tmpRow = $c;
$fieldsEvalCode
$fieldsEvalCodes
$evPrim = $result;
"""
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -827,4 +827,49 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {

checkEvaluation(cast(Literal.create(input, from), to), input)
}

test("SPARK-22500: cast for struct should not generate codes beyond 64KB") {
val N = 1000
val M = 250

val from1 = new StructType(
(1 to N).map(i => StructField(s"s$i", StringType)).toArray)
val to1 = new StructType(
(1 to N).map(i => StructField(s"i$i", IntegerType)).toArray)
val input1 = Row.fromSeq((1 to N).map(i => i.toString))
val output1 = Row.fromSeq((1 to N))
checkEvaluation(cast(Literal.create(input1, from1), to1), output1)

val from2 = new StructType(
(1 to N).map(i => StructField(s"a$i", ArrayType(StringType, containsNull = false))).toArray)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd expect something like

val from2 = new StructType(
  (1 to N).map(i => StructField(s"s$i", from1)).toArray)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or just test this case.

val to2 = new StructType(
(1 to N).map(i => StructField(s"i$i", ArrayType(IntegerType, containsNull = true))).toArray)
val input2 = Row.fromSeq((1 to N).map(_ => Seq("456", "true", "78.9")))
val output2 = Row.fromSeq((1 to N).map(_ => Seq(456, null, 78)))
checkEvaluation(cast(Literal.create(input2, from2), to2), output2)

val from3 = new StructType(
(1 to N).map(i => StructField(s"s$i",
StructType(Seq(StructField("l$i", IntegerType, nullable = true))))).toArray)
val to3 = new StructType(
(1 to N).map(i => StructField(s"s$i",
StructType(Seq(StructField("l$i", LongType, nullable = true))))).toArray)
val input3 = Row.fromSeq((1 to N).map(i => Row(i)))
val output3 = Row.fromSeq((1 to N).map(i => Row(i.toLong)))
checkEvaluation(cast(Literal.create(input3, from3), to3), output3)

val fromInner = new StructType(
(1 to M).map(i => StructField(s"s$i", DoubleType)).toArray)
val toInner = new StructType(
(1 to M).map(i => StructField(s"i$i", IntegerType)).toArray)
val inputInner = Row.fromSeq((1 to M).map(i => i + 0.5))
val outputInner = Row.fromSeq((1 to M))
val fromOuter = new StructType(
(1 to M).map(i => StructField(s"s$i", fromInner)).toArray)
val toOuter = new StructType(
(1 to M).map(i => StructField(s"s$i", toInner)).toArray)
val inputOuter = Row.fromSeq((1 to M).map(_ => inputInner))
val outputOuter = Row.fromSeq((1 to M).map(_ => outputInner))
checkEvaluation(cast(Literal.create(inputOuter, fromOuter), toOuter), outputOuter)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this case is good enough to cover all the above cases?

}
}