Skip to content

Commit dfcdb23

Browse files
committed
address comments
1 parent cecea8c commit dfcdb23

1 file changed

Lines changed: 5 additions & 0 deletions

File tree

sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2657,6 +2657,9 @@ class Dataset[T] private[sql](
26572657
*/
26582658
def toLocalIterator(): java.util.Iterator[T] = {
26592659
withAction("toLocalIterator", queryExecution) { plan =>
2660+
// This projection writes output to a `InternalRow`, which means applying this projection is
2661+
// not thread-safe. Here we create the projection inside this method to make `Dataset`
2662+
// thread-safe.
26602663
val objProj = GenerateSafeProjection.generate(deserializer :: Nil)
26612664
plan.executeToIterator().map { row =>
26622665
// The row returned by SafeProjection is `SpecificInternalRow`, which ignore the data type
@@ -3103,6 +3106,8 @@ class Dataset[T] private[sql](
31033106
* Collect all elements from a spark plan.
31043107
*/
31053108
private def collectFromPlan(plan: SparkPlan): Array[T] = {
3109+
// This projection writes output to a `InternalRow`, which means applying this projection is not
3110+
// thread-safe. Here we create the projection inside this method to make `Dataset` thread-safe.
31063111
val objProj = GenerateSafeProjection.generate(deserializer :: Nil)
31073112
plan.executeCollect().map { row =>
31083113
// The row returned by SafeProjection is `SpecificInternalRow`, which ignore the data type

0 commit comments

Comments
 (0)