File tree Expand file tree Collapse file tree
sql/core/src/main/scala/org/apache/spark/sql Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -2657,6 +2657,9 @@ class Dataset[T] private[sql](
26572657 */
26582658 def toLocalIterator (): java.util.Iterator [T ] = {
26592659 withAction(" toLocalIterator" , queryExecution) { plan =>
2660+ // This projection writes output to a `InternalRow`, which means applying this projection is
2661+ // not thread-safe. Here we create the projection inside this method to make `Dataset`
2662+ // thread-safe.
26602663 val objProj = GenerateSafeProjection .generate(deserializer :: Nil )
26612664 plan.executeToIterator().map { row =>
26622665 // The row returned by SafeProjection is `SpecificInternalRow`, which ignore the data type
@@ -3103,6 +3106,8 @@ class Dataset[T] private[sql](
31033106 * Collect all elements from a spark plan.
31043107 */
31053108 private def collectFromPlan (plan : SparkPlan ): Array [T ] = {
3109+ // This projection writes output to a `InternalRow`, which means applying this projection is not
3110+ // thread-safe. Here we create the projection inside this method to make `Dataset` thread-safe.
31063111 val objProj = GenerateSafeProjection .generate(deserializer :: Nil )
31073112 plan.executeCollect().map { row =>
31083113 // The row returned by SafeProjection is `SpecificInternalRow`, which ignore the data type
You can’t perform that action at this time.
0 commit comments