Skip to content

Commit afce06d

Browse files
xinrong-mengvicennial
authored andcommitted
[SPARK-40307][PYTHON] Introduce Arrow-optimized Python UDFs
### What changes were proposed in this pull request? Introduce Arrow-optimized Python UDFs. Please refer to [design](https://docs.google.com/document/d/e/2PACX-1vQxFyrMqFM3zhDhKlczrl9ONixk56cVXUwDXK0MMx4Vv2kH3oo-tWYoujhrGbCXTF78CSD2kZtnhnrQ/pub) for design details and micro benchmarks. There are two ways to enable/disable the Arrow optimization for Python UDFs: - the Spark configuration `spark.sql.execution.pythonUDF.arrow.enabled`, disabled by default. - the `useArrow` parameter of the `udf` function, None by default. The Spark configuration takes effect only when `useArrow` is None. Otherwise, `useArrow` decides whether a specific user-defined function is optimized by Arrow or not. The reason why we introduce these two ways is to provide both a convenient, per-Spark-session control and a finer-grained, per-UDF control of the Arrow optimization for Python UDFs. ### Why are the changes needed? Python user-defined function (UDF) enables users to run arbitrary code against PySpark columns. It uses Pickle for (de)serialization and executes row by row. One major performance bottleneck of Python UDFs is (de)serialization, that is, the data interchanging between the worker JVM and the spawned Python subprocess which actually executes the UDF. The PR proposes a better alternative to handle the (de)serialization: Arrow, which is used in the (de)serialization of Pandas UDF already. #### Benchmark The micro benchmarks are conducted in a cluster with 1 driver (i3.2xlarge), 2 workers (i3.2xlarge). An i3.2xlarge machine has 61 GB Memory, 8 Cores. The datasets used in the benchmarks are generated and sized 5 GB, 10 GB, 20 GB and 40 GB. As shown below, Arrow-optimized Python UDFs are **~1.4x** faster than non-Arrow-optimized Python UDFs. ![image](https://user-images.githubusercontent.com/47337188/210927609-e402e46f-20ee-43d6-9965-32a38d99fdd3.png) ![image](https://user-images.githubusercontent.com/47337188/210927614-4ac8db7f-083f-41b0-8f9d-efab2e8523b2.png) Please refer to [design](https://docs.google.com/document/d/e/2PACX-1vQxFyrMqFM3zhDhKlczrl9ONixk56cVXUwDXK0MMx4Vv2kH3oo-tWYoujhrGbCXTF78CSD2kZtnhnrQ/pub) for details. ### Does this PR introduce _any_ user-facing change? No, since the Arrow optimization for Python UDFs is disabled by default. ### How was this patch tested? Unit tests. Below is the script to generate the result table when the Arrow's type coercion is needed, as in the [docstring](https://github.com/apache/spark/pull/39384/files#diff-2df611ab00519d2d67e5fc20960bd5a6bd76ecd6f7d56cd50d8befd6ce30081bR96-R111) of `_create_py_udf` . ``` import sys import array import datetime from decimal import Decimal from pyspark.sql import Row from pyspark.sql.types import * from pyspark.sql.functions import udf data = [ None, True, 1, "a", datetime.date(1970, 1, 1), datetime.datetime(1970, 1, 1, 0, 0), 1.0, array.array("i", [1]), [1], (1,), bytearray([65, 66, 67]), Decimal(1), {"a": 1}, ] types = [ BooleanType(), ByteType(), ShortType(), IntegerType(), LongType(), StringType(), DateType(), TimestampType(), FloatType(), DoubleType(), BinaryType(), DecimalType(10, 0), ] df = spark.range(1) results = [] count = 0 total = len(types) * len(data) spark.sparkContext.setLogLevel("FATAL") for t in types: result = [] for v in data: try: row = df.select(udf(lambda _: v, t)("id")).first() ret_str = repr(row[0]) except Exception: ret_str = "X" result.append(ret_str) progress = "SQL Type: [%s]\n Python Value: [%s(%s)]\n Result Python Value: [%s]" % ( t.simpleString(), str(v), type(v).__name__, ret_str) count += 1 print("%s/%s:\n %s" % (count, total, progress)) results.append([t.simpleString()] + list(map(str, result))) schema = ["SQL Type \\ Python Value(Type)"] + list(map(lambda v: "%s(%s)" % (str(v), type(v).__name__), data)) strings = spark.createDataFrame(results, schema=schema)._jdf.showString(20, 20, False) print("\n".join(map(lambda line: " # %s # noqa" % line, strings.strip().split("\n")))) ``` Closes apache#39384 from xinrong-meng/arrow_py_udf_init. Authored-by: Xinrong Meng <xinrong@apache.org> Signed-off-by: Xinrong Meng <xinrong@apache.org>
1 parent b03a243 commit afce06d

6 files changed

Lines changed: 315 additions & 6 deletions

File tree

dev/sparktestsupport/modules.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,7 @@ def __hash__(self):
467467
"pyspark.sql.observation",
468468
# unittests
469469
"pyspark.sql.tests.test_arrow",
470+
"pyspark.sql.tests.test_arrow_python_udf",
470471
"pyspark.sql.tests.test_catalog",
471472
"pyspark.sql.tests.test_column",
472473
"pyspark.sql.tests.test_conf",

python/pyspark/sql/functions.py

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
from pyspark.sql.types import ArrayType, DataType, StringType, StructType, _from_numpy_type
4545

4646
# Keep UserDefinedFunction import for backwards compatible import; moved in SPARK-22409
47-
from pyspark.sql.udf import UserDefinedFunction, _create_udf # noqa: F401
47+
from pyspark.sql.udf import UserDefinedFunction, _create_py_udf # noqa: F401
4848

4949
# Keep pandas_udf and PandasUDFType import for backwards compatible import; moved in SPARK-28264
5050
from pyspark.sql.pandas.functions import pandas_udf, PandasUDFType # noqa: F401
@@ -9980,14 +9980,19 @@ def unwrap_udt(col: "ColumnOrName") -> Column:
99809980

99819981
@overload
99829982
def udf(
9983-
f: Callable[..., Any], returnType: "DataTypeOrString" = StringType()
9983+
f: Callable[..., Any],
9984+
returnType: "DataTypeOrString" = StringType(),
9985+
*,
9986+
useArrow: Optional[bool] = None,
99849987
) -> "UserDefinedFunctionLike":
99859988
...
99869989

99879990

99889991
@overload
99899992
def udf(
99909993
f: Optional["DataTypeOrString"] = None,
9994+
*,
9995+
useArrow: Optional[bool] = None,
99919996
) -> Callable[[Callable[..., Any]], "UserDefinedFunctionLike"]:
99929997
...
99939998

@@ -9996,13 +10001,16 @@ def udf(
999610001
def udf(
999710002
*,
999810003
returnType: "DataTypeOrString" = StringType(),
10004+
useArrow: Optional[bool] = None,
999910005
) -> Callable[[Callable[..., Any]], "UserDefinedFunctionLike"]:
1000010006
...
1000110007

1000210008

1000310009
def udf(
1000410010
f: Optional[Union[Callable[..., Any], "DataTypeOrString"]] = None,
1000510011
returnType: "DataTypeOrString" = StringType(),
10012+
*,
10013+
useArrow: Optional[bool] = None,
1000610014
) -> Union["UserDefinedFunctionLike", Callable[[Callable[..., Any]], "UserDefinedFunctionLike"]]:
1000710015
"""Creates a user defined function (UDF).
1000810016
@@ -10015,6 +10023,9 @@ def udf(
1001510023
returnType : :class:`pyspark.sql.types.DataType` or str
1001610024
the return type of the user-defined function. The value can be either a
1001710025
:class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
10026+
useArrow : bool or None
10027+
whether to use Arrow to optimize the (de)serialization. When it is None, the
10028+
Spark config "spark.sql.execution.pythonUDF.arrow.enabled" takes effect.
1001810029
1001910030
Examples
1002010031
--------
@@ -10093,10 +10104,15 @@ def udf(
1009310104
# for decorator use it as a returnType
1009410105
return_type = f or returnType
1009510106
return functools.partial(
10096-
_create_udf, returnType=return_type, evalType=PythonEvalType.SQL_BATCHED_UDF
10107+
_create_py_udf,
10108+
returnType=return_type,
10109+
evalType=PythonEvalType.SQL_BATCHED_UDF,
10110+
useArrow=useArrow,
1009710111
)
1009810112
else:
10099-
return _create_udf(f=f, returnType=returnType, evalType=PythonEvalType.SQL_BATCHED_UDF)
10113+
return _create_py_udf(
10114+
f=f, returnType=returnType, evalType=PythonEvalType.SQL_BATCHED_UDF, useArrow=useArrow
10115+
)
1010010116

1010110117

1010210118
def _test() -> None:
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
import unittest
19+
20+
from pyspark.sql.functions import udf
21+
from pyspark.sql.tests.test_udf import BaseUDFTests
22+
from pyspark.testing.sqlutils import (
23+
have_pandas,
24+
have_pyarrow,
25+
pandas_requirement_message,
26+
pyarrow_requirement_message,
27+
ReusedSQLTestCase,
28+
)
29+
30+
31+
@unittest.skipIf(
32+
not have_pandas or not have_pyarrow, pandas_requirement_message or pyarrow_requirement_message
33+
)
34+
class PythonUDFArrowTests(BaseUDFTests, ReusedSQLTestCase):
35+
@classmethod
36+
def setUpClass(cls):
37+
super(PythonUDFArrowTests, cls).setUpClass()
38+
cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled", "true")
39+
40+
@unittest.skip("Unrelated test, and it fails when it runs duplicatedly.")
41+
def test_broadcast_in_udf(self):
42+
super(PythonUDFArrowTests, self).test_broadcast_in_udf()
43+
44+
@unittest.skip("Unrelated test, and it fails when it runs duplicatedly.")
45+
def test_register_java_function(self):
46+
super(PythonUDFArrowTests, self).test_register_java_function()
47+
48+
@unittest.skip("Unrelated test, and it fails when it runs duplicatedly.")
49+
def test_register_java_udaf(self):
50+
super(PythonUDFArrowTests, self).test_register_java_udaf()
51+
52+
@unittest.skip("Struct input types are not supported with Arrow optimization")
53+
def test_udf_input_serialization_valuecompare_disabled(self):
54+
super(PythonUDFArrowTests, self).test_udf_input_serialization_valuecompare_disabled()
55+
56+
def test_nested_input_error(self):
57+
with self.assertRaisesRegexp(
58+
Exception, "NotImplementedError: Struct input type are not supported"
59+
):
60+
self.spark.range(1).selectExpr("struct(1, 2) as struct").select(
61+
udf(lambda x: x)("struct")
62+
).collect()
63+
64+
def test_complex_input_types(self):
65+
row = (
66+
self.spark.range(1)
67+
.selectExpr("array(1, 2, 3) as array", "map('a', 'b') as map")
68+
.select(
69+
udf(lambda x: str(x))("array"),
70+
udf(lambda x: str(x))("map"),
71+
)
72+
.first()
73+
)
74+
75+
# The input is NumPy array when the optimization is on.
76+
self.assertEquals(row[0], "[1 2 3]")
77+
self.assertEquals(row[1], "{'a': 'b'}")
78+
79+
def test_use_arrow(self):
80+
# useArrow=True
81+
row_true = (
82+
self.spark.range(1)
83+
.selectExpr(
84+
"array(1, 2, 3) as array",
85+
)
86+
.select(
87+
udf(lambda x: str(x), useArrow=True)("array"),
88+
)
89+
.first()
90+
)
91+
92+
# useArrow=None
93+
row_none = (
94+
self.spark.range(1)
95+
.selectExpr(
96+
"array(1, 2, 3) as array",
97+
)
98+
.select(
99+
udf(lambda x: str(x), useArrow=None)("array"),
100+
)
101+
.first()
102+
)
103+
104+
# The input is a NumPy array when the Arrow optimization is on.
105+
self.assertEquals(row_true[0], row_none[0]) # "[1 2 3]"
106+
107+
# useArrow=False
108+
row_false = (
109+
self.spark.range(1)
110+
.selectExpr(
111+
"array(1, 2, 3) as array",
112+
)
113+
.select(
114+
udf(lambda x: str(x), useArrow=False)("array"),
115+
)
116+
.first()
117+
)
118+
self.assertEquals(row_false[0], "[1, 2, 3]")
119+
120+
121+
if __name__ == "__main__":
122+
from pyspark.sql.tests.test_arrow_python_udf import * # noqa: F401
123+
124+
try:
125+
import xmlrunner
126+
127+
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
128+
except ImportError:
129+
testRunner = None
130+
unittest.main(testRunner=testRunner, verbosity=2)

python/pyspark/sql/tests/test_udf.py

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
from pyspark.testing.utils import QuietTest
4444

4545

46-
class UDFTests(ReusedSQLTestCase):
46+
class BaseUDFTests(object):
4747
def test_udf_with_callable(self):
4848
d = [Row(number=i, squared=i**2) for i in range(10)]
4949
rdd = self.sc.parallelize(d)
@@ -804,6 +804,54 @@ def test_udf_with_rand(self):
804804
)
805805

806806

807+
class UDFTests(BaseUDFTests, ReusedSQLTestCase):
808+
@classmethod
809+
def setUpClass(cls):
810+
super(BaseUDFTests, cls).setUpClass()
811+
cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled", "false")
812+
813+
814+
def test_use_arrow(self):
815+
# useArrow=True
816+
row_true = (
817+
self.spark.range(1)
818+
.selectExpr(
819+
"array(1, 2, 3) as array",
820+
)
821+
.select(
822+
udf(lambda x: str(x), useArrow=True)("array"),
823+
)
824+
.first()
825+
)
826+
# The input is a NumPy array when the Arrow optimization is on.
827+
self.assertEquals(row_true[0], "[1 2 3]")
828+
829+
# useArrow=None
830+
row_none = (
831+
self.spark.range(1)
832+
.selectExpr(
833+
"array(1, 2, 3) as array",
834+
)
835+
.select(
836+
udf(lambda x: str(x), useArrow=None)("array"),
837+
)
838+
.first()
839+
)
840+
841+
# useArrow=False
842+
row_false = (
843+
self.spark.range(1)
844+
.selectExpr(
845+
"array(1, 2, 3) as array",
846+
)
847+
.select(
848+
udf(lambda x: str(x), useArrow=False)("array"),
849+
)
850+
.first()
851+
)
852+
self.assertEquals(row_false[0], row_none[0]) # "[1, 2, 3]"
853+
854+
807855
class UDFInitializationTests(unittest.TestCase):
808856
def tearDown(self):
809857
if SparkSession._instantiatedSession is not None:

0 commit comments

Comments
 (0)