[Typing] Paddle 的 CI 中引入 mypy 对于 API 中 docstring 的示例代码的类型检查

paddle/scripts/paddle_build.sh

@@ -3525,6 +3525,26 @@ function exec_samplecode_test() {
     fi
 }
 
+function exec_type_hints() {
+    if [ -d "${PADDLE_ROOT}/build/pr_whl" ];then
+        pip install ${PADDLE_ROOT}/build/pr_whl/*.whl
+    else
+        echo "WARNING: PR wheel is not found. Use develop wheel !!!"
+        pip install ${PADDLE_ROOT}/build/python/dist/*.whl
+    fi
+
+    python -c "import paddle;print(paddle.__version__);paddle.version.show()"
+
+    cd ${PADDLE_ROOT}/tools
+
+    python type_hints.py --debug; type_hints_error=$?
+
+    if [ "$type_hints_error" != "0" ];then
+      echo "Example code type checking failed" >&2
+      exit 5
+    fi
+}
+
 
 function collect_ccache_hits() {
     ccache -s
@@ -3569,6 +3589,9 @@ function summary_check_problems() {
     set +x
     local example_code=$1
     local example_info=$2
+    local type_hints_code=$3
+    local type_hints_info=$4
+
     if [ $example_code -ne 0 ];then
         echo "==============================================================================="
         echo "*****Example code error***** Please fix the error listed in the information:"
@@ -3587,6 +3610,25 @@ function summary_check_problems() {
         echo "*****Example code PASS*****"
         echo "==============================================================================="
     fi
+
+    if [ $type_hints_code -ne 0 ];then
+        echo "==============================================================================="
+        echo "*****Example code type checking error***** Please fix the error listed in the information:"
+        echo "==============================================================================="
+        echo "$type_hints_info"
+        echo "==============================================================================="
+        echo "*****Example code type checking FAIL*****"
+        echo "==============================================================================="
+        exit $type_hints_code
+    else
+        echo "==============================================================================="
+        echo "*****Example code type checking info*****"
+        echo "==============================================================================="
+        echo "$type_hints_info"
+        echo "==============================================================================="
+        echo "*****Example code type checking PASS*****"
+        echo "==============================================================================="
+    fi
     set -x
 }
 
@@ -4274,7 +4316,12 @@ function main() {
         fi
         { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
         example_code=$?
-        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+        { type_hints_info=$(exec_type_hints 2>&1 1>&3 3>/dev/null); } 3>&1
+        type_hints_code=$?
+
+        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"  $type_hints_code "$type_hints_info"
+
         assert_api_spec_approvals
         ;;
       build_and_check_cpu)
@@ -4294,7 +4341,12 @@ function main() {
         fi
         { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
         example_code=$?
-        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+        { type_hints_info=$(exec_type_hints 2>&1 1>&3 3>/dev/null); } 3>&1
+        type_hints_code=$?
+
+        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}" $type_hints_code "$type_hints_info"
+
         assert_api_spec_approvals
         ;;
       check_whl_size)
@@ -4538,7 +4590,11 @@ function main() {
       api_example)
         { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
         example_code=$?
-        summary_check_problems $example_code "$example_info"
+
+        { type_hints_info=$(exec_type_hints 2>&1 1>&3 3>/dev/null); } 3>&1
+        type_hints_code=$?
+
+        summary_check_problems $example_code "$example_info" $type_hints_code "$type_hints_info"
         ;;
       test_op_benchmark)
         test_op_benchmark

python/paddle/tensor/math.py

@@ -15,6 +15,8 @@
 math functions
 """
 
+from __future__ import annotations
+
 import math
 import warnings
 
@@ -212,7 +214,9 @@ def log_(x, name=None):
         return _C_ops.log_(x)
 
 
-def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
+def scale(
+    x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None
+) -> int:
     """
     Scale operator.
 
@@ -322,7 +326,7 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
         return helper.append_activation(out)
 
 
-def stanh(x, scale_a=0.67, scale_b=1.7159, name=None):
+def stanh(x, scale_a: int = 1, scale_b=1.7159, name=None):
     r"""
 
     stanh activation.

python/setup.py.in

@@ -1073,6 +1073,9 @@ if '${WITH_CPP_DIST}' == 'ON':
     paddle_lib_test_dir = '${PADDLE_LIB_TEST_DIR}'
     install_cpp_dist_and_build_test(paddle_install_dir, paddle_lib_test_dir)
 
+# type hints
+package_data['paddle'] = package_data.get('paddle', []) + ['py.typed']
+package_data['paddle.tensor'] = package_data.get('paddle.tensor', []) + ['*.pyi']
 
 with redirect_stdout():
     setup(name='${PACKAGE_NAME}',

python/unittest_py/requirements.txt

@@ -19,3 +19,4 @@ wandb>=0.13 ; python_version<"3.12"
 xlsxwriter==3.0.9
 xdoctest==1.1.1
 ubelt==1.3.3 # just for xdoctest
+mypy==1.10.0

setup.py

@@ -1302,6 +1302,13 @@ def get_package_data_and_package_dir():
         ext_modules = []
     elif sys.platform == 'darwin':
         ext_modules = []
+
+    # type hints
+    package_data['paddle'] = package_data.get('paddle', []) + ['py.typed']
+    package_data['paddle.tensor'] = package_data.get('paddle.tensor', []) + [
+        '*.pyi'
+    ]
+
     return package_data, package_dir, ext_modules
 
 

tools/mypy.ini

@@ -0,0 +1,26 @@
+[mypy]
+python_version = 3.8
+
+cache_dir = .mypy_cache
+
+# Miscellaneous strictness flags
+allow_redefinition = True
+local_partial_types = True
+strict = False
+
+# Untyped definitions and calls
+check_untyped_defs = True
+
+# Import discovery
+follow_imports = normal
+
+# Miscellaneous
+warn_unused_configs = True
+
+# Configuring warnings
+warn_redundant_casts = True
+warn_unused_ignores = True
+warn_no_return = True
+
+# Configuring error messages
+show_column_numbers = True

tools/sampcd_processor_utils.py

@@ -334,16 +334,22 @@ def get_api_md5(path):
     if not os.path.isfile(API_spec):
         return api_md5
     pat = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})')
+
+    # insert ArgSpec for changing the API's type annotation can trigger the CI
     patArgSpec = re.compile(
-        r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})'
+        r'^(paddle[^,]+)\s+\((ArgSpec.*),.*document\W*([0-9a-z]{32})'
     )
+
     with open(API_spec) as f:
         for line in f.readlines():
             mo = pat.search(line)
-            if not mo:
-                mo = patArgSpec.search(line)
+
             if mo:
                 api_md5[mo.group(1)] = mo.group(2)
+            else:
+                mo = patArgSpec.search(line)
+                api_md5[mo.group(1)] = f'{mo.group(2)}, {mo.group(3)}'
+
     return api_md5
 
 

tools/test_sampcd_processor.py

@@ -103,19 +103,23 @@ def tearDown(self):
     def test_get_api_md5(self):
         res = get_api_md5('paddle/fluid/API_PR.spec')
         self.assertEqual(
-            "ff0f188c95030158cc6398d2a6c55one", res['paddle.one_plus_one']
+            "ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ff0f188c95030158cc6398d2a6c55one",
+            res['paddle.one_plus_one'],
         )
         self.assertEqual(
-            "ff0f188c95030158cc6398d2a6c55two", res['paddle.two_plus_two']
+            "ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ff0f188c95030158cc6398d2a6c55two",
+            res['paddle.two_plus_two'],
         )
         self.assertEqual(
-            "ff0f188c95030158cc6398d2a6cthree", res['paddle.three_plus_three']
+            "ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ff0f188c95030158cc6398d2a6cthree",
+            res['paddle.three_plus_three'],
         )
         self.assertEqual(
             "ff0f188c95030158cc6398d2a6c5four", res['paddle.four_plus_four']
         )
         self.assertEqual(
-            "ff0f188c95030158cc6398d2a6c5five", res['paddle.five_plus_five']
+            "ArgSpec(), ff0f188c95030158cc6398d2a6c5five",
+            res['paddle.five_plus_five'],
         )
 
 
@@ -302,8 +306,8 @@ def test_global_exec(self):
                     >>> import paddle
                     >>> a = paddle.to_tensor(.2)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                    0.20000000)
             """,
             'set_default': """
             placeholder
@@ -319,8 +323,8 @@ def test_global_exec(self):
                     >>> paddle.set_default_dtype('float64')
                     >>> a = paddle.to_tensor(.2)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float64, place=Place(cpu), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float64, place=Place(cpu), stop_gradient=True,
+                    0.20000000)
             """,
             'after_set_default': """
             placeholder
@@ -335,8 +339,8 @@ def test_global_exec(self):
                     >>> import paddle
                     >>> a = paddle.to_tensor(.2)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                    0.20000000)
             """,
         }
 
@@ -509,10 +513,10 @@ def test_patch_xdoctest(self):
                     >>> import paddle
                     >>> paddle.device.set_device('gpu')
                     >>> a = paddle.to_tensor(.2)
-                    >>> # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True, [0.20000000])
+                    >>> # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True, 0.20000000)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+                    0.20000000)
 
             """,
             'cpu_to_cpu': """
@@ -528,10 +532,10 @@ def test_patch_xdoctest(self):
                     >>> import paddle
                     >>> paddle.device.set_device('cpu')
                     >>> a = paddle.to_tensor(.2)
-                    >>> # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True, [0.20000000])
+                    >>> # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True, 0.20000000)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                    0.20000000)
 
             """,
             'gpu_to_cpu': """
@@ -547,10 +551,10 @@ def test_patch_xdoctest(self):
                     >>> import paddle
                     >>> paddle.device.set_device('gpu')
                     >>> a = paddle.to_tensor(.2)
-                    >>> # Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True, [0.20000000])
+                    >>> # Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True, 0.20000000)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                    0.20000000)
 
             """,
             'cpu_to_gpu': """
@@ -566,10 +570,10 @@ def test_patch_xdoctest(self):
                     >>> import paddle
                     >>> paddle.device.set_device('cpu')
                     >>> a = paddle.to_tensor(.2)
-                    >>> # Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True, [0.20000000])
+                    >>> # Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True, 0.20000000)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-                    [0.20000000])
+                    Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+                    0.20000000)
             """,
             'gpu_to_cpu_array': """
             placeholder
@@ -701,8 +705,8 @@ def test_patch_xdoctest(self):
                     >>> paddle.device.set_device('gpu')
                     >>> a = paddle.to_tensor(.123456789)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-                    [0.123456780])
+                    Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+                    0.123456780)
 
             """,
             'cpu_to_cpu': """
@@ -719,8 +723,8 @@ def test_patch_xdoctest(self):
                     >>> paddle.device.set_device('cpu')
                     >>> a = paddle.to_tensor(.123456789)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                    [0.123456780])
+                    Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                    0.123456780)
 
             """,
             'gpu_to_cpu': """
@@ -737,8 +741,8 @@ def test_patch_xdoctest(self):
                     >>> paddle.device.set_device('gpu')
                     >>> a = paddle.to_tensor(.123456789)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True,
-                    [0.123456780])
+                    Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
+                    0.123456780)
 
             """,
             'cpu_to_gpu': """
@@ -755,8 +759,8 @@ def test_patch_xdoctest(self):
                     >>> paddle.device.set_device('cpu')
                     >>> a = paddle.to_tensor(.123456789)
                     >>> print(a)
-                    Tensor(shape=[1], dtype=float32, place=Place(gpu:0), stop_gradient=True,
-                    [0.123456780])
+                    Tensor(shape=[], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+                    0.123456780)
             """,
             'gpu_to_cpu_array': """
             placeholder
@@ -2046,7 +2050,7 @@ def test_timeout(self):
 
     def test_bad_statements(self):
         docstrings_to_test = {
-            'bad_fluid': """
+            'good_fluid': """
             this is docstring...
 
             Examples:
@@ -2191,9 +2195,9 @@ def test_bad_statements(self):
             tr_10,
         ) = test_results
 
-        self.assertIn('bad_fluid', tr_0.name)
-        self.assertTrue(tr_0.badstatement)
-        self.assertFalse(tr_0.passed)
+        self.assertIn('good_fluid', tr_0.name)
+        self.assertFalse(tr_0.badstatement)
+        self.assertTrue(tr_0.passed)
 
         self.assertIn('bad_fluid_from', tr_1.name)
         self.assertTrue(tr_1.badstatement)