[Typing] Paddle 的 CI 中引入 mypy 对于 API 中 docstring 的示例代码的类型检查

paddle/scripts/paddle_build.sh

@@ -3514,17 +3514,59 @@ function exec_samplecode_test() {
 
     cd ${PADDLE_ROOT}/tools
     if [ "$1" = "cpu" ] ; then
-        python sampcd_processor.py --debug --mode cpu; example_error=$?
+        python sampcd_processor.py --mode cpu; example_error=$?
     elif [ "$1" = "gpu" ] ; then
         SAMPLE_CODE_EXEC_THREADS=${SAMPLE_CODE_EXEC_THREADS:-2}
-        python sampcd_processor.py --threads=${SAMPLE_CODE_EXEC_THREADS} --debug --mode gpu; example_error=$?
+        python sampcd_processor.py --threads=${SAMPLE_CODE_EXEC_THREADS} --mode gpu; example_error=$?
     fi
     if [ "$example_error" != "0" ];then
       echo "Code instance execution failed" >&2
       exit 5
     fi
 }
 
+function run_type_checking() {
+    set +x
+
+    # check pr title
+    TITLE_CHECK=`curl -s https://github.com/PaddlePaddle/Paddle/pull/${GIT_PR_ID} | grep "<title>" | grep -i "type checking" || true`
+
+    if [[ ${TITLE_CHECK} ]]; then
+        set -x
+        return 0
+    else
+        set -x
+        return 1
+    fi
+}
+
+function exec_type_checking() {
+    if [ -d "${PADDLE_ROOT}/build/pr_whl" ];then
+        pip install ${PADDLE_ROOT}/build/pr_whl/*.whl
+    else
+        echo "WARNING: PR wheel is not found. Use develop wheel !!!"
+        pip install ${PADDLE_ROOT}/build/python/dist/*.whl
+    fi
+
+    python -c "import paddle;print(paddle.__version__);paddle.version.show()"
+
+    cd ${PADDLE_ROOT}/tools
+
+    # check all sample code
+    TITLE_CHECK_ALL=`curl -s https://github.com/PaddlePaddle/Paddle/pull/${GIT_PR_ID} | grep "<title>" | grep -i "type checking all" || true`
+
+    if [[ ${TITLE_CHECK_ALL} ]]; then
+        python type_checking.py --full-test; type_checking_error=$?
+    else
+        python type_checking.py; type_checking_error=$?
+    fi
+
+    if [ "$type_checking_error" != "0" ];then
+      echo "Example code type checking failed" >&2
+      exit 5
+    fi
+}
+
 
 function collect_ccache_hits() {
     ccache -s
@@ -3565,10 +3607,11 @@ function test_model_benchmark() {
     bash ${PADDLE_ROOT}/tools/test_model_benchmark.sh
 }
 
-function summary_check_problems() {
+function summary_check_example_code_problems() {
     set +x
     local example_code=$1
     local example_info=$2
+
     if [ $example_code -ne 0 ];then
         echo "==============================================================================="
         echo "*****Example code error***** Please fix the error listed in the information:"
@@ -3591,6 +3634,33 @@ function summary_check_problems() {
 }
 
 
+function summary_type_checking_problems() {
+    set +x
+    local type_checking_code=$1
+    local type_checking_info=$2
+
+    if [ $type_checking_code -ne 0 ];then
+        echo "==============================================================================="
+        echo "*****Example code type checking error***** Please fix the error listed in the information:"
+        echo "==============================================================================="
+        echo "$type_checking_info"
+        echo "==============================================================================="
+        echo "*****Example code type checking FAIL*****"
+        echo "==============================================================================="
+        exit $type_checking_code
+    else
+        echo "==============================================================================="
+        echo "*****Example code type checking info*****"
+        echo "==============================================================================="
+        echo "$type_checking_info"
+        echo "==============================================================================="
+        echo "*****Example code type checking PASS*****"
+        echo "==============================================================================="
+    fi
+    set -x
+}
+
+
 function reuse_so_cache() {
     get_html="https://api.github.com/repos/PaddlePaddle/Paddle"
     curl -X GET ${get_html}/commits -H "authorization: token ${GITHUB_API_TOKEN}" >tmp.txt
@@ -4274,7 +4344,18 @@ function main() {
         fi
         { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
         example_code=$?
-        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+        summary_check_example_code_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+        run_type_checking
+        run_tc=$?
+        if [[ ${run_tc} -eq 0 ]]; then
+            { type_checking_info=$(exec_type_checking 2>&1 1>&3 3>/dev/null); } 3>&1
+            type_checking_code=$?
+
+            summary_type_checking_problems $type_checking_code "$type_checking_info"
+        fi
+
         assert_api_spec_approvals
         ;;
       build_and_check_cpu)
@@ -4294,7 +4375,18 @@ function main() {
         fi
         { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
         example_code=$?
-        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+        summary_check_example_code_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+        run_type_checking
+        run_tc=$?
+        if [[ ${run_tc} -eq 0 ]]; then
+            { type_checking_info=$(exec_type_checking 2>&1 1>&3 3>/dev/null); } 3>&1
+            type_checking_code=$?
+
+            summary_type_checking_problems $type_checking_code "$type_checking_info"
+        fi
+
         assert_api_spec_approvals
         ;;
       check_whl_size)
@@ -4538,7 +4630,17 @@ function main() {
       api_example)
         { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
         example_code=$?
-        summary_check_problems $example_code "$example_info"
+
+        summary_check_example_code_problems $example_code "$example_info"
+
+        run_type_checking
+        run_tc=$?
+        if [[ ${run_tc} -eq 0 ]]; then
+            { type_checking_info=$(exec_type_checking 2>&1 1>&3 3>/dev/null); } 3>&1
+            type_checking_code=$?
+
+            summary_type_checking_problems $type_checking_code "$type_checking_info"
+        fi
         ;;
       test_op_benchmark)
         test_op_benchmark

pyproject.toml

@@ -130,3 +130,23 @@ known-first-party = ["paddle"]
 "test/dygraph_to_static/test_loop.py" = ["C416", "F821"]
 # Ignore unnecessary lambda in dy2st unittest test_lambda
 "test/dygraph_to_static/test_lambda.py" = ["PLC3002"]
+
+[tool.mypy]
+python_version = "3.8"
+cache_dir = ".mypy_cache"
+# Miscellaneous strictness flags
+allow_redefinition = true
+local_partial_types = true
+strict = false
+# Untyped definitions and calls
+check_untyped_defs = true
+# Import discovery
+follow_imports = "normal"
+# Miscellaneous
+warn_unused_configs = true
+# Configuring warnings
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+# Configuring error messages
+show_column_numbers = true

python/paddle/tensor/math.py

@@ -15,6 +15,8 @@
 math functions
 """
 
+from __future__ import annotations
+
 import math
 import warnings
 
@@ -212,7 +214,10 @@ def log_(x, name=None):
         return _C_ops.log_(x)
 
 
-def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
+# TODO(megemini): type checking
+def scale(
+    x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None
+) -> int:
     """
     Scale operator.
 
@@ -322,7 +327,7 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
         return helper.append_activation(out)
 
 
-def stanh(x, scale_a=0.67, scale_b=1.7159, name=None):
+def stanh(x, scale_a: int = 1, scale_b=1.7159, name=None):
     r"""
 
     stanh activation.

python/setup.py.in

@@ -1073,6 +1073,9 @@ if '${WITH_CPP_DIST}' == 'ON':
     paddle_lib_test_dir = '${PADDLE_LIB_TEST_DIR}'
     install_cpp_dist_and_build_test(paddle_install_dir, paddle_lib_test_dir)
 
+# type hints
+package_data['paddle'] = package_data.get('paddle', []) + ['py.typed']
+package_data['paddle.tensor'] = package_data.get('paddle.tensor', []) + ['*.pyi']
 
 with redirect_stdout():
     setup(name='${PACKAGE_NAME}',

python/unittest_py/requirements.txt

@@ -19,3 +19,4 @@ wandb>=0.13 ; python_version<"3.12"
 xlsxwriter==3.0.9
 xdoctest==1.1.1
 ubelt==1.3.3 # just for xdoctest
+mypy==1.10.0

setup.py

@@ -1302,6 +1302,13 @@ def get_package_data_and_package_dir():
         ext_modules = []
     elif sys.platform == 'darwin':
         ext_modules = []
+
+    # type hints
+    package_data['paddle'] = package_data.get('paddle', []) + ['py.typed']
+    package_data['paddle.tensor'] = package_data.get('paddle.tensor', []) + [
+        '*.pyi'
+    ]
+
     return package_data, package_dir, ext_modules
 
 

tools/sampcd_processor_utils.py

@@ -48,6 +48,12 @@
 API_DIFF_SPEC_FN = 'dev_pr_diff_api.spec'
 TEST_TIMEOUT = 10
 
+PAT_API_SPEC_MEMBER = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})')
+# insert ArgSpec for changing the API's type annotation can trigger the CI
+PAT_API_SPEC_SIGNATURE = re.compile(
+    r'^(paddle[^,]+)\s+\((ArgSpec.*),.*document\W*([0-9a-z]{32})'
+)
+
 
 class Result:
     # name/key for result
@@ -333,17 +339,17 @@ def get_api_md5(path):
     API_spec = os.path.abspath(os.path.join(os.getcwd(), "..", path))
     if not os.path.isfile(API_spec):
         return api_md5
-    pat = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})')
-    patArgSpec = re.compile(
-        r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})'
-    )
+
     with open(API_spec) as f:
         for line in f.readlines():
-            mo = pat.search(line)
-            if not mo:
-                mo = patArgSpec.search(line)
+            mo = PAT_API_SPEC_MEMBER.search(line)
+
             if mo:
                 api_md5[mo.group(1)] = mo.group(2)
+            else:
+                mo = PAT_API_SPEC_SIGNATURE.search(line)
+                api_md5[mo.group(1)] = f'{mo.group(2)}, {mo.group(3)}'
+
     return api_md5