[Typing] Paddle 的 CI 中引入 mypy 对于 API 中 docstring 的示例代码的类型检查 (#63901)

Author: megemini

paddle/scripts/paddle_build.sh

@@ -3488,7 +3488,6 @@ function build_document_preview() {
     sh /paddle/tools/document_preview.sh ${PORT}
 }
 
-
 # origin name: example
 function exec_samplecode_test() {
     if [ -d "${PADDLE_ROOT}/build/pr_whl" ];then
@@ -3502,17 +3501,86 @@ function exec_samplecode_test() {
 
     cd ${PADDLE_ROOT}/tools
     if [ "$1" = "cpu" ] ; then
-        python sampcd_processor.py --debug --mode cpu; example_error=$?
+        python sampcd_processor.py --mode cpu; example_error=$?
     elif [ "$1" = "gpu" ] ; then
         SAMPLE_CODE_EXEC_THREADS=${SAMPLE_CODE_EXEC_THREADS:-2}
-        python sampcd_processor.py --threads=${SAMPLE_CODE_EXEC_THREADS} --debug --mode gpu; example_error=$?
+        python sampcd_processor.py --threads=${SAMPLE_CODE_EXEC_THREADS} --mode gpu; example_error=$?
     fi
     if [ "$example_error" != "0" ];then
       echo "Code instance execution failed" >&2
       exit 5
     fi
 }
 
+function need_type_checking() {
+    set +x
+
+    # check pr title
+    TITLE_CHECK=`curl -s https://github.com/PaddlePaddle/Paddle/pull/${GIT_PR_ID} | grep "<title>" | grep -i "typing" || true`
+
+    if [[ ${TITLE_CHECK} ]]; then
+        set -x
+        return 0
+    else
+        set -x
+        return 1
+    fi
+}
+
+function exec_type_checking() {
+    if [ -d "${PADDLE_ROOT}/build/pr_whl" ];then
+        pip install ${PADDLE_ROOT}/build/pr_whl/*.whl
+    else
+        echo "WARNING: PR wheel is not found. Use develop wheel !!!"
+        pip install ${PADDLE_ROOT}/build/python/dist/*.whl
+    fi
+
+    python -c "import paddle;print(paddle.__version__);paddle.version.show()"
+
+    cd ${PADDLE_ROOT}/tools
+    
+    # check all sample code
+    TITLE_CHECK_ALL=`curl -s https://github.com/PaddlePaddle/Paddle/pull/${GIT_PR_ID} | grep "<title>" | grep -i "typing all" || true`
+
+    if [[ ${TITLE_CHECK_ALL} ]]; then
+        python type_checking.py --full-test; type_checking_error=$?
+    else
+        python type_checking.py; type_checking_error=$?
+    fi
+
+    if [ "$type_checking_error" != "0" ];then
+      echo "Example code type checking failed" >&2
+      exit 5
+    fi
+}
+
+
+function exec_samplecode_checking() {
+    example_info_gpu=""
+    example_code_gpu=0
+    if [ "${WITH_GPU}" == "ON" ] ; then
+        { example_info_gpu=$(exec_samplecode_test gpu 2>&1 1>&3 3>/dev/null); } 3>&1
+        example_code_gpu=$?
+    fi
+    { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
+    example_code=$?
+
+    # TODO(megemini): type_checkding should be default after type annotation been done.
+    need_type_checking
+    type_checking_status=$?
+
+    if [[ ${type_checking_status} -eq 0 ]]; then
+        { type_checking_info=$(exec_type_checking 2>&1 1>&3 3>/dev/null); } 3>&1
+        type_checking_code=$?
+    fi
+
+    summary_check_example_code_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+
+    if [[ ${type_checking_status} -eq 0 ]]; then
+        summary_type_checking_problems $type_checking_code "$type_checking_info"
+    fi
+}
+
 
 function collect_ccache_hits() {
     ccache -s
@@ -3553,10 +3621,11 @@ function test_model_benchmark() {
     bash ${PADDLE_ROOT}/tools/test_model_benchmark.sh
 }
 
-function summary_check_problems() {
+function summary_check_example_code_problems() {
     set +x
     local example_code=$1
     local example_info=$2
+
     if [ $example_code -ne 0 ];then
         echo "==============================================================================="
         echo "*****Example code error***** Please fix the error listed in the information:"
@@ -3579,6 +3648,33 @@ function summary_check_problems() {
 }
 
 
+function summary_type_checking_problems() {
+    set +x
+    local type_checking_code=$1
+    local type_checking_info=$2
+
+    if [ $type_checking_code -ne 0 ];then
+        echo "==============================================================================="
+        echo "*****Example code type checking error***** Please fix the error listed in the information:"
+        echo "==============================================================================="
+        echo "$type_checking_info"
+        echo "==============================================================================="
+        echo "*****Example code type checking FAIL*****"
+        echo "==============================================================================="
+        exit $type_checking_code
+    else
+        echo "==============================================================================="
+        echo "*****Example code type checking info*****"
+        echo "==============================================================================="
+        echo "$type_checking_info"
+        echo "==============================================================================="
+        echo "*****Example code type checking PASS*****"
+        echo "==============================================================================="
+    fi
+    set -x
+}
+
+
 function reuse_so_cache() {
     get_html="https://api.github.com/repos/PaddlePaddle/Paddle"
     curl -X GET ${get_html}/commits -H "authorization: token ${GITHUB_API_TOKEN}" >tmp.txt
@@ -4262,15 +4358,7 @@ function main() {
         check_sequence_op_unittest
         generate_api_spec ${PYTHON_ABI:-""} "PR"
         set +e
-        example_info_gpu=""
-        example_code_gpu=0
-        if [ "${WITH_GPU}" == "ON" ] ; then
-            { example_info_gpu=$(exec_samplecode_test gpu 2>&1 1>&3 3>/dev/null); } 3>&1
-            example_code_gpu=$?
-        fi
-        { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
-        example_code=$?
-        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+        exec_samplecode_checking
         assert_api_spec_approvals
         ;;
       build_and_check_cpu)
@@ -4282,15 +4370,7 @@ function main() {
         ;;
       build_and_check_gpu)
         set +e
-        example_info_gpu=""
-        example_code_gpu=0
-        if [ "${WITH_GPU}" == "ON" ] ; then
-            { example_info_gpu=$(exec_samplecode_test gpu 2>&1 1>&3 3>/dev/null); } 3>&1
-            example_code_gpu=$?
-        fi
-        { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
-        example_code=$?
-        summary_check_problems $[${example_code_gpu} + ${example_code}] "${example_info_gpu}\n${example_info}"
+        exec_samplecode_checking
         assert_api_spec_approvals
         ;;
       check_whl_size)
@@ -4533,11 +4613,6 @@ function main() {
         build ${parallel_number}
         build_document_preview
         ;;
-      api_example)
-        { example_info=$(exec_samplecode_test cpu 2>&1 1>&3 3>/dev/null); } 3>&1
-        example_code=$?
-        summary_check_problems $example_code "$example_info"
-        ;;
       test_op_benchmark)
         test_op_benchmark
         ;;

pyproject.toml

@@ -131,3 +131,32 @@ known-first-party = ["paddle"]
 "test/dygraph_to_static/test_loop.py" = ["C416", "F821"]
 # Ignore unnecessary lambda in dy2st unittest test_lambda
 "test/dygraph_to_static/test_lambda.py" = ["PLC3002"]
+
+[tool.mypy]
+python_version = "3.8"
+cache_dir = ".mypy_cache"
+# Miscellaneous strictness flags
+allow_redefinition = true
+local_partial_types = true
+strict = false
+# Untyped definitions and calls
+check_untyped_defs = true
+# Import discovery
+follow_imports = "normal"
+# Miscellaneous
+warn_unused_configs = true
+# Configuring warnings
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+# Configuring error messages
+show_column_numbers = true
+
+[[tool.mypy.overrides]]
+module = [
+    "astor",
+    "cv2",
+    "scipy",
+    "xlsxwriter"
+]
+ignore_missing_imports = true

python/unittest_py/requirements.txt

@@ -19,3 +19,4 @@ wandb>=0.13 ; python_version<"3.12"
 xlsxwriter==3.0.9
 xdoctest==1.1.1
 ubelt==1.3.3 # just for xdoctest
+mypy==1.10.0

tools/sampcd_processor_utils.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import annotations
+
 import argparse
 import inspect
 import logging
@@ -48,6 +50,12 @@
 API_DIFF_SPEC_FN = 'dev_pr_diff_api.spec'
 TEST_TIMEOUT = 10
 
+PAT_API_SPEC_MEMBER = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})')
+# insert ArgSpec for changing the API's type annotation can trigger the CI
+PAT_API_SPEC_SIGNATURE = re.compile(
+    r'^(paddle[^,]+)\s+\((ArgSpec.*),.*document\W*([0-9a-z]{32})'
+)
+
 
 class Result:
     # name/key for result
@@ -66,7 +74,7 @@ class Result:
     order: int = 0
 
     @classmethod
-    def msg(cls, count: int, env: typing.Set) -> str:
+    def msg(cls, count: int, env: set) -> str:
         """Message for logging with api `count` and running `env`."""
         raise NotImplementedError
 
@@ -85,8 +93,8 @@ class MetaResult(type):
     def __new__(
         mcs,
         name: str,
-        bases: typing.Tuple[type, ...],
-        namespace: typing.Dict[str, typing.Any],
+        bases: tuple[type, ...],
+        namespace: dict[str, typing.Any],
     ) -> type:
         cls = super().__new__(mcs, name, bases, namespace)
         if issubclass(cls, Result):
@@ -104,7 +112,7 @@ def get(mcs, name: str) -> type:
         return mcs.__cls_map.get(name)
 
     @classmethod
-    def cls_map(mcs) -> typing.Dict[str, Result]:
+    def cls_map(mcs) -> dict[str, Result]:
         return mcs.__cls_map
 
 
@@ -290,7 +298,7 @@ def prepare(self, test_capacity: set) -> None:
         """
         pass
 
-    def run(self, api_name: str, docstring: str) -> typing.List[TestResult]:
+    def run(self, api_name: str, docstring: str) -> list[TestResult]:
         """Extract codeblocks from docstring, and run the test.
         Run only one docstring at a time.
 
@@ -304,7 +312,7 @@ def run(self, api_name: str, docstring: str) -> typing.List[TestResult]:
         raise NotImplementedError
 
     def print_summary(
-        self, test_results: typing.List[TestResult], whl_error: typing.List[str]
+        self, test_results: list[TestResult], whl_error: list[str]
     ) -> None:
         """Post process test results and print test summary.
 
@@ -333,17 +341,17 @@ def get_api_md5(path):
     API_spec = os.path.abspath(os.path.join(os.getcwd(), "..", path))
     if not os.path.isfile(API_spec):
         return api_md5
-    pat = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})')
-    patArgSpec = re.compile(
-        r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})'
-    )
+
     with open(API_spec) as f:
         for line in f.readlines():
-            mo = pat.search(line)
-            if not mo:
-                mo = patArgSpec.search(line)
+            mo = PAT_API_SPEC_MEMBER.search(line)
+
             if mo:
                 api_md5[mo.group(1)] = mo.group(2)
+            else:
+                mo = PAT_API_SPEC_SIGNATURE.search(line)
+                api_md5[mo.group(1)] = f'{mo.group(2)}, {mo.group(3)}'
+
     return api_md5
 
 
@@ -397,18 +405,6 @@ def get_full_api_from_pr_spec():
         get_full_api_by_walk()
 
 
-def get_full_api():
-    """
-    get all the apis
-    """
-    global API_DIFF_SPEC_FN  # readonly
-    from print_signatures import get_all_api_from_modulelist
-
-    member_dict = get_all_api_from_modulelist()
-    with open(API_DIFF_SPEC_FN, 'w') as f:
-        f.write("\n".join(member_dict.keys()))
-
-
 def extract_code_blocks_from_docstr(docstr, google_style=True):
     """
     extract code-blocks from the given docstring.
@@ -599,9 +595,16 @@ def get_test_capacity(run_on_device="cpu"):
     return sample_code_test_capacity
 
 
-def get_docstring(full_test=False):
+def get_docstring(
+    full_test: bool = False,
+    filter_api: typing.Callable[[str], bool] | None = None,
+):
     '''
     this function will get the docstring for test.
+
+    Args:
+        full_test, get all api
+        filter_api, a function that filter api, if `True` then skip add to `docstrings_to_test`.
     '''
     import paddle
     import paddle.static.quantization  # noqa: F401
@@ -616,6 +619,9 @@ def get_docstring(full_test=False):
     with open(API_DIFF_SPEC_FN) as f:
         for line in f.readlines():
             api = line.replace('\n', '')
+            if filter_api is not None and filter_api(api.strip()):
+                continue
+
             try:
                 api_obj = eval(api)
             except AttributeError:
@@ -637,7 +643,7 @@ def get_docstring(full_test=False):
     return docstrings_to_test, whl_error
 
 
-def check_old_style(docstrings_to_test: typing.Dict[str, str]):
+def check_old_style(docstrings_to_test: dict[str, str]):
     old_style_apis = []
     for api_name, raw_docstring in docstrings_to_test.items():
         for codeblock in extract_code_blocks_from_docstr(
@@ -715,8 +721,8 @@ def exec_gen_doc():
 
 
 def get_test_results(
-    doctester: DocTester, docstrings_to_test: typing.Dict[str, str]
-) -> typing.List[TestResult]:
+    doctester: DocTester, docstrings_to_test: dict[str, str]
+) -> list[TestResult]:
     """Get test results from doctester with docstrings to test."""
     _test_style = (
         doctester.style