add new api ci check file (#33609)

Author: zhiboniu

python/paddle/__init__.py

@@ -501,7 +501,7 @@
            'log10',
            'concat',
            'check_shape',
-           'trunc'
+           'trunc',
            'digamma',
            'standard_normal'
 ]

python/paddle/vision/datasets/__init__.py

@@ -22,7 +22,7 @@
 from .voc2012 import VOC2012  # noqa: F401
 
 __all__ = [ #noqa
-    'DatasetFolder'
+    'DatasetFolder',
     'ImageFolder',
     'MNIST',
     'FashionMNIST',

tools/print_signatures.py

@@ -62,96 +62,6 @@ def md5(doc):
     return md5sum
 
 
-def get_functools_partial_spec(func):
-    func_str = func.func.__name__
-    args = func.args
-    keywords = func.keywords
-    return '{}(args={}, keywords={})'.format(func_str, args, keywords)
-
-
-def format_spec(spec):
-    args = spec.args
-    varargs = spec.varargs
-    keywords = spec.keywords
-    defaults = spec.defaults
-    if defaults is not None:
-        defaults = list(defaults)
-        for idx, item in enumerate(defaults):
-            if not isinstance(item, functools.partial):
-                continue
-
-            defaults[idx] = get_functools_partial_spec(item)
-
-        defaults = tuple(defaults)
-
-    return 'ArgSpec(args={}, varargs={}, keywords={}, defaults={})'.format(
-        args, varargs, keywords, defaults)
-
-
-def queue_dict(member, cur_name):
-    if cur_name != 'paddle':
-        try:
-            eval(cur_name)
-        except (AttributeError, NameError, SyntaxError) as e:
-            print(
-                "Error({}) occurred when `eval({})`, discard it.".format(
-                    str(e), cur_name),
-                file=sys.stderr)
-            return
-
-    if (inspect.isclass(member) or inspect.isfunction(member) or
-            inspect.ismethod(member)) and hasattr(
-                member, '__module__') and hasattr(member, '__name__'):
-        args = member.__module__ + "." + member.__name__
-        try:
-            eval(args)
-        except (AttributeError, NameError, SyntaxError) as e:
-            print(
-                "Error({}) occurred when `eval({})`, discard it for {}.".format(
-                    str(e), args, cur_name),
-                file=sys.stderr)
-            return
-    else:
-        try:
-            args = inspect.getargspec(member)
-            has_type_error = False
-        except TypeError:  # special for PyBind method
-            args = "  ".join([
-                line.strip() for line in pydoc.render_doc(member).split('\n')
-                if "->" in line
-            ])
-            has_type_error = True
-
-        if not has_type_error:
-            args = format_spec(args)
-
-    doc_md5 = md5(member.__doc__)
-    member_dict[cur_name] = "({}, ('document', '{}'))".format(args, doc_md5)
-
-
-def visit_member(parent_name, member, member_name=None):
-    if member_name:
-        cur_name = ".".join([parent_name, member_name])
-    else:
-        cur_name = ".".join([parent_name, member.__name__])
-    if inspect.isclass(member):
-        queue_dict(member, cur_name)
-        for name, value in inspect.getmembers(member):
-            if hasattr(value, '__name__') and not name.startswith("_"):
-                visit_member(cur_name, value)
-    elif inspect.ismethoddescriptor(member):
-        return
-    elif inspect.isbuiltin(member):
-        return
-    elif callable(member):
-        queue_dict(member, cur_name)
-    elif inspect.isgetsetdescriptor(member):
-        return
-    else:
-        raise RuntimeError("Unsupported generate signature of member, type {0}".
-                           format(str(type(member))))
-
-
 def is_primitive(instance):
     int_types = (int, )
     pritimitive_types = int_types + (float, str)
@@ -167,6 +77,13 @@ def is_primitive(instance):
         return False
 
 
+ErrorSet = set()
+IdSet = set()
+skiplist = [
+    'paddle.vision.datasets.DatasetFolderImageFolder', 'paddle.truncdigamma'
+]
+
+
 def visit_all_module(mod):
     mod_name = mod.__name__
     if mod_name != 'paddle' and not mod_name.startswith('paddle.'):
@@ -177,37 +94,36 @@ def visit_all_module(mod):
 
     if mod in visited_modules:
         return
-
     visited_modules.add(mod)
+
+    member_names = dir(mod)
     if hasattr(mod, "__all__"):
-        member_names = (name for name in mod.__all__
-                        if not name.startswith("_"))
-    elif mod_name == 'paddle':
-        member_names = dir(mod)
-    else:
-        return
+        member_names += mod.__all__
     for member_name in member_names:
-        instance = getattr(mod, member_name, None)
-        if instance is None:
+        if member_name.startswith('__'):
             continue
-
-        if is_primitive(instance):
-            continue
-
-        if not hasattr(instance, "__name__"):
-            continue
-
-        if inspect.ismodule(instance):
-            visit_all_module(instance)
-        else:
-            if member_name != instance.__name__:
-                print(
-                    "Found alias API, alias name is: {}, original name is: {}".
-                    format(member_name, instance.__name__),
-                    file=sys.stderr)
-                visit_member(mod.__name__, instance, member_name)
+        cur_name = mod_name + '.' + member_name
+        try:
+            instance = getattr(mod, member_name)
+            if inspect.ismodule(instance):
+                visit_all_module(instance)
             else:
-                visit_member(mod.__name__, instance)
+                doc_md5 = md5(instance.__doc__)
+                instance_id = id(instance)
+                if instance_id in IdSet:
+                    continue
+                IdSet.add(instance_id)
+                member_dict[cur_name] = "({}, ('document', '{}'))".format(
+                    cur_name, doc_md5)
+                if hasattr(instance,
+                           '__name__') and member_name != instance.__name__:
+                    print(
+                        "Found alias API, alias name is: {}, original name is: {}".
+                        format(member_name, instance.__name__),
+                        file=sys.stderr)
+        except:
+            if not cur_name in ErrorSet and not cur_name in skiplist:
+                ErrorSet.add(cur_name)
 
 
 # all from gen_doc.py
@@ -306,28 +222,22 @@ def process_module(m, attr="__all__"):
 
 
 def get_all_api_from_modulelist():
-    modulelist = [
-        paddle, paddle.amp, paddle.nn, paddle.nn.functional,
-        paddle.nn.initializer, paddle.nn.utils, paddle.static, paddle.static.nn,
-        paddle.io, paddle.jit, paddle.metric, paddle.distribution,
-        paddle.optimizer, paddle.optimizer.lr, paddle.regularizer, paddle.text,
-        paddle.utils, paddle.utils.download, paddle.utils.profiler,
-        paddle.utils.cpp_extension, paddle.sysconfig, paddle.vision,
-        paddle.distributed, paddle.distributed.fleet,
-        paddle.distributed.fleet.utils, paddle.distributed.parallel,
-        paddle.distributed.utils, paddle.callbacks, paddle.hub, paddle.autograd
-    ]
+    modulelist = [paddle]
     for m in modulelist:
         visit_all_module(m)
 
     return member_dict
 
 
 if __name__ == '__main__':
-    # modules = sys.argv[1].split(",")
-    # for m in modules:
-    #    visit_all_module(importlib.import_module(m))
     get_all_api_from_modulelist()
 
     for name in member_dict:
         print(name, member_dict[name])
+    if len(ErrorSet) == 0:
+        sys.exit(0)
+    for erroritem in ErrorSet:
+        print(
+            "Error, new function {} is unreachable".format(erroritem),
+            file=sys.stderr)
+    sys.exit(1)