From 6cdd0fd12a4bb41b299ae76cc5636566737eb55f Mon Sep 17 00:00:00 2001 From: Wang Huan Date: Thu, 26 Aug 2021 07:49:28 +0000 Subject: [PATCH 1/6] fix count_api_without_core_ops, test=develop --- python/paddle/fluid/layers/detection.py | 2 +- python/paddle/fluid/layers/loss.py | 2 +- tools/count_api_without_core_ops.py | 60 +++++++++++++++---------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index d66943d34eb7ba..5425a9eeb02d67 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -3751,7 +3751,7 @@ def distribute_fpn_proposals(fpn_rois, refer_level=4, refer_scale=224) """ - num_lvl = max_level - min_level + 1 + num_lvl = max_level - min_level + 1 if in_dygraph_mode(): assert rois_num is not None, "rois_num should not be None in dygraph mode." diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index 0954fe7f548d31..3cf0bd2d5f89e9 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -1107,7 +1107,7 @@ def sampled_softmax_with_cross_entropy(logits, out = fluid.layers.sampled_softmax_with_cross_entropy( logits=fc, label=label, num_samples=25) """ - helper = LayerHelper('sample_logits', **locals()) + helper = LayerHelper('sample_logits', **locals()) samples = customized_samples if use_customized_samples else helper.create_variable_for_type_inference( dtype='int64') probabilities = customized_probabilities if use_customized_samples else helper.create_variable_for_type_inference( diff --git a/tools/count_api_without_core_ops.py b/tools/count_api_without_core_ops.py index e84a03d93e9bbf..c8c012f4274eab 100644 --- a/tools/count_api_without_core_ops.py +++ b/tools/count_api_without_core_ops.py @@ -51,13 +51,19 @@ def split_with_and_without_core_ops(member, cur_name): if cur_name in omitted_list: return + if member.__doc__.find(':api_attr: Static Graph') != -1: + return + + if cur_name.find('ParamBase') != -1 or cur_name.find('Parameter') != -1 or cur_name.find('Variable') != -1 or cur_name.find('control_flow') != -1 or cur_name.find('contrib.mixed_precision') != -1: + return + if inspect.isclass(member): pass else: try: source = inspect.getsource(member) if source.find('append_op') != -1: - if source.find('core.ops') != -1: + if source.find('core.ops') != -1 or source.find('_C_ops') != -1: api_with_ops.append(cur_name) else: api_without_ops.append(cur_name) @@ -116,8 +122,12 @@ def is_primitive(instance): else: return False +ErrorSet = set() +IdSet = set() +skiplist = [] +visited_modules = set() -def visit_all_module(mod, visited, func): +def visit_all_module(mod, func): mod_name = mod.__name__ if mod_name != 'paddle' and not mod_name.startswith('paddle.'): return @@ -125,30 +135,32 @@ def visit_all_module(mod, visited, func): if mod_name.startswith('paddle.fluid.core'): return - if mod in visited: + if mod in visited_modules: return + visited_modules.add(mod) - visited.add(mod) - - for member_name in ( - name - for name in (mod.__all__ if hasattr(mod, "__all__") else dir(mod)) - if not name.startswith("_")): - instance = getattr(mod, member_name, None) - if instance is None: - continue - - if is_primitive(instance): + member_names = dir(mod) + if hasattr(mod, "__all__"): + member_names += mod.__all__ + for member_name in member_names: + if member_name.startswith('_'): continue - - if not hasattr(instance, "__name__"): + cur_name = mod_name + '.' + member_name + if cur_name in skiplist: continue - - if inspect.ismodule(instance): - visit_all_module(instance, visited, func) - else: - visit_member(mod.__name__, instance, func) - + try: + instance = getattr(mod, member_name) + if inspect.ismodule(instance): + visit_all_module(instance, func) + else: + instance_id = id(instance) + if instance_id in IdSet: + continue + IdSet.add(instance_id) + visit_member(mod.__name__, instance, func) + except: + if not cur_name in ErrorSet and not cur_name in skiplist: + ErrorSet.add(cur_name) def get_apis_with_and_without_core_ops(modules): global api_with_ops, api_without_ops @@ -156,7 +168,7 @@ def get_apis_with_and_without_core_ops(modules): api_without_ops = [] for m in modules: visit_all_module( - importlib.import_module(m), set(), split_with_and_without_core_ops) + importlib.import_module(m), split_with_and_without_core_ops) return api_with_ops, api_without_ops @@ -164,7 +176,7 @@ def get_api_source_desc(modules): global func_dict func_dict = collections.OrderedDict() for m in modules: - visit_all_module(importlib.import_module(m), set(), get_md5_of_func) + visit_all_module(importlib.import_module(m), get_md5_of_func) return func_dict From 07d5b1123613986e669a12b3c0c87a483d8aecd4 Mon Sep 17 00:00:00 2001 From: Wang Huan Date: Thu, 26 Aug 2021 07:50:58 +0000 Subject: [PATCH 2/6] fix count_api_without_core_ops, test=develop --- tools/count_api_without_core_ops.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/count_api_without_core_ops.py b/tools/count_api_without_core_ops.py index c8c012f4274eab..a2093e34fbacbf 100644 --- a/tools/count_api_without_core_ops.py +++ b/tools/count_api_without_core_ops.py @@ -54,7 +54,11 @@ def split_with_and_without_core_ops(member, cur_name): if member.__doc__.find(':api_attr: Static Graph') != -1: return - if cur_name.find('ParamBase') != -1 or cur_name.find('Parameter') != -1 or cur_name.find('Variable') != -1 or cur_name.find('control_flow') != -1 or cur_name.find('contrib.mixed_precision') != -1: + if cur_name.find('ParamBase') != -1 or cur_name.find( + 'Parameter') != -1 or cur_name.find( + 'Variable') != -1 or cur_name.find( + 'control_flow') != -1 or cur_name.find( + 'contrib.mixed_precision') != -1: return if inspect.isclass(member): @@ -122,11 +126,13 @@ def is_primitive(instance): else: return False + ErrorSet = set() IdSet = set() skiplist = [] visited_modules = set() + def visit_all_module(mod, func): mod_name = mod.__name__ if mod_name != 'paddle' and not mod_name.startswith('paddle.'): @@ -162,6 +168,7 @@ def visit_all_module(mod, func): if not cur_name in ErrorSet and not cur_name in skiplist: ErrorSet.add(cur_name) + def get_apis_with_and_without_core_ops(modules): global api_with_ops, api_without_ops api_with_ops = [] From 04c2684a8b7a1577bca6be36066aab1724f8457e Mon Sep 17 00:00:00 2001 From: Wang Huan Date: Thu, 26 Aug 2021 10:40:15 +0000 Subject: [PATCH 3/6] refine, test=develop --- python/paddle/fluid/layers/detection.py | 4 ++-- python/paddle/fluid/layers/loss.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 5425a9eeb02d67..4c064a217dbbbc 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -3751,10 +3751,10 @@ def distribute_fpn_proposals(fpn_rois, refer_level=4, refer_scale=224) """ - num_lvl = max_level - min_level + 1 + num_lvl = max_level - min_level + 1 if in_dygraph_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." + assert rois_num is not None, "rois_num should not be sdf None in dygraph mode." attrs = ('min_level', min_level, 'max_level', max_level, 'refer_level', refer_level, 'refer_scale', refer_scale) multi_rois, restore_ind, rois_num_per_level = _C_ops.distribute_fpn_proposals( diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index 3cf0bd2d5f89e9..f23a07ea79bd18 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -1107,7 +1107,8 @@ def sampled_softmax_with_cross_entropy(logits, out = fluid.layers.sampled_softmax_with_cross_entropy( logits=fc, label=label, num_samples=25) """ - helper = LayerHelper('sample_logits', **locals()) + print("test") + helper = LayerHelper('sample_logits', **locals()) samples = customized_samples if use_customized_samples else helper.create_variable_for_type_inference( dtype='int64') probabilities = customized_probabilities if use_customized_samples else helper.create_variable_for_type_inference( From f305104a400971466cef0bf1fffd3f60de8081a0 Mon Sep 17 00:00:00 2001 From: Wang Huan Date: Fri, 27 Aug 2021 01:59:11 +0000 Subject: [PATCH 4/6] remove test code, test=develop --- python/paddle/fluid/layers/loss.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index f23a07ea79bd18..0954fe7f548d31 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -1107,7 +1107,6 @@ def sampled_softmax_with_cross_entropy(logits, out = fluid.layers.sampled_softmax_with_cross_entropy( logits=fc, label=label, num_samples=25) """ - print("test") helper = LayerHelper('sample_logits', **locals()) samples = customized_samples if use_customized_samples else helper.create_variable_for_type_inference( dtype='int64') From fcdbcfbd8f7eedcac866ca63572857ce3485bfe9 Mon Sep 17 00:00:00 2001 From: Wang Huan Date: Fri, 27 Aug 2021 02:50:31 +0000 Subject: [PATCH 5/6] remove test, test=develop --- python/paddle/fluid/layers/detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 4c064a217dbbbc..d66943d34eb7ba 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -3754,7 +3754,7 @@ def distribute_fpn_proposals(fpn_rois, num_lvl = max_level - min_level + 1 if in_dygraph_mode(): - assert rois_num is not None, "rois_num should not be sdf None in dygraph mode." + assert rois_num is not None, "rois_num should not be None in dygraph mode." attrs = ('min_level', min_level, 'max_level', max_level, 'refer_level', refer_level, 'refer_scale', refer_scale) multi_rois, restore_ind, rois_num_per_level = _C_ops.distribute_fpn_proposals( From cb5e8aff74546884bfd72279d74691ba84bcb3ec Mon Sep 17 00:00:00 2001 From: Wang Huan Date: Fri, 27 Aug 2021 03:03:08 +0000 Subject: [PATCH 6/6] modify check_api_approvals.sh, test=develop --- tools/check_api_approvals.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_api_approvals.sh b/tools/check_api_approvals.sh index c0a0b754d5191c..9391f8cdc01433 100644 --- a/tools/check_api_approvals.sh +++ b/tools/check_api_approvals.sh @@ -63,7 +63,7 @@ fi api_src_spec_diff=`python ${PADDLE_ROOT}/tools/check_api_source_without_core_ops.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.source.md5 ${PADDLE_ROOT}/paddle/fluid/API_PR.source.md5` if [ "$api_src_spec_diff" != "" ]; then echo_line="APIs without core.ops: \n${api_src_spec_diff}\n" - echo_line="${echo_line}You must have one RD (zhiqiu (Recommend) or phlrain) approval for the api change for the opreator-related api without 'core.ops'.\n" + echo_line="${echo_line}You must have one RD (zhiqiu (Recommend) or phlrain) approval for the api change for the opreator-related api without '_C_ops'.\n" echo_line="${echo_line}For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/paddle_api_development_manual.md]\n" check_approval 1 6888866 43953930 fi