From 8e1f70bf87800fcebd8ce79eb250d5b2d8f058f1 Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Thu, 20 May 2021 14:48:33 +0800 Subject: [PATCH 1/8] Tensor Parallel: bugfix bias is_distributed attr --- python/paddle/distributed/collective.py | 4 ++++ 1 file changed, 4 insertions(+) mode change 100644 => 100755 python/paddle/distributed/collective.py diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py old mode 100644 new mode 100755 index e28ef1e94b1a4c..79dd3ce06800d5 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -999,6 +999,10 @@ def _parallel_linear(x, main_block = paddle.static.default_main_program().global_block() startup_block.vars[linear.weight.name].is_distributed = True main_block.vars[linear.weight.name].is_distributed = True + # set is_distributed for splited bias + if axis == 1 and linear._bias_attr != False: + startup_block.vars[linear.bias.name].is_distributed = True + main_block.vars[linear.bias.name].is_distributed = True if not gather_out: return linear_out From 4d465abef23717e87b149084313875a4103c78ce Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Thu, 20 May 2021 14:50:07 +0800 Subject: [PATCH 2/8] AMP-Tensor Parallel: c_identity to gray list --- python/paddle/fluid/contrib/mixed_precision/fp16_lists.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 python/paddle/fluid/contrib/mixed_precision/fp16_lists.py diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py old mode 100644 new mode 100755 index f940f6a3143a09..2913d99ee6b217 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py @@ -145,6 +145,7 @@ def _update_list(self): 'sign', 'cast', 'fused_bn_add_activation', + 'c_identity', } # The set of ops that don't support fp16 calculation From 7f3ac7684bd388761529b1646353bc7d8186cc56 Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Thu, 20 May 2021 16:18:34 +0800 Subject: [PATCH 3/8] revise file mode --- python/paddle/distributed/collective.py | 0 python/paddle/fluid/contrib/mixed_precision/fp16_lists.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 python/paddle/distributed/collective.py mode change 100755 => 100644 python/paddle/fluid/contrib/mixed_precision/fp16_lists.py diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py old mode 100755 new mode 100644 diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py old mode 100755 new mode 100644 From 2a4449e9bd932cccad3ae477af9859a1e09a92da Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Thu, 20 May 2021 16:46:18 +0800 Subject: [PATCH 4/8] doc: fix spell typos in sharding config --- python/paddle/distributed/fleet/base/distributed_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index f9cd623afef76a..0a989fe90f96a6 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -814,7 +814,7 @@ def sharding_configs(self): "sharding_segment_strategy": "segment_broadcast_MB", "segment_broadcast_MB": 32, "sharding_degree": 8, - "sharding_degree": 2, + "dp_degree": 2, "gradient_merge_acc_step": 4, } """ From 7bc90e4da1d2388f7c0409e6795cdab241efa1b0 Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Fri, 21 May 2021 18:15:35 +0800 Subject: [PATCH 5/8] update code comment --- python/paddle/distributed/collective.py | 7 +++++++ 1 file changed, 7 insertions(+) mode change 100644 => 100755 python/paddle/distributed/collective.py diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py old mode 100644 new mode 100755 index 68525bfa764c0a..d5b0e813cc01a7 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -977,6 +977,11 @@ def _parallel_linear(x, group=None): """ Parallel Linear + + axis the dimension of the parameter of linear layer. + axis = 0: the row dimension + axid = 1: the col dimension + """ if group is not None and not group.is_member(): return @@ -1009,6 +1014,8 @@ def _parallel_linear(x, startup_block.vars[linear.weight.name].is_distributed = True main_block.vars[linear.weight.name].is_distributed = True # set is_distributed for splited bias + # if a linear layer is splited by row, each rank would hold a complete bias + # if a linear layer is splited by col, the bias would also be split into each rank as its weight if axis == 1 and linear._bias_attr != False: startup_block.vars[linear.bias.name].is_distributed = True main_block.vars[linear.bias.name].is_distributed = True From cef1fb9477dba776bf4aae3ff88e779e20e6e29e Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Mon, 24 May 2021 11:22:22 +0800 Subject: [PATCH 6/8] Tensor parallel: update unitest --- python/paddle/distributed/collective.py | 2 +- .../fluid/tests/unittests/column_parallel_linear_api.py | 2 +- .../paddle/fluid/tests/unittests/row_parallel_linear_api.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 python/paddle/fluid/tests/unittests/column_parallel_linear_api.py mode change 100644 => 100755 python/paddle/fluid/tests/unittests/row_parallel_linear_api.py diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index d5b0e813cc01a7..4f3a6f4768933d 100755 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -1014,7 +1014,7 @@ def _parallel_linear(x, startup_block.vars[linear.weight.name].is_distributed = True main_block.vars[linear.weight.name].is_distributed = True # set is_distributed for splited bias - # if a linear layer is splited by row, each rank would hold a complete bias + # if a linear layer is splited by row, each rank would hold a complete bias and they should be the same in each rank. # if a linear layer is splited by col, the bias would also be split into each rank as its weight if axis == 1 and linear._bias_attr != False: startup_block.vars[linear.bias.name].is_distributed = True diff --git a/python/paddle/fluid/tests/unittests/column_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/column_parallel_linear_api.py old mode 100644 new mode 100755 index cfe70cf2922392..815018dc4b2f4e --- a/python/paddle/fluid/tests/unittests/column_parallel_linear_api.py +++ b/python/paddle/fluid/tests/unittests/column_parallel_linear_api.py @@ -69,7 +69,7 @@ def get_model(self, main_prog, startup_program, rank): axis=1, num_partitions=2, weight_attr=param_attr, - bias_attr=False, ) + bias_attr=True, ) return [linear_out] diff --git a/python/paddle/fluid/tests/unittests/row_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/row_parallel_linear_api.py old mode 100644 new mode 100755 index a62e3c05508a16..a24c0874482113 --- a/python/paddle/fluid/tests/unittests/row_parallel_linear_api.py +++ b/python/paddle/fluid/tests/unittests/row_parallel_linear_api.py @@ -65,12 +65,12 @@ def get_model(self, main_prog, startup_program, rank): linear_out = paddle.distributed.split( data, - size=(1000, 8), + size=(1000, 16), operation='linear', axis=0, num_partitions=2, weight_attr=param_attr, - bias_attr=False, ) + bias_attr=True, ) return [linear_out] From ae93c8a8c29eac956d153ad79afd66d085e3edb2 Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Mon, 24 May 2021 20:22:40 +0800 Subject: [PATCH 7/8] update unitest --- .../paddle/fluid/tests/unittests/test_collective_api_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 python/paddle/fluid/tests/unittests/test_collective_api_base.py diff --git a/python/paddle/fluid/tests/unittests/test_collective_api_base.py b/python/paddle/fluid/tests/unittests/test_collective_api_base.py old mode 100644 new mode 100755 index e6693b676cf643..f0c042eb7e95b6 --- a/python/paddle/fluid/tests/unittests/test_collective_api_base.py +++ b/python/paddle/fluid/tests/unittests/test_collective_api_base.py @@ -154,7 +154,10 @@ def _run_cluster(self, model_file, envs): #update environment env0.update(envs) env1.update(envs) - tr_cmd = "%s %s" + if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': + tr_cmd = "%s -m coverage run --branch -p %s" + else: + tr_cmd = "%s %s" tr0_cmd = tr_cmd % (self._python_interp, model_file) tr1_cmd = tr_cmd % (self._python_interp, model_file) tr0_pipe = open("/tmp/tr0_err_%d.log" % os.getpid(), "w") From ffe80170929a3486b8cb9ca5d99b5fa85273bde8 Mon Sep 17 00:00:00 2001 From: JZ-LIANG Date: Tue, 25 May 2021 14:32:05 +0800 Subject: [PATCH 8/8] revise file permission --- python/paddle/distributed/collective.py | 0 python/paddle/distributed/fleet/base/distributed_strategy.py | 0 python/paddle/fluid/tests/unittests/column_parallel_linear_api.py | 0 python/paddle/fluid/tests/unittests/row_parallel_linear_api.py | 0 python/paddle/fluid/tests/unittests/test_collective_api_base.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 python/paddle/distributed/collective.py mode change 100755 => 100644 python/paddle/distributed/fleet/base/distributed_strategy.py mode change 100755 => 100644 python/paddle/fluid/tests/unittests/column_parallel_linear_api.py mode change 100755 => 100644 python/paddle/fluid/tests/unittests/row_parallel_linear_api.py mode change 100755 => 100644 python/paddle/fluid/tests/unittests/test_collective_api_base.py diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py old mode 100755 new mode 100644 diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py old mode 100755 new mode 100644 diff --git a/python/paddle/fluid/tests/unittests/column_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/column_parallel_linear_api.py old mode 100755 new mode 100644 diff --git a/python/paddle/fluid/tests/unittests/row_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/row_parallel_linear_api.py old mode 100755 new mode 100644 diff --git a/python/paddle/fluid/tests/unittests/test_collective_api_base.py b/python/paddle/fluid/tests/unittests/test_collective_api_base.py old mode 100755 new mode 100644