From e32d5966ff5a46d7d11313702dbf89fa7992cef9 Mon Sep 17 00:00:00 2001 From: WangXi Date: Tue, 24 Aug 2021 22:54:04 +0800 Subject: [PATCH] fix npu found_finite in hybrid --- .../fleet/meta_optimizers/sharding_optimizer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index c94bd572f05878..a76a70cdcab3df 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -371,8 +371,11 @@ def _adapt_amp_clip_without_sharding(self): # FIXME(wangxi): mp should prune duplicated param_grads when calc # amp inf_var & clip global_norm_var - FP16Utils.sync_amp_check_nan_inf(main_block, - [self.mp_ring_id, self.pp_ring_id]) + rings = [self.mp_ring_id, self.pp_ring_id] + # FIXME(wangxi): some problem with NPU found_finite, need sync with DP + if core.is_compiled_with_npu(): + rings += [self.dp_ring_id] + FP16Utils.sync_amp_check_nan_inf(main_block, rings) gradientclip_helper = GradientClipHelper(None) gradientclip_helper.sync_global_norm(