fix

Author: wangxicoding

python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py

@@ -428,6 +428,9 @@ def minimize_impl(self,
 
         self._adapt_amp_clip_without_sharding()
 
+        # loss div dp_degree
+        self._insert_loss_grad_scale_op()
+
         self._apply_optimize_offload_pass()
 
         # step6: (optional) sharding gradient merge
@@ -561,11 +564,6 @@ def _init_pipeline_comm(self, startup_block):
     def _init_comm(self):
         # sync var
         startup_block = self._startup_program.global_block()
-        self.startup_prog_sync_var = startup_block.create_var(
-            name="startup_prog_sync_var",
-            shape=[1],
-            dtype=core.VarDesc.VarType.INT32,
-            persistable=False)
 
         # mp ring
         if self.mp_degree > 1:
@@ -1281,7 +1279,8 @@ def _initialization_broadcast(self):
         this funtion is to ensure the initialization between dp group to be 
         identical when hybrid-dp is used.
         """
-        if not self.hybrid_dp: return
+        if not self.hybrid_dp:
+            return
 
         startup_block = self._startup_program.global_block()