improve append_optimizer_op

Author: zhangting2020

python/paddle/fluid/tests/unittests/test_momentum_op.py

@@ -614,14 +614,14 @@ def test_momentum_static(self):
 
 
 class TestFusedMomentumWithDecayAPI(unittest.TestCase):
-    def get_program(self, weight_attr):
+    def get_program(self, weight_attr, bias_attr=False):
         main_program = paddle.static.Program()
         startup_program = paddle.static.Program()
         with paddle.static.program_guard(
                 main_program=main_program, startup_program=startup_program):
             x = paddle.static.data(name='x', shape=[10, 10])
             linear = paddle.nn.Linear(
-                10, 10, weight_attr=weight_attr, bias_attr=False)
+                10, 10, weight_attr=weight_attr, bias_attr=bias_attr)
             out = linear(x)
             loss = paddle.mean(out)
             optimizer = paddle.optimizer.Momentum(
@@ -637,7 +637,7 @@ def test_param_has_l2decay(self):
             name="weight",
             initializer=paddle.nn.initializer.Constant(value=0.5),
             regularizer=paddle.regularizer.L2Decay(0.1))
-        program = self.get_program(weight_attr)
+        program = self.get_program(weight_attr, bias_attr=False)
         ops = program.global_block().ops
 
         self.assertEqual(ops[-1].attr('regularization_method'), 'l2_decay')
@@ -652,21 +652,30 @@ def test_param_has_l1decay(self):
             name="weight",
             initializer=paddle.nn.initializer.Constant(value=0.5),
             regularizer=paddle.regularizer.L1Decay(0.1))
-        program = self.get_program(weight_attr)
+        bias_attr = paddle.ParamAttr(
+            name="bias",
+            initializer=paddle.nn.initializer.Constant(value=0.),
+            regularizer=None)
+        program = self.get_program(weight_attr, bias_attr)
         ops = program.global_block().ops
-        self.assertEqual(ops[-1].attr('regularization_method'), '')
-        self.assertEqual(ops[-1].attr('regularization_coeff'), 0)
-        self.assertEqual(ops[-2].type, 'sum')
-        self.assertEqual(ops[-3].type, 'scale')
-        self.assertEqual(ops[-4].type, 'sign')
 
-    def test_param_regularizer_is_none(self):
+        self.assertEqual(ops[-1].type, 'momentum')
+        self.assertEqual(ops[-2].type, 'momentum')
+        self.assertEqual(ops[-3].type, 'sum')
+        self.assertEqual(ops[-4].type, 'scale')
+        self.assertEqual(ops[-5].type, 'sign')
+        self.assertEqual(ops[-6].type, 'matmul_grad')
+        if 'weight' in ops[-1].input('Param'):
+            self.assertEqual(ops[-1].attr('regularization_method'), '')
+            self.assertEqual(ops[-1].attr('regularization_coeff'), 0)
+        if 'bias' in ops[-2].input('Param'):
+            self.assertEqual(ops[-2].attr('regularization_method'), 'l2_decay')
+            self.assertEqual(ops[-2].attr('regularization_coeff'),
+                             np.float32(0.5))
+
+    def test_param_has_no_regularizer(self):
         paddle.enable_static()
-        weight_attr = paddle.ParamAttr(
-            name="weight",
-            initializer=paddle.nn.initializer.Constant(value=0.5),
-            regularizer=None)
-        program = self.get_program(weight_attr)
+        program = self.get_program(weight_attr=None)
         ops = program.global_block().ops
         self.assertEqual(ops[-1].attr('regularization_method'), 'l2_decay')
         self.assertEqual(ops[-1].attr('regularization_coeff'), np.float32(0.5))

python/paddle/optimizer/momentum.py

@@ -274,16 +274,19 @@ def _append_optimize_op(self, block, param_and_grad):
                                              param_and_grad[0])
         lr = self._create_param_lr(param_and_grad)
 
+        # For fusion of momentum and l2decay 
         param = param_and_grad[0]
+        regularization_method = self._regularization_method
+        regularization_coeff = self._regularization_coeff
         if hasattr(param, 'regularizer'):
             # we skip param's l2decay before, so fuse it with momentum here.
             if isinstance(param.regularizer, L2DecayRegularizer):
-                self._regularization_method = "l2_decay"
-                self._regularization_coeff = param.regularizer._regularization_coeff
+                regularization_method = "l2_decay"
+                regularization_coeff = param.regularizer._regularization_coeff
             # the param's regularization has been done before, we avoid do l2decay in momentum.
             elif param.regularizer is not None:
-                self._regularization_method = ""
-                self._regularization_coeff = 0
+                regularization_method = ""
+                regularization_coeff = 0
 
         if framework.in_dygraph_mode():
             if isinstance(param_and_grad, dict):
@@ -292,8 +295,8 @@ def _append_optimize_op(self, block, param_and_grad):
                 param_and_grad[0], param_and_grad[1], velocity_acc, lr,
                 param_and_grad[0], velocity_acc, 'mu', self._momentum,
                 'use_nesterov', self._use_nesterov, 'regularization_method',
-                self._regularization_method, 'regularization_coeff',
-                self._regularization_coeff)
+                regularization_method, 'regularization_coeff',
+                regularization_coeff)
             return None
 
         find_master = self._multi_precision and param_and_grad[
@@ -304,8 +307,8 @@ def _append_optimize_op(self, block, param_and_grad):
         attrs = {
             "mu": self._momentum,
             "use_nesterov": self._use_nesterov,
-            "regularization_method": self._regularization_method,
-            "regularization_coeff": self._regularization_coeff,
+            "regularization_method": regularization_method,
+            "regularization_coeff": regularization_coeff,
             "multi_precision": find_master,
             "rescale_grad": self._rescale_grad
         }