add flag FLAGS_cudnn_deterministic

Author: zhangbo9674

python/paddle/fluid/tests/unittests/test_imperative_auto_mixed_precision.py

@@ -286,10 +286,8 @@ def train_resnet(self,
                      use_save_load=True):
         seed = 90
 
-        EPOCH_NUM = 4  # 设置外层循环次数
-
         batch_size = train_parameters["batch_size"]
-        batch_num = 1
+        batch_num = 4
 
         paddle.seed(seed)
         paddle.framework.random._manual_program_seed(seed)
@@ -322,54 +320,51 @@ def train_resnet(self,
             train_loader.set_sample_list_generator(train_reader)
             train_reader = train_loader
 
-        for epoch_id in range(EPOCH_NUM):
-            for batch_id, data in enumerate(train_reader()):
-                if batch_id >= batch_num:
-                    break
-                if use_data_loader:
-                    img, label = data
-                else:
-                    dy_x_data = np.array(
-                        [x[0].reshape(3, 224, 224)
-                         for x in data]).astype('float32')
-                    if len(np.array([x[1] for x in data]).astype(
-                            'int64')) != batch_size:
-                        continue
-                    y_data = np.array(
-                        [x[1] for x in data]).astype('int64').reshape(-1, 1)
-
-                    img = paddle.to_tensor(dy_x_data)
-                    label = paddle.to_tensor(y_data)
-                label.stop_gradient = True
+        for batch_id, data in enumerate(train_reader()):
+            if batch_id >= batch_num:
+                break
+            if use_data_loader:
+                img, label = data
+            else:
+                dy_x_data = np.array([x[0].reshape(3, 224, 224)
+                                      for x in data]).astype('float32')
+                if len(np.array([x[1]
+                                 for x in data]).astype('int64')) != batch_size:
+                    continue
+                y_data = np.array(
+                    [x[1] for x in data]).astype('int64').reshape(-1, 1)
 
-                with paddle.amp.auto_cast(enable=enable_amp):
-                    out = resnet(img)
+                img = paddle.to_tensor(dy_x_data)
+                label = paddle.to_tensor(y_data)
+            label.stop_gradient = True
 
-                loss = paddle.nn.functional.cross_entropy(
-                    input=out, label=label)
-                avg_loss = paddle.mean(x=loss)
+            with paddle.amp.auto_cast(enable=enable_amp):
+                out = resnet(img)
 
-                dy_out = avg_loss.numpy()
+            loss = paddle.nn.functional.cross_entropy(input=out, label=label)
+            avg_loss = paddle.mean(x=loss)
 
-                scaled_loss = scaler.scale(avg_loss)
-                scaled_loss.backward()
+            dy_out = avg_loss.numpy()
 
-                scaler.minimize(optimizer, scaled_loss)
+            scaled_loss = scaler.scale(avg_loss)
+            scaled_loss.backward()
 
-                dy_grad_value = {}
-                for param in resnet.parameters():
-                    if param.trainable:
-                        np_array = np.array(param._grad_ivar().value()
-                                            .get_tensor())
-                        dy_grad_value[param.name + fluid.core.grad_var_suffix(
-                        )] = np_array
+            scaler.minimize(optimizer, scaled_loss)
 
-                resnet.clear_gradients()
+            dy_grad_value = {}
+            for param in resnet.parameters():
+                if param.trainable:
+                    np_array = np.array(param._grad_ivar().value().get_tensor())
+                    dy_grad_value[param.name + fluid.core.grad_var_suffix(
+                    )] = np_array
 
-                dy_param_value = {}
-                for param in resnet.parameters():
-                    dy_param_value[param.name] = param.numpy()
-            if use_save_load and epoch_id == 2:
+            resnet.clear_gradients()
+
+            dy_param_value = {}
+            for param in resnet.parameters():
+                dy_param_value[param.name] = param.numpy()
+
+            if use_save_load and batch_id == 2:
                 paddle.save(scaler.state_dict(), 'ResNet_model.pdparams')
                 dict_load = paddle.load('ResNet_model.pdparams')
                 scaler.load_state_dict(dict_load)
@@ -378,15 +373,16 @@ def train_resnet(self,
         return dy_out, dy_param_value, dy_grad_value
 
     def test_with_state_dict(self):
+        if fluid.core.is_compiled_with_cuda():
+            fluid.set_flags({"FLAGS_cudnn_deterministic": True})
         with fluid.dygraph.guard():
             out_use_state_dict = self.train_resnet(
                 enable_amp=True, use_data_loader=True, use_save_load=True)
             out_no_state_dict = self.train_resnet(
                 enable_amp=True, use_data_loader=True, use_save_load=False)
         print('save_load:', out_use_state_dict[0], out_no_state_dict[0])
         self.assertTrue(
-            np.allclose(
-                out_use_state_dict[0], out_no_state_dict[0], atol=1.e-2))
+            np.allclose(out_use_state_dict[0], out_no_state_dict[0]))
 
 
 class TestResnet2(unittest.TestCase):
@@ -479,13 +475,17 @@ def train_resnet(self, enable_amp=True, use_data_loader=False):
         return dy_out, dy_param_value, dy_grad_value
 
     def test_resnet(self):
+        if fluid.core.is_compiled_with_cuda():
+            fluid.set_flags({"FLAGS_cudnn_deterministic": True})
         with fluid.dygraph.guard():
             out_fp32 = self.train_resnet(enable_amp=False)
             out_amp = self.train_resnet(enable_amp=True)
         print(out_fp32[0], out_amp[0])
         self.assertTrue(np.allclose(out_fp32[0], out_amp[0], atol=1.e-2))
 
     def test_with_data_loader(self):
+        if fluid.core.is_compiled_with_cuda():
+            fluid.set_flags({"FLAGS_cudnn_deterministic": True})
         with fluid.dygraph.guard():
             out_fp32 = self.train_resnet(enable_amp=False, use_data_loader=True)
             out_amp = self.train_resnet(enable_amp=True, use_data_loader=True)
@@ -566,6 +566,8 @@ def train_resnet(self, enable_amp=True):
         return dy_out, dy_param_value, dy_grad_value
 
     def test_resnet(self):
+        if fluid.core.is_compiled_with_cuda():
+            fluid.set_flags({"FLAGS_cudnn_deterministic": True})
         out_fp32 = self.train_resnet(enable_amp=False)
         out_amp = self.train_resnet(enable_amp=True)
         print(out_fp32[0], out_amp[0])