From 9310407a6b1f1fafb26c10e2a5250f456dd9bda0 Mon Sep 17 00:00:00 2001 From: sandyhouse Date: Thu, 28 Jan 2021 10:37:52 +0800 Subject: [PATCH 1/3] update, test=develop --- paddle/fluid/framework/distributed_strategy.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 7cf8d55aeeb1d9..c5bdac3fcce71c 100644 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -139,9 +139,9 @@ message DistributedStrategy { optional bool fuse_all_reduce_ops = 18 [ default = true ]; optional int32 fuse_grad_size_in_MB = 19 [ default = 32 ]; optional float fuse_grad_size_in_TFLOPS = 20 [ default = 50 ]; - optional bool cudnn_exhaustive_search = 21 [ default = true ]; + optional bool cudnn_exhaustive_search = 21 [ default = false ]; optional int32 conv_workspace_size_limit = 22 [ default = 4000 ]; - optional bool cudnn_batchnorm_spatial_persistent = 23 [ default = true ]; + optional bool cudnn_batchnorm_spatial_persistent = 23 [ default = false ]; optional bool adaptive_localsgd = 24 [ default = false ]; optional bool fp16_allreduce = 25 [ default = false ]; optional bool sharding = 26 [ default = false ]; From c568613fa0d747827410d53898b2135661b6d9ab Mon Sep 17 00:00:00 2001 From: sandyhouse Date: Thu, 4 Feb 2021 21:57:15 +0800 Subject: [PATCH 2/3] update, test=develop --- .../fleet/base/distributed_strategy.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index f7a28f15e9b70b..ed511dfdd73b75 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -115,6 +115,28 @@ def __init__(self): """ self.strategy = distributed_strategy_pb2.DistributedStrategy() + + # Set the default values of the following flags to the ones set by users + key = 'FLAGS_cudnn_batchnorm_spatial_persistent' + if core.globals().is_public(key): + self.strategy.cudnn_batchnorm_spatial_persistent = bool( + core.globals()[key]) + key = 'FLAGS_conv_workspace_size_limit' + if core.globals().is_public(key): + self.strategy.conv_workspace_size_limit = int(core.globals()[key]) + key = 'FLAGS_cudnn_exhaustive_search' + if core.globals().is_public(key): + self.strategy.cudnn_exhaustive_search = bool(core.globals()[key]) + key = 'FLAGS_sync_nccl_allreduce' + if core.globals().is_public(key): + self.strategy.sync_nccl_allreduce = bool(core.globals()[key]) + key = 'FLAGS_fuse_parameter_memory_size' + if core.globals().is_public(key): + self.strategy.fuse_grad_size_in_MB = int(core.globals()[key]) + key = 'FLAGS_fuse_parameter_groups_size' + if core.globals().is_public(key): + self.strategy.fuse_grad_size_in_TFLOPS = int(core.globals()[key]) + self.__lock_attr = True def __setattr__(self, key, value): From 4887fc67a8114b700b0b444674db77bd646ca132 Mon Sep 17 00:00:00 2001 From: sandyhouse Date: Tue, 23 Feb 2021 14:55:07 +0800 Subject: [PATCH 3/3] update, test=develop --- .../paddle/distributed/fleet/base/distributed_strategy.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index f3fe5d398e8fd8..f79013d7347c00 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -133,12 +133,6 @@ def __init__(self): key = 'FLAGS_sync_nccl_allreduce' if core.globals().is_public(key): self.strategy.sync_nccl_allreduce = bool(core.globals()[key]) - key = 'FLAGS_fuse_parameter_memory_size' - if core.globals().is_public(key): - self.strategy.fuse_grad_size_in_MB = int(core.globals()[key]) - key = 'FLAGS_fuse_parameter_groups_size' - if core.globals().is_public(key): - self.strategy.fuse_grad_size_in_TFLOPS = int(core.globals()[key]) self.__lock_attr = True