From 7e31c611828f482039f261a1ee921bbe6671c1f7 Mon Sep 17 00:00:00 2001
From: Yibing Liu <liuyibing01@baidu.com>
Date: Fri, 24 May 2019 02:38:19 +0000
Subject: [PATCH] Fix trust ratio in lamb

test=develop
---
 paddle/fluid/operators/optimizers/lamb_op.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/operators/optimizers/lamb_op.h b/paddle/fluid/operators/optimizers/lamb_op.h
index 4fbe486cb1d68f..72a9093859cdf5 100644
--- a/paddle/fluid/operators/optimizers/lamb_op.h
+++ b/paddle/fluid/operators/optimizers/lamb_op.h
@@ -174,10 +174,11 @@ struct LambParamUpateFunctor {
 
   inline HOSTDEVICE void operator()(size_t i) const {
     T lr = *lr_;
-    T p_norm = *param_norm_;
-    T tr_div_norm = *trust_ratio_div_norm_;
+    T p = *param_norm_;
+    T t = *trust_ratio_div_norm_;
 
-    lr *= p_norm / tr_div_norm;
+    T r = (p > 0 && t > 0) ? p / t : 1.0;
+    lr *= r;
     param_out_[i] = param_[i] - lr * trust_ratio_div_[i];
   }
 };