Skip to content

fix set_grad_ivar bug of Tensor.backward #34819

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 12, 2021
Merged

fix set_grad_ivar bug of Tensor.backward #34819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions paddle/fluid/imperative/gradient_accumulator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,12 @@ void TensorAdd(const framework::Variable& src, framework::Variable* dst) {
auto data_type = src_tensor.type();
auto place = src_tensor.place();

PADDLE_ENFORCE_EQ(dst_tensor->type(), data_type,
platform::errors::PreconditionNotMet(
"The data type of source tensor and destination tensor "
"should be equal, Otherwise, the calculation results "
"will be incorrect."));

#define PADDLE_TENSOR_ADD(cpp_type) \
if (data_type == framework::DataTypeTrait<cpp_type>::DataType()) { \
TensorAddFunctor<cpp_type> func( \
Expand Down Expand Up @@ -422,9 +428,9 @@ void GradientAccumulator::AccumulateGrad() {
auto* src = inner_var_->MutableVar();
auto* dst = var_->MutableVar();
if (!var_->IsEmpty()) {
VLOG(6) << "Leaf Gradient Var(" << var_->Name()
<< ") has been calculated by previous graph, will accumulate on "
"previous graph.";
VLOG(6) << "Leaf Var(" << var_->Name()
<< ")'s Gradient has been initizlized, will accumulate on "
"previous gradient.";
if (dst->IsType<framework::LoDTensor>()) {
if (src->IsType<framework::LoDTensor>()) {
TensorAdd(*src, dst);
Expand All @@ -444,8 +450,9 @@ void GradientAccumulator::AccumulateGrad() {
"Only support LoDTensor and SelectedRows for gradient var"));
}
} else {
VLOG(6) << "Leaf Gradient Var(" << var_->Name()
<< ") has not been initialized, not accumulate. Just move";
VLOG(6)
<< "Leaf Var(" << var_->Name()
<< ")'s Gradient has not been initialized, not accumulate. Just move";
*(dst) = std::move(*src);
var_->SetType(inner_var_->Type());
var_->SetDataType(inner_var_->DataType());
Expand Down
87 changes: 64 additions & 23 deletions paddle/fluid/imperative/layer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,32 +277,73 @@ std::shared_ptr<VarBase> VarBase::NewVarBase(const platform::Place& dst_place,
}

void VarBase::CopyFrom(const VarBase& src, const bool blocking) {
if (SharedVar()->IsEmpty()) {
VLOG(3) << "deep copy Variable from " << src.Name() << " to " << Name();
SetPersistable(src.Persistable());
if (src.SharedVar()->IsEmpty()) {
return;
}

VLOG(3) << "Deep copy Tensor from " << src.Name() << " to " << Name();
if (Var().IsInitialized()) {
PADDLE_ENFORCE_EQ(DataType(), src.DataType(),
platform::errors::PreconditionNotMet(
"Tensor %s has different data type with Tensor %s, "
"Tensor Copy cannot be performed!",
Name(), src.Name()));
PADDLE_ENFORCE_EQ(Type(), src.Type(),
platform::errors::PreconditionNotMet(
"Tensor %s has different type with Tensor %s, Tensor "
"Copy cannot be performed!",
Name(), src.Name()));
} else {
SetDataType(src.DataType());
SetType(src.Type());
SetOverridedStopGradient(src.OverridedStopGradient());
if (!src.SharedVar()->IsEmpty()) {
const platform::Place& place = src.Place();
if (src.Var().IsType<framework::LoDTensor>()) {
auto& src_tensor = src.Var().Get<framework::LoDTensor>();
auto* dst_tensor = MutableVar()->GetMutable<framework::LoDTensor>();
dst_tensor->set_lod(src_tensor.lod());
framework::TensorCopy(src_tensor, place, dst_tensor);
} else if (src.Var().IsType<framework::SelectedRows>()) {
auto& src_selected_rows = src.Var().Get<framework::SelectedRows>();
auto* dst_selected_rows =
MutableVar()->GetMutable<framework::SelectedRows>();
dst_selected_rows->set_height(src_selected_rows.height());
dst_selected_rows->set_rows(src_selected_rows.rows());
framework::TensorCopy(src_selected_rows.value(), place,
dst_selected_rows->mutable_value());
}
if (blocking) {
platform::DeviceContextPool::Instance().Get(place)->Wait();
}
SetPersistable(src.Persistable());
InnerSetOverridedStopGradient(src.OverridedStopGradient());
}

platform::Place place = src.Place();
if (src.Var().IsType<framework::LoDTensor>()) {
auto& src_tensor = src.Var().Get<framework::LoDTensor>();
auto* dst_tensor = MutableVar()->GetMutable<framework::LoDTensor>();
if (dst_tensor && dst_tensor->IsInitialized()) {
PADDLE_ENFORCE_EQ(dst_tensor->dims(), src_tensor.dims(),
platform::errors::PreconditionNotMet(
"Tensor %s has different dims with Tensor %s, "
"Tensor Copy cannot be performed!",
Name(), src.Name()));
PADDLE_ENFORCE_EQ(dst_tensor->lod(), src_tensor.lod(),
platform::errors::PreconditionNotMet(
"Tensor %s has different dims with Tensor %s, "
"Tensor Copy cannot be performed!",
Name(), src.Name()));
place = Place();
} else {
dst_tensor->set_lod(src_tensor.lod());
dst_tensor->Resize(src_tensor.dims());
}
framework::TensorCopy(src_tensor, place, dst_tensor);
} else if (src.Var().IsType<framework::SelectedRows>()) {
auto& src_selected_rows = src.Var().Get<framework::SelectedRows>();
auto* dst_selected_rows =
MutableVar()->GetMutable<framework::SelectedRows>();
dst_selected_rows->set_height(src_selected_rows.height());
dst_selected_rows->set_rows(src_selected_rows.rows());

auto& src_tensor = src_selected_rows.value();
auto* dst_tensor = dst_selected_rows->mutable_value();
if (dst_tensor && dst_tensor->IsInitialized()) {
PADDLE_ENFORCE_EQ(dst_tensor->dims(), src_tensor.dims(),
platform::errors::PreconditionNotMet(
"Tensor %s has different dims with Tensor %s, "
"Tensor Copy cannot be performed!",
Name(), src.Name()));
place = Place();
} else {
dst_tensor->Resize(src_tensor.dims());
}
framework::TensorCopy(src_tensor, place, dst_tensor);
}
if (blocking) {
platform::DeviceContextPool::Instance().Get(place)->Wait();
}
}

Expand Down
11 changes: 8 additions & 3 deletions paddle/fluid/imperative/layer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ class VarBase {

void SetGradVarBase(const VarBase& grad_var) {
MutableGradVarBase()->CopyFrom(grad_var, true);
MutableGradVarBase()->SharedVar()->SetIsEmpty(false);
}

const std::shared_ptr<VarBase>& MutableGradVarBase() {
Expand Down Expand Up @@ -142,6 +143,8 @@ class VarBase {
return grad_var_->MutableVar();
}

bool IsLeaf() const { return var_->IsLeaf(); }

void SetOverridedStopGradient(bool stop_gradient) {
var_->SetOverridedStopGradient(stop_gradient);
if (grad_var_) {
Expand All @@ -151,17 +154,19 @@ class VarBase {

bool OverridedStopGradient() const { return var_->OverridedStopGradient(); }

bool IsLeaf() const { return var_->IsLeaf(); }

void InnerSetOverridedStopGradient(bool stop_gradient) {
if (var_->InnerOverridedStopGradient() == -1) {
if (InnerOverridedStopGradient() == -1) {
var_->InnerSetOverridedStopGradient(stop_gradient);
if (grad_var_) {
grad_var_->InnerSetOverridedStopGradient(stop_gradient);
}
}
}

int InnerOverridedStopGradient() const {
return var_->InnerOverridedStopGradient();
}

void SetPersistable(bool persistable) { var_->SetPersistable(persistable); }

bool Persistable() const { return var_->Persistable(); }
Expand Down
12 changes: 10 additions & 2 deletions python/paddle/fluid/tests/unittests/test_imperative_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ def forward(self, inputs):
class MLP(fluid.Layer):
def __init__(self, input_size):
super(MLP, self).__init__()
self._linear1 = None
self._linear1 = Linear(
input_size,
3,
Expand Down Expand Up @@ -607,12 +606,21 @@ def test_mlp(sort_sum_gradient):

mlp2.clear_gradients()
self.assertTrue(np.array_equal(clear_loss.grad.numpy(), [1]))
if ((batch_id + 1) % 10) == 0:
if ((batch_id + 1) % 10) % 2 == 0:
mlp1.clear_gradients()
expected_weight1_grad = 0.
expected_bias1_grad = 0.
expected_weight2_grad = 0.
expected_bias2_grad = 0.
elif ((batch_id + 1) % 10) % 2 == 1:
mlp1.clear_gradients()
mlp1._linear1.weight._set_grad_ivar(
paddle.ones([input_size, 3]))
mlp1._linear2.weight._set_grad_ivar(paddle.ones([3, 4]))
expected_weight1_grad = 1.
expected_bias1_grad = 0.
expected_weight2_grad = 1.
expected_bias2_grad = 0.

with fluid.dygraph.guard():
test_single_api(False)
Expand Down