From 1f5b08a4714afcc7345c9602e2c5f34b99d34ac3 Mon Sep 17 00:00:00 2001 From: zmxdream Date: Mon, 18 Oct 2021 21:00:53 +0800 Subject: [PATCH 1/3] bug fix for DeserializeSelectedRows. test=develop --- paddle/fluid/distributed/service/brpc_utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/distributed/service/brpc_utils.cc b/paddle/fluid/distributed/service/brpc_utils.cc index a356b77e73733e..376e820cb7a741 100644 --- a/paddle/fluid/distributed/service/brpc_utils.cc +++ b/paddle/fluid/distributed/service/brpc_utils.cc @@ -273,8 +273,8 @@ void DeserializeSelectedRows(framework::Variable* var, const VarMsg& msg, auto* slr = var->GetMutable(); framework::Tensor* tensor = slr->mutable_value(); slr->set_height(msg.slr_height()); - std::vector tmp_rows(msg.slr_height()); - memcpy(&tmp_rows[0], msg.data().data(), msg.slr_height() * sizeof(int64_t)); + std::vector tmp_rows(msg.dims()[0]); + memcpy(tmp_rows.data(), msg.data().data(), msg.dims()[0] * sizeof(int64_t)); slr->set_rows(tmp_rows); std::vector vec_dim; for (auto& x : msg.dims()) { From d7780165477a410f03d6f5d3acbe22be4c929309 Mon Sep 17 00:00:00 2001 From: zmxdream Date: Tue, 19 Oct 2021 17:16:41 +0800 Subject: [PATCH 2/3] fix bug for SerializeSelectedRows. test=develop --- .../fluid/distributed/service/brpc_utils.cc | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/paddle/fluid/distributed/service/brpc_utils.cc b/paddle/fluid/distributed/service/brpc_utils.cc index 376e820cb7a741..d88d42e85c0d89 100644 --- a/paddle/fluid/distributed/service/brpc_utils.cc +++ b/paddle/fluid/distributed/service/brpc_utils.cc @@ -138,23 +138,14 @@ void SerializeSelectedRows(framework::Variable* var, var_data->clear(); var_data->resize(rows->size() * sizeof(int64_t)); char* data_ptr = const_cast(var_data->data()); - - if (platform::is_cpu_place(tensor->place())) { - memcpy(data_ptr, &(*rows)[0], rows->size() * sizeof(int64_t)); - } else { -#ifdef PADDLE_WITH_CUDA - auto stream = - reinterpret_cast(ctx).stream(); - memory::Copy(platform::CPUPlace(), data_ptr, - BOOST_GET_CONST(platform::CUDAPlace, tensor->place()), - &(*rows)[0], rows->size() * sizeof(int64_t), stream); -#endif - } + // the type of rows is framework::Vector, get data by [] + // will trigger data copy operation, if the data is in GPU + // it will be copied to CPU + memcpy(data_ptr, &((*rows)[0]), rows->size() * sizeof(int64_t)); var_msg->set_data_type(static_cast(tensor->type())); for (auto& dim : framework::vectorize(tensor->dims())) { var_msg->add_dims(dim); } - // IO Buffer if (platform::is_cpu_place(tensor->place())) { auto data_len = tensor->numel() * framework::SizeOfType(tensor->type()); @@ -273,6 +264,10 @@ void DeserializeSelectedRows(framework::Variable* var, const VarMsg& msg, auto* slr = var->GetMutable(); framework::Tensor* tensor = slr->mutable_value(); slr->set_height(msg.slr_height()); + // now, the rows of SelectedRows is in CPU + // if current device is CPU, it's ok + // if current device is GPU, the row data is in CPU and the tensor data is GPU + // but it doesn't matter because sum op for selected rows will copy rows to GPU std::vector tmp_rows(msg.dims()[0]); memcpy(tmp_rows.data(), msg.data().data(), msg.dims()[0] * sizeof(int64_t)); slr->set_rows(tmp_rows); From 0d980dc8d61ad83fa0582e020185e022662eed04 Mon Sep 17 00:00:00 2001 From: zmxdream Date: Tue, 19 Oct 2021 22:15:02 +0800 Subject: [PATCH 3/3] update. test=develop --- paddle/fluid/distributed/service/brpc_utils.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/paddle/fluid/distributed/service/brpc_utils.cc b/paddle/fluid/distributed/service/brpc_utils.cc index d88d42e85c0d89..92dcde99cccb0b 100644 --- a/paddle/fluid/distributed/service/brpc_utils.cc +++ b/paddle/fluid/distributed/service/brpc_utils.cc @@ -138,9 +138,6 @@ void SerializeSelectedRows(framework::Variable* var, var_data->clear(); var_data->resize(rows->size() * sizeof(int64_t)); char* data_ptr = const_cast(var_data->data()); - // the type of rows is framework::Vector, get data by [] - // will trigger data copy operation, if the data is in GPU - // it will be copied to CPU memcpy(data_ptr, &((*rows)[0]), rows->size() * sizeof(int64_t)); var_msg->set_data_type(static_cast(tensor->type())); for (auto& dim : framework::vectorize(tensor->dims())) { @@ -264,10 +261,6 @@ void DeserializeSelectedRows(framework::Variable* var, const VarMsg& msg, auto* slr = var->GetMutable(); framework::Tensor* tensor = slr->mutable_value(); slr->set_height(msg.slr_height()); - // now, the rows of SelectedRows is in CPU - // if current device is CPU, it's ok - // if current device is GPU, the row data is in CPU and the tensor data is GPU - // but it doesn't matter because sum op for selected rows will copy rows to GPU std::vector tmp_rows(msg.dims()[0]); memcpy(tmp_rows.data(), msg.data().data(), msg.dims()[0] * sizeof(int64_t)); slr->set_rows(tmp_rows);