From 3582d3f9c77655994f6bbb736b3bad12df7892ce Mon Sep 17 00:00:00 2001 From: liuwei1031 Date: Wed, 10 Apr 2019 07:04:26 +0000 Subject: [PATCH 1/4] disable memory_optimize and inpalce strategy by default, test=develop --- paddle/fluid/framework/details/build_strategy.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/framework/details/build_strategy.h b/paddle/fluid/framework/details/build_strategy.h index 8aa444a30c0f7f..121d4a27cd30ab 100644 --- a/paddle/fluid/framework/details/build_strategy.h +++ b/paddle/fluid/framework/details/build_strategy.h @@ -83,11 +83,11 @@ struct BuildStrategy { bool sync_batch_norm_{false}; - bool memory_optimize_{true}; - // TODO(dzhwinter): - // make enable_inplace, memory_optimize_ - // memory_early_delete_ true by default - bool enable_inplace_{true}; + // FIXME(liuwei1031) disable memory_optimzie and enable_inplace in 1.4 + // to open them by default, we need to solve the fetch variable issue + bool memory_optimize_{false}; + + bool enable_inplace_{false}; bool enable_sequential_execution_{false}; From f80b21e3f61bfd28073becbaca20f0cf5c4fecf3 Mon Sep 17 00:00:00 2001 From: liuwei1031 Date: Wed, 10 Apr 2019 12:14:39 +0000 Subject: [PATCH 2/4] fix security issue http://newicafe.baidu.com:80/issue/PaddleSec-3/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-8/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-12/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-32/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-35/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-37/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-40/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-43/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-44/show?from=page http://newicafe.baidu.com:80/issue/PaddleSec-45/show?from=page test=develop --- paddle/fluid/framework/op_desc.cc | 1 + .../fluid/inference/api/analysis_predictor.cc | 3 ++ paddle/fluid/inference/api/api.cc | 1 + paddle/fluid/inference/api/api_impl.cc | 5 ++ .../tests/api/analyzer_seq_conv1_tester.cc | 1 + paddle/fluid/operators/detection/gpc.cc | 5 ++ .../fluid/operators/squared_l2_distance_op.h | 48 +++++++++---------- paddle/fluid/string/piece.cc | 13 ++++- 8 files changed, 51 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/framework/op_desc.cc b/paddle/fluid/framework/op_desc.cc index e6f5b15af8cd44..1ea93b7638a85e 100644 --- a/paddle/fluid/framework/op_desc.cc +++ b/paddle/fluid/framework/op_desc.cc @@ -241,6 +241,7 @@ OpDesc::OpDesc(const std::string &type, const VariableNameMap &inputs, outputs_ = outputs; attrs_ = attrs; need_update_ = true; + block_ = nullptr; } OpDesc::OpDesc(const OpDesc &other, BlockDesc *block) { diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 6942604b0723f8..0155609a029664 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -259,6 +259,9 @@ bool AnalysisPredictor::SetFeed(const std::vector &inputs, return false; } + PADDLE_ENFORCE_NOT_NULL(input_ptr); + PADDLE_ENFORCE_NOT_NULL(inputs[i].data.data()); + if (platform::is_cpu_place(place_)) { // TODO(panyx0718): Init LoDTensor from existing memcpy to save a copy. std::memcpy(static_cast(input_ptr), inputs[i].data.data(), diff --git a/paddle/fluid/inference/api/api.cc b/paddle/fluid/inference/api/api.cc index 7d57b6ec74468d..82185437d1ce75 100644 --- a/paddle/fluid/inference/api/api.cc +++ b/paddle/fluid/inference/api/api.cc @@ -54,6 +54,7 @@ PaddleBuf &PaddleBuf::operator=(const PaddleBuf &other) { memory_owned_ = other.memory_owned_; } else { Resize(other.length()); + PADDLE_ENFORCE_NOT_NULL(other.data()); memcpy(data_, other.data(), other.length()); length_ = other.length(); memory_owned_ = true; diff --git a/paddle/fluid/inference/api/api_impl.cc b/paddle/fluid/inference/api/api_impl.cc index 54f40563c3662a..56996c5cff88f5 100644 --- a/paddle/fluid/inference/api/api_impl.cc +++ b/paddle/fluid/inference/api/api_impl.cc @@ -169,6 +169,7 @@ std::unique_ptr NativePaddlePredictor::Clone() { std::unique_ptr cls(new NativePaddlePredictor(config_)); // Hot fix the bug that result diff in multi-thread. // TODO(Superjomn) re-implement a real clone here. + PADDLE_ENFORCE_NOT_NULL(dynamic_cast(cls.get())); if (!dynamic_cast(cls.get())->Init(nullptr)) { LOG(ERROR) << "fail to call Init"; return nullptr; @@ -210,6 +211,8 @@ bool NativePaddlePredictor::SetFeed(const std::vector &inputs, return false; } + PADDLE_ENFORCE_NOT_NULL(input_ptr); + PADDLE_ENFORCE_NOT_NULL(inputs[i].data.data()); if (platform::is_cpu_place(place_)) { // TODO(panyx0718): Init LoDTensor from existing memcpy to save a copy. std::memcpy(static_cast(input_ptr), inputs[i].data.data(), @@ -316,6 +319,8 @@ std::unique_ptr CreatePaddlePredictor< } std::unique_ptr predictor(new NativePaddlePredictor(config)); + PADDLE_ENFORCE_NOT_NULL( + dynamic_cast(predictor.get())); if (!dynamic_cast(predictor.get())->Init(nullptr)) { return nullptr; } diff --git a/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc b/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc index 9f23b9f037bcae..5ee848c3cfa211 100644 --- a/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc +++ b/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc @@ -47,6 +47,7 @@ struct DataRecord { num_lines++; std::vector data; split(line, '\t', &data); + PADDLE_ENFORCE(data.size() >= 4); // load title1 data std::vector title1_data; split_to_int64(data[0], ' ', &title1_data); diff --git a/paddle/fluid/operators/detection/gpc.cc b/paddle/fluid/operators/detection/gpc.cc index 7c0823c0487d39..f46aaf7d0a7b2d 100644 --- a/paddle/fluid/operators/detection/gpc.cc +++ b/paddle/fluid/operators/detection/gpc.cc @@ -24,6 +24,7 @@ **/ #include "paddle/fluid/operators/detection/gpc.h" +#include "paddle/fluid/platform/enforce.h" namespace gpc { @@ -689,6 +690,7 @@ static bbox *create_contour_bboxes(gpc_polygon *p) { gpc_malloc(box, p->num_contours * sizeof(bbox), const_cast("Bounding box creation")); + PADDLE_ENFORCE_NOT_NULL(box); /* Construct contour bounding boxes */ for (c = 0; c < p->num_contours; c++) { @@ -852,6 +854,7 @@ void gpc_add_contour(gpc_polygon *p, gpc_vertex_list *new_contour, int hole) { /* Create an extended hole array */ gpc_malloc(extended_hole, (p->num_contours + 1) * sizeof(int), const_cast("contour hole addition")); + PADDLE_ENFORCE_NOT_NULL(extended_hole); /* Create an extended contour array */ gpc_malloc(extended_contour, @@ -969,6 +972,7 @@ void gpc_polygon_clip(gpc_op op, gpc_polygon *subj, gpc_polygon *clip, /* Build scanbeam table from scanbeam tree */ gpc_malloc(sbt, sbt_entries * sizeof(double), const_cast("sbt creation")); + PADDLE_ENFORCE_NOT_NULL(sbt); build_sbt(&scanbeam, sbt, sbtree); scanbeam = 0; free_sbtree(&sbtree); @@ -1604,6 +1608,7 @@ void gpc_tristrip_clip(gpc_op op, gpc_polygon *subj, gpc_polygon *clip, /* Build scanbeam table from scanbeam tree */ gpc_malloc(sbt, sbt_entries * sizeof(double), const_cast("sbt creation")); + PADDLE_ENFORCE_NOT_NULL(sbt); build_sbt(&scanbeam, sbt, sbtree); scanbeam = 0; free_sbtree(&sbtree); diff --git a/paddle/fluid/operators/squared_l2_distance_op.h b/paddle/fluid/operators/squared_l2_distance_op.h index e0133d33e6a840..12a8f05b5a6034 100644 --- a/paddle/fluid/operators/squared_l2_distance_op.h +++ b/paddle/fluid/operators/squared_l2_distance_op.h @@ -77,6 +77,9 @@ class SquaredL2DistanceGradKernel : public framework::OpKernel { auto* x_g = context.Output(framework::GradVarName("X")); auto* y_g = context.Output(framework::GradVarName("Y")); + PADDLE_ENFORCE_NOT_NULL(x_g); + PADDLE_ENFORCE_NOT_NULL(y_g); + auto sub_result = EigenMatrix::From(*in0); auto out_grad = EigenMatrix::From(*in1); @@ -92,31 +95,28 @@ class SquaredL2DistanceGradKernel : public framework::OpKernel { // propagate back to input auto& eigen_place = *context.template device_context().eigen_device(); - if (x_g) { - x_g->mutable_data(context.GetPlace()); - // eigen matrix - auto x_grad = - EigenMatrix::From(*x_g, framework::make_ddim({x_dims[0], cols})); - // dimensions are same with subResult - x_grad.device(eigen_place) = grad_mat; - } - if (y_g) { - y_g->mutable_data(context.GetPlace()); - - PADDLE_ENFORCE_GE(sub_result.dimensions()[0], y_dims[0], - "First dimension of gradient must be greater or " - "equal than first dimension of target."); - - if (sub_result.dimensions()[0] == y_dims[0]) { - auto y_grad = - EigenMatrix::From(*y_g, framework::make_ddim({y_dims[0], cols})); - y_grad.device(eigen_place) = -1 * grad_mat; - } else { - auto col_sum_res = -1 * (grad_mat.sum(Eigen::array({{0}}))); - auto y_grad = EigenVector::Flatten(*y_g); - y_grad.device(eigen_place) = col_sum_res; - } + x_g->mutable_data(context.GetPlace()); + // eigen matrix + auto x_grad = + EigenMatrix::From(*x_g, framework::make_ddim({x_dims[0], cols})); + // dimensions are same with subResult + x_grad.device(eigen_place) = grad_mat; + + y_g->mutable_data(context.GetPlace()); + + PADDLE_ENFORCE_GE(sub_result.dimensions()[0], y_dims[0], + "First dimension of gradient must be greater or " + "equal than first dimension of target."); + + if (sub_result.dimensions()[0] == y_dims[0]) { + auto y_grad = + EigenMatrix::From(*y_g, framework::make_ddim({y_dims[0], cols})); + y_grad.device(eigen_place) = -1 * grad_mat; + } else { + auto col_sum_res = -1 * (grad_mat.sum(Eigen::array({{0}}))); + auto y_grad = EigenVector::Flatten(*y_g); + y_grad.device(eigen_place) = col_sum_res; } } }; diff --git a/paddle/fluid/string/piece.cc b/paddle/fluid/string/piece.cc index 8e8cfb0e913894..2657732540d55b 100644 --- a/paddle/fluid/string/piece.cc +++ b/paddle/fluid/string/piece.cc @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/string/piece.h" - #include #include #include #include +#include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/string/piece.h" + namespace paddle { namespace string { @@ -41,6 +42,8 @@ char Piece::operator[](size_t n) const { int Compare(Piece a, Piece b) { const size_t min_len = (a.len() < b.len()) ? a.len() : b.len(); + PADDLE_ENFORCE_NOT_NULL(a.data()); + PADDLE_ENFORCE_NOT_NULL(b.data()); int r = memcmp(a.data(), b.data(), min_len); if (r == 0) { if (a.len() < b.len()) @@ -52,6 +55,8 @@ int Compare(Piece a, Piece b) { } bool operator==(Piece x, Piece y) { + PADDLE_ENFORCE_NOT_NULL(x.data()); + PADDLE_ENFORCE_NOT_NULL(y.data()); return ((x.len() == y.len()) && (x.data() == y.data() || memcmp(x.data(), y.data(), x.len()) == 0)); } @@ -65,10 +70,14 @@ bool operator<=(Piece x, Piece y) { return Compare(x, y) <= 0; } bool operator>=(Piece x, Piece y) { return Compare(x, y) >= 0; } bool HasPrefix(Piece s, Piece x) { + PADDLE_ENFORCE_NOT_NULL(s.data()); + PADDLE_ENFORCE_NOT_NULL(x.data()); return ((s.len() >= x.len()) && (memcmp(s.data(), x.data(), x.len()) == 0)); } bool HasSuffix(Piece s, Piece x) { + PADDLE_ENFORCE_NOT_NULL(s.data()); + PADDLE_ENFORCE_NOT_NULL(x.data()); return ((s.len() >= x.len()) && (memcmp(s.data() + (s.len() - x.len()), x.data(), x.len()) == 0)); } From cb4a06fed98bffe27095c425ae8e8310504a5288 Mon Sep 17 00:00:00 2001 From: liuwei1031 Date: Wed, 10 Apr 2019 17:06:50 +0000 Subject: [PATCH 3/4] revert piece.cc, test=develop --- paddle/fluid/string/piece.cc | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/paddle/fluid/string/piece.cc b/paddle/fluid/string/piece.cc index 2657732540d55b..8e8cfb0e913894 100644 --- a/paddle/fluid/string/piece.cc +++ b/paddle/fluid/string/piece.cc @@ -12,15 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "paddle/fluid/string/piece.h" + #include #include #include #include -#include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/string/piece.h" - namespace paddle { namespace string { @@ -42,8 +41,6 @@ char Piece::operator[](size_t n) const { int Compare(Piece a, Piece b) { const size_t min_len = (a.len() < b.len()) ? a.len() : b.len(); - PADDLE_ENFORCE_NOT_NULL(a.data()); - PADDLE_ENFORCE_NOT_NULL(b.data()); int r = memcmp(a.data(), b.data(), min_len); if (r == 0) { if (a.len() < b.len()) @@ -55,8 +52,6 @@ int Compare(Piece a, Piece b) { } bool operator==(Piece x, Piece y) { - PADDLE_ENFORCE_NOT_NULL(x.data()); - PADDLE_ENFORCE_NOT_NULL(y.data()); return ((x.len() == y.len()) && (x.data() == y.data() || memcmp(x.data(), y.data(), x.len()) == 0)); } @@ -70,14 +65,10 @@ bool operator<=(Piece x, Piece y) { return Compare(x, y) <= 0; } bool operator>=(Piece x, Piece y) { return Compare(x, y) >= 0; } bool HasPrefix(Piece s, Piece x) { - PADDLE_ENFORCE_NOT_NULL(s.data()); - PADDLE_ENFORCE_NOT_NULL(x.data()); return ((s.len() >= x.len()) && (memcmp(s.data(), x.data(), x.len()) == 0)); } bool HasSuffix(Piece s, Piece x) { - PADDLE_ENFORCE_NOT_NULL(s.data()); - PADDLE_ENFORCE_NOT_NULL(x.data()); return ((s.len() >= x.len()) && (memcmp(s.data() + (s.len() - x.len()), x.data(), x.len()) == 0)); } From 5cf616ab6e1cc95d87131a807e3b39449021e6bf Mon Sep 17 00:00:00 2001 From: liuwei1031 Date: Thu, 11 Apr 2019 06:19:09 +0000 Subject: [PATCH 4/4] adjust api.cc,test=develop --- paddle/fluid/inference/api/api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/inference/api/api.cc b/paddle/fluid/inference/api/api.cc index 82185437d1ce75..fc2d7b48c2a1f8 100644 --- a/paddle/fluid/inference/api/api.cc +++ b/paddle/fluid/inference/api/api.cc @@ -54,7 +54,7 @@ PaddleBuf &PaddleBuf::operator=(const PaddleBuf &other) { memory_owned_ = other.memory_owned_; } else { Resize(other.length()); - PADDLE_ENFORCE_NOT_NULL(other.data()); + PADDLE_ENFORCE(!(other.length() > 0 && other.data() == nullptr)); memcpy(data_, other.data(), other.length()); length_ = other.length(); memory_owned_ = true;