From 08376e385c4a042fb2231cf14fb1c6e563aea35c Mon Sep 17 00:00:00 2001 From: a6802739 Date: Wed, 12 Jun 2019 19:52:45 +0800 Subject: [PATCH 1/3] fix some bug when merge sparse embedding parameters 1. fix the bug that out_put_var in SaveSelectedRows would be empty string 2. use merge_sparse_lookup_table to replace sum op for load_persistables_for_inference 3. fix the bug in _clone_var_in_block_ when the var is SELECTED_ROWS. test=develop --- paddle/fluid/operators/save_op.h | 9 +++++--- .../fluid/contrib/utils/lookup_table_utils.py | 9 +++++++- python/paddle/fluid/io.py | 22 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/paddle/fluid/operators/save_op.h b/paddle/fluid/operators/save_op.h index b41c70560812c5..b59421cb9e08e3 100644 --- a/paddle/fluid/operators/save_op.h +++ b/paddle/fluid/operators/save_op.h @@ -102,16 +102,19 @@ class SaveOpKernel : public framework::OpKernel { void SaveSelectedRows(const framework::ExecutionContext &ctx, const platform::Place &place, const framework::Variable *var) const { - framework::Variable *out_put_var = ctx.OutputVar(LOOKUP_TABLE_PATH); - auto file_path = ctx.Attr("file_path"); auto overwrite = ctx.Attr("overwrite"); std::string filename = file_path; + VLOG(4) << "SaveSelectedRows output file_path: " << file_path; + framework::Variable *out_put_var = ctx.scope().FindVar(LOOKUP_TABLE_PATH); if (out_put_var != nullptr) { auto *lt_var = out_put_var->GetMutable(); - filename = *lt_var; + if (lt_var->length() > 0) { + VLOG(4) << "SaveSelectedRows output var name: " << *lt_var; + filename = *lt_var; + } } if (FileExists(filename) && !overwrite) { diff --git a/python/paddle/fluid/contrib/utils/lookup_table_utils.py b/python/paddle/fluid/contrib/utils/lookup_table_utils.py index b15ee94f63512d..74466613b48e55 100644 --- a/python/paddle/fluid/contrib/utils/lookup_table_utils.py +++ b/python/paddle/fluid/contrib/utils/lookup_table_utils.py @@ -363,7 +363,14 @@ def _load_lookup_table_vars(executor, dirname, main_program, }) sums.append(param_var) global_block.append_op( - type='sum', inputs={"X": sums}, outputs={'Out': emb_var}, attrs={}) + type='merge_sparse_lookup_table', inputs={"X": sums}, outputs={'Out': emb_var}, attrs={}) + global_block.append_op( + type='save', + inputs={"X": [emb_var]}, + outputs={}, + attrs={ + 'file_path': os.path.join(lookup_table_dirname, emb_var.name) + }) global_block.append_op(type='delete_var', inputs={'X': sums}) executor.run(convert_program) diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 64afa7e856fb88..a0573881b79ba8 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -86,13 +86,21 @@ def is_persistable(var): def _clone_var_in_block_(block, var): assert isinstance(var, Variable) - return block.create_var( - name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True) + else: + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + persistable=True) def save_vars(executor, From a5b092fdfff6871233973b692c4f1e23ec3ee531 Mon Sep 17 00:00:00 2001 From: a6802739 Date: Mon, 17 Jun 2019 14:59:11 +0800 Subject: [PATCH 2/3] fix code style problem, test=develop --- python/paddle/fluid/contrib/utils/lookup_table_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/contrib/utils/lookup_table_utils.py b/python/paddle/fluid/contrib/utils/lookup_table_utils.py index 74466613b48e55..6e6e17e80a3890 100644 --- a/python/paddle/fluid/contrib/utils/lookup_table_utils.py +++ b/python/paddle/fluid/contrib/utils/lookup_table_utils.py @@ -363,7 +363,10 @@ def _load_lookup_table_vars(executor, dirname, main_program, }) sums.append(param_var) global_block.append_op( - type='merge_sparse_lookup_table', inputs={"X": sums}, outputs={'Out': emb_var}, attrs={}) + type='merge_sparse_lookup_table', + inputs={"X": sums}, + outputs={'Out': emb_var}, + attrs={}) global_block.append_op( type='save', inputs={"X": [emb_var]}, From 5d1a78d152d633050a748f889fd38814dfa00249 Mon Sep 17 00:00:00 2001 From: a6802739 Date: Fri, 21 Jun 2019 10:51:39 +0800 Subject: [PATCH 3/3] remove the space at the end of the line for code style, test=develop --- python/paddle/fluid/contrib/utils/lookup_table_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/contrib/utils/lookup_table_utils.py b/python/paddle/fluid/contrib/utils/lookup_table_utils.py index 6e6e17e80a3890..402850e70ab269 100644 --- a/python/paddle/fluid/contrib/utils/lookup_table_utils.py +++ b/python/paddle/fluid/contrib/utils/lookup_table_utils.py @@ -363,9 +363,9 @@ def _load_lookup_table_vars(executor, dirname, main_program, }) sums.append(param_var) global_block.append_op( - type='merge_sparse_lookup_table', - inputs={"X": sums}, - outputs={'Out': emb_var}, + type='merge_sparse_lookup_table', + inputs={"X": sums}, + outputs={'Out': emb_var}, attrs={}) global_block.append_op( type='save',