Skip to content

[feat]: split model support partition variables #263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[feat]: split model support partition variables #263

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b0ea0f3
support split partition variables
tiankongdeguiji Aug 3, 2022
064f536
Merge branch 'master' into features/split_model_with_partition
tiankongdeguiji Aug 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions easy_rec/python/tools/split_model_pai.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def extract_sub_graph(graph_def, dest_nodes, variable_protos):
node_seq = {}
seq = 0
nodes_to_keep = set()
variables_to_keep = set()
variables_to_keep = dict()

for node in graph_def.node:
n = _node_name(node.name)
Expand All @@ -81,11 +81,11 @@ def extract_sub_graph(graph_def, dest_nodes, variable_protos):
n = next_to_visit[0]

if n in variable_protos:
Copy link
Collaborator

@chengmengli06 chengmengli06 Aug 8, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add a unit test for partitioned variables

proto = variable_protos[n]
proto = variable_protos[n][0]
next_to_visit.append(_node_name(proto.initial_value_name))
next_to_visit.append(_node_name(proto.initializer_name))
next_to_visit.append(_node_name(proto.snapshot_name))
variables_to_keep.add(proto.variable_name)
variables_to_keep[proto.variable_name] = (proto, variable_protos[n][1])

del next_to_visit[0]
if n in nodes_to_keep:
Expand Down Expand Up @@ -137,7 +137,7 @@ def load_meta_graph_def(model_dir):
tf.logging.info('%s' % proto.variable_name)
variable_node_name = _node_name(proto.variable_name)
if variable_node_name not in variable_protos:
variable_protos[variable_node_name] = proto
variable_protos[variable_node_name] = (proto, key)

# parse signature info for SavedModel
for sig_name in signatures:
Expand Down Expand Up @@ -197,8 +197,9 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names,
with sess.graph.as_default():
graph = ops.get_default_graph()
importer.import_graph_def(inference_graph, name='')
for name in variables_to_keep:
variable = graph.get_tensor_by_name(name)
for name, (proto, key) in variables_to_keep.items():
from_proto = ops.get_from_proto_function(key)
variable = from_proto(proto)
graph.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, variable)
saver = tf_saver.Saver()
saver.restore(sess, get_variables_path(model_dir))
Expand Down