Skip to content

export_model tweak #1277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zphang merged 5 commits into from
Feb 6, 2021
Merged

export_model tweak #1277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d61ec03
export_model tweak
zphang Feb 6, 2021
1dd4896
add hf_pretrained_model_name_or_path to saved config
zphang Feb 6, 2021
9ff4763
remove model_type form export_model
zphang Feb 6, 2021
8298a29
update test
zphang Feb 6, 2021
49be2c8
remote unused var
zphang Feb 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions examples/notebooks/jiant_Basic_Example.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,8 +310,8 @@
" jiant_task_container_config_path=\"./run_configs/mrpc_run_config.json\",\n",
" output_dir=\"./runs/mrpc\",\n",
" hf_pretrained_model_name_or_path=\"roberta-base\",\n",
" model_path=\"./models/roberta-base/model/roberta.p\",\n",
" model_config_path=\"./models/roberta-base/model/roberta.json\",\n",
" model_path=\"./models/roberta-base/model/model.p\",\n",
" model_config_path=\"./models/roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down
4 changes: 2 additions & 2 deletions examples/notebooks/jiant_EdgeProbing_Example.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3147,8 +3147,8 @@
" jiant_task_container_config_path=\"./run_configs/semeval_run_config.json\",\n",
" output_dir=\"./runs/semeval\",\n",
" hf_pretrained_model_name_or_path=\"roberta-base\",\n",
" model_path=\"./models/roberta-base/model/roberta.p\",\n",
" model_config_path=\"./models/roberta-base/model/roberta.json\",\n",
" model_path=\"./models/roberta-base/model/model.p\",\n",
" model_config_path=\"./models/roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down
4 changes: 2 additions & 2 deletions examples/notebooks/jiant_MNLI_Diagnostic_Example.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,8 @@
" jiant_task_container_config_path=\"./run_configs/jiant_run_config.json\",\n",
" output_dir=\"./runs/run1\",\n",
" hf_pretrained_model_name_or_path=\"roberta-base\",\n",
" model_path=\"./models/roberta-base/model/roberta.p\",\n",
" model_config_path=\"./models/roberta-base/model/roberta.json\",\n",
" model_path=\"./models/roberta-base/model/model.p\",\n",
" model_config_path=\"./models/roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down
4 changes: 2 additions & 2 deletions examples/notebooks/jiant_Multi_Task_Example.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,8 @@
" jiant_task_container_config_path=\"./run_configs/jiant_run_config.json\",\n",
" output_dir=\"./runs/run1\",\n",
" hf_pretrained_model_name_or_path=\"roberta-base\",\n",
" model_path=\"./models/roberta-base/model/roberta.p\",\n",
" model_config_path=\"./models/roberta-base/model/roberta.json\",\n",
" model_path=\"./models/roberta-base/model/model.p\",\n",
" model_config_path=\"./models/roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down
6 changes: 3 additions & 3 deletions examples/notebooks/jiant_STILTs_Example.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,8 +368,8 @@
" jiant_task_container_config_path=\"./run_configs/mnli_run_config.json\",\n",
" output_dir=\"./runs/mnli\",\n",
" hf_pretrained_model_name_or_path=\"roberta-base\",\n",
" model_path=\"./models/roberta-base/model/roberta.p\",\n",
" model_config_path=\"./models/roberta-base/model/roberta.json\",\n",
" model_path=\"./models/roberta-base/model/modle.p\",\n",
" model_config_path=\"./models/roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down Expand Up @@ -406,7 +406,7 @@
" hf_pretrained_model_name_or_path=\"roberta-base\",\n",
" model_path=\"./runs/mnli/best_model.p\", # Loading the best model\n",
" model_load_mode=\"partial\",\n",
" model_config_path=\"./models/roberta-base/model/roberta.json\",\n",
" model_config_path=\"./models/roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down
4 changes: 2 additions & 2 deletions examples/notebooks/jiant_XNLI_Example.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,8 +384,8 @@
" jiant_task_container_config_path=\"./run_configs/jiant_run_config.json\",\n",
" output_dir=\"./runs/run1\",\n",
" hf_pretrained_model_name_or_path=\"xlm-roberta-base\",\n",
" model_path=\"./models/xlm-roberta-base/model/xlm-roberta.p\",\n",
" model_config_path=\"./models/xlm-roberta-base/model/xlm-roberta.json\",\n",
" model_path=\"./models/xlm-roberta-base/model/model.p\",\n",
" model_config_path=\"./models/xlm-roberta-base/model/config.json\",\n",
" learning_rate=1e-5,\n",
" eval_every_steps=500,\n",
" do_train=True,\n",
Expand Down
7 changes: 3 additions & 4 deletions jiant/proj/main/export_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,10 @@ def export_model(
output_base_path: Base path to save output to
"""
model = AutoModelForPreTraining.from_pretrained(hf_pretrained_model_name_or_path)
model_type = model.config_class.model_type

model_fol_path = os.path.join(output_base_path, "model")
model_path = os.path.join(model_fol_path, f"{model_type}.p")
model_config_path = os.path.join(model_fol_path, f"{model_type}.json")
model_path = os.path.join(model_fol_path, "model.p")
model_config_path = os.path.join(model_fol_path, "config.json")
tokenizer_fol_path = os.path.join(output_base_path, "tokenizer")

os.makedirs(tokenizer_fol_path, exist_ok=True)
Expand All @@ -56,7 +55,7 @@ def export_model(
tokenizer = AutoTokenizer.from_pretrained(hf_pretrained_model_name_or_path)
tokenizer.save_pretrained(tokenizer_fol_path)
config = {
"model_type": model_type,
"hf_pretrained_model_name_or_path": hf_pretrained_model_name_or_path,
"model_path": model_path,
"model_config_path": model_config_path,
}
Expand Down
6 changes: 3 additions & 3 deletions tests/proj/main/test_export_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ def test_export_model(tmp_path, model_type, model_class, hf_pretrained_model_nam
output_base_path=tmp_path,
)
read_config = py_io.read_json(os.path.join(tmp_path, f"config.json"))
assert read_config["model_type"] == model_type
assert read_config["model_path"] == os.path.join(tmp_path, "model", f"{model_type}.p")
assert read_config["model_config_path"] == os.path.join(tmp_path, "model", f"{model_type}.json")
assert read_config["hf_pretrained_model_name_or_path"] == hf_pretrained_model_name_or_path
assert read_config["model_path"] == os.path.join(tmp_path, "model", "model.p")
assert read_config["model_config_path"] == os.path.join(tmp_path, "model", "config.json")