Skip to content

Set no_merge_train_val in *_tune.yml #309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Eleven1Liu merged 6 commits into from
May 15, 2023
Merged

Set no_merge_train_val in *_tune.yml #309

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e528c41
args.no_merge_train_val -> config.no_merge_train_val
Eleven1Liu May 13, 2023
bd37e3d
Merge branch 'master' into retrain_args
Eleven1Liu May 13, 2023
8c22c65
trainer.checkpoint_callback.*_model is '' when running else
Eleven1Liu May 13, 2023
eec86cc
Use best_model_path instead of best_checkpoint path
Eleven1Liu May 13, 2023
728872a
use last_model_path
Eleven1Liu May 15, 2023
e7c9578
best_model_path is used before assignment
Eleven1Liu May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions search_params.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import json
import logging
import os
import time
from datetime import datetime
from pathlib import Path

Expand Down Expand Up @@ -234,22 +233,22 @@ def retrain_best_model(exp_name, best_config, best_log_dir, merge_train_val):
logging.info(f"Re-training with best config: \n{best_config}")
trainer = TorchTrainer(config=best_config, **data)
trainer.train()
best_model_path = trainer.checkpoint_callback.last_model_path
else:
# If not merging training and validation data, load the best result from tune experiments.
logging.info(f"Loading best model with best config: \n{best_config}")
trainer = TorchTrainer(config=best_config, **data)
best_checkpoint = os.path.join(best_log_dir, "best_model.ckpt")
last_checkpoint = os.path.join(best_log_dir, "last.ckpt")
trainer._setup_model(checkpoint_path=best_checkpoint)
os.popen(f"cp {best_checkpoint} {os.path.join(checkpoint_dir, 'best_model.ckpt')}")
best_model_path = os.path.join(checkpoint_dir, 'best_model.ckpt')
os.popen(f"cp {best_checkpoint} {best_model_path}")
os.popen(f"cp {last_checkpoint} {os.path.join(checkpoint_dir, 'last.ckpt')}")

if "test" in data["datasets"]:
test_results = trainer.test()
logging.info(f"Test results after re-training: {test_results}")
logging.info(
f"Best model saved to {trainer.checkpoint_callback.best_model_path or trainer.checkpoint_callback.last_model_path}."
)
logging.info(f"Best model saved to {best_model_path}.")


def main():
Expand Down Expand Up @@ -341,7 +340,7 @@ def main():
# Save best model after parameter search.
best_config = analysis.get_best_config(f"val_{config.val_metric}", config.mode, scope="all")
best_log_dir = analysis.get_best_logdir(f"val_{config.val_metric}", config.mode, scope="all")
retrain_best_model(exp_name, best_config, best_log_dir, merge_train_val=not args.no_merge_train_val)
retrain_best_model(exp_name, best_config, best_log_dir, merge_train_val=not config.no_merge_train_val)


if __name__ == "__main__":
Expand Down