Skip to content

Commit be17a70

Browse files
authored
Merge pull request #20 from pyiron/cmd
Implement command line interface for remote submission
2 parents 7402cbb + 8ac7341 commit be17a70

File tree

8 files changed

+467
-13
lines changed

8 files changed

+467
-13
lines changed

.travis.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ matrix:
1212
- conda info -a
1313
- conda config --set always_yes yes --set changeps1 no
1414
- conda update -q conda
15-
- conda install -y -c conda-forge python=${PYTHONVER} coverage defusedxml pandas yaml jinja2
15+
- conda install -y -c conda-forge python=${PYTHONVER} coverage defusedxml pandas yaml jinja2 paramiko tqdm
1616
- pip install --pre .
1717
script:
1818
- coverage run -m unittest discover tests
@@ -28,7 +28,7 @@ matrix:
2828
- conda info -a
2929
- conda config --set always_yes yes --set changeps1 no
3030
- conda update -q conda
31-
- conda install -y -c conda-forge python=${PYTHONVER} coverage coveralls defusedxml pandas yaml jinja2
31+
- conda install -y -c conda-forge python=${PYTHONVER} coverage coveralls defusedxml pandas yaml jinja2 paramiko tqdm
3232
- pip install --pre .
3333
script:
3434
- coverage run -m unittest discover tests
@@ -63,7 +63,7 @@ matrix:
6363
- conda info -a
6464
- conda config --set always_yes yes --set changeps1 no
6565
- conda update -q conda
66-
- conda install -y -c conda-forge python=${PYTHONVER} coverage defusedxml pandas yaml jinja2
66+
- conda install -y -c conda-forge python=${PYTHONVER} coverage defusedxml pandas yaml jinja2 paramiko tqdm
6767
- pip install --pre .
6868
script:
6969
- coverage run -m unittest discover tests
@@ -79,7 +79,7 @@ matrix:
7979
- conda info -a
8080
- conda config --set always_yes yes --set changeps1 no
8181
- conda update -q conda
82-
- conda install -y -c conda-forge python=${PYTHONVER} coverage defusedxml pandas yaml jinja2
82+
- conda install -y -c conda-forge python=${PYTHONVER} coverage defusedxml pandas yaml jinja2 paramiko tqdm
8383
- pip install --pre .
8484
script:
8585
- coverage run -m unittest discover tests

appveyor.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ install:
3939
- '"%CONDA%\Scripts\activate.bat"'
4040
- "conda config --set always_yes yes --set changeps1 no"
4141
- "conda update -q conda"
42-
- 'conda install -c conda-forge coverage defusedxml "pandas>=0.23" yaml jinja2'
42+
- 'conda install -c conda-forge coverage defusedxml "pandas>=0.23" yaml jinja2 paramiko tqdm'
4343
- "conda info -a"
4444
- "pip install --pre ."
4545

azure-pipelines.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
- bash: echo "##vso[task.prependpath]/usr/share/miniconda/bin"
2525
displayName: Add conda to PATH
2626

27-
- bash: conda create -n pysqa -q --yes -c conda-forge python=$(python.version) coverage defusedxml pandas yaml jinja2
27+
- bash: conda create -n pysqa -q --yes -c conda-forge python=$(python.version) coverage defusedxml pandas yaml jinja2 paramiko tqdm
2828
displayName: conda install
2929

3030
- bash: |
@@ -55,7 +55,7 @@ jobs:
5555
- bash: sudo chown -R $USER $CONDA
5656
displayName: Take ownership of conda installation
5757

58-
- bash: conda create -n pysqa -q --yes -c conda-forge python=$(python.version) coverage defusedxml pandas yaml jinja2
58+
- bash: conda create -n pysqa -q --yes -c conda-forge python=$(python.version) coverage defusedxml pandas yaml jinja2 paramiko tqdm
5959
displayName: conda install
6060

6161
- bash: |
@@ -86,7 +86,7 @@ jobs:
8686
- bash: echo "##vso[task.prependpath]$CONDA/bin"
8787
displayName: Add conda to PATH
8888

89-
- script: conda create -n pysqa -q --yes -c conda-forge python=$(python.version) coverage defusedxml pandas yaml jinja2
89+
- script: conda create -n pysqa -q --yes -c conda-forge python=$(python.version) coverage defusedxml pandas yaml jinja2 paramiko tqdm
9090
displayName: conda install
9191

9292
- script: |

pysqa/basic.py

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,19 @@ def __init__(self, config, directory="~/.queues"):
6161
elif self._config["queue_type"] == "GENT":
6262
class_name = "GentCommands"
6363
module_name = "pysqa.wrapper.gent"
64+
elif self._config["queue_type"] == "REMOTE":
65+
class_name = None
66+
module_name = None
6467
else:
6568
raise ValueError()
66-
self._commands = getattr(importlib.import_module(module_name), class_name)()
69+
if self._config["queue_type"] != "REMOTE":
70+
self._commands = getattr(importlib.import_module(module_name), class_name)()
6771
self._queues = Queues(self.queue_list)
72+
self._remote_flag = False
73+
74+
@property
75+
def remote_flag(self):
76+
return self._remote_flag
6877

6978
@property
7079
def config(self):
@@ -239,6 +248,18 @@ def get_status_of_jobs(self, process_id_lst):
239248
results_lst.append("finished")
240249
return results_lst
241250

251+
def get_job_from_remote(self, working_directory, delete_remote=False):
252+
"""
253+
Get the results of the calculation - this is necessary when the calculation was executed on a remote host.
254+
"""
255+
pass
256+
257+
def convert_path_to_remote(self, path):
258+
pass
259+
260+
def transfer_file(self, file, transfer_back=False, delete_remote=False):
261+
pass
262+
242263
def check_queue_parameters(
243264
self, queue, cores=1, run_time_max=None, memory_max=None, active_queue=None
244265
):
@@ -416,8 +437,9 @@ def _load_templates(queue_lst_dict, directory="."):
416437
directory (str):
417438
"""
418439
for queue_dict in queue_lst_dict.values():
419-
with open(os.path.join(directory, queue_dict["script"]), "r") as f:
420-
queue_dict["template"] = Template(f.read())
440+
if "script" in queue_dict.keys():
441+
with open(os.path.join(directory, queue_dict["script"]), "r") as f:
442+
queue_dict["template"] = Template(f.read())
421443

422444
@staticmethod
423445
def _value_error_if_none(value):

pysqa/cmd.py

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import sys
2+
import os
3+
import json
4+
import getopt
5+
from pysqa.queueadapter import QueueAdapter
6+
7+
8+
def command_line(argv):
9+
"""
10+
Parse the command line arguments.
11+
12+
Args:
13+
argv: Command line arguments
14+
15+
"""
16+
directory = "~/.queues"
17+
queue = None,
18+
job_name = None,
19+
working_directory = None,
20+
cores = None,
21+
memory_max = None,
22+
run_time_max = None,
23+
command = None,
24+
job_id = None
25+
try:
26+
opts, args = getopt.getopt(
27+
argv, "f:pq:j:w:n:m:t:c:ri:dslh", ["config_directory=", "submit", "queue=", "job_name=",
28+
"working_directory=", "cores=", "memory=", "run_time=",
29+
"command=", "reservation", "id", "delete", "status", "list", "help"]
30+
)
31+
except getopt.GetoptError:
32+
print("cmd.py help")
33+
sys.exit()
34+
else:
35+
mode_submit = False
36+
mode_delete = False
37+
mode_reservation = False
38+
mode_status = False
39+
mode_list = False
40+
for opt, arg in opts:
41+
if opt in ("-f", "--config_directory"):
42+
directory = arg
43+
elif opt in ("-p", "--submit"):
44+
mode_submit = True
45+
elif opt in ("-q", "--queue"):
46+
queue = arg
47+
elif opt in ("-j", "--job_name"):
48+
job_name = arg
49+
elif opt in ("-w", "--working_directory"):
50+
working_directory = arg
51+
elif opt in ("-n", "--cores"):
52+
cores = int(arg)
53+
elif opt in ("-m", "--memory"):
54+
memory_max = arg
55+
elif opt in ("-t", "--run_time"):
56+
run_time_max = arg
57+
elif opt in ("-c", "--command"):
58+
command = arg
59+
elif opt in ("-r", "--reservation"):
60+
mode_reservation = arg
61+
elif opt in ("-i", "--id"):
62+
job_id = int(arg)
63+
elif opt in ("-d", "--delete"):
64+
mode_delete = True
65+
elif opt in ("-d", "--status"):
66+
mode_status = True
67+
elif opt in ("-l", "--list"):
68+
mode_list = True
69+
elif opt in ("-h", "--help"):
70+
print("cmd.py help ... coming soon.")
71+
sys.exit()
72+
if mode_submit or mode_delete or mode_reservation or mode_status:
73+
qa = QueueAdapter(directory=directory)
74+
if mode_submit:
75+
print(qa.submit_job(
76+
queue=queue,
77+
job_name=job_name,
78+
working_directory=working_directory,
79+
cores=cores,
80+
memory_max=memory_max,
81+
run_time_max=run_time_max,
82+
command=command
83+
))
84+
elif mode_delete:
85+
print(qa.delete_job(process_id=job_id))
86+
elif mode_reservation:
87+
print(qa.enable_reservation(process_id=job_id))
88+
elif mode_status:
89+
print(json.dumps(qa.get_queue_status().to_dict(orient='list')))
90+
elif mode_list:
91+
working_directory = os.path.abspath(os.path.expanduser(working_directory))
92+
remote_dirs, remote_files = [], []
93+
for p, folder, files in os.walk(working_directory):
94+
remote_dirs.append(p)
95+
remote_files += [os.path.join(p, f) for f in files]
96+
print(json.dumps({'dirs': remote_dirs, 'files': remote_files}))
97+
sys.exit()
98+
99+
100+
if __name__ == "__main__":
101+
command_line(sys.argv[1:])

pysqa/queueadapter.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import yaml
66
from pysqa.basic import BasisQueueAdapter
77
from pysqa.modular import ModularQueueAdapter
8+
from pysqa.remote import RemoteQueueAdapter
89

910
__author__ = "Jan Janssen"
1011
__copyright__ = "Copyright 2019, Jan Janssen"
@@ -48,6 +49,8 @@ def __init__(self, directory="~/.queues"):
4849
self._adapter = BasisQueueAdapter(config=config, directory=directory)
4950
elif config["queue_type"] in ["GENT"]:
5051
self._adapter = ModularQueueAdapter(config=config, directory=directory)
52+
elif config["queue_type"] in ["REMOTE"]:
53+
self._adapter = RemoteQueueAdapter(config=config, directory=directory)
5154
else:
5255
raise ValueError
5356

@@ -60,6 +63,15 @@ def config(self):
6063
"""
6164
return self._adapter.config
6265

66+
@property
67+
def remote_flag(self):
68+
"""
69+
70+
Returns:
71+
bool:
72+
"""
73+
return self._adapter.remote_flag
74+
6375
@property
6476
def queue_list(self):
6577
"""
@@ -127,6 +139,40 @@ def enable_reservation(self, process_id):
127139
"""
128140
return self._adapter.enable_reservation(process_id=process_id)
129141

142+
def get_job_from_remote(self, working_directory, delete_remote=False):
143+
"""
144+
Get the results of the calculation - this is necessary when the calculation was executed on a remote host.
145+
146+
Args:
147+
working_directory (str):
148+
delete_remote (bool):
149+
"""
150+
self._adapter.get_job_from_remote(working_directory=working_directory, delete_remote=delete_remote)
151+
152+
def transfer_file_to_remote(self, file, transfer_back=False, delete_remote=False):
153+
"""
154+
155+
Args:
156+
file (str):
157+
transfer_back (bool):
158+
delete_remote (bool):
159+
160+
Returns:
161+
str:
162+
"""
163+
self._adapter.transfer_file(file=file, transfer_back=transfer_back, delete_remote=delete_remote)
164+
165+
def convert_path_to_remote(self, path):
166+
"""
167+
168+
Args:
169+
path (str):
170+
171+
Returns:
172+
str:
173+
"""
174+
return self._adapter.convert_path_to_remote(path=path)
175+
130176
def delete_job(self, process_id):
131177
"""
132178

0 commit comments

Comments
 (0)