From 409302f349306176fd07849361c400ee0788a352 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 3 May 2023 13:37:47 -0600 Subject: [PATCH 1/5] Add command line test --- pysqa/cmd.py | 6 +++-- pysqa/queueadapter.py | 16 ++++++++----- tests/test_cmd.py | 56 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 tests/test_cmd.py diff --git a/pysqa/cmd.py b/pysqa/cmd.py index 4a1c589f..b8f6bb10 100644 --- a/pysqa/cmd.py +++ b/pysqa/cmd.py @@ -3,14 +3,16 @@ import json import getopt from pysqa.queueadapter import QueueAdapter +from pysqa.utils.execute import execute_command -def command_line(argv): +def command_line(argv, execute_command=execute_command): """ Parse the command line arguments. Args: argv: Command line arguments + execute_command: function to comunicate with shell process """ directory = "~/.queues" @@ -94,7 +96,7 @@ def command_line(argv): print("cmd.py help ... coming soon.") sys.exit() if mode_submit or mode_delete or mode_reservation or mode_status: - qa = QueueAdapter(directory=directory) + qa = QueueAdapter(directory=directory, execute_command=execute_command) if mode_submit: print( qa.submit_job( diff --git a/pysqa/queueadapter.py b/pysqa/queueadapter.py index 114d7d36..6ffc1271 100644 --- a/pysqa/queueadapter.py +++ b/pysqa/queueadapter.py @@ -6,6 +6,7 @@ from pysqa.ext.modular import ModularQueueAdapter from pysqa.ext.remote import RemoteQueueAdapter from pysqa.utils.config import read_config +from pysqa.utils.execute import execute_command __author__ = "Jan Janssen" __copyright__ = "Copyright 2019, Jan Janssen" @@ -44,14 +45,16 @@ class QueueAdapter(object): Queues available for auto completion QueueAdapter().queues. returns the queue name. """ - def __init__(self, directory="~/.queues"): + def __init__(self, directory="~/.queues", execute_command=execute_command): queue_yaml = os.path.join(directory, "queue.yaml") clusters_yaml = os.path.join(directory, "clusters.yaml") self._adapter = None if os.path.exists(queue_yaml): self._queue_dict = { "default": set_queue_adapter( - config=read_config(file_name=queue_yaml), directory=directory + config=read_config(file_name=queue_yaml), + directory=directory, + execute_command=execute_command ) } primary_queue = "default" @@ -61,6 +64,7 @@ def __init__(self, directory="~/.queues"): k: set_queue_adapter( config=read_config(file_name=os.path.join(directory, v)), directory=directory, + execute_command=execute_command ) for k, v in config["cluster"].items() } @@ -296,7 +300,7 @@ def check_queue_parameters( ) -def set_queue_adapter(config, directory): +def set_queue_adapter(config, directory, execute_command=execute_command): """ Initialize the queue adapter @@ -305,10 +309,10 @@ def set_queue_adapter(config, directory): directory (str): directory which contains the queue configurations """ if config["queue_type"] in ["SGE", "TORQUE", "SLURM", "LSF", "MOAB"]: - return BasisQueueAdapter(config=config, directory=directory) + return BasisQueueAdapter(config=config, directory=directory, execute_command=execute_command) elif config["queue_type"] in ["GENT"]: - return ModularQueueAdapter(config=config, directory=directory) + return ModularQueueAdapter(config=config, directory=directory, execute_command=execute_command) elif config["queue_type"] in ["REMOTE"]: - return RemoteQueueAdapter(config=config, directory=directory) + return RemoteQueueAdapter(config=config, directory=directory, execute_command=execute_command) else: raise ValueError diff --git a/tests/test_cmd.py b/tests/test_cmd.py new file mode 100644 index 00000000..fa61e3c0 --- /dev/null +++ b/tests/test_cmd.py @@ -0,0 +1,56 @@ +import os +import unittest +from pysqa.cmd import command_line + + +class TestCMD(unittest.TestCase): + def test_help(self): + with self.assertRaises(SystemExit): + command_line(["--help"]) + + def test_wrong_option(self): + with self.assertRaises(SystemExit): + command_line(["--error"]) + + def test_submit(self): + def execute_command( + commands, + working_directory=None, + split_output=True, + shell=False, + error_filename="pysqa.err", + ): + return "1\n" + + with self.assertRaises(SystemExit): + command_line( + [ + "--config_directory", "config/slurm", + "--submit", + "--queue", "slurm", + "--job_name", "test", + "--working_directory", ".", + "--cores", "2", + "--memory", "1GB", + "--run_time", "10", + "--command", "echo hello" + ], + execute_command=execute_command + ) + with open("run_queue.sh") as f: + output = f.readlines() + content = [ + '#!/bin/bash\n', + '#SBATCH --output=time.out\n', + '#SBATCH --job-name=test\n', + '#SBATCH --chdir=.\n', + '#SBATCH --get-user-env=L\n', + '#SBATCH --partition=slurm\n', + '#SBATCH --time=4320\n', + '#SBATCH --mem=1GBG\n', + '#SBATCH --cpus-per-task=10\n', + '\n', + 'echo hello' + ] + self.assertEqual(output, content) + os.remove("run_queue.sh") From 2949668e4689ec065b7a7c9e146e5e5b3e68668a Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Wed, 3 May 2023 19:39:49 +0000 Subject: [PATCH 2/5] Format black --- pysqa/queueadapter.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pysqa/queueadapter.py b/pysqa/queueadapter.py index 6ffc1271..95a02dfb 100644 --- a/pysqa/queueadapter.py +++ b/pysqa/queueadapter.py @@ -54,7 +54,7 @@ def __init__(self, directory="~/.queues", execute_command=execute_command): "default": set_queue_adapter( config=read_config(file_name=queue_yaml), directory=directory, - execute_command=execute_command + execute_command=execute_command, ) } primary_queue = "default" @@ -64,7 +64,7 @@ def __init__(self, directory="~/.queues", execute_command=execute_command): k: set_queue_adapter( config=read_config(file_name=os.path.join(directory, v)), directory=directory, - execute_command=execute_command + execute_command=execute_command, ) for k, v in config["cluster"].items() } @@ -309,10 +309,16 @@ def set_queue_adapter(config, directory, execute_command=execute_command): directory (str): directory which contains the queue configurations """ if config["queue_type"] in ["SGE", "TORQUE", "SLURM", "LSF", "MOAB"]: - return BasisQueueAdapter(config=config, directory=directory, execute_command=execute_command) + return BasisQueueAdapter( + config=config, directory=directory, execute_command=execute_command + ) elif config["queue_type"] in ["GENT"]: - return ModularQueueAdapter(config=config, directory=directory, execute_command=execute_command) + return ModularQueueAdapter( + config=config, directory=directory, execute_command=execute_command + ) elif config["queue_type"] in ["REMOTE"]: - return RemoteQueueAdapter(config=config, directory=directory, execute_command=execute_command) + return RemoteQueueAdapter( + config=config, directory=directory, execute_command=execute_command + ) else: raise ValueError From 16dd8456adfb48e83fd41c5ea2c2dd8a10adf674 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 3 May 2023 13:42:18 -0600 Subject: [PATCH 3/5] fixes --- pysqa/ext/modular.py | 5 +++-- pysqa/ext/remote.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pysqa/ext/modular.py b/pysqa/ext/modular.py index d1e703a6..2ccb412f 100644 --- a/pysqa/ext/modular.py +++ b/pysqa/ext/modular.py @@ -3,11 +3,12 @@ import pandas from pysqa.utils.basic import BasisQueueAdapter +from pysqa.utils.execute import execute_command class ModularQueueAdapter(BasisQueueAdapter): - def __init__(self, config, directory="~/.queues"): - super(ModularQueueAdapter, self).__init__(config=config, directory=directory) + def __init__(self, config, directory="~/.queues", execute_command=execute_command): + super(ModularQueueAdapter, self).__init__(config=config, directory=directory, execute_command=execute_command) self._queue_to_cluster_dict = { k: v["cluster"] for k, v in self._config["queues"].items() } diff --git a/pysqa/ext/remote.py b/pysqa/ext/remote.py index 6e3e731a..1c1fe75c 100644 --- a/pysqa/ext/remote.py +++ b/pysqa/ext/remote.py @@ -8,11 +8,12 @@ import warnings from tqdm import tqdm from pysqa.utils.basic import BasisQueueAdapter +from pysqa.utils.execute import execute_command class RemoteQueueAdapter(BasisQueueAdapter): - def __init__(self, config, directory="~/.queues"): - super(RemoteQueueAdapter, self).__init__(config=config, directory=directory) + def __init__(self, config, directory="~/.queues", execute_command=execute_command): + super(RemoteQueueAdapter, self).__init__(config=config, directory=directory, execute_command=execute_command) self._ssh_host = config["ssh_host"] self._ssh_username = config["ssh_username"] self._ssh_known_hosts = os.path.abspath( From 0139e6991cb582463fa5f4b23dd8397ea155d2b7 Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Wed, 3 May 2023 19:43:06 +0000 Subject: [PATCH 4/5] Format black --- pysqa/ext/modular.py | 4 +++- pysqa/ext/remote.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pysqa/ext/modular.py b/pysqa/ext/modular.py index 2ccb412f..02a861fe 100644 --- a/pysqa/ext/modular.py +++ b/pysqa/ext/modular.py @@ -8,7 +8,9 @@ class ModularQueueAdapter(BasisQueueAdapter): def __init__(self, config, directory="~/.queues", execute_command=execute_command): - super(ModularQueueAdapter, self).__init__(config=config, directory=directory, execute_command=execute_command) + super(ModularQueueAdapter, self).__init__( + config=config, directory=directory, execute_command=execute_command + ) self._queue_to_cluster_dict = { k: v["cluster"] for k, v in self._config["queues"].items() } diff --git a/pysqa/ext/remote.py b/pysqa/ext/remote.py index 1c1fe75c..ef9e7b33 100644 --- a/pysqa/ext/remote.py +++ b/pysqa/ext/remote.py @@ -13,7 +13,9 @@ class RemoteQueueAdapter(BasisQueueAdapter): def __init__(self, config, directory="~/.queues", execute_command=execute_command): - super(RemoteQueueAdapter, self).__init__(config=config, directory=directory, execute_command=execute_command) + super(RemoteQueueAdapter, self).__init__( + config=config, directory=directory, execute_command=execute_command + ) self._ssh_host = config["ssh_host"] self._ssh_username = config["ssh_username"] self._ssh_known_hosts = os.path.abspath( From b6305a5edf7041f06fa03335e73a1677ae7da8e6 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 3 May 2023 13:47:15 -0600 Subject: [PATCH 5/5] define config directory relative to own directory --- tests/test_cmd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index fa61e3c0..5d53f530 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -22,10 +22,11 @@ def execute_command( ): return "1\n" + test_dir = os.path.abspath(os.path.dirname(__file__)) with self.assertRaises(SystemExit): command_line( [ - "--config_directory", "config/slurm", + "--config_directory", os.path.join(test_dir, "config", "slurm"), "--submit", "--queue", "slurm", "--job_name", "test",