Move shell out commands into separate library

Coupled_Drivers/cice_driver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2023-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2023-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
@@ -26,14 +26,15 @@
 import re
 import datetime
 import time
-import subprocess
 import time2days
 import inc_days
 import common
 import error
 import dr_env_lib.cice_def
 import dr_env_lib.env_lib
 
+from mocilib import shellout
+
 
 def __expand_array(short_array):
     '''

Coupled_Drivers/common.py

@@ -296,6 +296,82 @@ def __exec_subproc_true_shell(cmd, verbose=True):
     return process.returncode, output
 
 
+def exec_subproc_timeout(cmd, timeout_sec=10):
+    '''
+    Execute a given shell command with a timeout. Takes a list containing
+    the commands to be run, and an integer timeout_sec for how long to
+    wait for the command to run. Returns the return code from the process
+    and the standard out from the command or 'None' if the command times out.
+
+    This function is now DEPRECATED in favour of the exec_subprocess function
+    used in mocilib/shellout.py
+    '''
+    process = subprocess.Popen(cmd, shell=False,
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    timer = threading.Timer(timeout_sec, process.kill)
+    try:
+        timer.start()
+        stdout, err = process.communicate()
+        if err:
+            sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error)
+        rcode = process.returncode
+    finally:
+        timer.cancel()
+    if sys.version_info[0] >= 3:
+        output = stdout.decode()
+    else:
+        output = stdout
+    return rcode, output
+
+
+def exec_subproc(cmd, verbose=True):
+    '''
+    Execute given shell command. Takes a list containing the commands to be
+    run, and a logical verbose which if set to true will write the output of
+    the command to stdout.
+
+    This function is now DEPRECATED in favour of the exec_subprocess function
+    used in mocilib/shellout.py
+    '''
+    process = subprocess.Popen(cmd, shell=False,
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    output, err = process.communicate()
+    if verbose and output:
+        sys.stdout.write('[SUBPROCESS OUTPUT] %s\n' % output)
+    if err:
+        sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error)
+    if sys.version_info[0] >= 3:
+        output = output.decode()
+    return process.returncode, output
+
+
+def __exec_subproc_true_shell(cmd, verbose=True):
+    '''
+    Execute given shell command, with shell=True. Only use this function if
+    exec_subproc does not work correctly. Takes a list containing the commands
+    to be run, and a logical verbose which if set to true will write the
+    output of the command to stdout.
+
+    This function is now DEPRECATED in favour of the exec_subprocess function
+    used in mocilib/shellout.py
+    '''
+    process = subprocess.Popen(cmd, shell=True,
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    output, err = process.communicate()
+    if verbose and output:
+        sys.stdout.write('[SUBPROCESS OUTPUT] %s\n' % output)
+    if err:
+        sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error)
+    if sys.version_info[0] >= 3:
+        output = output.decode()
+    return process.returncode, output
+
 def _calculate_ppn_values(nproc, nodes):
     '''
     Calculates number of processes per node and numa node for launch

Coupled_Drivers/cpmip_utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2023-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2023-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy

Coupled_Drivers/cpmip_xios.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2021-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2021-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy

Coupled_Drivers/mct_driver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2021-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2021-2026 Met Office. All rights reserved.
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
  of the code, the use, duplication or disclosure of it is strictly
@@ -28,6 +28,9 @@
 import dr_env_lib.mct_def
 import dr_env_lib.env_lib
 import cpmip_controller
+
+from mocilib import shellout
+
 try:
     import f90nml
 except ImportError:

Coupled_Drivers/nemo_driver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2023-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2023-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
@@ -29,6 +29,8 @@
 import common
 import error
 
+from mocilib import shellout
+
 try:
     import cf_units
 except ImportError:

Coupled_Drivers/rivers_driver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2025 Met Office. All rights reserved.
+ (C) Crown copyright 2025-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
@@ -26,6 +26,8 @@
 import error
 import dr_env_lib.rivers_def
 import dr_env_lib.env_lib
+
+from mocilib import shellout
 try:
     import f90nml
 except ImportError:

Coupled_Drivers/si3_controller.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2021-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2021-2026 Met Office. All rights reserved.
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
  of the code, the use, duplication or disclosure of it is strictly
@@ -26,6 +26,8 @@
 import dr_env_lib.ocn_cont_def
 import dr_env_lib.env_lib
 
+from mocilib import shellout
+
 def _check_si3nl_envar(envar_container):
     '''
     Get the si3 namelist file exists

Coupled_Drivers/top_controller.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2021-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2021-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
@@ -73,6 +73,8 @@
 import dr_env_lib.ocn_cont_def
 import dr_env_lib.env_lib
 
+from mocilib import shellout
+
 # Define errors for the TOP controller only
 SERIAL_MODE_ERROR = 99
 

Coupled_Drivers/unittests/test_cpmip_utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2023-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2023-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy

Coupled_Drivers/unittests/test_cpmip_xios.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2021-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2021-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy

Coupled_Drivers/unittests/test_rivers_driver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2025 Met Office. All rights reserved.
+ (C) Crown copyright 2025-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy

Coupled_Drivers/write_namcouple.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2021-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2021-2026 Met Office. All rights reserved.
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
  of the code, the use, duplication or disclosure of it is strictly
@@ -25,6 +25,8 @@
 import write_namcouple_fields
 import write_namcouple_header
 
+from mocilib import shellout
+
 # Dictionary containing the RMP mappings
 RMP_MAPPING = {'Bc':'BICUBIC',
                'Bi':'BILINEA',

Coupled_Drivers/xios_driver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 '''
 *****************************COPYRIGHT******************************
- (C) Crown copyright 2023-2025 Met Office. All rights reserved.
+ (C) Crown copyright 2023-2026 Met Office. All rights reserved.
 
  Use, duplication or disclosure of this code is subject to the restrictions
  as set forth in the licence. If no licence has been raised with this copy
@@ -27,6 +27,8 @@
 import dr_env_lib.xios_def
 import dr_env_lib.env_lib
 
+from mocilib import shellout
+
 def _copy_iodef_custom(xios_evar):
     '''
     If a custom iodef file exists, copy this to the required input filename

Postprocessing/common/utils.py

@@ -26,6 +26,8 @@
 import subprocess
 import timer
 
+from mocilib import shellout
+
 
 globals()['debug_mode'] = None
 globals()['debug_ok'] = True

Postprocessing/unittests/test_shellout.py

@@ -0,0 +1,33 @@
+# -----------------------------------------------------------------------------
+# (C) Crown copyright Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+# -----------------------------------------------------------------------------
+
+import unittest
+from mocilib import shellout
+from hypothesis import given, strategies as st
+
+class ExecTets(unittest.TestCase):
+    ''' Unit tests for executing shellout commands'''
+
+    def test_semicolon_commands(self):
+        cmd = "echo Hello There;echo General Kenobi"
+        _,rcode = shellout._exec_subprocess(cmd=cmd)
+        assert rcode == 0
+
+    def test_and_commands(self):
+        cmd ="echo Hello There&&echo General Kenobi"
+        _,rcode = shellout._exec_subprocess(cmd=cmd)
+        assert rcode == 0
+
+    @given(st.text())
+    def test_called_process_error(self,directory):
+        cmd = f"ls /{directory}"
+        _,rcode = shellout._exec_subprocess(cmd=cmd)
+        assert rcode != 0
+
+    def test_timeout_expired(self):
+        cmd = "sleep 15"
+        _,rcode = shellout._exec_subprocess(cmd=cmd,timeout=10)
+        assert rcode != 0

mocilib/__init__.py

@@ -0,0 +1,2 @@
+___all__ = ["shellout"]
+from . import shellout

mocilib/shellout.py

@@ -0,0 +1,51 @@
+# -----------------------------------------------------------------------------
+# (C) Crown copyright Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+# -----------------------------------------------------------------------------
+
+import subprocess
+import os
+import sys
+import shlex
+
+
+def _exec_subprocess(cmd, verbose=False, timeout=None ,current_working_directory=os.getcwd()):
+    """
+    Execute a given shell command
+
+    :param cmd: The command to be executed given as a string
+    :param verbose: A boolean value to determine if the stdout
+    stream is displayed during the runtime.
+    :param current_working_directory: The directory in which the
+    command should be executed.
+    """
+
+    cmd = shlex.split(cmd)
+
+    try:
+
+        output = subprocess.run(
+            cmd,
+            capture_output=True,
+            cwd=current_working_directory,
+            timeout=timeout,
+            check=True
+        )
+        rcode = output.returncode
+        output_message = output.stdout.decode()
+
+        if verbose and output:
+            sys.stdout.write(f"[DEBUG]{output.stdout}\n")
+        if output.stderr and output.returncode != 0:
+            sys.stderr.write(f"[ERROR] {output.stderr}\n")
+
+    except subprocess.CalledProcessError as exc:
+        output_message = exc.stdout.decode() if exc.stdout else ""
+        rcode = exc.returncode
+
+    except subprocess.TimeoutExpired as exc:
+        output_message = exc.stdout.decode() if exc.stdout else ""
+        rcode = exc.returncode
+
+    return rcode,output_message