check presence of current working directory at the end of run_shell_cmd and try to return to original working directory if non-existent

easybuild/tools/run.py

@@ -505,6 +505,21 @@ def to_cmd_str(cmd):
         if fail_on_error:
             raise_run_shell_cmd_error(res)
 
+    # check that we still are in a sane environment after command execution
+    # safeguard against commands that leave behind the system in a borked state
+    try:
+        os.getcwd()
+    except FileNotFoundError:
+        try:
+            warn_msg = (
+                f"Shell command `{cmd_str}` completed successfully but left system in a broken state. "
+                f"Changing back to initial working directory: {res.work_dir}"
+            )
+            _log.warning(warn_msg)
+            os.chdir(res.work_dir)
+        except OSError as err:
+            raise EasyBuildError(f"Failed to return to {res.work_dir} after executing command `{cmd_str}`: {err}")
+
     if with_hooks:
         run_hook_kwargs = {
             'exit_code': res.exit_code,

test/framework/run.py

@@ -24,7 +24,7 @@
 # along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
 # #
 """
-Unit tests for filetools.py
+Unit tests for run.py
 
 @author: Toon Willems (Ghent University)
 @author: Kenneth Hoste (Ghent University)
@@ -44,7 +44,7 @@
 import time
 from concurrent.futures import ThreadPoolExecutor
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
-from unittest import TextTestRunner
+from unittest import TextTestRunner, mock
 from easybuild.base.fancylogger import setLogLevelDebug
 
 import easybuild.tools.asyncprocess as asyncprocess
@@ -500,22 +500,38 @@ def test_run_shell_cmd_work_dir(self):
         """
         Test running shell command in specific directory with run_shell_cmd function.
         """
-        orig_wd = os.getcwd()
-        self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
-
         test_dir = os.path.join(self.test_prefix, 'test')
+        test_workdir = os.path.join(self.test_prefix, 'test', 'workdir')
         for fn in ('foo.txt', 'bar.txt'):
-            write_file(os.path.join(test_dir, fn), 'test')
+            write_file(os.path.join(test_workdir, fn), 'test')
+
+        os.chdir(test_dir)
+        orig_wd = os.getcwd()
+        self.assertFalse(os.path.samefile(orig_wd, self.test_prefix))
 
         cmd = "ls | sort"
+
+        # working directory is not explicitly defined
         with self.mocked_stdout_stderr():
-            res = run_shell_cmd(cmd, work_dir=test_dir)
+            res = run_shell_cmd(cmd)
+
+        self.assertEqual(res.cmd, cmd)
+        self.assertEqual(res.exit_code, 0)
+        self.assertEqual(res.output, 'workdir\n')
+        self.assertEqual(res.stderr, None)
+        self.assertEqual(res.work_dir, orig_wd)
+
+        self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
+
+        # working directory is explicitly defined
+        with self.mocked_stdout_stderr():
+            res = run_shell_cmd(cmd, work_dir=test_workdir)
 
         self.assertEqual(res.cmd, cmd)
         self.assertEqual(res.exit_code, 0)
         self.assertEqual(res.output, 'bar.txt\nfoo.txt\n')
         self.assertEqual(res.stderr, None)
-        self.assertEqual(res.work_dir, test_dir)
+        self.assertEqual(res.work_dir, test_workdir)
 
         self.assertTrue(os.path.samefile(orig_wd, os.getcwd()))
 
@@ -1848,12 +1864,62 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
             stdout = self.get_stdout()
 
         expected_stdout = '\n'.join([
-            "pre-run hook 'make' in %s" % cwd,
+            f"pre-run hook 'make' in {cwd}",
             "post-run hook 'echo make' (exit code: 0, output: 'make\n')",
             '',
         ])
         self.assertEqual(stdout, expected_stdout)
 
+        # Test commands that destroy directories inside initial working directory
+        workdir = os.path.join(self.test_prefix, 'workdir')
+        sub_workdir = os.path.join(workdir, 'subworkdir')
+
+        # 1. test destruction of CWD which is a subdirectory inside original working directory
+        cmd_workdir_rm = "echo 'Command that jumps to subdir and removes it' && "
+        cmd_workdir_rm += f"cd {sub_workdir} && pwd && rm -rf {sub_workdir} && "
+        cmd_workdir_rm += "echo 'Working sub-directory removed.'"
+        expected_stdout_patterns = [
+            rf"pre-run hook '{cmd_workdir_rm}' in {workdir}",
+            (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, "
+             rf"output: 'Command that jumps to subdir.*\n{sub_workdir}\nWorking sub-directory removed\..*'\)"),
+        ]
+        expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns]
+
+        # 1.a. in a robust system
+        mkdir(sub_workdir, parents=True)
+        with self.mocked_stdout_stderr():
+            run_shell_cmd(cmd_workdir_rm, work_dir=workdir)
+            stdout = self.get_stdout()
+
+            for regex in expected_stdout_patterns:
+                self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}")
+
+        # 1.b. in a flaky system that ends up in a broken environment after execution
+        mkdir(sub_workdir, parents=True)
+        with self.mocked_stdout_stderr():
+            with mock.patch('os.getcwd') as mock_getcwd:
+                mock_getcwd.side_effect = FileNotFoundError()
+                run_shell_cmd(cmd_workdir_rm, work_dir=workdir)
+                stdout = self.get_stdout()
+
+                for regex in expected_stdout_patterns:
+                    self.assertTrue(regex.search(stdout), f"Pattern '{regex.pattern}' should be found in: {stdout}")
+
+        # 2. test destruction of CWD which is main working directory passed to run_shell_cmd
+        cmd_workdir_rm = "echo 'Command that removes working directory' && pwd && "
+        cmd_workdir_rm += f"rm -rf {workdir} && echo 'Working directory removed.'"
+        expected_stdout_patterns = [
+            rf"pre-run hook '{cmd_workdir_rm}' in {workdir}",
+            (rf"post-run hook '{cmd_workdir_rm}' \(exit code: 0, "
+             rf"output: 'Command that removes working.*\n{workdir}\nWorking directory removed\..*'\)"),
+        ]
+        expected_stdout_patterns = [re.compile(pattern, re.S) for pattern in expected_stdout_patterns]
+
+        mkdir(workdir)
+        with self.mocked_stdout_stderr():
+            error_pattern = rf"Failed to return to {workdir} after executing command"
+            self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd_workdir_rm, work_dir=workdir)
+
 
 def suite():
     """ returns all the testcases in this module """