Add --keep-going option to fail at the end not at first failing installation + use proper exit code as determined by main function

easybuild/main.py

@@ -276,7 +276,7 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state
 
     # Loop over each item in the EasyStack file, each time updating the config
     # This is because each item in an EasyStack file can have options associated with it
-    do_cleanup = True
+    is_successful = True
     for (path, ec_opts) in easystack.ec_opt_tuples:
         _log.debug("Starting build for %s" % path)
 
@@ -313,10 +313,10 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state
         modtool = modules_tool(testing=testing)
 
         # Process actual item in the EasyStack file
-        do_cleanup &= process_eb_args([path], eb_go, cfg_settings, modtool, testing, init_session_state,
-                                      hooks, do_build)
+        is_successful &= process_eb_args([path], eb_go, cfg_settings, modtool, testing, init_session_state,
+                                         hooks, do_build)
 
-    return do_cleanup
+    return is_successful
 
 
 def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session_state, hooks, do_build):
@@ -339,7 +339,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
 
     global _log
     # Unpack cfg_settings
-    (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
+    (build_specs, _log, _logfile, robot_path, search_query, _eb_tmpdir, try_to_generate,
      from_pr_list, tweaked_ecs_paths) = cfg_settings
 
     # determine easybuild-easyconfigs package install path
@@ -589,7 +589,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     # build software, will exit when errors occurs (except when testing)
     start_time = datetime.now()
     if not testing or (testing and do_build):
-        exit_on_failure = not (options.dump_test_report or options.upload_test_report)
+        exit_on_failure = not any((options.dump_test_report, options.upload_test_report, options.keep_going))
 
         with rich_live_cm():
             run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs])
@@ -629,14 +629,16 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     return overall_success
 
 
-def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None):
+def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None) -> EasyBuildExit:
     """
     Main function: parse command line options, and act accordingly.
     :param args: command line arguments to use
     :param logfile: log file to use
     :param do_build: whether or not to actually perform the build
     :param testing: enable testing mode
     :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None)
+
+    :return: error code
     """
     if prepared_cfg_data is None or any([args, logfile, testing]):
         init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing)
@@ -646,8 +648,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
     options, orig_paths = eb_go.options, eb_go.args
 
     global _log
-    (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
-     from_pr_list, tweaked_ecs_paths) = cfg_settings
+    (_build_specs, _log, logfile, _robot_path, search_query, eb_tmpdir, _try_to_generate,
+     _from_pr_list, _tweaked_ecs_paths) = cfg_settings
 
     # compare running Framework and EasyBlocks versions
     if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION:
@@ -773,15 +775,19 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
                 "The following arguments will be ignored:",
             ] + orig_paths)
             print_warning(msg)
-        do_cleanup = process_easystack(options.easystack, args, logfile, testing, init_session_state, do_build)
+        is_successful = process_easystack(options.easystack, args, logfile, testing, init_session_state, do_build)
     else:
-        do_cleanup = process_eb_args(orig_paths, eb_go, cfg_settings, modtool, testing, init_session_state,
-                                     hooks, do_build)
+        is_successful = process_eb_args(orig_paths, eb_go, cfg_settings, modtool, testing, init_session_state,
+                                        hooks, do_build)
 
     # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
     stop_logging(logfile, logtostdout=options.logtostdout)
-    if do_cleanup:
+    if is_successful:
         cleanup(logfile, eb_tmpdir, testing, silent=options.terse)
+    if options.dump_test_report or options.upload_test_report:
+        # Generation test reports is successful even when software failed to build
+        return EasyBuildExit.SUCCESS
+    return EasyBuildExit.SUCCESS if is_successful else EasyBuildExit.ERROR
 
 
 def prepare_main(args=None, logfile=None, testing=None):
@@ -820,7 +826,7 @@ def main_with_hooks(args=None):
     hooks = load_hooks(eb_go.options.hooks)
 
     try:
-        main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
+        sys.exit(int(main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings))))
     except EasyBuildError as err:
         run_hook(FAIL, hooks, args=[err])
         print_error(err.msg, exit_on_error=True, exit_code=err.exit_code)

easybuild/tools/build_log.py

@@ -264,7 +264,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.
             if tmp_logdir and not os.path.exists(tmp_logdir):
                 try:
                     os.makedirs(tmp_logdir)
-                except (IOError, OSError) as err:
+                except OSError as err:
                     raise EasyBuildError("Failed to create temporary log directory %s: %s", tmp_logdir, err)
 
             # mkstemp returns (fd,filename), fd is from os.open, not regular open!
@@ -397,9 +397,8 @@ def print_error(msg, *args, **kwargs):
     if exitCode is not None:
         _init_easybuildlog.deprecated("'exitCode' option in print_error function is replaced with 'exit_code'", '6.0')
 
-    # use 1 as defaut exit code
     if exit_code is None:
-        exit_code = 1
+        exit_code = EasyBuildExit.ERROR
 
     log = kwargs.pop('log', None)
     opt_parser = kwargs.pop('opt_parser', None)
@@ -413,7 +412,7 @@ def print_error(msg, *args, **kwargs):
             if opt_parser:
                 opt_parser.print_shorthelp()
             sys.stderr.write("ERROR: %s\n" % msg)
-        sys.exit(exit_code)
+        sys.exit(int(exit_code))
     elif log is not None:
         raise EasyBuildError(msg)
 

easybuild/tools/config.py

@@ -328,6 +328,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'ignore_test_failure',
         'install_latest_eb_release',
         'keep_debug_symbols',
+        'keep_going',
         'logtostdout',
         'minimal_toolchains',
         'module_only',

easybuild/tools/options.py

@@ -808,6 +808,8 @@ def github_options(self):
             'close-pr-msg': ("Custom close message for pull request closed with --close-pr; ", str, 'store', None),
             'close-pr-reasons': ("Close reason for pull request closed with --close-pr; "
                                  "supported values: %s" % ", ".join(VALID_CLOSE_PR_REASONS), str, 'store', None),
+            'keep-going': ("Continue installation of remaining software after a failed installation. "
+                           "Implied by --dump-test-report and --upload-test-report", None, 'store_true', False),
             'list-prs': ("List pull requests", str, 'store_or_None',
                          ",".join([DEFAULT_LIST_PR_STATE, DEFAULT_LIST_PR_ORDER, DEFAULT_LIST_PR_DIREC]),
                          {'metavar': 'STATE,ORDER,DIRECTION'}),

test/framework/options.py

@@ -122,7 +122,7 @@ def tearDown(self):
 
     def purge_environment(self):
         """Remove any leftover easybuild variables"""
-        for var in os.environ.keys():
+        for var in list(os.environ):
             # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files!
             if var.startswith('EASYBUILD_') and var != 'EASYBUILD_IGNORECONFIGFILES':
                 del os.environ[var]
@@ -3239,8 +3239,8 @@ def test_http_header_fields_urlpat(self):
         mentionhdr = 'Custom HTTP header field set: %s'
         mentionfile = 'File included in parse_http_header_fields_urlpat: %s'
 
-        def run_and_assert(args, msg, words_expected=None, words_unexpected=None):
-            stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)
+        def run_and_assert(args, _msg, words_expected=None, words_unexpected=None):
+            stdout, _stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)
             if words_expected is not None:
                 self._assert_regexs(words_expected, stdout)
             if words_unexpected is not None:
@@ -5258,7 +5258,7 @@ def test_show_config(self):
             'EASYBUILD_SOURCEPATH',
             'EASYBUILD_SOURCEPATH_DATA',
         ]
-        for key in os.environ.keys():
+        for key in list(os.environ):
             if key.startswith('EASYBUILD_') and key not in retained_eb_env_vars:
                 del os.environ[key]
 
@@ -6451,7 +6451,7 @@ def test_force_download(self):
             '--force-download',
             '--sourcepath=%s' % self.test_prefix,
         ]
-        stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True)
+        _stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True)
         regex = re.compile(r"^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$")
         self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr))
 
@@ -6735,6 +6735,43 @@ def test_sanity_check_only(self):
         import easybuild.easyblocks.generic.toy_extension
         reload(easybuild.easyblocks.generic.toy_extension)
 
+    def test_keep_going(self):
+        """Test use of --keep-going."""
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += '\nsources=["toy-0.0.tar.gz"]'
+        write_file(test_ec, test_ec_txt + '\nversion="broken"\npreconfigopts = "false && "')
+        test_ec2 = os.path.join(self.test_prefix, 'test2.eb')
+        write_file(test_ec2, test_ec_txt + '\nversion="working"')
+
+        args = [test_ec, test_ec2, '--rebuild']
+        with self.mocked_stdout_stderr():
+            outtxt, exit_code, error_thrown = self.eb_main(args, do_build=True, return_error=True,
+                                                           return_exit_code=True)
+        self.assertIn("Installation of test.eb failed", str(error_thrown))
+        self.assertNotEqual(exit_code, 0)
+        self.assertRegex(outtxt, r'\[FAILED\] *toy/broken')
+        self.assertRegex(outtxt, r'\[SKIPPED\] *toy/working')
+
+        args.append('--keep-going')
+        with self.mocked_stdout_stderr():
+            outtxt, exit_code = self.eb_main(args, do_build=True, raise_error=True,
+                                             return_exit_code=True)
+        self.assertNotEqual(exit_code, 0)
+        self.assertRegex(outtxt, r'\[FAILED\] *toy/broken')
+        self.assertRegex(outtxt, r'\[SUCCESS\] *toy/working')
+
+        args.append(f"--dump-test-report={os.path.join(tempfile.gettempdir(), 'report.md')}")
+        with self.mocked_stdout_stderr():
+            outtxt, exit_code = self.eb_main(args, do_build=True, raise_error=True,
+                                             return_exit_code=True)
+        self.assertEqual(exit_code, 0)  # Creating a test report returns success even on build failure
+        self.assertRegex(outtxt, r'\[FAILED\] *toy/broken')
+        self.assertRegex(outtxt, r'\[SUCCESS\] *toy/working')
+
     def test_skip_extensions(self):
         """Test use of --skip-extensions."""
         topdir = os.path.abspath(os.path.dirname(__file__))

test/framework/utilities.py

@@ -295,8 +295,9 @@ def reset_modulepath(self, modpaths):
             self.modtool.add_module_path(modpath, set_mod_paths=False)
         self.modtool.set_mod_paths()
 
-    def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False,
-                reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True, clear_caches=True):
+    def eb_main(self, args, do_build=False, return_error=False, return_exit_code=False, logfile=None, verbose=False,
+                raise_error=False, reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True,
+                clear_caches=True):
         """Helper method to call EasyBuild main function."""
 
         cleanup(clear_caches=clear_caches)
@@ -315,21 +316,24 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos
 
         env_before = copy.deepcopy(os.environ)
 
+        exit_code = eb_build_log.EasyBuildExit.ERROR
         try:
             if '--fetch' in args:
                 # The config sets modules_tool to None if --fetch is specified,
                 # so do the same here to keep the behavior consistent
                 modtool = None
             else:
                 modtool = self.modtool
-            main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool)
+            exit_code = main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool)
         except SystemExit as err:
             if raise_systemexit:
                 raise err
         except Exception as err:
             myerr = err
             if verbose:
                 print("err: %s" % err)
+            if isinstance(err, eb_build_log.EasyBuildError):
+                exit_code = err.exit_code
 
         if logfile and os.path.exists(logfile):
             logtxt = read_file(logfile)
@@ -352,9 +356,12 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos
             raise myerr
 
         if return_error:
+            if return_exit_code:
+                return logtxt, exit_code, myerr
             return logtxt, myerr
-        else:
-            return logtxt
+        if return_exit_code:
+            return logtxt, exit_code
+        return logtxt
 
     def setup_hierarchical_modules(self):
         """Setup hierarchical modules to run tests on."""