diff --git a/easybuild/main.py b/easybuild/main.py index 1af389f2d8..8f8ea3fc7b 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -276,7 +276,7 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state # Loop over each item in the EasyStack file, each time updating the config # This is because each item in an EasyStack file can have options associated with it - do_cleanup = True + is_successful = True for (path, ec_opts) in easystack.ec_opt_tuples: _log.debug("Starting build for %s" % path) @@ -313,10 +313,10 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state modtool = modules_tool(testing=testing) # Process actual item in the EasyStack file - do_cleanup &= process_eb_args([path], eb_go, cfg_settings, modtool, testing, init_session_state, - hooks, do_build) + is_successful &= process_eb_args([path], eb_go, cfg_settings, modtool, testing, init_session_state, + hooks, do_build) - return do_cleanup + return is_successful def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session_state, hooks, do_build): @@ -339,7 +339,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session global _log # Unpack cfg_settings - (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, + (build_specs, _log, _logfile, robot_path, search_query, _eb_tmpdir, try_to_generate, from_pr_list, tweaked_ecs_paths) = cfg_settings # determine easybuild-easyconfigs package install path @@ -589,7 +589,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # build software, will exit when errors occurs (except when testing) start_time = datetime.now() if not testing or (testing and do_build): - exit_on_failure = not (options.dump_test_report or options.upload_test_report) + exit_on_failure = not any((options.dump_test_report, options.upload_test_report, options.keep_going)) with rich_live_cm(): run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs]) @@ -629,7 +629,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None) -> EasyBuildExit: """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use @@ -637,6 +637,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr :param do_build: whether or not to actually perform the build :param testing: enable testing mode :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None) + + :return: error code """ if prepared_cfg_data is None or any([args, logfile, testing]): init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing) @@ -646,8 +648,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options, orig_paths = eb_go.options, eb_go.args global _log - (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, - from_pr_list, tweaked_ecs_paths) = cfg_settings + (_build_specs, _log, logfile, _robot_path, search_query, eb_tmpdir, _try_to_generate, + _from_pr_list, _tweaked_ecs_paths) = cfg_settings # compare running Framework and EasyBlocks versions if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION: @@ -773,15 +775,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr "The following arguments will be ignored:", ] + orig_paths) print_warning(msg) - do_cleanup = process_easystack(options.easystack, args, logfile, testing, init_session_state, do_build) + is_successful = process_easystack(options.easystack, args, logfile, testing, init_session_state, do_build) else: - do_cleanup = process_eb_args(orig_paths, eb_go, cfg_settings, modtool, testing, init_session_state, - hooks, do_build) + is_successful = process_eb_args(orig_paths, eb_go, cfg_settings, modtool, testing, init_session_state, + hooks, do_build) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) - if do_cleanup: + if is_successful: cleanup(logfile, eb_tmpdir, testing, silent=options.terse) + return EasyBuildExit.SUCCESS if is_successful else EasyBuildExit.ERROR def prepare_main(args=None, logfile=None, testing=None): @@ -823,7 +826,8 @@ def main_with_hooks(args=None): hooks = load_hooks(eb_go.options.hooks) try: - main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) + exit_code: EasyBuildExit = main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) + sys.exit(int(exit_code)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg, exit_on_error=True, exit_code=err.exit_code) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 1e962a339d..ecc9e812cf 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -271,7 +271,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger. if tmp_logdir and not os.path.exists(tmp_logdir): try: os.makedirs(tmp_logdir) - except (IOError, OSError) as err: + except OSError as err: raise EasyBuildError("Failed to create temporary log directory %s: %s", tmp_logdir, err) # mkstemp returns (fd,filename), fd is from os.open, not regular open! @@ -404,9 +404,8 @@ def print_error(msg, *args, **kwargs): if exitCode is not None: _init_easybuildlog.deprecated("'exitCode' option in print_error function is replaced with 'exit_code'", '6.0') - # use 1 as defaut exit code if exit_code is None: - exit_code = 1 + exit_code = EasyBuildExit.ERROR log = kwargs.pop('log', None) opt_parser = kwargs.pop('opt_parser', None) @@ -420,7 +419,7 @@ def print_error(msg, *args, **kwargs): if opt_parser: opt_parser.print_shorthelp() sys.stderr.write("ERROR: %s\n" % msg) - sys.exit(exit_code) + sys.exit(int(exit_code)) elif log is not None: raise EasyBuildError(msg) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 4bcebbe158..b27d238bd5 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -328,6 +328,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'ignore_test_failure', 'install_latest_eb_release', 'keep_debug_symbols', + 'keep_going', 'logtostdout', 'minimal_toolchains', 'module_only', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 2ef90dbf05..9ad3fff414 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -808,6 +808,8 @@ def github_options(self): 'close-pr-msg': ("Custom close message for pull request closed with --close-pr; ", str, 'store', None), 'close-pr-reasons': ("Close reason for pull request closed with --close-pr; " "supported values: %s" % ", ".join(VALID_CLOSE_PR_REASONS), str, 'store', None), + 'keep-going': ("Continue installation of remaining software after a failed installation. " + "Implied by --dump-test-report and --upload-test-report", None, 'store_true', False), 'list-prs': ("List pull requests", str, 'store_or_None', ",".join([DEFAULT_LIST_PR_STATE, DEFAULT_LIST_PR_ORDER, DEFAULT_LIST_PR_DIREC]), {'metavar': 'STATE,ORDER,DIRECTION'}), diff --git a/test/framework/options.py b/test/framework/options.py index 321062231b..4ab45a5dc2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3232,8 +3232,8 @@ def test_http_header_fields_urlpat(self): mentionhdr = 'Custom HTTP header field set: %s' mentionfile = 'File included in parse_http_header_fields_urlpat: %s' - def run_and_assert(args, msg, words_expected=None, words_unexpected=None): - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + def run_and_assert(args, _msg, words_expected=None, words_unexpected=None): + stdout, _stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) if words_expected is not None: self.assert_multi_regex(words_expected, stdout) if words_unexpected is not None: @@ -6435,7 +6435,7 @@ def test_force_download(self): '--force-download', '--sourcepath=%s' % self.test_prefix, ] - stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True) + _stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, verbose=True, strip=True) regex = re.compile(r"^WARNING: Found file toy-0.0.tar.gz at .*, but re-downloading it anyway\.\.\.$") self.assertTrue(regex.match(stderr), "Pattern '%s' matches: %s" % (regex.pattern, stderr)) @@ -6719,6 +6719,43 @@ def test_sanity_check_only(self): import easybuild.easyblocks.generic.toy_extension reload(easybuild.easyblocks.generic.toy_extension) + def test_keep_going(self): + """Test use of --keep-going.""" + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nsources=["toy-0.0.tar.gz"]' + write_file(test_ec, test_ec_txt + '\nversion="broken"\npreconfigopts = "false && "') + test_ec2 = os.path.join(self.test_prefix, 'test2.eb') + write_file(test_ec2, test_ec_txt + '\nversion="working"') + + args = [test_ec, test_ec2, '--rebuild'] + with self.mocked_stdout_stderr(): + outtxt, exit_code, error_thrown = self.eb_main(args, do_build=True, return_error=True, + return_exit_code=True) + self.assertIn("Installation of test.eb failed", str(error_thrown)) + self.assertNotEqual(exit_code, 0) + self.assertRegex(outtxt, r'\[FAILED\] *toy/broken') + self.assertRegex(outtxt, r'\[SKIPPED\] *toy/working') + + args.append('--keep-going') + with self.mocked_stdout_stderr(): + outtxt, exit_code = self.eb_main(args, do_build=True, raise_error=True, + return_exit_code=True) + self.assertNotEqual(exit_code, 0) + self.assertRegex(outtxt, r'\[FAILED\] *toy/broken') + self.assertRegex(outtxt, r'\[SUCCESS\] *toy/working') + + args.append(f"--dump-test-report={os.path.join(tempfile.gettempdir(), 'report.md')}") + with self.mocked_stdout_stderr(): + outtxt, exit_code = self.eb_main(args, do_build=True, raise_error=True, + return_exit_code=True) + self.assertEqual(exit_code, 1) # Return failure also when creating a test report + self.assertRegex(outtxt, r'\[FAILED\] *toy/broken') + self.assertRegex(outtxt, r'\[SUCCESS\] *toy/working') + def test_skip_extensions(self): """Test use of --skip-extensions.""" topdir = os.path.abspath(os.path.dirname(__file__)) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 97dbc4cd9c..1807085d75 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -305,8 +305,9 @@ def reset_modulepath(self, modpaths): self.modtool.add_module_path(modpath, set_mod_paths=False) self.modtool.set_mod_paths() - def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False, - reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True, clear_caches=True): + def eb_main(self, args, do_build=False, return_error=False, return_exit_code=False, logfile=None, verbose=False, + raise_error=False, reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True, + clear_caches=True): """Helper method to call EasyBuild main function.""" cleanup(clear_caches=clear_caches) @@ -325,6 +326,7 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos env_before = copy.deepcopy(os.environ) + exit_code = eb_build_log.EasyBuildExit.ERROR try: if '--fetch' in args: # The config sets modules_tool to None if --fetch is specified, @@ -332,7 +334,7 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos modtool = None else: modtool = self.modtool - main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) + exit_code = main(args=main_args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) except SystemExit as err: if raise_systemexit: raise err @@ -340,6 +342,8 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos myerr = err if verbose: print("err: %s" % err) + if isinstance(err, eb_build_log.EasyBuildError): + exit_code = err.exit_code if logfile and os.path.exists(logfile): logtxt = read_file(logfile) @@ -362,9 +366,12 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos raise myerr if return_error: + if return_exit_code: + return logtxt, exit_code, myerr return logtxt, myerr - else: - return logtxt + if return_exit_code: + return logtxt, exit_code + return logtxt def setup_hierarchical_modules(self): """Setup hierarchical modules to run tests on."""