diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a5d21ba80c..88da255d1d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,7 +70,7 @@ from easybuild.framework.extension import Extension, resolve_exts_filter_template from easybuild.tools import LooseVersion, config from easybuild.tools.build_details import get_build_stats -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES @@ -678,7 +678,10 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % src_fn) else: - raise EasyBuildError('Checksum verification for extension source %s failed', src_fn) + raise EasyBuildError( + 'Checksum verification for extension source %s failed', src_fn, + exit_code=EasyBuildExit.FAIL_CHECKSUM + ) # locate extension patches (if any), and verify checksums ext_patches = ext_options.get('patches', []) @@ -713,8 +716,10 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % patch_fn) else: - raise EasyBuildError("Checksum verification for extension patch %s failed", - patch_fn) + raise EasyBuildError( + "Checksum verification for extension patch %s failed", patch_fn, + exit_code=EasyBuildExit.FAIL_CHECKSUM + ) else: self.log.debug('No patches found for extension %s.' % ext_name) @@ -785,12 +790,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No return fullpath except IOError as err: + msg = f"Downloading file {filename} from url {url} to {fullpath} failed: {err}" if not warning_only: - raise EasyBuildError("Downloading file %s " - "from url %s to %s failed: %s", filename, url, fullpath, err) + raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_DOWNLOAD) else: - self.log.warning("Downloading file %s " - "from url %s to %s failed: %s", filename, url, fullpath, err) + self.log.warning(msg) return None else: @@ -863,118 +867,124 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if self.dry_run: self.dry_run_msg(" * %s found at %s", filename, foundfile) return foundfile - elif no_download: + + if no_download: if self.dry_run: self.dry_run_msg(" * %s (MISSING)", filename) return filename - else: - if not warning_only: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) - else: - self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... " - "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) - return None - elif git_config: - return get_source_tarball_from_git(filename, targetdir, git_config) - else: - # try and download source files from specified source URLs - if urls: - source_urls = urls[:] - else: - source_urls = [] - source_urls.extend(self.cfg['source_urls']) - # Add additional URLs as configured. - for url in build_option("extra_source_urls"): - url += "/" + name_letter + "/" + location - source_urls.append(url) + failedpaths_msg = "\n * ".join([""]+failedpaths) + file_notfound_msg = ( + f"Couldn't find file '{filename}' anywhere, and downloading it is disabled... " + f"Paths attempted (in order): {failedpaths_msg}" + ) - mkdir(targetdir, parents=True) + if warning_only: + self.log.warning(file_notfound_msg) + return None - for url in source_urls: + raise EasyBuildError(file_notfound_msg, exit_code=EasyBuildExit.MISSING_SOURCES) - if extension: - targetpath = os.path.join(targetdir, "extensions", filename) - else: - targetpath = os.path.join(targetdir, filename) + if git_config: + return get_source_tarball_from_git(filename, targetdir, git_config) - url_filename = download_filename or filename + # try and download source files from specified source URLs + if urls: + source_urls = urls[:] + else: + source_urls = [] + source_urls.extend(self.cfg['source_urls']) - if isinstance(url, str): - if url[-1] in ['=', '/']: - fullurl = "%s%s" % (url, url_filename) - else: - fullurl = "%s/%s" % (url, url_filename) - elif isinstance(url, tuple): - # URLs that require a suffix, e.g., SourceForge download links - # e.g. http://sourceforge.net/projects/math-atlas/files/Stable/3.8.4/atlas3.8.4.tar.bz2/download - fullurl = "%s/%s/%s" % (url[0], url_filename, url[1]) - else: - self.log.warning("Source URL %s is of unknown type, so ignoring it." % url) - continue + # Add additional URLs as configured. + for url in build_option("extra_source_urls"): + url += "/" + name_letter + "/" + location + source_urls.append(url) - # PyPI URLs may need to be converted due to change in format of these URLs, - # cfr. https://bitbucket.org/pypa/pypi/issues/438 - if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl): - alt_url = derive_alt_pypi_url(fullurl) - if alt_url: - _log.debug("Using alternative PyPI URL for %s: %s", fullurl, alt_url) - fullurl = alt_url - else: - _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original", - fullurl) + mkdir(targetdir, parents=True) - if self.dry_run: - self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath) - if extension and urls: - # extensions typically have custom source URLs specified, only mention first - self.dry_run_msg(" (from %s, ...)", fullurl) - downloaded = True + for url in source_urls: - else: - self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath)) - downloaded = False - try: - if download_file(filename, fullurl, targetpath): - downloaded = True + if extension: + targetpath = os.path.join(targetdir, "extensions", filename) + else: + targetpath = os.path.join(targetdir, filename) - except IOError as err: - self.log.debug("Failed to download %s from %s: %s" % (filename, url, err)) - failedpaths.append(fullurl) - continue + url_filename = download_filename or filename - if downloaded: - # if fetching from source URL worked, we're done - self.log.info("Successfully downloaded source file %s from %s" % (filename, fullurl)) - return targetpath + if isinstance(url, str): + if url[-1] in ['=', '/']: + fullurl = "%s%s" % (url, url_filename) else: - failedpaths.append(fullurl) + fullurl = "%s/%s" % (url, url_filename) + elif isinstance(url, tuple): + # URLs that require a suffix, e.g., SourceForge download links + # e.g. http://sourceforge.net/projects/math-atlas/files/Stable/3.8.4/atlas3.8.4.tar.bz2/download + fullurl = "%s/%s/%s" % (url[0], url_filename, url[1]) + else: + self.log.warning("Source URL %s is of unknown type, so ignoring it." % url) + continue + + # PyPI URLs may need to be converted due to change in format of these URLs, + # cfr. https://bitbucket.org/pypa/pypi/issues/438 + if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl): + alt_url = derive_alt_pypi_url(fullurl) + if alt_url: + _log.debug("Using alternative PyPI URL for %s: %s", fullurl, alt_url) + fullurl = alt_url + else: + _log.debug("Failed to derive alternative PyPI URL for %s, so retaining the original", + fullurl) if self.dry_run: - self.dry_run_msg(" * %s (MISSING)", filename) - return filename + self.dry_run_msg(" * %s will be downloaded to %s", filename, targetpath) + if extension and urls: + # extensions typically have custom source URLs specified, only mention first + self.dry_run_msg(" (from %s, ...)", fullurl) + downloaded = True + else: - error_msg = "Couldn't find file %s anywhere, " - if download_instructions is None: - download_instructions = self.cfg['download_instructions'] - if download_instructions is not None and download_instructions != "": - msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n' - msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths()) - print_msg(msg, prefix=False, stderr=True) - error_msg += "please follow the download instructions above, and make the file available " - error_msg += "in the active source path (%s)" % ':'.join(source_paths()) - else: - # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf') - failedpaths_msg = ', '.join(failedpaths).replace('%', '%%') - error_msg += "and downloading it didn't work either... " - error_msg += "Paths attempted (in order): %s " % failedpaths_msg + self.log.debug("Trying to download file %s from %s to %s ..." % (filename, fullurl, targetpath)) + downloaded = False + try: + if download_file(filename, fullurl, targetpath): + downloaded = True - if not warning_only: - raise EasyBuildError(error_msg, filename) - else: - self.log.warning(error_msg, filename) - return None + except IOError as err: + self.log.debug("Failed to download %s from %s: %s" % (filename, url, err)) + failedpaths.append(fullurl) + continue + + if downloaded: + # if fetching from source URL worked, we're done + self.log.info("Successfully downloaded source file %s from %s" % (filename, fullurl)) + return targetpath + else: + failedpaths.append(fullurl) + + if self.dry_run: + self.dry_run_msg(" * %s (MISSING)", filename) + return filename + else: + error_msg = "Couldn't find file %s anywhere, " + if download_instructions is None: + download_instructions = self.cfg['download_instructions'] + if download_instructions is not None and download_instructions != "": + msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n' + msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths()) + print_msg(msg, prefix=False, stderr=True) + error_msg += "please follow the download instructions above, and make the file available " + error_msg += "in the active source path (%s)" % ':'.join(source_paths()) + else: + # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf') + failedpaths_msg = ', '.join(failedpaths).replace('%', '%%') + error_msg += "and downloading it didn't work either... " + error_msg += "Paths attempted (in order): %s " % failedpaths_msg + + if not warning_only: + raise EasyBuildError(error_msg, filename, exit_code=EasyBuildExit.FAIL_DOWNLOAD) + else: + self.log.warning(error_msg, filename) + return None # # GETTER/SETTER UTILITY FUNCTIONS @@ -1805,7 +1815,7 @@ def skip_extensions_sequential(self, exts_filter): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) res = run_shell_cmd(cmd, stdin=stdin, fail_on_error=False, hidden=True) self.log.info(f"exts_filter result for {ext_inst.name}: exit code {res.exit_code}; output: {res.output}") - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: print_msg(f"skipping extension {ext_inst.name}", silent=self.silent, log=self.log) else: self.log.info(f"Not skipping {ext_inst.name}") @@ -1841,7 +1851,7 @@ def skip_extensions_parallel(self, exts_filter): idx = res.task_id ext_name = self.ext_instances[idx].name self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: print_msg(f"skipping extension {ext_name}", log=self.log) installed_exts_ids.append(idx) @@ -1900,7 +1910,6 @@ def install_extensions_sequential(self, install=True): exts_cnt = len(self.ext_instances) for idx, ext in enumerate(self.ext_instances): - self.log.info("Starting extension %s", ext.name) run_hook(SINGLE_EXTENSION, self.hooks, pre_step_hook=True, args=[ext]) @@ -2003,7 +2012,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_task.done(): res = ext.async_cmd_task.result() - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: self.log.info(f"Installation of extension {ext.name} completed!") # run post-install method for extension from same working dir as installation of extension cwd = change_dir(res.work_dir) @@ -2472,7 +2481,10 @@ def checksum_step(self): elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % fil['name']) else: - raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum']) + raise EasyBuildError( + "Checksum verification for %s using %s failed.", fil['path'], fil['checksum'], + exit_code=EasyBuildExit.FAIL_CHECKSUM + ) def check_checksums_for(self, ent, sub='', source_cnt=None): """ @@ -2794,7 +2806,6 @@ def init_ext_instances(self): exts_cnt = len(self.exts) self.update_exts_progress_bar("creating internal datastructures for extensions") - for idx, ext in enumerate(self.exts): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) @@ -3187,7 +3198,7 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): if check_readelf_rpath: fail_msg = None res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False, hidden=True) - if res.exit_code: + if res.exit_code != EasyBuildExit.SUCCESS: fail_msg = f"Failed to run 'readelf -d {path}': {res.output}" elif not readelf_rpath_regex.search(res.output): fail_msg = f"No '(RPATH)' found in 'readelf -d' output for {path}: {out}" @@ -3622,6 +3633,7 @@ def xs2str(xs): if not found: sanity_check_fail_msg = "no %s found at %s in %s" % (typ, xs2str(xs), self.installdir) self.sanity_check_fail_msgs.append(sanity_check_fail_msg) + self.exit_code = EasyBuildExit.FAIL_SANITY_CHECK self.log.warning("Sanity check: %s", sanity_check_fail_msg) trace_msg("%s %s found: %s" % (typ, xs2str(xs), ('FAILED', 'OK')[found])) @@ -3644,14 +3656,15 @@ def xs2str(xs): trace_msg(f"running command '{cmd}' ...") res = run_shell_cmd(cmd, fail_on_error=False, hidden=True) - if res.exit_code != 0: - fail_msg = f"sanity check command {cmd} exited with code {res.exit_code} (output: {res.output})" + if res.exit_code != EasyBuildExit.SUCCESS: + fail_msg = f"sanity check command {cmd} failed with exit code {res.exit_code} (output: {res.output})" self.sanity_check_fail_msgs.append(fail_msg) + self.exit_code = EasyBuildExit.FAIL_SANITY_CHECK self.log.warning(f"Sanity check: {fail_msg}") else: self.log.info(f"sanity check command {cmd} ran successfully! (output: {res.output})") - cmd_result_str = ('FAILED', 'OK')[res.exit_code == 0] + cmd_result_str = ('FAILED', 'OK')[res.exit_code == EasyBuildExit.SUCCESS] trace_msg(f"result for command '{cmd}': {cmd_result_str}") # also run sanity check for extensions (unless we are an extension ourselves) @@ -3692,7 +3705,10 @@ def xs2str(xs): # pass or fail if self.sanity_check_fail_msgs: - raise EasyBuildError("Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs)) + raise EasyBuildError( + "Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs), + exit_code=EasyBuildExit.FAIL_SANITY_CHECK, + ) else: self.log.debug("Sanity check passed!") @@ -3794,8 +3810,13 @@ def make_module_step(self, fake=False): for line in txt.split('\n'): self.dry_run_msg(INDENT_4SPACES + line) else: - write_file(mod_filepath, txt) - self.log.info("Module file %s written: %s", mod_filepath, txt) + try: + write_file(mod_filepath, txt) + self.log.info("Module file %s written: %s", mod_filepath, txt) + except EasyBuildError: + raise EasyBuildError( + f"Unable to write Module file {mod_filepath}", exit_code=EasyBuildExit.FAIL_MODULE_WRITE + ) # if backup module file is there, print diff with newly generated module file if self.mod_file_backup and not fake: @@ -4172,9 +4193,15 @@ def run_all_steps(self, run_test_cases): self.run_step(step_name, step_methods) except RunShellCmdError as err: err.print() - ec_path = os.path.basename(self.cfg.path) - error_msg = f"shell command '{err.cmd_name} ...' failed in {step_name} step for {ec_path}" - raise EasyBuildError(error_msg) + error_msg = ( + f"shell command '{err.cmd_name} ...' failed with exit code {err.exit_code} " + f"in {step_name} step for {os.path.basename(self.cfg.path)}" + ) + try: + step_exit_code = EasyBuildExit[f"FAIL_{step_name.upper()}_STEP"] + except KeyError: + step_exit_code = EasyBuildExit.ERROR + raise EasyBuildError(error_msg, exit_code=step_exit_code) from err finally: if not self.dry_run: step_duration = datetime.now() - start_time @@ -4276,6 +4303,7 @@ def build_and_install_one(ecdict, init_env): # build easyconfig error_msg = '(no error)' + exit_code = None # timing info start_time = time.time() try: @@ -4314,6 +4342,10 @@ def build_and_install_one(ecdict, init_env): except EasyBuildError as err: error_msg = err.msg + try: + exit_code = err.exit_code + except AttributeError: + exit_code = EasyBuildExit.ERROR result = False ended = 'ended' @@ -4431,11 +4463,13 @@ def ensure_writable_log_dir(log_dir): success = True summary = 'COMPLETED' succ = 'successfully' + exit_code = EasyBuildExit.SUCCESS else: # build failed success = False summary = 'FAILED' succ = "unsuccessfully: " + error_msg + exit_code = EasyBuildExit.ERROR if exit_code is None else exit_code # cleanup logs app.close_log() @@ -4463,7 +4497,7 @@ def ensure_writable_log_dir(log_dir): del app - return (success, application_log, error_msg) + return (success, application_log, error_msg, exit_code) def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 28fa4c6c84..0d54443517 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -65,7 +65,7 @@ from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme @@ -907,9 +907,12 @@ def validate_os_deps(self): not_found.append(dep) if not_found: - raise EasyBuildError("One or more OS dependencies were not found: %s", not_found) - else: - self.log.info("OS dependencies ok: %s" % self['osdependencies']) + raise EasyBuildError( + "One or more OS dependencies were not found: %s", not_found, + exit_code=EasyBuildExit.MISSING_SYSTEM_DEPENDENCY + ) + + self.log.info("OS dependencies ok: %s" % self['osdependencies']) return True @@ -1272,7 +1275,10 @@ def _validate(self, attr, values): # private method if values is None: values = [] if self[attr] and self[attr] not in values: - raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) + raise EasyBuildError( + "%s provided '%s' is not valid: %s", attr, self[attr], values, + exit_code=EasyBuildExit.VALUE_ERROR + ) def probe_external_module_metadata(self, mod_name, existing_metadata=None): """ @@ -1922,12 +1928,20 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error error_re = re.compile(r"No module named '?.*/?%s'?" % modname) _log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern) if error_re.match(str(err)): + # Missing easyblock type of error if error_on_missing_easyblock: - raise EasyBuildError("No software-specific easyblock '%s' found for %s", class_name, name) - elif error_on_failed_import: - raise EasyBuildError("Failed to import %s easyblock: %s", class_name, err) + raise EasyBuildError( + "No software-specific easyblock '%s' found for %s", class_name, name, + exit_code=EasyBuildExit.MISSING_EASYBLOCK + ) from err else: - _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err)) + # Broken import + if error_on_failed_import: + raise EasyBuildError( + "Failed to import %s easyblock: %s", class_name, err, + exit_code=EasyBuildExit.EASYBLOCK_ERROR + ) from err + _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err)) if cls is not None: _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')", @@ -1941,7 +1955,10 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error # simply reraise rather than wrapping it into another error raise err except Exception as err: - raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err) + raise EasyBuildError( + "Failed to obtain class for %s easyblock (not available?): %s", easyblock, err, + exit_code=EasyBuildExit.EASYBLOCK_ERROR + ) def get_module_path(name, generic=None, decode=True): @@ -2086,7 +2103,11 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, try: ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden) except EasyBuildError as err: - raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg) + try: + exit_code = err.exit_code + except AttributeError: + exit_code = EasyBuildExit.EASYCONFIG_ERROR + raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code) name = ec['name'] diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 8be2362e09..9c832bd196 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files @@ -409,7 +409,7 @@ def parse_easyconfigs(paths, validate=True): # keep track of whether any files were generated generated_ecs |= generated if not os.path.exists(path): - raise EasyBuildError("Can't find path %s", path) + raise EasyBuildError("Can't find path %s", path, exit_code=EasyBuildExit.MISSING_EASYCONFIG) try: ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) for ec_file in ec_files: diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 90b9abaac2..0005f24428 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.easyconfig import resolve_template from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict -from easybuild.tools.build_log import EasyBuildError, raise_nosupport +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, raise_nosupport from easybuild.tools.filetools import change_dir from easybuild.tools.run import run_shell_cmd @@ -302,7 +302,7 @@ def sanity_check_step(self): cmd, stdin = resolve_exts_filter_template(exts_filter, self) cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) - if cmd_res.exit_code: + if cmd_res.exit_code != EasyBuildExit.SUCCESS: if stdin: fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin) else: diff --git a/easybuild/main.py b/easybuild/main.py index f07119955a..83836d22c3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -133,10 +133,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) + (ec_res['success'], app_log, err_msg, err_code) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log if not ec_res['success']: - ec_res['err'] = EasyBuildError(err) + ec_res['err'] = EasyBuildError(err_msg, exit_code=err_code) except Exception as err: # purposely catch all exceptions ec_res['success'] = False @@ -174,7 +174,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): if not isinstance(ec_res['err'], EasyBuildError): raise ec_res['err'] else: - raise EasyBuildError(test_msg) + raise EasyBuildError(test_msg, exit_code=err_code) res.append((ec, ec_res)) @@ -779,7 +779,7 @@ def main_with_hooks(args=None): try: init_session_state, eb_go, cfg_settings = prepare_main(args=args) except EasyBuildError as err: - print_error(err.msg) + print_error(err.msg, exit_code=err.exit_code) hooks = load_hooks(eb_go.options.hooks) @@ -787,7 +787,7 @@ def main_with_hooks(args=None): main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) - print_error(err.msg, exit_on_error=True, exit_code=1) + print_error(err.msg, exit_on_error=True, exit_code=err.exit_code) except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 4880415151..fac06ecbbc 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -40,12 +40,12 @@ import tempfile from copy import copy from datetime import datetime +from enum import IntEnum from easybuild.base import fancylogger from easybuild.base.exceptions import LoggedException from easybuild.tools.version import VERSION, this_is_easybuild - # EasyBuild message prefix EB_MSG_PREFIX = "==" @@ -71,6 +71,55 @@ logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL') +class EasyBuildExit(IntEnum): + """ + Table of exit codes + """ + SUCCESS = 0 + ERROR = 1 + # core errors + OPTION_ERROR = 2 + VALUE_ERROR = 3 + MISSING_EASYCONFIG = 4 + EASYCONFIG_ERROR = 5 + MISSING_EASYBLOCK = 6 + EASYBLOCK_ERROR = 7 + MODULE_ERROR = 8 + # step errors in order of execution + FAIL_FETCH_STEP = 10 + FAIL_READY_STEP = 11 + FAIL_SOURCE_STEP = 12 + FAIL_PATCH_STEP = 13 + FAIL_PREPARE_STEP = 14 + FAIL_CONFIGURE_STEP = 15 + FAIL_BUILD_STEP = 16 + FAIL_TEST_STEP = 17 + FAIL_INSTALL_STEP = 18 + FAIL_EXTENSIONS_STEP = 19 + FAIL_POST_ITER_STEP = 20 + FAIL_POST_PROC_STEP = 21 + FAIL_SANITY_CHECK_STEP = 22 + FAIL_CLEANUP_STEP = 23 + FAIL_MODULE_STEP = 24 + FAIL_PERMISSIONS_STEP = 25 + FAIL_PACKAGE_STEP = 26 + FAIL_TEST_CASES_STEP = 27 + # errors on missing things + MISSING_SOURCES = 30 + MISSING_DEPENDENCY = 31 + MISSING_SYSTEM_DEPENDENCY = 32 + MISSING_EB_DEPENDENCY = 33 + # errors on specific task failures + FAIL_SYSTEM_CHECK = 40 + FAIL_DOWNLOAD = 41 + FAIL_CHECKSUM = 42 + FAIL_EXTRACT = 43 + FAIL_PATCH_APPLY = 44 + FAIL_SANITY_CHECK = 45 + FAIL_MODULE_WRITE = 46 + FAIL_GITHUB = 47 + + class EasyBuildError(LoggedException): """ EasyBuildError is thrown when EasyBuild runs into something horribly wrong. @@ -80,12 +129,13 @@ class EasyBuildError(LoggedException): # always include location where error was raised from, even under 'python -O' INCLUDE_LOCATION = True - def __init__(self, msg, *args): + def __init__(self, msg, *args, exit_code=EasyBuildExit.ERROR, **kwargs): """Constructor: initialise EasyBuildError instance.""" if args: msg = msg % args - LoggedException.__init__(self, msg) + LoggedException.__init__(self, msg, exit_code=exit_code, **kwargs) self.msg = msg + self.exit_code = exit_code def __str__(self): """Return string representation of this EasyBuildError instance.""" diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 80f5b5bc75..13f9722bce 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -50,7 +50,7 @@ from easybuild.base import fancylogger from easybuild.base.frozendict import FrozenDictKnownKeys from easybuild.base.wrapper import create_base_metaclass -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit try: import rich # noqa @@ -506,7 +506,10 @@ def get_items_check_required(self): """ missing = [x for x in self.KNOWN_KEYS if x not in self] if len(missing) > 0: - raise EasyBuildError("Cannot determine value for configuration variables %s. Please specify it.", missing) + raise EasyBuildError( + "Cannot determine value for configuration variables %s. Please specify it.", ', '.join(missing), + exit_code=EasyBuildExit.OPTION_ERROR + ) return self.items() @@ -539,7 +542,10 @@ def init(options, config_options_dict): tmpdict['sourcepath'] = sourcepath.split(':') _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath'])) elif not isinstance(sourcepath, (tuple, list)): - raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath) + raise EasyBuildError( + "Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath, + exit_code=EasyBuildExit.OPTION_ERROR + ) # initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True) @@ -623,7 +629,7 @@ def build_option(key, **kwargs): error_msg = "Undefined build option: '%s'. " % key error_msg += "Make sure you have set up the EasyBuild configuration using set_up_configuration() " error_msg += "(from easybuild.tools.options) in case you're not using EasyBuild via the 'eb' CLI." - raise EasyBuildError(error_msg) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR) def update_build_option(key, value): @@ -688,7 +694,10 @@ def install_path(typ=None): known_types = ['modules', 'software'] if typ not in known_types: - raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types)) + raise EasyBuildError( + "Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types), + exit_code=EasyBuildExit.OPTION_ERROR + ) variables = ConfigurationVariables() @@ -780,7 +789,10 @@ def get_output_style(): output_style = OUTPUT_STYLE_BASIC if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH: - raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH) + raise EasyBuildError( + "Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH, + exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY + ) return output_style @@ -805,8 +817,10 @@ def log_file_format(return_directory=False, ec=None, date=None, timestamp=None): logfile_format = ConfigurationVariables()['logfile_format'] if not isinstance(logfile_format, tuple) or len(logfile_format) != 2: - raise EasyBuildError("Incorrect log file format specification, should be 2-tuple (, ): %s", - logfile_format) + raise EasyBuildError( + "Incorrect log file format specification, should be 2-tuple (, ): %s", logfile_format, + exit_code=EasyBuildExit.OPTION_ERROR + ) idx = int(not return_directory) res = ConfigurationVariables()['logfile_format'][idx] % { @@ -913,7 +927,10 @@ def find_last_log(curlog): sorted_paths = [p for (_, p) in sorted(paths)] except OSError as err: - raise EasyBuildError("Failed to locate/select/order log files matching '%s': %s", glob_pattern, err) + raise EasyBuildError( + "Failed to locate/select/order log files matching '%s': %s", glob_pattern, err, + exit_code=EasyBuildExit.OPTION_ERROR + ) try: # log of current session is typically listed last, should be taken into account diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 690cdd52a2..c8c5c059f9 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -29,7 +29,7 @@ import os import re -from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg from easybuild.tools.containers.singularity import SingularityContainer from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS @@ -49,7 +49,7 @@ def apptainer_version(): """Get Apptainer version.""" version_cmd = "apptainer --version" res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) - if res.exit_code: + if res.exit_code != EasyBuildExit.SUCCESS: raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index be8573f8c9..19151a9c78 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -34,7 +34,7 @@ import re from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS from easybuild.tools.config import build_option, container_path @@ -163,7 +163,7 @@ def singularity_version(): """Get Singularity version.""" version_cmd = "singularity --version" res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) - if res.exit_code: + if res.exit_code != EasyBuildExit.SUCCESS: raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index f2ae68d133..c36af49a8f 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -34,7 +34,7 @@ from functools import reduce from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg from easybuild.tools.filetools import which from easybuild.tools.run import run_shell_cmd @@ -77,7 +77,7 @@ def check_tool(tool_name, min_tool_version=None): version_cmd = f"{tool_name} --version" res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) - if res.exit_code: + if res.exit_code != EasyBuildExit.SUCCESS: raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}") regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index cb3649eb85..72d4f36b76 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -62,7 +62,8 @@ from easybuild.base import fancylogger # import build_log must stay, to use of EasyBuildLog -from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR +from easybuild.tools.build_log import dry_run_msg, print_msg, print_warning from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.hooks import load_source @@ -854,7 +855,10 @@ def download_file(filename, url, path, forced=False, trace=True): if error_re.match(str(err)): switch_to_requests = True except Exception as err: - raise EasyBuildError("Unexpected error occurred when trying to download %s to %s: %s", url, path, err) + raise EasyBuildError( + "Unexpected error occurred when trying to download %s to %s: %s", url, path, err, + exit_code=EasyBuildExit.FAIL_DOWNLOAD + ) if not downloaded and attempt_cnt < max_attempts: _log.info("Attempt %d of downloading %s to %s failed, trying again..." % (attempt_cnt, url, path)) @@ -1069,7 +1073,10 @@ def locate_files(files, paths, ignore_subdirs=None): if files_to_find: filenames = ', '.join([f for (_, f) in files_to_find]) paths = ', '.join(paths) - raise EasyBuildError("One or more files not found: %s (search paths: %s)", filenames, paths) + raise EasyBuildError( + "One or more files not found: %s (search paths: %s)", filenames, paths, + exit_code=EasyBuildExit.MISSING_EASYCONFIG + ) return [os.path.abspath(f) for f in files] @@ -1524,8 +1531,10 @@ def create_patch_info(patch_spec): else: patch_info['copy'] = patch_arg else: - raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", - str(patch_spec)) + raise EasyBuildError( + "Wrong patch spec '%s', only int/string are supported as 2nd element", str(patch_spec), + exit_code=EasyBuildExit.EASYCONFIG_ERROR + ) elif isinstance(patch_spec, str): validate_patch_spec(patch_spec) @@ -1536,13 +1545,17 @@ def create_patch_info(patch_spec): if key in valid_keys: patch_info[key] = patch_spec[key] else: - raise EasyBuildError("Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)", - str(patch_spec), key, valid_keys) + raise EasyBuildError( + "Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)", + str(patch_spec), key, valid_keys, exit_code=EasyBuildExit.EASYCONFIG_ERROR + ) # Dict must contain at least the patchfile name if 'name' not in patch_info.keys(): - raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied", - str(patch_spec)) + raise EasyBuildError( + "Wrong patch spec '%s', when using a dict 'name' entry must be supplied", str(patch_spec), + exit_code=EasyBuildExit.EASYCONFIG_ERROR + ) if 'copy' not in patch_info.keys(): validate_patch_spec(patch_info['name']) else: @@ -1551,9 +1564,11 @@ def create_patch_info(patch_spec): "this implies you want to copy a file to the 'copy' location)", str(patch_spec)) else: - error_msg = "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict " \ - "(with possible keys %s): %s" % (valid_keys, patch_spec) - raise EasyBuildError(error_msg) + error_msg = ( + "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict " + f"(with possible keys {valid_keys}): {patch_spec}" + ) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.EASYCONFIG_ERROR) return patch_info @@ -1561,8 +1576,10 @@ def create_patch_info(patch_spec): def validate_patch_spec(patch_spec): allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS] if not any(patch_spec.endswith(x) for x in allowed_patch_exts): - raise EasyBuildError("Wrong patch spec (%s), extension type should be any of %s." % - (patch_spec, ', '.join(allowed_patch_exts))) + raise EasyBuildError( + "Wrong patch spec (%s), extension type should be any of %s.", patch_spec, ', '.join(allowed_patch_exts), + exit_code=EasyBuildExit.EASYCONFIG_ERROR + ) def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False): @@ -1653,7 +1670,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False if res.exit_code: msg = f"Couldn't apply patch file {patch_file}. " msg += f"Process exited with code {res.exit_code}: {res.output}" - raise EasyBuildError(msg) + raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_PATCH_APPLY) return True @@ -2688,7 +2705,7 @@ def get_source_tarball_from_git(filename, target_dir, git_config): work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir res = run_shell_cmd(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True) - if res.exit_code != 0 or tag not in res.output.splitlines(): + if res.exit_code != EasyBuildExit.SUCCESS or tag not in res.output.splitlines(): msg = f"Tag {tag} was not downloaded in the first try due to {url}/{repo_name} containing a branch" msg += f" with the same name. You might want to alert the maintainers of {repo_name} about that issue." print_warning(msg) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 13acf16cba..f39a82d203 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -54,7 +54,7 @@ from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files from easybuild.tools.filetools import det_patched_files, download_file, extract_file @@ -208,7 +208,7 @@ def listdir(self, path): return listing[1] else: self.log.warning("error: %s" % str(listing)) - raise EasyBuildError("Invalid response from github (I/O error)") + raise EasyBuildError("Invalid response from github (I/O error)", exit_code=EasyBuildExit.FAIL_GITHUB) def walk(self, top=None, topdown=True): """ @@ -315,9 +315,12 @@ def github_api_put_request(request_f, github_user=None, token=None, **kwargs): if status == 200: _log.info("Put request successful: %s", data['message']) elif status in [405, 409]: - raise EasyBuildError("FAILED: %s", data['message']) + raise EasyBuildError("FAILED: %s", data['message'], exit_code=EasyBuildExit.FAIL_GITHUB) else: - raise EasyBuildError("FAILED: %s", data.get('message', "(unknown reason)")) + raise EasyBuildError( + "FAILED: %s", data.get('message', "(unknown reason)"), + exit_code=EasyBuildExit.FAIL_GITHUB + ) _log.debug("get request result for %s: status: %d, data: %s", url.url, status, data) return (status, data) @@ -339,20 +342,23 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= status, data = github_api_get_request(lambda x: x.repos[account][repo].branches, github_user=github_user, token=token, per_page=GITHUB_MAX_PER_PAGE) if status != HTTP_STATUS_OK: - raise EasyBuildError("Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)", - branch, account, repo, status, data) + raise EasyBuildError( + "Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)", + branch, account, repo, status, data, exit_code=EasyBuildExit.FAIL_GITHUB + ) res = None for entry in data: - if entry[u'name'] == branch: + if entry['name'] == branch: res = entry['commit']['sha'] break if res is None: - error_msg = "No branch with name %s found in repo %s/%s" % (branch, account, repo) + error_msg = f"No branch with name {branch} found in repo {account}/{repo}" if len(data) >= GITHUB_MAX_PER_PAGE: - error_msg += "; only %d branches were checked (too many branches in %s/%s?)" % (len(data), account, repo) - raise EasyBuildError(error_msg + ': ' + ', '.join([x[u'name'] for x in data])) + error_msg += f"; only {len(data)} branches were checked (too many branches in {account}/{repo}?)" + error_msg += ": " + ", ".join([x['name'] for x in data]) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_GITHUB) return res @@ -387,7 +393,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun else: error_msg = r"Specified commit SHA %s for downloading %s/%s is not valid, " error_msg += "must be full SHA-1 (40 chars)" - raise EasyBuildError(error_msg, commit, account, repo) + raise EasyBuildError(error_msg, commit, account, repo, exit_code=EasyBuildExit.VALUE_ERROR) extracted_dir_name = '%s-%s' % (repo, commit) base_name = '%s.tar.gz' % commit @@ -397,7 +403,9 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun base_name = '%s.tar.gz' % branch latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user) else: - raise EasyBuildError("Either branch or commit should be specified in download_repo") + raise EasyBuildError( + "Either branch or commit should be specified in download_repo", exit_code=EasyBuildExit.VALUE_ERROR + ) expected_path = os.path.join(path, extracted_dir_name) latest_sha_path = os.path.join(expected_path, 'latest-sha') @@ -415,7 +423,10 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) downloaded_path = download_file(base_name, url, target_path, forced=True, trace=False) if downloaded_path is None: - raise EasyBuildError("Failed to download tarball for %s/%s commit %s", account, repo, commit) + raise EasyBuildError( + "Failed to download tarball for %s/%s commit %s", account, repo, commit, + exit_code=EasyBuildExit.FAIL_DOWNLOAD + ) else: _log.debug("%s downloaded to %s, extracting now", base_name, path) @@ -429,7 +440,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun error_msg += "at branch " + branch elif commit: error_msg += "at commit " + commit - raise EasyBuildError(error_msg) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_EXTRACT) write_file(latest_sha_path, latest_commit_sha, forced=True) @@ -491,13 +502,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", - pr, extra_ec_paths) + raise EasyBuildError( + "Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths, + exit_code=EasyBuildExit.FAIL_GITHUB + ) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) else: - raise EasyBuildError("Unknown repo: %s" % github_repo) + raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR) if path is None: path = tempfile.mkdtemp() @@ -513,7 +526,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi elif github_repo == GITHUB_EASYBLOCKS_REPO: easyfiles = 'easyblocks' else: - raise EasyBuildError("Don't know how to fetch files from repo %s", github_repo) + raise EasyBuildError( + "Don't know how to fetch files from repo %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR + ) subdir = os.path.join('easybuild', easyfiles) @@ -587,7 +602,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if os.path.exists(full_path): files.append(full_path) else: - raise EasyBuildError("Couldn't find path to patched file %s", full_path) + raise EasyBuildError( + "Couldn't find path to patched file %s", full_path, exit_code=EasyBuildExit.OPTION_ERROR + ) if github_repo == GITHUB_EASYCONFIGS_REPO: ver_file = os.path.join(final_path, 'setup.py') @@ -666,15 +683,17 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", - commit, extra_ec_paths) + raise EasyBuildError( + "Failed to isolate path for commit %s from list of commit paths: %s", + commit, extra_ec_paths, exit_code=EasyBuildExit.FAIL_GITHUB + ) else: path = os.path.join(tempfile.gettempdir(), 'ecs_commit_' + commit) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) else: - raise EasyBuildError("Unknown repo: %s" % github_repo) + raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR) # if no files are specified, determine which files are touched in commit if not files: @@ -688,7 +707,10 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) _log.debug("List of patched files for commit %s: %s", commit, files) else: - raise EasyBuildError("Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo) + raise EasyBuildError( + "Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo, + exit_code=EasyBuildExit.FAIL_GITHUB + ) # download tarball for specific commit repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account) @@ -698,7 +720,7 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, elif github_repo == GITHUB_EASYBLOCKS_REPO: files_subdir = 'easybuild/easyblocks/' else: - raise EasyBuildError("Unknown repo: %s" % github_repo) + raise EasyBuildError("Unknown repo: %s", github_repo, exit_code=EasyBuildExit.OPTION_ERROR) # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path mkdir(path, parents=True) @@ -777,7 +799,9 @@ def create_gist(txt, fn, descr=None, github_user=None, github_token=None): status, data = g.gists.post(body=body) if status != HTTP_STATUS_CREATED: - raise EasyBuildError("Failed to create gist; status %s, data: %s", status, data) + raise EasyBuildError( + "Failed to create gist; status %s, data: %s", status, data, exit_code=EasyBuildExit.FAIL_GITHUB + ) return data['html_url'] @@ -792,7 +816,9 @@ def delete_gist(gist_id, github_user=None, github_token=None): status, data = gh.gists[gist_id].delete() if status != HTTP_STATUS_NO_CONTENT: - raise EasyBuildError("Failed to delete gist with ID %s: status %s, data: %s", status, data) + raise EasyBuildError( + "Failed to delete gist with ID %s: status %s, data: %s", status, data, exit_code=EasyBuildExit.FAIL_GITHUB + ) def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCONFIGS_REPO, github_user=None): @@ -801,7 +827,10 @@ def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCO try: issue = int(issue) except ValueError as err: - raise EasyBuildError("Failed to parse specified pull request number '%s' as an int: %s; ", issue, err) + raise EasyBuildError( + "Failed to parse specified pull request number '%s' as an int: %s; ", issue, err, + exit_code=EasyBuildExit.OPTION_ERROR + ) dry_run = build_option('dry_run') or build_option('extended_dry_run') @@ -818,7 +847,10 @@ def post_comment_in_issue(issue, txt, account=GITHUB_EB_MAIN, repo=GITHUB_EASYCO status, data = pr_url.comments.post(body={'body': txt}) if not status == HTTP_STATUS_CREATED: - raise EasyBuildError("Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data) + raise EasyBuildError( + "Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data, + exit_code=EasyBuildExit.FAIL_GITHUB + ) def init_repo(path, repo_name, silent=False): @@ -844,13 +876,15 @@ def init_repo(path, repo_name, silent=False): workrepo = git.Repo(workdir) workrepo.clone(repo_path) except GitCommandError as err: - raise EasyBuildError("Failed to clone git repo at %s: %s", workdir, err) + raise EasyBuildError( + "Failed to clone git repo at %s: %s", workdir, err, exit_code=EasyBuildExit.FAIL_GITHUB + ) # initalize repo in repo_path try: repo = git.Repo.init(repo_path) except GitCommandError as err: - raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err) + raise EasyBuildError("Failed to init git repo at %s: %s", repo_path, err, exit_code=EasyBuildExit.FAIL_GITHUB) _log.debug("temporary git working directory ready at %s", repo_path) @@ -870,7 +904,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa _log.debug("Cloning from %s", github_url) if target_account is None: - raise EasyBuildError("target_account not specified in setup_repo_from!") + raise EasyBuildError("target_account not specified in setup_repo_from!", exit_code=EasyBuildExit.OPTION_ERROR) # salt to use for names of remotes/branches that are created salt = ''.join(random.choice(ascii_letters) for _ in range(5)) @@ -879,7 +913,7 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa origin = git_repo.create_remote(remote_name, github_url) if not origin.exists(): - raise EasyBuildError("%s does not exist?", github_url) + raise EasyBuildError("%s does not exist?", github_url, exit_code=EasyBuildExit.FAIL_GITHUB) # git fetch # can't use --depth to only fetch a shallow copy, since pushing to another repo from a shallow copy doesn't work @@ -888,21 +922,32 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa try: res = origin.fetch() except GitCommandError as err: - raise EasyBuildError("Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err) + raise EasyBuildError( + "Failed to fetch branch '%s' from %s: %s", branch_name, github_url, err, + exit_code=EasyBuildExit.FAIL_GITHUB + ) if res: if res[0].flags & res[0].ERROR: - raise EasyBuildError("Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note) + raise EasyBuildError( + "Fetching branch '%s' from remote %s failed: %s", branch_name, origin, res[0].note, + exit_code=EasyBuildExit.FAIL_GITHUB + ) else: _log.debug("Fetched branch '%s' from remote %s (note: %s)", branch_name, origin, res[0].note) else: - raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin) + raise EasyBuildError( + "Fetching branch '%s' from remote %s failed: empty result", branch_name, origin, + exit_code=EasyBuildExit.FAIL_GITHUB + ) # git checkout -b ; git pull try: origin_branch = getattr(origin.refs, branch_name) except AttributeError: - raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url) + raise EasyBuildError( + "Branch '%s' not found at %s", branch_name, github_url, exit_code=EasyBuildExit.FAIL_GITHUB + ) _log.debug("Checking out branch '%s' from remote %s", branch_name, github_url) try: @@ -913,7 +958,10 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa try: origin_branch.checkout(b=alt_branch, force=True) except GitCommandError as err: - raise EasyBuildError("Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err) + raise EasyBuildError( + "Failed to check out branch '%s' from repo at %s: %s", alt_branch, github_url, err, + exit_code=EasyBuildExit.FAIL_GITHUB + ) return remote_name @@ -949,7 +997,7 @@ def setup_repo(git_repo, target_account, target_repo, branch_name, silent=False, if res: return res else: - raise EasyBuildError('\n'.join(errors)) + raise EasyBuildError('\n'.join(errors), exit_code=EasyBuildExit.FAIL_GITHUB) @only_if_module_is_available('git', pkgname='GitPython') @@ -981,14 +1029,20 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ ec_paths.append(path) if non_existing_paths: - raise EasyBuildError("One or more non-existing paths specified: %s", ', '.join(non_existing_paths)) + raise EasyBuildError( + "One or more non-existing paths specified: %s", ', '.join(non_existing_paths), + exit_code=EasyBuildExit.OPTION_ERROR + ) if not any(paths.values()): - raise EasyBuildError("No paths specified") + raise EasyBuildError("No paths specified", exit_code=EasyBuildExit.OPTION_ERROR) pr_target_repo = det_pr_target_repo(paths) if pr_target_repo is None: - raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!") + raise EasyBuildError( + "Failed to determine target repository, please specify it via --pr-target-repo!", + exit_code=EasyBuildExit.OPTION_ERROR + ) # initialize repository git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') @@ -996,7 +1050,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ repo_path = os.path.join(git_working_dir, pr_target_repo) if pr_target_repo not in [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO]: - raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo) + raise EasyBuildError( + "Don't know how to create/update a pull request to the %s repository", pr_target_repo, + exit_code=EasyBuildExit.OPTION_ERROR + ) if start_account is None: start_account = build_option('pr_target_account') @@ -1007,7 +1064,9 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ target_account = build_option('github_org') or build_option('github_user') if target_account is None: - raise EasyBuildError("--github-org or --github-user must be specified!") + raise EasyBuildError( + "--github-org or --github-user must be specified!", exit_code=EasyBuildExit.OPTION_ERROR + ) # if branch to start from is specified, we're updating an existing PR start_branch = build_option('pr_target_branch') @@ -1038,8 +1097,11 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and all(file_info['new']): commit_msg = "adding easyblocks: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) else: - raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " - "modifying/deleting files or targeting the framework repo.") + raise EasyBuildError( + "A meaningful commit message must be specified via --pr-commit-msg when " + "modifying/deleting files or targeting the framework repo.", + exit_code=EasyBuildExit.OPTION_ERROR + ) # figure out to which software name patches relate, and copy them to the right place if paths['patch_files']: @@ -1060,7 +1122,10 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if len(hits) == 1: deleted_paths.append(hits[0]) else: - raise EasyBuildError("Path doesn't exist or file to delete isn't found in target branch: %s", fn) + raise EasyBuildError( + "Path doesn't exist or file to delete isn't found in target branch: %s", fn, + exit_code=EasyBuildExit.OPTION_ERROR + ) dep_info = { 'ecs': [], @@ -1117,8 +1182,11 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ diff_stat = git_repo.git.diff(cached=True, stat=True) if not diff_stat: - raise EasyBuildError("No changed files found when comparing to current develop branch. " - "Refused to make empty pull request.") + raise EasyBuildError( + f"No changed files found when comparing to current {start_branch} branch. " + "Refused to make empty pull request.", + exit_code=EasyBuildExit.FAIL_GITHUB + ) # commit git_repo.index.commit(commit_msg) @@ -1149,7 +1217,9 @@ def create_remote(git_repo, account, repo, https=False): try: remote = git_repo.create_remote(remote_name, github_url) except GitCommandError as err: - raise EasyBuildError("Failed to create remote %s for %s: %s", remote_name, github_url, err) + raise EasyBuildError( + "Failed to create remote %s for %s: %s", remote_name, github_url, err, exit_code=EasyBuildExit.FAIL_GITHUB + ) return remote @@ -1164,7 +1234,9 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): :param branch: name of branch to push """ if target_account is None: - raise EasyBuildError("target_account not specified in push_branch_to_github!") + raise EasyBuildError( + "target_account not specified in push_branch_to_github!", exit_code=EasyBuildExit.OPTION_ERROR + ) # push to GitHub remote = create_remote(git_repo, target_account, target_repo) @@ -1181,17 +1253,24 @@ def push_branch_to_github(git_repo, target_account, target_repo, branch): try: res = remote.push(branch) except GitCommandError as err: - raise EasyBuildError("Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err) + raise EasyBuildError( + "Failed to push branch '%s' to GitHub (%s): %s", branch, github_url, err, + exit_code=EasyBuildExit.FAIL_GITHUB + ) if res: if res[0].ERROR & res[0].flags: - raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: %s", - branch, remote, github_url, res[0].summary) + raise EasyBuildError( + "Pushing branch '%s' to remote %s (%s) failed: %s", branch, remote, github_url, res[0].summary, + exit_code=EasyBuildExit.FAIL_GITHUB + ) else: _log.debug("Pushed branch %s to remote %s (%s): %s", branch, remote, github_url, res[0].summary) else: - raise EasyBuildError("Pushing branch '%s' to remote %s (%s) failed: empty result", - branch, remote, github_url) + raise EasyBuildError( + "Pushing branch '%s' to remote %s (%s) failed: empty result", branch, remote, github_url, + exit_code=EasyBuildExit.FAIL_GITHUB + ) def is_patch_for(patch_name, ec): @@ -1248,7 +1327,10 @@ def det_patch_specs(patch_paths, file_info, ec_dirs): patch_specs.append((patch_path, soft_name)) else: # still nothing found - raise EasyBuildError("Failed to determine software name to which patch file %s relates", patch_path) + raise EasyBuildError( + "Failed to determine software name to which patch file %s relates", patch_path, + exit_code=EasyBuildExit.OPTION_ERROR + ) return patch_specs @@ -1515,7 +1597,7 @@ def close_pr(pr, motivation_msg=None): """ github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub user must be specified to use --close-pr") + raise EasyBuildError("GitHub user must be specified to use --close-pr", exit_code=EasyBuildExit.OPTION_ERROR) pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO @@ -1523,7 +1605,10 @@ def close_pr(pr, motivation_msg=None): pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) if pr_data['state'] == GITHUB_STATE_CLOSED: - raise EasyBuildError("PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo) + raise EasyBuildError( + "PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo, + exit_code=EasyBuildExit.OPTION_ERROR + ) pr_owner = pr_data['user']['login'] msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_owner) @@ -1540,8 +1625,10 @@ def close_pr(pr, motivation_msg=None): possible_reasons = reasons_for_closing(pr_data) if not possible_reasons: - raise EasyBuildError("No reason specified and none found from PR data, " - "please use --close-pr-reasons or --close-pr-msg") + raise EasyBuildError( + "No reason specified and none found from PR data, please use --close-pr-reasons or --close-pr-msg", + exit_code=EasyBuildExit.OPTION_ERROR + ) else: motivation_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in possible_reasons]) print_msg("\nNo reason specified but found possible reasons: %s.\n" % motivation_msg, prefix=False) @@ -1560,18 +1647,26 @@ def close_pr(pr, motivation_msg=None): else: github_token = fetch_github_token(github_user) if github_token is None: - raise EasyBuildError("GitHub token for user '%s' must be available to use --close-pr", github_user) + raise EasyBuildError( + "GitHub token for user '%s' must be available to use --close-pr", github_user, + exit_code=EasyBuildExit.FAIL_GITHUB + ) g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) pull_url = g.repos[pr_target_account][pr_target_repo].pulls[pr] body = {'state': 'closed'} status, data = pull_url.post(body=body) if not status == HTTP_STATUS_OK: - raise EasyBuildError("Failed to close PR #%s; status %s, data: %s", pr, status, data) + raise EasyBuildError( + "Failed to close PR #%s; status %s, data: %s", pr, status, data, exit_code=EasyBuildExit.FAIL_GITHUB + ) if reopen: body = {'state': 'open'} status, data = pull_url.post(body=body) if not status == HTTP_STATUS_OK: - raise EasyBuildError("Failed to reopen PR #%s; status %s, data: %s", pr, status, data) + raise EasyBuildError( + "Failed to reopen PR #%s; status %s, data: %s", pr, status, data, + exit_code=EasyBuildExit.FAIL_GITHUB + ) def list_prs(params, per_page=GITHUB_MAX_PER_PAGE, github_user=None): @@ -1607,7 +1702,7 @@ def merge_pr(pr): """ github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub user must be specified to use --merge-pr") + raise EasyBuildError("GitHub user must be specified to use --merge-pr", exit_code=EasyBuildExit.OPTION_ERROR) pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO @@ -1619,16 +1714,16 @@ def merge_pr(pr): msg += "\nPR title: %s\n\n" % pr_data['title'] print_msg(msg, prefix=False) if pr_data['user']['login'] == github_user: - raise EasyBuildError("Please do not merge your own PRs!") + raise EasyBuildError("Please do not merge your own PRs!", exit_code=EasyBuildExit.OPTION_ERROR) force = build_option('force') dry_run = build_option('dry_run') or build_option('extended_dry_run') if not dry_run: if pr_data['merged']: - raise EasyBuildError("This PR is already merged.") + raise EasyBuildError("This PR is already merged.", exit_code=EasyBuildExit.OPTION_ERROR) elif pr_data['state'] == GITHUB_STATE_CLOSED: - raise EasyBuildError("This PR is closed.") + raise EasyBuildError("This PR is closed.", exit_code=EasyBuildExit.OPTION_ERROR) def merge_url(gh): """Utility function to fetch merge URL for a specific PR.""" @@ -1712,7 +1807,10 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH): """ pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO if pr_target_repo != GITHUB_EASYCONFIGS_REPO: - raise EasyBuildError("Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet") + raise EasyBuildError( + "Adding labels to PRs for repositories other than easyconfigs hasn't been implemented yet", + exit_code=EasyBuildExit.OPTION_ERROR + ) tmpdir = tempfile.mkdtemp() @@ -1820,11 +1918,17 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, # fetch GitHub token (required to perform actions on GitHub) github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub user must be specified to open a pull request") + raise EasyBuildError( + "GitHub user must be specified to open a pull request", + exit_code=EasyBuildExit.OPTION_ERROR + ) github_token = fetch_github_token(github_user) if github_token is None: - raise EasyBuildError("GitHub token for user '%s' must be available to open a pull request", github_user) + raise EasyBuildError( + "GitHub token for user '%s' must be available to open a pull request", github_user, + exit_code=EasyBuildExit.FAIL_GITHUB + ) # GitHub organisation or GitHub user where branch is located github_account = build_option('github_org') or github_user @@ -1886,7 +1990,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, print_msg('\n'.join(msg), log=_log) else: - raise EasyBuildError("No changes in '%s' branch compared to current 'develop' branch!", branch_name) + raise EasyBuildError( + f"No changes in '{branch_name}' branch compared to current '{pr_target_branch}' branch!", + exit_code=EasyBuildExit.FAIL_GITHUB + ) # copy repo while target branch is still checked out tmpdir = tempfile.mkdtemp() @@ -1919,8 +2026,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, title = "new easyblock%s for %s" % (plural, (', '.join(file_info['eb_names']))) if title is None: - raise EasyBuildError("Don't know how to make a PR title for this PR. " - "Please include a title (use --pr-title)") + raise EasyBuildError( + "Don't know how to make a PR title for this PR. Please include a title (use --pr-title)", + exit_code=EasyBuildExit.FAIL_GITHUB + ) full_descr = "(created using `eb --new-pr`)\n" if descr is not None: @@ -1957,7 +2066,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, } status, data = pulls_url.post(body=body) if not status == HTTP_STATUS_CREATED: - raise EasyBuildError("Failed to open PR for branch %s; status %s, data: %s", branch_name, status, data) + raise EasyBuildError( + "Failed to open PR for branch %s; status %s, data: %s", branch_name, status, data, + exit_code=EasyBuildExit.FAIL_GITHUB + ) print_msg("Opened pull request: %s" % data['html_url'], log=_log, prefix=False) @@ -1995,8 +2107,10 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): for key in patch.keys(): patch_info[key] = patch[key] if 'name' not in patch_info.keys(): - raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied", - str(patch)) + raise EasyBuildError( + "Wrong patch spec '%s', when using a dict 'name' entry must be supplied", str(patch), + exit_code=EasyBuildExit.EASYCONFIG_ERROR + ) patch = patch_info['name'] if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), @@ -2015,7 +2129,7 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None): github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub username (--github-user) must be specified!") + raise EasyBuildError("GitHub username (--github-user) must be specified!", exit_code=EasyBuildExit.OPTION_ERROR) pr_target_account = build_option('pr_target_account') if pr_target_repo is None: @@ -2087,7 +2201,10 @@ def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None) commit_msg = build_option('pr_commit_msg') if commit_msg is None: - raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when using --update-pr") + raise EasyBuildError( + "A meaningful commit message must be specified via --pr-commit-msg when using --update-pr", + exit_code=EasyBuildExit.OPTION_ERROR + ) if github_account is None: github_account = build_option('github_user') or build_option('github_org') @@ -2118,7 +2235,10 @@ def update_pr(pr_id, paths, ecs, commit_msg=None): pr_target_repo = det_pr_target_repo(paths) if pr_target_repo is None: - raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!") + raise EasyBuildError( + "Failed to determine target repository, please specify it via --pr-target-repo!", + exit_code=EasyBuildExit.OPTION_ERROR + ) github_account, branch_name = det_account_branch_for_pr(pr_id, pr_target_repo=pr_target_repo) @@ -2180,7 +2300,9 @@ def check_github(): print_msg("OK\n", log=_log, prefix=False) else: print_msg("FAIL (%s)", ', '.join(online_state), log=_log, prefix=False) - raise EasyBuildError("checking status of GitHub integration must be done online") + raise EasyBuildError( + "checking status of GitHub integration must be done online", exit_code=EasyBuildExit.FAIL_GITHUB + ) # GitHub user print_msg("* GitHub user...", log=_log, prefix=False, newline=False) @@ -2414,7 +2536,9 @@ def install_github_token(github_user, silent=False): :param silent: keep quiet (don't print any messages) """ if github_user is None: - raise EasyBuildError("GitHub user must be specified to install GitHub token") + raise EasyBuildError( + "GitHub user must be specified to install GitHub token", exit_code=EasyBuildExit.OPTION_ERROR + ) # check if there's a token available already current_token = fetch_github_token(github_user) @@ -2425,8 +2549,10 @@ def install_github_token(github_user, silent=False): msg = "WARNING: overwriting installed token '%s' for user '%s'..." % (current_token, github_user) print_msg(msg, prefix=False, silent=silent) else: - raise EasyBuildError("Installed token '%s' found for user '%s', not overwriting it without --force", - current_token, github_user) + raise EasyBuildError( + "Installed token '%s' found for user '%s', not overwriting it without --force", + current_token, github_user, exit_code=EasyBuildExit.OPTION_ERROR + ) # get token to install token = getpass.getpass(prompt="Token: ").strip() @@ -2437,7 +2563,10 @@ def install_github_token(github_user, silent=False): if valid_token: print_msg("Token seems to be valid, installing it.", prefix=False, silent=silent) else: - raise EasyBuildError("Token validation failed, not installing it. Please verify your token and try again.") + raise EasyBuildError( + "Token validation failed, not installing it. Please verify your token and try again.", + exit_code=EasyBuildExit.FAIL_GITHUB + ) # install token keyring.set_password(KEYRING_GITHUB_TOKEN, github_user, token) @@ -2511,7 +2640,7 @@ def find_easybuild_easyconfig(github_user=None): if file_versions: fn = sorted(file_versions)[-1][1] else: - raise EasyBuildError("Couldn't find any EasyBuild easyconfigs") + raise EasyBuildError("Couldn't find any EasyBuild easyconfigs", exit_code=EasyBuildExit.MISSING_EASYCONFIG) eb_file = os.path.join(eb_parent_path, fn) return eb_file @@ -2538,8 +2667,11 @@ def check_suites_url(gh): # first check combined commit status (set by e.g. Travis CI) status, commit_status_data = github_api_get_request(commit_status_url, github_user) if status != HTTP_STATUS_OK: - raise EasyBuildError("Failed to get status of commit %s from %s/%s (status: %d %s)", - commit_sha, account, repo, status, commit_status_data) + raise EasyBuildError( + "Failed to get status of commit %s from %s/%s (status: %d %s)", + commit_sha, account, repo, status, commit_status_data, + exit_code=EasyBuildExit.FAIL_GITHUB + ) commit_status_count = commit_status_data['total_count'] combined_commit_status = commit_status_data['state'] @@ -2587,7 +2719,9 @@ def check_suites_url(gh): break else: app_name = check_suite_data.get('app', {}).get('name', 'UNKNOWN') - raise EasyBuildError("Unknown check suite status set by %s: '%s'", app_name, status) + raise EasyBuildError( + "Unknown check suite status set by %s: '%s'", app_name, status, exit_code=EasyBuildExit.FAIL_GITHUB + ) return result @@ -2605,13 +2739,16 @@ def pr_url(gh): try: status, pr_data = github_api_get_request(pr_url, github_user, **parameters) except HTTPError as err: - raise EasyBuildError("Failed to get data for PR #%d from %s/%s (%s)\n" - "Please check PR #, account and repo.", - pr, pr_target_account, pr_target_repo, err) + raise EasyBuildError( + "Failed to get data for PR #%d from %s/%s (%s)\nPlease check PR #, account and repo.", + pr, pr_target_account, pr_target_repo, err, exit_code=EasyBuildExit.FAIL_GITHUB + ) if status != HTTP_STATUS_OK: - raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", - pr, pr_target_account, pr_target_repo, status, pr_data) + raise EasyBuildError( + "Failed to get data for PR #%d from %s/%s (status: %d %s)", + pr, pr_target_account, pr_target_repo, status, pr_data, exit_code=EasyBuildExit.FAIL_GITHUB + ) if full: # also fetch status of last commit @@ -2625,8 +2762,10 @@ def comments_url(gh): status, comments_data = github_api_get_request(comments_url, github_user, **parameters) if status != HTTP_STATUS_OK: - raise EasyBuildError("Failed to get comments for PR #%d from %s/%s (status: %d %s)", - pr, pr_target_account, pr_target_repo, status, comments_data) + raise EasyBuildError( + "Failed to get comments for PR #%d from %s/%s (status: %d %s)", + pr, pr_target_account, pr_target_repo, status, comments_data, exit_code=EasyBuildExit.FAIL_GITHUB + ) pr_data['issue_comments'] = comments_data # also fetch reviews @@ -2636,8 +2775,10 @@ def reviews_url(gh): status, reviews_data = github_api_get_request(reviews_url, github_user, **parameters) if status != HTTP_STATUS_OK: - raise EasyBuildError("Failed to get reviews for PR #%d from %s/%s (status: %d %s)", - pr, pr_target_account, pr_target_repo, status, reviews_data) + raise EasyBuildError( + "Failed to get reviews for PR #%d from %s/%s (status: %d %s)", + pr, pr_target_account, pr_target_repo, status, reviews_data, exit_code=EasyBuildExit.FAIL_GITHUB + ) pr_data['reviews'] = reviews_data return pr_data, pr_url @@ -2684,7 +2825,9 @@ def sync_pr_with_develop(pr_id): """Sync pull request with specified ID with current develop branch.""" github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub user must be specified to use --sync-pr-with-develop") + raise EasyBuildError( + "GitHub user must be specified to use --sync-pr-with-develop", exit_code=EasyBuildExit.OPTION_ERROR + ) target_account = build_option('pr_target_account') target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO @@ -2707,7 +2850,9 @@ def sync_branch_with_develop(branch_name): """Sync branch with specified name with current develop branch.""" github_user = build_option('github_user') if github_user is None: - raise EasyBuildError("GitHub user must be specified to use --sync-branch-with-develop") + raise EasyBuildError( + "GitHub user must be specified to use --sync-branch-with-develop", exit_code=EasyBuildExit.OPTION_ERROR + ) target_account = build_option('pr_target_account') target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index bbf0c5449e..a4e1f565ae 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -44,7 +44,7 @@ from easybuild.base import fancylogger from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS from easybuild.tools.config import build_option, get_modules_tool, install_path @@ -303,9 +303,9 @@ def check_module_function(self, allow_mismatch=False, regex=None): if self.testing: # grab 'module' function definition from environment if it's there; only during testing try: - output, exit_code = os.environ['module'], 0 + output, exit_code = os.environ['module'], EasyBuildExit.SUCCESS except KeyError: - output, exit_code = None, 1 + output, exit_code = None, EasyBuildExit.FAIL_SYSTEM_CHECK else: cmd = "type module" res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True, output_file=False) @@ -316,7 +316,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): mod_cmd_re = re.compile(regex, re.M) mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.NAME) - if exit_code == 0: + if exit_code == EasyBuildExit.SUCCESS: if mod_cmd_re.search(output): self.log.debug("Found pattern '%s' in defined 'module' function." % mod_cmd_re.pattern) else: @@ -826,7 +826,7 @@ def run_module(self, *args, **kwargs): self.log.debug("Output of module command '%s': stdout: %s; stderr: %s", cmd, stdout, stderr) # also catch and check exit code - if kwargs.get('check_exit_code', True) and res.exit_code != 0: + if kwargs.get('check_exit_code', True) and res.exit_code != EasyBuildExit.SUCCESS: raise EasyBuildError("Module command '%s' failed with exit code %s; stderr: %s; stdout: %s", cmd, res.exit_code, stderr, stdout) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8921ce482c..740e66af37 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for from easybuild.toolchains.compiler.systemcompiler import TC_CONSTANT_SYSTEM from easybuild.tools import LooseVersion, build_log, run # build_log should always stay there, to ensure EasyBuildLog -from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError +from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, EasyBuildExit from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CHECKSUM_PRIORITY_CHOICES, DEFAULT_CHECKSUM_PRIORITY from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES @@ -915,7 +915,10 @@ def validate(self): error_msgs.append(error_msg % (cuda_cc_regex.pattern, ', '.join(faulty_cuda_ccs))) if error_msgs: - raise EasyBuildError("Found problems validating the options: %s", '\n'.join(error_msgs)) + raise EasyBuildError( + "Found problems validating the options: %s", '\n'.join(error_msgs), + exit_code=EasyBuildExit.OPTION_ERROR + ) def postprocess(self): """Do some postprocessing, in particular print stuff""" @@ -1003,15 +1006,20 @@ def _postprocess_optarch(self): n_parts = len(optarch_parts) map_char_cnts = [p.count(OPTARCH_MAP_CHAR) for p in optarch_parts] if (n_parts > 1 and any(c != 1 for c in map_char_cnts)) or (n_parts == 1 and map_char_cnts[0] > 1): - raise EasyBuildError("The optarch option has an incorrect syntax: %s", self.options.optarch) + raise EasyBuildError( + "The optarch option has an incorrect syntax: %s", self.options.optarch, + exit_code=EasyBuildExit.OPTION_ERROR + ) else: # if there are options for different compilers, we set up a dict if OPTARCH_MAP_CHAR in optarch_parts[0]: optarch_dict = {} for compiler, compiler_opt in [p.split(OPTARCH_MAP_CHAR) for p in optarch_parts]: if compiler in optarch_dict: - raise EasyBuildError("The optarch option contains duplicated entries for compiler %s: %s", - compiler, self.options.optarch) + raise EasyBuildError( + "The optarch option contains duplicated entries for compiler %s: %s", + compiler, self.options.optarch, exit_code=EasyBuildExit.OPTION_ERROR + ) else: optarch_dict[compiler] = compiler_opt self.options.optarch = optarch_dict @@ -1023,13 +1031,17 @@ def _postprocess_optarch(self): def _postprocess_close_pr_reasons(self): """Postprocess --close-pr-reasons options""" if self.options.close_pr_msg: - raise EasyBuildError("Please either specify predefined reasons with --close-pr-reasons or " + - "a custom message with--close-pr-msg") + raise EasyBuildError( + "Please either select a reason with --close-pr-reasons or add a custom message with--close-pr-msg", + exit_code=EasyBuildExit.OPTION_ERROR + ) reasons = self.options.close_pr_reasons.split(',') if any([reason not in VALID_CLOSE_PR_REASONS.keys() for reason in reasons]): - raise EasyBuildError("Argument to --close-pr_reasons must be a comma separated list of valid reasons " + - "among %s" % VALID_CLOSE_PR_REASONS.keys()) + raise EasyBuildError( + "Argument to --close-pr_reasons must be a comma separated list of valid reasons among %s", + VALID_CLOSE_PR_REASONS.keys(), exit_code=EasyBuildExit.OPTION_ERROR + ) self.options.close_pr_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in reasons]) def _postprocess_list_prs(self): @@ -1038,18 +1050,30 @@ def _postprocess_list_prs(self): nparts = len(list_pr_parts) if nparts > 3: - raise EasyBuildError("Argument to --list-prs must be in the format 'state[,order[,direction]]") + raise EasyBuildError( + "Argument to --list-prs must be in the format 'state[,order[,direction]]", + exit_code=EasyBuildExit.OPTION_ERROR + ) list_pr_state = list_pr_parts[0] list_pr_order = list_pr_parts[1] if nparts > 1 else DEFAULT_LIST_PR_ORDER list_pr_direc = list_pr_parts[2] if nparts > 2 else DEFAULT_LIST_PR_DIREC if list_pr_state not in GITHUB_PR_STATES: - raise EasyBuildError("1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES) + raise EasyBuildError( + "1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES, + exit_code=EasyBuildExit.OPTION_ERROR + ) if list_pr_order not in GITHUB_PR_ORDERS: - raise EasyBuildError("2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS) + raise EasyBuildError( + "2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS, + exit_code=EasyBuildExit.OPTION_ERROR + ) if list_pr_direc not in GITHUB_PR_DIRECTIONS: - raise EasyBuildError("3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS) + raise EasyBuildError( + "3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS, + exit_code=EasyBuildExit.OPTION_ERROR + ) self.options.list_prs = (list_pr_state, list_pr_order, list_pr_direc) @@ -1071,41 +1095,62 @@ def _postprocess_checks(self): # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.include_easyblocks_from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: - raise EasyBuildError("Required support for using GitHub API is not available (see warnings)") + raise EasyBuildError( + "Required support for using GitHub API is not available (see warnings)", + exit_code=EasyBuildExit.FAIL_GITHUB + ) # using Lua module syntax only makes sense when modules tool being used is Lmod if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: error_msg = "Generating Lua module files requires Lmod as modules tool; " mod_syntaxes = ', '.join(sorted(avail_module_generators().keys())) error_msg += "use --module-syntax to specify a different module syntax to use (%s)" % mod_syntaxes - raise EasyBuildError(error_msg) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR) # check whether specified action --detect-loaded-modules is valid if self.options.detect_loaded_modules not in LOADED_MODULES_ACTIONS: error_msg = "Unknown action specified to --detect-loaded-modules: %s (known values: %s)" - raise EasyBuildError(error_msg % (self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS))) + raise EasyBuildError( + error_msg, self.options.detect_loaded_modules, ', '.join(LOADED_MODULES_ACTIONS), + exit_code=EasyBuildExit.OPTION_ERROR + ) # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: - raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available") + raise EasyBuildError( + "Python 'keyring' module required for obtaining GitHub token is not available", + exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY + ) if self.options.github_user is None: - raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token") + raise EasyBuildError( + "No GitHub user name provided, required for fetching GitHub token", + exit_code=EasyBuildExit.FAIL_GITHUB + ) token = fetch_github_token(self.options.github_user) if token is None: - raise EasyBuildError("Failed to obtain required GitHub token for user '%s'" % self.options.github_user) + raise EasyBuildError( + "Failed to obtain required GitHub token for user '%s'", self.options.github_user, + exit_code=EasyBuildExit.FAIL_GITHUB + ) # make sure autopep8 is available when it needs to be if self.options.dump_autopep8: if not HAVE_AUTOPEP8: - raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") + raise EasyBuildError( + "Python 'autopep8' module required to reformat dumped easyconfigs as requested", + exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY + ) # if a path is specified to --sysroot, it must exist if self.options.sysroot: if os.path.exists(self.options.sysroot): self.log.info("Specified sysroot '%s' exists: OK", self.options.sysroot) else: - raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot) + raise EasyBuildError( + "Specified sysroot '%s' does not exist!", self.options.sysroot, + exit_code=EasyBuildExit.OPTION_ERROR + ) # 'software_commit' is specific to a particular software package and cannot be used with 'robot' if self.options.software_commit: @@ -1137,7 +1182,10 @@ def _ensure_abs_path(self, opt_name): setattr(self.options, opt_name, abs_paths) else: error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)" - raise EasyBuildError(error_msg, opt_name, type(opt_val)) + raise EasyBuildError( + error_msg, opt_name, type(opt_val), + exit_code=EasyBuildExit.OPTION_ERROR + ) def _postprocess_config(self): """Postprocessing of configuration options""" @@ -1197,7 +1245,10 @@ def _postprocess_config(self): self.args.append(robot_arg) self.options.robot = [] else: - raise EasyBuildError("Argument passed to --robot is not an existing directory: %s", robot_arg) + raise EasyBuildError( + "Argument passed to --robot is not an existing directory: %s", robot_arg, + exit_code=EasyBuildExit.OPTION_ERROR + ) # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver @@ -1523,7 +1574,11 @@ def parse_options(args=None, with_include=True): go_args=eb_args, error_env_options=True, error_env_option_method=raise_easybuilderror, with_include=with_include) except EasyBuildError as err: - raise EasyBuildError("Failed to parse configuration options: %s" % err) + try: + exit_code = err.exit_code + except AttributeError: + exit_code = EasyBuildExit.OPTION_ERROR + raise EasyBuildError("Failed to parse configuration options: %s", err, exit_code=exit_code) return eb_go @@ -1533,12 +1588,15 @@ def check_options(options): Check configuration options, some combinations are not allowed. """ if options.from_commit and options.from_pr: - raise EasyBuildError("--from-commit and --from-pr should not be used together, pick one") + raise EasyBuildError( + "--from-commit and --from-pr should not be used together, pick one", + exit_code=EasyBuildExit.OPTION_ERROR + ) if options.include_easyblocks_from_commit and options.include_easyblocks_from_pr: error_msg = "--include-easyblocks-from-commit and --include-easyblocks-from-pr " error_msg += "should not be used together, pick one" - raise EasyBuildError(error_msg) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR) def check_root_usage(allow_use_as_root=False): @@ -1582,7 +1640,10 @@ def check_included_multiple(included_easyblocks_from, source): try: easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr] except ValueError: - raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s") + raise EasyBuildError( + "Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s", + exit_code=EasyBuildExit.OPTION_ERROR + ) for easyblock_pr in easyblock_prs: easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) @@ -1677,12 +1738,18 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r try: from_prs = [int(x) for x in eb_go.options.from_pr] except ValueError: - raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + raise EasyBuildError( + "Argument to --from-pr must be a comma separated list of PR #s.", + exit_code=EasyBuildExit.OPTION_ERROR + ) try: review_pr = (lambda x: int(x) if x else None)(eb_go.options.review_pr) except ValueError: - raise EasyBuildError("Argument to --review-pr must be an integer PR #.") + raise EasyBuildError( + "Argument to --review-pr must be an integer PR #.", + exit_code=EasyBuildExit.OPTION_ERROR + ) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs @@ -1857,7 +1924,10 @@ def parse_external_modules_metadata(cfgs): paths.extend(res) else: # if there are no matches, we report an error to avoid silently ignores faulty paths - raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", cfg) + raise EasyBuildError( + "Specified path for file with external modules metadata does not exist: %s", cfg, + exit_code=EasyBuildExit.OPTION_ERROR + ) cfgs = paths # use external modules metadata configuration files that are available by default, unless others are specified @@ -1889,7 +1959,10 @@ def parse_external_modules_metadata(cfgs): try: parsed_metadata.merge(ConfigObj(cfg)) except ConfigObjError as err: - raise EasyBuildError("Failed to parse %s with external modules metadata: %s", cfg, err) + raise EasyBuildError( + "Failed to parse %s with external modules metadata: %s", cfg, err, + exit_code=EasyBuildExit.MODULE_ERROR + ) known_metadata_keys = ['name', 'prefix', 'version'] unknown_keys = {} @@ -1910,14 +1983,17 @@ def parse_external_modules_metadata(cfgs): # if both names and versions are available, lists must be of same length names, versions = entry.get('name'), entry.get('version') if names is not None and versions is not None and len(names) != len(versions): - raise EasyBuildError("Different length for lists of names/versions in metadata for external module %s: " - "names: %s; versions: %s", mod, names, versions) + raise EasyBuildError( + "Different length for lists of names/versions in metadata for external module %s: ; " + "names: %s; versions: %s", mod, names, versions, + exit_code=EasyBuildExit.MODULE_ERROR + ) if unknown_keys: error_msg = "Found metadata entries with unknown keys:" for mod in sorted(unknown_keys.keys()): error_msg += "\n* %s: %s" % (mod, ', '.join(sorted(unknown_keys[mod]))) - raise EasyBuildError(error_msg) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MODULE_ERROR) _log.debug("External modules metadata: %s", parsed_metadata) return parsed_metadata diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index edef824fbc..7c2a87a5a9 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -43,7 +43,7 @@ from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, process_easyconfig from easybuild.framework.easyconfig.easyconfig import robot_find_easyconfig, verify_easyconfig_filename from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit from easybuild.tools.config import build_option from easybuild.tools.filetools import det_common_path_prefix, get_cwd, search_file from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS @@ -325,7 +325,7 @@ def raise_error_missing_deps(missing_deps, extra_msg=None): error_msg = "Missing dependencies: %s" % mod_names if extra_msg: error_msg += ' (%s)' % extra_msg - raise EasyBuildError(error_msg) + raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MISSING_DEPENDENCY) def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b9bb2ed8c8..68b3bd215a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -64,7 +64,8 @@ from threading import get_ident as get_thread_id from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError, CWD_NOTFOUND_ERROR, dry_run_msg, print_msg, time_str_since +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR +from easybuild.tools.build_log import dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -574,7 +575,7 @@ def to_cmd_str(cmd): else: _log.info(f"Output of '{cmd_name} ...' shell command (stdout + stderr):\n{res.output}") - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: _log.info(f"Shell command completed successfully (see output above): {cmd_str}") else: _log.warning(f"Shell command FAILED (exit code {res.exit_code}, see output above): {cmd_str}") diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 2fdd9c4ba8..07b828f110 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -61,7 +61,7 @@ pass from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning from easybuild.tools.config import IGNORE from easybuild.tools.filetools import is_readable, read_file, which from easybuild.tools.run import run_shell_cmd, subprocess_popen_text @@ -312,7 +312,7 @@ def get_total_memory(): cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: memtotal = int(res.output.strip()) // (1024**2) if memtotal is None: @@ -396,7 +396,7 @@ def get_cpu_vendor(): res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) out = res.output.strip() - if res.exit_code == 0 and out in VENDOR_IDS: + if res.exit_code == EasyBuildExit.SUCCESS and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: @@ -404,7 +404,7 @@ def get_cpu_vendor(): res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) out = res.output.strip().split(' ')[0] - if res.exit_code == 0 and out in CPU_VENDORS: + if res.exit_code == EasyBuildExit.SUCCESS and out in CPU_VENDORS: vendor = out _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) @@ -506,7 +506,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -553,7 +553,7 @@ def get_cpu_speed(): res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) out = res.output.strip() cpu_freq = None - if res.exit_code == 0 and out: + if res.exit_code == EasyBuildExit.SUCCESS and out: # returns clock frequency in cycles/sec, but we want MHz cpu_freq = float(out) // (1000 ** 2) @@ -600,7 +600,7 @@ def get_cpu_features(): _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: cpu_feat.extend(res.output.strip().lower().split()) cpu_feat.sort() @@ -628,7 +628,7 @@ def get_gpu_info(): _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) nvidia_gpu_info.setdefault(line, 0) @@ -647,14 +647,14 @@ def get_gpu_info(): _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] amd_card_model = line.split(',')[2] @@ -873,7 +873,7 @@ def check_os_dependency(dep): ]) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False, stream_output=False) - found = res.exit_code == 0 + found = res.exit_code == EasyBuildExit.SUCCESS if found: break @@ -887,7 +887,7 @@ def check_os_dependency(dep): res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False, stream_output=False) try: - found = (res.exit_code == 0 and int(res.output.strip()) > 0) + found = (res.exit_code == EasyBuildExit.SUCCESS and int(res.output.strip()) > 0) except ValueError: # Returned something else than an int -> Error found = False @@ -902,7 +902,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): """ res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) - if not ignore_ec and res.exit_code: + if not ignore_ec and res.exit_code != EasyBuildExit.SUCCESS: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN else: @@ -916,7 +916,7 @@ def get_gcc_version(): res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, output_file=False, stream_output=False) gcc_ver = None - if res.exit_code: + if res.exit_code != EasyBuildExit.SUCCESS: _log.warning("Failed to determine the version of GCC: %s", res.output) gcc_ver = UNKNOWN @@ -971,7 +971,7 @@ def get_linked_libs_raw(path): """ res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False) - if res.exit_code: + if res.exit_code != EasyBuildExit.SUCCESS: fail_msg = "Failed to run 'file %s': %s" % (path, res.output) _log.warning(fail_msg) @@ -1006,7 +1006,7 @@ def get_linked_libs_raw(path): # like printing 'not a dynamic executable' when not enough memory is available # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111) res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False) - if res.exit_code == 0: + if res.exit_code == EasyBuildExit.SUCCESS: linked_libs_out = res.output else: fail_msg = "Determining linked libraries for %s via '%s' failed! Output: '%s'" @@ -1194,8 +1194,9 @@ def get_default_parallelism(): else: maxuserproc = int(res.output) except ValueError as err: - raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", - res.exit_code, res.output, err) + raise EasyBuildError( + "Failed to determine max user processes (%s, %s): %s", res.exit_code, res.output, err + ) # assume 6 processes per build thread + 15 overhead par_guess = (maxuserproc - 15) // 6 if par_guess < par: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e7218b2273..505a1c1d2f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1803,7 +1803,7 @@ def test_obtain_file(self): # test no_download option urls = ['file://%s' % tmpdir_subdir] - error_pattern = "Couldn't find file toy-0.0.tar.gz anywhere, and downloading it is disabled" + error_pattern = "Couldn't find file 'toy-0.0.tar.gz' anywhere, and downloading it is disabled" with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, eb.obtain_file, toy_tarball, urls=urls, alt_location='alt_toy', no_download=True) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 38a0595b6c..afe0b86bef 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1456,7 +1456,7 @@ def test_toy_extension_extract_cmd(self): ]) write_file(test_ec, test_ec_txt) - error_pattern = r"shell command 'unzip \.\.\.' failed in extensions step for test.eb" + error_pattern = r"shell command 'unzip \.\.\.' failed with exit code 9 in extensions step for test.eb" with self.mocked_stdout_stderr(): # for now, we expect subprocess.CalledProcessError, but eventually 'run' function will # do proper error reporting