diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 56e82e0e26..bd96bc7e8b 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -62,7 +62,7 @@ ['name', 'version', 'versionprefix', 'versionsuffix'], ['homepage', 'description'], ['toolchain', 'toolchainopts'], - ['sources', 'source_urls'], + ['source_urls', 'sources'], ['patches'], DEPENDENCY_PARAMETERS, ['osdependencies'], diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index aad7ac2073..d0c507585b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -50,12 +50,13 @@ from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file +from easybuild.tools.filetools import find_easyconfigs, find_extension, is_patch_file, read_file, resolve_path +from easybuild.tools.filetools import which, write_file from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo -from easybuild.tools.modules import modules_tool +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.multidiff import multidiff from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME @@ -204,11 +205,12 @@ def mk_node_name(spec): # build directed graph dgr = digraph() dgr.add_nodes(all_nodes) + edge_attrs = [('style', 'dotted'), ('color', 'blue'), ('arrowhead', 'diamond')] for spec in specs: for dep in spec['ec'].all_dependencies: dgr.add_edge((spec['module'], dep)) if dep in spec['ec'].build_dependencies: - dgr.add_edge_attributes((spec['module'], dep), attrs=[('style','dotted'), ('color','blue'), ('arrowhead','diamond')]) + dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs) _dep_graph_dump(dgr, filename) @@ -699,3 +701,214 @@ def avail_easyblocks(): easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks + + +def parse_param_value(string): + """Parse specified string as an easyconfig parameter value.""" + + def split_one(item, sep=','): + """Helper function to split of first part in an item, using given separator.""" + parts = item.split(sep) + return (parts[0], sep.join(parts[1:])) + + param, value = None, None + + # separators + list_sep = ';' + tuple_sep = ',' + dict_key_val_sep = ':' + + # determine list of names of known easyblocks, so we can descriminate an easyblock name + easyblock_names = [e['class'] for e in avail_easyblocks().values()] + _log.debug("List of names for known easyblocks: %s", sorted(easyblock_names)) + + # regular expression to recognise a version + version_regex = re.compile('^[0-9][0-9.-]') + + # first check whether easyconfig parameter name is specified as '=' + if re.match('^[a-z_]+=', string): + param, string = split_one(string, sep='=') + _log.info("Found (raw) value for '%s' easyconfig parameter: %s", param, string) + + # check if the value is most likely a dictionary ':[;:]' + if re.match('^[a-z_]+' + dict_key_val_sep, string): + _log.info("String value '%s' represents a dictionary value", string) + value = {} + for item in string.split(list_sep): + if dict_key_val_sep in item: + key, val = split_one(item, sep=dict_key_val_sep) + # recurse to obtain parsed value for this key + value[key] = parse_param_value(val)[1] + else: + raise EasyBuildError("Wrong format for dictionary item '%s', should be ':= 3 and param is None: + param, value = 'description', string + + else: + value = string + + if param: + _log.info("Found value for '%s' easyconfig parameter: %s", param, value) + else: + _log.info("Found value for unknown easyconfig parameter: %s", value) + + return (param, value) + + +def create_new_easyconfig(path, args): + """Create new easyconfig file based on specified information.""" + + _log.experimental("Generating easyconfig using 'eb --new'") + + specs = {} + + # handle values that start with 'http' first + # try and discriminate between homepage and source URL + http_args = [arg for arg in args if arg.startswith('http')] + + # first, check and see if we have a full download URL provided as an argument + for arg in http_args: + maybe_filename = os.path.basename(arg) + ext = find_extension(maybe_filename, raise_error=False) + if ext: + specs['source_urls'] = [os.path.dirname(arg)] + # try to recognise downloading of source tarballs by commit ID + if re.search('^[0-9a-f]+\.', maybe_filename): + specs['sources'] = [{ + 'download_filename': maybe_filename, + 'filename': '%(name)s-%(version)s' + ext, + }] + else: + specs['sources'] = [maybe_filename] + http_args.remove(arg) + args.remove(arg) + break + + for arg in http_args: + if specs.get('homepage') is None: + # homepage is more like to be of form https://example.com, i.e. top-level domain + # if source_urls is already set, then this URL is likely to be a value for homepage + if '/' not in arg.split('://')[-1] or specs.get('source_urls'): + specs['homepage'] = arg + args.remove(arg) + # go to next iteration to avoid also using this value for 'source_urls' + continue + + if specs.get('source_urls') is None: + specs['source_urls'] = [arg] + args.remove(arg) + if specs.get('homepage') is None: + specs['homepage'] = arg + + # iterate over provided arguments, and try to figure out what they specify + for arg in args: + + key, val = parse_param_value(arg) + + # first argument is assumed to be the software name + if specs.get('name') is None: + specs['name'] = arg + + elif key and specs.get(key) is None: + + if key in ['builddeps', 'deps', 'source_urls', 'sources']: + if not isinstance(val, list): + val = [val] + + if key in ['builddeps', 'deps']: + key = key.replace('deps', 'dependencies') + + specs[key] = val + + # toolchain is usually specified as /, e.g. intel/2018a + elif isinstance(val, basestring) and '/' in val and specs.get('toolchain') is None: + tc_name, tc_ver = val.split('/') + specs['toolchain'] = {'name': tc_name, 'version': tc_ver} + + else: + print_warning("Unhandled argument: %s" % quote_str(arg)) + + # make sure that at least name, version and toolchain are known + missing = [p for p in ['name', 'toolchain', 'version'] if specs.get(p) is None] + if missing: + raise EasyBuildError("One or more required parameters are not specified: %s", ', '.join(missing)) + + # inject educated guesses for some important easyconfig parameters that are not specified + educated_guesses = { + 'description': "This is an example description.", + 'homepage': 'https://example.com', + + 'easyblock': 'ConfigureMake', + 'moduleclass': 'tools', + 'sanity_check_paths': {'files': [os.path.join('bin', specs['name'])], 'dirs': []}, + + 'source_urls': ['%(homepage)s'], + 'sources': ['%(name)s-%(version)s.tar.gz'], + } + for key in educated_guesses: + if key not in specs: + specs[key] = educated_guesses[key] + print_warning("No value found for '%s' parameter, injected dummy value: %s" % (key, quote_str(specs[key]))) + + # create EasyConfig instance and dump easyconfig file to current directory + ec_raw = '\n'.join("%s = %s" % (key, quote_str(specs[key])) for key in specs) + + ec, err = None, None + try: + ec = EasyConfig(None, rawtxt=ec_raw) + except EasyBuildError as err: + print_warning("Problem occured when parsing generated easyconfig file: %s" % err) + + if err: + print_msg(ec_raw + '\n', prefix=False) + raise EasyBuildError("Easyconfig file with raw contents shown above NOT created because of errors: %s", err) + else: + full_ec_ver = det_full_ec_version(specs) + fp = os.path.join(path, '%s-%s.eb' % (specs['name'], full_ec_ver)) + + ec.dump(fp) + return fp diff --git a/easybuild/main.py b/easybuild/main.py index 85eba47f6f..cd916a1643 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -49,14 +49,14 @@ from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph +from easybuild.framework.easyconfig.tools import categorize_files_by_type, create_new_easyconfig, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, edit_file, read_file, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import list_prs, new_pr, merge_pr, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook @@ -173,6 +173,73 @@ def run_contrib_style_checks(ecs, check_contrib, check_style): return check_contrib or check_style +def handle_cat_copy_edit(filepaths, target=None, copy=None): + """Handle use of --cat, --copy and --edit.""" + + if copy is None: + copy = build_option('copy') + + res = [] + for orig_fp in filepaths: + if copy: + # if target location is an existing directory, retain filename + # if not, assume last part of specific location is filename + if os.path.isdir(target): + fp = os.path.join(target, os.path.basename(orig_fp)) + else: + fp = target + + if os.path.exists(fp) and not build_option('force'): + raise EasyBuildError("Not overwriting existing file %s without --force", fp) + + copy_file(orig_fp, fp) + res.append(fp) + else: + fp = orig_fp + + if build_option('edit'): + edit_file(fp) + + if build_option('show'): + print_msg("Contents of easyconfig file %s:\n" % fp) + print_msg(read_file(fp), prefix=False) + + return res + + +def handle_new(tmpdir, args): + """Handle use of --new.""" + tmpfp = create_new_easyconfig(tmpdir, args) + + # use current directory as default location to save generated file, in case no location is specified via --copy + res = handle_cat_copy_edit([tmpfp], target=build_option('copy') or '.', copy=True) + + print_msg("easyconfig file %s created!" % res[0]) + + +def handle_search(search_query, search_filename, search_short): + """Handle use of --search.""" + + copy_path = build_option('copy') + search_action = copy_path or build_option('show') or build_option('edit') + res = search_easyconfigs(search_query, short=search_short, filename_only=search_filename, + terse=build_option('terse'), return_hits=search_action) + + if search_action: + # only perform action(s) if there's a single search result, unless --force is used + search_action_limit = build_option('search_action_limit') or 1 + if len(res) > search_action_limit: + err_msg = "Found %d results which is more than search action limit (%d), so not performing search action(s)" + raise EasyBuildError(err_msg, len(res), search_action_limit) + + res = handle_cat_copy_edit(res, target=copy_path or '.') + + if res: + print_msg("copied easyconfig files:") + for path in res: + print_msg("* %s" % path, prefix=False) + + def clean_exit(logfile, tmpdir, testing, silent=False): """Small utility function to perform a clean exit.""" cleanup(logfile, tmpdir, testing, silent=silent) @@ -191,7 +258,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - options, orig_paths = eb_go.options, eb_go.args + options, args = eb_go.options, eb_go.args global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings @@ -220,8 +287,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # search for easyconfigs, if a query is specified if search_query: - search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, - terse=options.terse) + handle_search(search_query, options.search_filename, options.search_short) # GitHub options that warrant a silent cleanup & exit if options.check_github: @@ -246,6 +312,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif options.list_software: print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') + elif options.new: + handle_new(eb_tmpdir, args) + # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, @@ -254,6 +323,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): options.list_software, options.list_prs, options.merge_pr, + options.new, options.review_pr, options.terse, search_query, @@ -274,14 +344,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: - if orig_paths: + if args: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() - orig_paths.append(eb_file) + args.append(eb_file) - categorized_paths = categorize_files_by_type(orig_paths) + categorized_paths = categorize_files_by_type(args) # command line options that do not require any easyconfigs to be specified new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 041cf8a21b..378c153ecf 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -118,7 +118,7 @@ def experimental(self, msg, *args, **kwargs): msg = common_msg + ': ' + msg self.warning(msg, *args, **kwargs) else: - msg = common_msg + " (use --experimental option to enable): " + msg + msg = common_msg + " (use --experimental or -E to enable): " + msg raise EasyBuildError(msg, *args) def deprecated(self, msg, ver, max_ver=None, more_info=None, *args, **kwargs): diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 72fe9240ed..82626f0664 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -131,9 +131,11 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'container_image_format', 'container_image_name', 'container_tmpdir', + 'copy', 'download_timeout', 'dump_test_report', 'easyblock', + 'editor_command_template', 'extra_modules', 'filter_deps', 'filter_env_vars', @@ -168,6 +170,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_target_repo', 'rpath_filter', 'regtest_output_dir', + 'search_action_limit', 'skip', 'stop', 'subdir_user_modules', @@ -184,6 +187,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'debug', 'debug_lmod', 'dump_autopep8', + 'edit', 'enforce_checksums', 'extended_dry_run', 'experimental', @@ -207,6 +211,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'set_gid_bit', 'skip_test_cases', 'sticky_bit', + 'terse', 'trace', 'upload_test_report', 'update_modules_tool_cache', @@ -214,6 +219,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'use_f90cache', 'use_existing_modules', 'set_default_module', + 'show', ], True: [ 'cleanup_builddir', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 937d4e33db..e1071f63a4 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -46,6 +46,7 @@ import os import re import shutil +import subprocess import stat import sys import tempfile @@ -59,7 +60,7 @@ # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg from easybuild.tools.config import build_option -from easybuild.tools import run +from easybuild.tools.run import run_cmd try: import requests @@ -371,7 +372,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced if extra_options: cmd = "%s %s" % (cmd, extra_options) - run.run_cmd(cmd, simple=True, force_in_dry_run=forced) + run_cmd(cmd, simple=True, force_in_dry_run=forced) return find_base_dir() @@ -829,16 +830,23 @@ def get_local_dirs_purged(): return new_dir -def find_extension(filename): - """Find best match for filename extension.""" +def find_extension(filename, raise_error=True): + """ + Find best match for filename extension. + + :param raise_error: raise an error if no known extension was found in specified filename; + if False, just returns None if no known extension was found + """ # sort by length, so longest file extensions get preference suffixes = sorted(EXTRACT_CMDS.keys(), key=len, reverse=True) pat = r'(?P%s)$' % '|'.join([s.replace('.', '\\.') for s in suffixes]) res = re.search(pat, filename, flags=re.IGNORECASE) if res: ext = res.group('ext') - else: + elif raise_error: raise EasyBuildError('Unknown file type for file %s', filename) + else: + ext = None return ext @@ -1003,7 +1011,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): _log.debug("Using specified patch level %d for patch %s" % (level, patch_file)) patch_cmd = "patch -b -p%s -i %s" % (level, apatch) - out, ec = run.run_cmd(patch_cmd, simple=False, path=adest, log_ok=False, trace=False) + out, ec = run_cmd(patch_cmd, simple=False, path=adest, log_ok=False, trace=False) if ec: raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out) @@ -1360,7 +1368,7 @@ def move_logs(src_logfile, target_logfile): _log.info("Moved log file %s to %s" % (src_logfile, new_log_path)) if zip_log_cmd: - run.run_cmd("%s %s" % (zip_log_cmd, new_log_path)) + run_cmd("%s %s" % (zip_log_cmd, new_log_path)) _log.info("Zipped log %s using '%s'", new_log_path, zip_log_cmd) except (IOError, OSError), err: @@ -1525,21 +1533,6 @@ def decode_class_name(name): return decode_string(name) -def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): - """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead""" - _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): - """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead""" - _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead""" - _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - def det_size(path): """ Determine total size of given filepath (in bytes). @@ -1717,6 +1710,44 @@ def copy(paths, target_path, force_in_dry_run=False): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) +def edit_file(fp): + """Edit file at specified location.""" + + tmpl_editor_cmd = build_option('editor_command_template') + if not tmpl_editor_cmd: + raise EasyBuildError("No editor command template specified (see --editor-command-template)") + elif tmpl_editor_cmd.count('%s') != 1: + raise EasyBuildError("Editor command template should contain exactly one '%s' as placeholder for filename: '" + + tmpl_editor_cmd + "'") + + # read file before editing so we can detect whether or not any changes were made + txt = read_file(fp) + + # replace placeholder %s with path of file to edit + cmd = [] + for cmd_part in tmpl_editor_cmd.split(' '): + if cmd_part == '%s': + cmd.append(fp) + else: + cmd.append(cmd_part) + + print_msg("editing %s... " % fp, newline=False) + # can't use run_cmd here, just hangs without actually bringing up the editor... + try: + exit_code = subprocess.call(cmd) + except OSError as err: + raise EasyBuildError("Editor command '%s' failed: %s", ' '.join(cmd), err) + if exit_code: + raise EasyBuildError("Editor command '%s' failed with exit code %s", ' '.join(cmd), exit_code) + + if txt == read_file(fp): + done_msg = 'done (no changes)' + else: + done_msg = 'done (changes detected)' + + print_msg(done_msg, prefix=False) + + def get_source_tarball_from_git(filename, targetdir, git_config): """ Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it @@ -1773,7 +1804,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) # if a specific commit is asked for, check it out if commit: @@ -1781,11 +1812,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: checkout_cmd.extend(['&&', 'git', 'submodule', 'update']) - run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) # create an archive and delete the git repo directory tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) change_dir(cwd) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 932a5a8ecd..2243bcb548 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -367,7 +367,7 @@ def override_options(self): 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", - None, 'store_true', False), + None, 'store_true', False, 'E'), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " @@ -459,6 +459,7 @@ def config_options(self): mk_full_default_path('containerpath')), 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", 'strlist', 'store', None), + 'editor-command-template': ("Editor command template to use (used for --edit)", 'str', 'store', "vim %s"), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), @@ -560,6 +561,7 @@ def informative_options(self): None, 'store_true', False), 'search': ("Search for easyconfig files in the robot search path, print full paths", None, 'store', None, {'metavar': 'REGEX'}), + 'search-action-limit': ("Search hit count limit to perform search action(s)", int, 'store', None), 'search-filename': ("Search for easyconfig files in the robot search path, print only filenames", None, 'store', None, {'metavar': 'REGEX'}), 'search-short': ("Search for easyconfig files in the robot search path, print short paths", @@ -673,8 +675,13 @@ def easyconfig_options(self): descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = OrderedDict({ + 'copy': ("Copy easyconfig file(s) to specified location (works well with --search)", + None, 'store_or_None', '.', {'metavar': "PATH"}), + 'edit': ("Edit easyconfig file (using editor specified via --editor)", None, 'store_true', False), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), + 'new': ("Create a new easyconfig file based on specified information", None, 'store_true', False, 'n'), + 'show': ("Print contents of easyconfig file(s) (works well with --search)", None, 'store_true', False), }) self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='') diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 8e40328b60..9f569879fb 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -279,7 +279,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] - avail_modules = [m for m in avail_modules if not m in being_installed] + avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s' % easyconfigs) @@ -352,7 +352,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: - if not ec in easyconfigs + additional: + if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: @@ -379,8 +379,15 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): return ordered_ecs -def search_easyconfigs(query, short=False, filename_only=False, terse=False): - """Search for easyconfigs, if a query is provided.""" +def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_hits=False): + """ + Search for easyconfigs, if a query is provided. + + :param short: generate short output, by figuring out common prefix among hits + :param filename_only: only return filenames, not file paths + :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing + :param return_hits: return list of search hits, rather than printing them + """ search_path = build_option('robot_path') if not search_path: search_path = [os.getcwd()] @@ -390,11 +397,15 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): ignore_dirs = build_option('ignore_dirs') + # we need long full path results when returns list of hits + if return_hits: + filename_only, short, terse = False, False, True + # note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse, silent=True, filename_only=False) - # filter out archived easyconfigs, these are handled separately + # filter out archived easyconfigs, these are handled separately hits, archived_hits = [], [] for hit in _hits: if EASYCONFIGS_ARCHIVE_DIR in hit.split(os.path.sep): @@ -423,6 +434,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): if not terse: lines.extend(['', "Matching archived easyconfigs:", '']) lines.extend(tmpl % hit for hit in archived_hits) + hits.extend(archived_hits) elif not terse: cnt = len(archived_hits) lines.extend([ @@ -430,4 +442,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt, ]) - print '\n'.join(lines) + if return_hits: + return hits + else: + print '\n'.join(lines) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 45b33d3685..4f9b3f0d66 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -56,13 +56,13 @@ from easybuild.framework.easyconfig.style import check_easyconfigs_style from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs +from easybuild.framework.easyconfig.tools import parse_param_value from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates -from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, read_file, remove_file, symlink -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, read_file, remove_file, symlink, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata @@ -2357,6 +2357,59 @@ def test_deprecated(self): error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n\(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + def test_parse_param_value(self): + """Tests for parse_param_value function.""" + self.assertEqual(parse_param_value(''), (None, '')) + self.assertEqual(parse_param_value('test123'), (None, 'test123')) + + self.assertEqual(parse_param_value("this is a description"), ('description', "this is a description")) + + self.assertEqual(parse_param_value('1.2.3'), ('version', '1.2.3')) + + # names of source tarballs are recognized + for ext in ['tar.gz', 'gz', 'tar.bz2', 'bz2', 'zip']: + self.assertEqual(parse_param_value('toy-0.0.' + ext), ('sources', 'toy-0.0.' + ext)) + + # known easyblocks are recognized + for easyblock in ['ConfigureMake', 'Toolchain', 'EB_toy']: + self.assertEqual(parse_param_value(easyblock), ('easyblock', easyblock)) + + # list, tuple + self.assertEqual(parse_param_value('one,two,three,four'), (None, ('one', 'two', 'three', 'four'))) + self.assertEqual(parse_param_value('one,two,,four'), (None, ('one', 'two', 'four'))) + self.assertEqual(parse_param_value('one;two;three;four'), (None, ['one', 'two', 'three', 'four'])) + # list of tuples + self.assertEqual(parse_param_value('one,two;three,four'), (None, [('one', 'two'), ('three', 'four')])) + # dict with string values + self.assertEqual(parse_param_value('one:1;two:2;three:3'), (None, {'one': '1', 'two': '2', 'three': '3'})) + # dict with list values + expected = {'ones': ('1', '1', '1'), 'twos': ('2', '2'), 'threes': ('3',)} + self.assertEqual(parse_param_value('ones:1,1,1;twos:2,2;threes:3,'), (None, expected)) + # dict with a single entry, value can contain additional ':' characters, no problem there + self.assertEqual(parse_param_value('one:1:2:3'), (None, {'one': '1:2:3'})) + + # check error handling for dict with incorrect entry + error_pattern = "Wrong format for dictionary item 'two', should be ': 1) for '^gcc-4' search query + error_pattern = r"Found [0-9]* results which is more than search action limit \(1\), " + error_pattern += "so not performing search action\(s\)" + + copy = '--copy=%s' % self.test_prefix + search_actions_to_test = [ + [copy], + ['--edit'], + ['--show'], + [copy, '--show'], + [copy, '--edit'], + ['--edit', '--show'], + [copy, '--edit', '--show'], + ] + for search_actions in search_actions_to_test: + + args = ['--search=^gcc-4', '--editor-command-template=true %s'] + search_actions + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, testing=False, raise_error=True) + + args.append('--search-action-limit=100') + self.mock_stdout(True) + self.eb_main(args, testing=False, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + + ecs_dir = os.path.join(test_ecs, 'g', 'GCC') + + if any(a.startswith('--copy') for a in search_actions): + self.assertTrue("== copied easyconfig files:\n" in stdout) + for ec in test_gcc_ecs: + self.assertTrue("* %s/%s" % (self.test_prefix, ec) in stdout) + self.assertEqual(sorted(f for f in os.listdir(self.test_prefix) if f.startswith('GCC-')), test_gcc_ecs) + ecs_dir = self.test_prefix + else: + self.assertFalse("== copied easyconfig files" in stdout) + + if any(a.startswith('--edit') for a in search_actions): + for ec in test_gcc_ecs: + self.assertTrue("== editing %s/%s... done (no changes)" % (ecs_dir, ec) in stdout) + else: + self.assertFalse("== editing" in stdout) + + if any(a.startswith('--show') for a in search_actions): + for ec in test_gcc_ecs: + self.assertTrue("== Contents of easyconfig file %s/%s:" % (ecs_dir, ec) in stdout) + else: + self.assertFalse("== Contents of easyconfig file" in stdout) + + # clean up and recreate test dir + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # test with search query that only returns a single result + args = ['--search=^gcc-4.8.3', '--copy=%s' % self.test_prefix, '--edit', '--show', + '--editor-command-template=true %s'] + self.mock_stdout(True) + self.eb_main(args, testing=False, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + patterns = [ + r"^== editing %s/GCC-4.8.3.eb... done \(no changes\)$" % self.test_prefix, + r"^== Contents of easyconfig file %s/GCC-4.8.3.eb:$" % self.test_prefix, + r'^name = "GCC"$', + r"^version = '4.8.3'$", + r"^toolchain = .*dummy", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + self.assertTrue(stdout.strip().endswith("== copied easyconfig files:\n* %s/GCC-4.8.3.eb" % self.test_prefix)) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -1296,26 +1376,23 @@ def test_experimental(self): log = fancylogger.getLogger() - # force it to False - topt = EasyBuildOptions( - go_args=['--disable-experimental'], - ) - try: - log.experimental('x') - # sanity check, should never be reached if it works. - self.assertTrue(False, "Experimental logging should be disabled by setting the --disable-experimental option") - except easybuild.tools.build_log.EasyBuildError, err: - # check error message - self.assertTrue('Experimental functionality.' in str(err)) - - # toggle experimental - topt = EasyBuildOptions( - go_args=['--experimental'], - ) - try: - log.experimental('x') - except easybuild.tools.build_log.EasyBuildError, err: - self.assertTrue(False, 'Experimental logging should be allowed by the --experimental option.') + for experimental_opt in ['--experimental', '-E']: + # force it to False + topt = EasyBuildOptions(go_args=['--disable-experimental']) + try: + log.experimental('x') + # sanity check, should never be reached if it works. + self.assertTrue(False, "Experimental logging should be disabled via --disable-experimental") + except easybuild.tools.build_log.EasyBuildError, err: + # check error message + self.assertTrue('Experimental functionality.' in str(err)) + + # toggle experimental + EasyBuildOptions(go_args=[experimental_opt]) + try: + log.experimental('x') + except easybuild.tools.build_log.EasyBuildError, err: + self.assertTrue(False, 'Experimental logging should be allowed by the --experimental option.') # set it back easybuild.tools.build_log.EXPERIMENTAL = orig_value @@ -3805,6 +3882,282 @@ def test_enforce_checksums(self): error_pattern = "Missing checksum for toy-0.0.tar.gz" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def run_eb_new(self, args): + """Helper function to run 'eb --new' with specified arguments.""" + change_dir(self.test_prefix) + self.mock_stdout(True) + self.mock_stderr(True) + self.eb_main(['--new', '--experimental'] + args, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + return (stdout, stderr) + + def test_new_ec(self): + """Test creating new easyconfigs via 'eb --new'.""" + + # running without any arguments results in a clean error + error_pattern = "One or more required parameters are not specified: %s" + self.assertErrorRegex(EasyBuildError, error_pattern % "name, toolchain, version", self.run_eb_new, []) + + # partial specs also results in a clean error + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", self.run_eb_new, ['toy']) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", self.run_eb_new, ['toy', '1.2.3']) + self.assertErrorRegex(EasyBuildError, error_pattern % "version", self.run_eb_new, ['toy', 'foss/2018a']) + + # first argument is *always* the software name, even if it matches with a known easyblock (e.g. SCons) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", self.run_eb_new, ['ConfigureMake']) + + # when just the required specs are specified, a lot of warnings are spit out + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b']) + + ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') + self.assertTrue(os.path.exists(ec_fp)) + + expected_stdout = "== easyconfig file ./%s created!" + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'toy') + self.assertEqual(ec.version, '1.2.3') + self.assertEqual(ec['toolchain'], {'name': 'foss', 'version': '2018b'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + + warn_params = ['description', 'easyblock', 'homepage', 'moduleclass', 'sanity_check_paths', + 'sources', 'source_urls'] + for param in warn_params: + self.assertTrue("WARNING: No value found for '%s' parameter, injected dummy value" % param in stderr) + # no unused arguments + self.assertFalse("WARNING: Unhandled argument" in stderr) + + # --new doesn't blindly overwrite an existing easyconfig file + error_pattern = "Not overwriting existing file ./toy-1.2.3-foss-2018b.eb without --force" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, ['toy', '1.2.3', 'foss/2018b']) + + # works fine with --force + self.assertTrue(os.path.exists(ec_fp)) + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'moduleclass=bio', '--force']) + self.assertTrue(os.path.exists(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'toy') + self.assertEqual(ec.version, '1.2.3') + self.assertEqual(ec['toolchain'], {'name': 'foss', 'version': '2018b'}) + self.assertEqual(ec['moduleclass'], 'bio') + + remove_file(ec_fp) + + # warnings are printed for arguments that can't be matched to an easyconfig parameter + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertTrue('WARNING: Unhandled argument: "this_is_a_useless_value"' in stderr) + + # easyblock names are recognized + for easyblock in ['Toolchain', 'EB_toy']: + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', easyblock]) + + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertFalse("WARNING: Unhandled argument" in stderr) + + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], easyblock) + + remove_file(ec_fp) + + # check handling of description (first arguments with 3 or more spaces) + args = ['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"] + (stdout, stderr) = self.run_eb_new(args) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['description'], "this is a description") + self.assertTrue('WARNING: Unhandled argument: "not a description"' in stderr) + + remove_file(ec_fp) + + # check handling of sanity_check_paths (dict value with files/dir keys) + tests = [ + ('files:', {'files': [], 'dirs': []}), + ('dirs:', {'files': [], 'dirs': []}), + ('files:;dirs:', {'files': [], 'dirs': []}), + ('files:bin/foo', {'files': ['bin/foo'], 'dirs': []}), + ('dirs:include', {'files': [], 'dirs': ['include']}), + ('files:bin/foo,bin/bar', {'files': ['bin/foo', 'bin/bar'], 'dirs': []}), + ('dirs:bin,include,lib', {'files': [], 'dirs': ['bin', 'include', 'lib']}), + ('files:bin/foo;dirs:include', {'files': ['bin/foo'], 'dirs': ['include']}), + ('files:bin/foo,bin/bar;dirs:include,lib', {'files': ['bin/foo', 'bin/bar'], 'dirs': ['include', 'lib']}), + ] + for arg, expected in tests: + args = ['bar', '3.4.5', 'GCCcore/6.4.0', arg] + (stdout, stderr) = self.run_eb_new(args) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['sanity_check_paths'], expected) + + remove_file(ec_fp) + + # check handling of deps/builddeps + args = ['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4'] + (stdout, stderr) = self.run_eb_new(args) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertFalse("WARNING: Unhandled argument" in stderr) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + self.assertEqual(len(ec['dependencies']), 2) + self.assertEqual(ec['dependencies'][0]['name'], 'toy') + self.assertEqual(ec['dependencies'][0]['version'], '0.0') + self.assertEqual(ec['dependencies'][1]['name'], 'GCC') + self.assertEqual(ec['dependencies'][1]['version'], '4.9.2') + self.assertEqual(len(ec['builddependencies']), 1) + self.assertEqual(ec['builddependencies'][0]['name'], 'gzip') + self.assertEqual(ec['builddependencies'][0]['version'], '1.4') + + remove_file(ec_fp) + + # check handling of easyconfig parameters that are specified by =, which get preference + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', + 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertTrue('WARNING: Unhandled argument: "Toolchain"' in stderr) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], 'EB_toy') + self.assertEqual(ec['moduleclass'], 'lib') + + remove_file(ec_fp) + + # check handling of URLs: source_urls vs homepage + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', + 'https://example.com/files/bar-3.4.5.tar.gz', 'http://example.com/bar']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['homepage'], 'http://example.com/bar') + self.assertEqual(ec['source_urls'], ['https://example.com/files']) + self.assertEqual(ec['sources'], ['bar-3.4.5.tar.gz']) + + # using an unknown easyblock means trouble + error_pattern = "Easyconfig file with raw contents shown above NOT created because of errors: " + error_pattern += "'Failed to obtain class for NoSuchEasyBlock easyblock .*'" + args = ['bar', '3.4.5', 'foss/2018a', 'easyblock=NoSuchEasyBlock'] + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, args) + + def test_new_cat_copy_edit(self): + """Test combining --new with --show, --copy and --edit.""" + + args = ['toy', '1.2.3', 'foss/2018b'] + ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') + + # contents of easyconfig are printed when using --show, file still is created into current directory + (stdout, stderr) = self.run_eb_new(args + ['--show']) + self.assertTrue(os.path.exists(ec_fp)) + + patterns = [ + r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'", + r"^name = 'toy'$", + r"^version = '1.2.3'$", + r"^toolchain = {'name': 'foss', 'version': '2018b'}$", + r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + + # clean up test dir + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # check whether easyconfig file is copied when --copy is used + # --copy without an argument doesn't change anything, since --new always dumps easyconfig in currect dir + (stdout, stderr) = self.run_eb_new(args + ['--copy']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), "== easyconfig file ./toy-1.2.3-foss-2018b.eb created!") + + # --copy doesn't blindly overwrite an existing easyconfig file without --force + error_pattern = "Not overwriting existing file ./toy-1.2.3-foss-2018b.eb without --force" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, args + ['--copy']) + + # overwriting works fine with --force + write_file(ec_fp, '') + (stdout, stderr) = self.run_eb_new(args + ['--copy', '--force']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertTrue(read_file(ec_fp) != '') + + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # if as argument is passed to --copy, the easyconfig file is created there instead + # (and not in the default location) + copy_args = [ + os.path.join(self.test_prefix, 'test.eb'), # non-existing file in existing directory + os.path.join(self.test_prefix, 'asubdir', 'test.eb'), # non-existing file in non-existing directory + os.path.join(self.test_prefix, 'asubdir'), # existing (sub)directory, no filename specified + ] + for copy_arg in copy_args: + if os.path.isdir(copy_arg): + test_ec = os.path.join(copy_arg, 'toy-1.2.3-foss-2018b.eb') + else: + test_ec = copy_arg + + (stdout, stderr) = self.run_eb_new(args + ['--copy', copy_arg]) + self.assertFalse(os.path.exists(ec_fp)) + self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) + self.assertEqual(stdout.strip(), "== easyconfig file %s created!" % test_ec) + + remove_file(test_ec) + + # combination of --copy and --show also works + (stdout, stderr) = self.run_eb_new(args + ['--show', '--copy', copy_arg]) + self.assertFalse(os.path.exists(ec_fp)) + self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) + patterns[0] = r"== Contents of easyconfig file %s:\n\neasyblock = 'ConfigureMake'" % test_ec + patterns[-1] = r"== easyconfig file %s created!" % test_ec + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # throwing --edit in the mix + args.extend(['--edit', '--editor-command-template=true %s']) + (stdout, stderr) = self.run_eb_new(args + ['--show']) + self.assertTrue(os.path.exists(ec_fp)) + patterns[0] = r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'" + patterns[-1] = r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!" + patterns.insert(0, r"== editing ./toy-1.2.3-foss-2018b.eb... done \(no changes\)") + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/robot.py b/test/framework/robot.py index 93e137a36a..1e4f1e76ff 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -38,7 +38,6 @@ from unittest import TextTestRunner import easybuild.framework.easyconfig.easyconfig as ecec -import easybuild.framework.easyconfig.tools as ectools import easybuild.tools.build_log import easybuild.tools.robot as robot from easybuild.framework.easyconfig.easyconfig import _easyconfig_files_cache, process_easyconfig, EasyConfig @@ -64,7 +63,6 @@ ORIG_MODULES_TOOL = modules.modules_tool ORIG_ECEC_MODULES_TOOL = ecec.modules_tool -ORIG_ECTOOLS_MODULES_TOOL = ectools.modules_tool ORIG_MODULE_FUNCTION = os.environ.get('module', None) @@ -102,7 +100,6 @@ def install_mock_module(self): """Install MockModule as modules tool.""" # replace Modules class with something we have control over config.modules_tool = mock_module - ectools.modules_tool = mock_module ecec.modules_tool = mock_module robot.modules_tool = mock_module os.environ['module'] = "() { eval `/bin/echo $*`\n}" @@ -124,7 +121,6 @@ def tearDown(self): # restore original modules tool, it may have been tampered with config.modules_tool = ORIG_MODULES_TOOL - ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL ecec.modules_tool = ORIG_ECEC_MODULES_TOOL if ORIG_MODULE_FUNCTION is None: if 'module' in os.environ: