From d85c6ba7b67aade1275c1168bae78a23ec9c01b2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 15 Nov 2018 13:03:34 +0100 Subject: [PATCH 01/20] add support for 'eb --new' to simplify creating new easyconfig files from scratch --- .../framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/tools.py | 138 +++++++++++++++++- easybuild/main.py | 14 +- easybuild/tools/filetools.py | 13 +- easybuild/tools/options.py | 1 + test/framework/easyconfig.py | 124 +++++++++++++++- test/framework/robot.py | 4 - 7 files changed, 277 insertions(+), 19 deletions(-) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 56e82e0e26..bd96bc7e8b 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -62,7 +62,7 @@ ['name', 'version', 'versionprefix', 'versionsuffix'], ['homepage', 'description'], ['toolchain', 'toolchainopts'], - ['sources', 'source_urls'], + ['source_urls', 'sources'], ['patches'], DEPENDENCY_PARAMETERS, ['osdependencies'], diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index aad7ac2073..4814bc8d30 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -50,12 +50,13 @@ from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file +from easybuild.tools.filetools import find_easyconfigs, find_extension, is_patch_file, read_file, resolve_path +from easybuild.tools.filetools import which, write_file from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo -from easybuild.tools.modules import modules_tool +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.multidiff import multidiff from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME @@ -204,11 +205,12 @@ def mk_node_name(spec): # build directed graph dgr = digraph() dgr.add_nodes(all_nodes) + edge_attrs = [('style', 'dotted'), ('color', 'blue'), ('arrowhead', 'diamond')] for spec in specs: for dep in spec['ec'].all_dependencies: dgr.add_edge((spec['module'], dep)) if dep in spec['ec'].build_dependencies: - dgr.add_edge_attributes((spec['module'], dep), attrs=[('style','dotted'), ('color','blue'), ('arrowhead','diamond')]) + dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs) _dep_graph_dump(dgr, filename) @@ -699,3 +701,131 @@ def avail_easyblocks(): easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks + + +def create_new_easyconfig(args): + """Create new easyconfig file based on specified information.""" + + specs = {} + + # regular expression to recognise a version + version_regex = re.compile('^[0-9]') + + # try and discriminate between homepage and source URL + http_args = [arg for arg in args if arg.startswith('http')] + if http_args: + + # first, check and see if we have a full download URL provided as an argument + for arg in http_args: + maybe_filename = os.path.basename(arg) + ext = find_extension(maybe_filename, raise_error=False) + if ext: + specs['source_urls'] = [os.path.dirname(arg)] + # try to recognise downloading of source tarballs by commit ID + if re.search('^[0-9a-f]+\.', maybe_filename): + specs['sources'] = [{ + 'download_filename': maybe_filename, + 'filename': '%(name)s-%(version)s' + ext, + }] + else: + specs['sources'] = [maybe_filename] + http_args.remove(arg) + args.remove(arg) + break + + for arg in http_args: + if specs.get('homepage') is None: + # homepage is more like to be of form https://example.com, i.e. top-level domain + if '/' not in arg.split('://')[-1] or specs.get('source_urls'): + specs['homepage'] = arg + args.remove(arg) + # go to next iteration to avoid also using this value for 'source_urls' + continue + + if specs.get('source_urls') is None: + specs['source_urls'] = [arg] + args.remove(arg) + if specs.get('homepage') is None: + specs['homepage'] = arg + + # determine list of names of known easyblocks, so we can descriminate an easyblock name + easyblock_names = [e['class'] for e in avail_easyblocks().values()] + + # iterate over provided arguments, and try to figure out what they specify + for arg in args: + + # arguments that start with '=' are dealt with first + if re.match('^[a-z_]+=', arg): + key = arg.split('=')[0] + if key in ['builddeps', 'deps']: + deps = [] + for dep in arg.split('=')[-1].split(';'): + deps.append(tuple(dep.split(','))) + specs[key.replace('deps', 'dependencies')] = deps + else: + specs[key] = '='.join(arg.split('=')[1:]) + + # first argument is assumed to be the software name + elif specs.get('name') is None: + specs['name'] = arg + + elif version_regex.match(arg) and specs.get('version') is None: + specs['version'] = arg + + # toolchain is usually specified as /, e.g. intel/2018a + elif '/' in arg and specs.get('toolchain') is None: + tc_name, tc_ver = arg.split('/') + specs['toolchain'] = {'name': tc_name, 'version': tc_ver} + + elif arg in easyblock_names and specs.get('easyblock') is None: + specs['easyblock'] = arg + + elif arg.count(' ') >= 3 and specs.get('description') is None: + specs['description'] = arg + + elif find_extension(arg, raise_error=False): + specs['sources'] = [arg] + + else: + print_warning("Unhandled argument: %s" % quote_str(arg)) + + # make sure that at least name, version and toolchain are known + missing = [p for p in ['name', 'toolchain', 'version'] if specs.get(p) is None] + if missing: + raise EasyBuildError("One or more required parameters are not specified: %s", ', '.join(missing)) + + # inject educated guesses for some important easyconfig parameters that are not specified + educated_guesses = { + 'description': "This is an example description.", + 'homepage': 'https://example.com', + + 'easyblock': 'ConfigureMake', + 'moduleclass': 'tools', + 'sanity_check_paths': {'files': [os.path.join('bin', specs['name'])], 'dirs': []}, + + 'source_urls': ['%(homepage)s'], + 'sources': ['%(name)s-%(version)s.tar.gz'], + } + for key in educated_guesses: + if key not in specs: + specs[key] = educated_guesses[key] + print_warning("No value found for '%s' parameter, injected dummy value: %s" % (key, quote_str(specs[key]))) + + # create EasyConfig instance and dump easyconfig file to current directory + ec_raw = '\n'.join("%s = %s" % (key, quote_str(specs[key])) for key in specs) + + ec, err = None, None + try: + ec = EasyConfig(None, rawtxt=ec_raw) + except EasyBuildError as err: + print_warning("Problem occured when parsing generated easyconfig file: %s" % err) + + if ec: + full_ec_ver = det_full_ec_version(specs) + fn = '%s-%s.eb' % (specs['name'], full_ec_ver) + + ec.dump(fn) + print_msg("Easyconfig file %s created based on specified information!" % fn) + else: + print_msg(ec_raw + '\n', prefix=False) + raise EasyBuildError("Easyconfig file with raw contents shown above NOT created because of errors: %s", err) diff --git a/easybuild/main.py b/easybuild/main.py index 4de63d1166..ac91514e53 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -49,7 +49,7 @@ from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph +from easybuild.framework.easyconfig.tools import categorize_files_by_type, create_new_easyconfig, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak @@ -191,7 +191,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - options, orig_paths = eb_go.options, eb_go.args + options, args = eb_go.options, eb_go.args global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings @@ -243,6 +243,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif options.list_software: print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') + elif options.new: + create_new_easyconfig(args) + # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, @@ -250,6 +253,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): options.list_installed_software, options.list_software, options.merge_pr, + options.new, options.review_pr, options.terse, search_query, @@ -270,14 +274,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: - if orig_paths: + if args: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() - orig_paths.append(eb_file) + args.append(eb_file) - categorized_paths = categorize_files_by_type(orig_paths) + categorized_paths = categorize_files_by_type(args) # command line options that do not require any easyconfigs to be specified new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 937d4e33db..73a3ed7aec 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -829,16 +829,23 @@ def get_local_dirs_purged(): return new_dir -def find_extension(filename): - """Find best match for filename extension.""" +def find_extension(filename, raise_error=True): + """ + Find best match for filename extension. + + :param raise_error: raise an error if no known extension was found in specified filename; + if False, just returns None if no known extension was found + """ # sort by length, so longest file extensions get preference suffixes = sorted(EXTRACT_CMDS.keys(), key=len, reverse=True) pat = r'(?P%s)$' % '|'.join([s.replace('.', '\\.') for s in suffixes]) res = re.search(pat, filename, flags=re.IGNORECASE) if res: ext = res.group('ext') - else: + elif raise_error: raise EasyBuildError('Unknown file type for file %s', filename) + else: + ext = None return ext diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4df30e7baa..dd5899208c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -666,6 +666,7 @@ def easyconfig_options(self): opts = OrderedDict({ 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), + 'new': ("Create a new easyconfig file based on specified information", None, 'store_true', False), }) self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='') diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 0447b20c2d..0718483264 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -61,8 +61,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates -from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, read_file, remove_file, symlink -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file, remove_file, symlink +from easybuild.tools.filetools import write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata @@ -2357,6 +2357,126 @@ def test_deprecated(self): error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n\(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + def test_new_ec(self): + """Test creating new easyconfigs via 'eb --new'.""" + + def run_eb_new(args): + """Helper function to run 'eb --new' with specified arguments.""" + change_dir(self.test_prefix) + self.mock_stdout(True) + self.mock_stderr(True) + self.eb_main(['--new'] + args, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + return (stdout, stderr) + + # running without any arguments results in a clean error + error_pattern = "One or more required parameters are not specified: %s" + self.assertErrorRegex(EasyBuildError, error_pattern % "name, toolchain, version", run_eb_new, []) + + # partial specs also results in a clean error + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", run_eb_new, ['toy']) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", run_eb_new, ['toy', '1.2.3']) + self.assertErrorRegex(EasyBuildError, error_pattern % "version", run_eb_new, ['toy', 'foss/2018a']) + + # when just the required specs are specified, a lot of warnings are spit out + (stdout, stderr) = run_eb_new(['toy', '1.2.3', 'foss/2018b']) + + ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') + self.assertTrue(os.path.exists(ec_fp)) + + expected_stdout = "== Easyconfig file %s created based on specified information!" + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'toy') + self.assertEqual(ec.version, '1.2.3') + self.assertEqual(ec['toolchain'], {'name': 'foss', 'version': '2018b'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + + warn_params = ['description', 'easyblock', 'homepage', 'moduleclass', 'sanity_check_paths', + 'sources', 'source_urls'] + for param in warn_params: + self.assertTrue("WARNING: No value found for '%s' parameter, injected dummy value" % param in stderr) + # no unused arguments + self.assertFalse("WARNING: Unhandled argument" in stderr) + + # warnings are printed for arguments that can't be matched to an easyconfig parameter + (stdout, stderr) = run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertTrue('WARNING: Unhandled argument: "this_is_a_useless_value"' in stderr) + + # easyblock names are recognized + for easyblock in ['Toolchain', 'EB_toy']: + (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', easyblock]) + + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertFalse("WARNING: Unhandled argument" in stderr) + + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], easyblock) + + # check handling of description (first arguments with 3 or more spaces) + (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"]) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['description'], "this is a description") + self.assertTrue('WARNING: Unhandled argument: "not a description"' in stderr) + + # check handling of deps/builddeps + (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4']) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertFalse("WARNING: Unhandled argument" in stderr) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + self.assertEqual(len(ec['dependencies']), 2) + self.assertEqual(ec['dependencies'][0]['name'], 'toy') + self.assertEqual(ec['dependencies'][0]['version'], '0.0') + self.assertEqual(ec['dependencies'][1]['name'], 'GCC') + self.assertEqual(ec['dependencies'][1]['version'], '4.9.2') + self.assertEqual(len(ec['builddependencies']), 1) + self.assertEqual(ec['builddependencies'][0]['name'], 'gzip') + self.assertEqual(ec['builddependencies'][0]['version'], '1.4') + + # check handling of easyconfig parameters that are specified by =, which get preference + (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', + 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertTrue('WARNING: Unhandled argument: "Toolchain"' in stderr) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + + # using an unknown easyblock means trouble + error_pattern = "Easyconfig file with raw contents shown above NOT created because of errors: " + error_pattern += "'Failed to obtain class for NoSuchEasyBlock easyblock .*'" + args = ['bar', '3.4.5', 'foss/2018a', 'easyblock=NoSuchEasyBlock'] + self.assertErrorRegex(EasyBuildError, error_pattern, run_eb_new, args) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/robot.py b/test/framework/robot.py index e98f7a86cd..99c8e9617f 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -38,7 +38,6 @@ from unittest import TextTestRunner import easybuild.framework.easyconfig.easyconfig as ecec -import easybuild.framework.easyconfig.tools as ectools import easybuild.tools.build_log import easybuild.tools.robot as robot from easybuild.framework.easyconfig.easyconfig import _easyconfig_files_cache, process_easyconfig, EasyConfig @@ -64,7 +63,6 @@ ORIG_MODULES_TOOL = modules.modules_tool ORIG_ECEC_MODULES_TOOL = ecec.modules_tool -ORIG_ECTOOLS_MODULES_TOOL = ectools.modules_tool ORIG_MODULE_FUNCTION = os.environ.get('module', None) @@ -102,7 +100,6 @@ def install_mock_module(self): """Install MockModule as modules tool.""" # replace Modules class with something we have control over config.modules_tool = mock_module - ectools.modules_tool = mock_module ecec.modules_tool = mock_module robot.modules_tool = mock_module os.environ['module'] = "() { eval `/bin/echo $*`\n}" @@ -124,7 +121,6 @@ def tearDown(self): # restore original modules tool, it may have been tampered with config.modules_tool = ORIG_MODULES_TOOL - ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL ecec.modules_tool = ORIG_ECEC_MODULES_TOOL if ORIG_MODULE_FUNCTION is None: if 'module' in os.environ: From 7193739dc71afc335dda286c0fc73704e4469b29 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Nov 2018 22:02:09 +0100 Subject: [PATCH 02/20] make -E the short option for --experimental --- easybuild/tools/options.py | 2 +- test/framework/options.py | 37 +++++++++++++++++-------------------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index dd5899208c..61c4712fcf 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -361,7 +361,7 @@ def override_options(self): 'enforce-checksums': ("Enforce availability of checksums for all sources/patches, so they can be verified", None, 'store_true', False), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", - None, 'store_true', False), + None, 'store_true', False, 'E'), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " diff --git a/test/framework/options.py b/test/framework/options.py index 046394ae11..5ed8f7a469 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1294,26 +1294,23 @@ def test_experimental(self): log = fancylogger.getLogger() - # force it to False - topt = EasyBuildOptions( - go_args=['--disable-experimental'], - ) - try: - log.experimental('x') - # sanity check, should never be reached if it works. - self.assertTrue(False, "Experimental logging should be disabled by setting the --disable-experimental option") - except easybuild.tools.build_log.EasyBuildError, err: - # check error message - self.assertTrue('Experimental functionality.' in str(err)) - - # toggle experimental - topt = EasyBuildOptions( - go_args=['--experimental'], - ) - try: - log.experimental('x') - except easybuild.tools.build_log.EasyBuildError, err: - self.assertTrue(False, 'Experimental logging should be allowed by the --experimental option.') + for experimental_opt in ['--experimental', '-E']: + # force it to False + topt = EasyBuildOptions(go_args=['--disable-experimental']) + try: + log.experimental('x') + # sanity check, should never be reached if it works. + self.assertTrue(False, "Experimental logging should be disabled via --disable-experimental") + except easybuild.tools.build_log.EasyBuildError, err: + # check error message + self.assertTrue('Experimental functionality.' in str(err)) + + # toggle experimental + EasyBuildOptions(go_args=[experimental_opt]) + try: + log.experimental('x') + except easybuild.tools.build_log.EasyBuildError, err: + self.assertTrue(False, 'Experimental logging should be allowed by the --experimental option.') # set it back easybuild.tools.build_log.EXPERIMENTAL = orig_value From ed2e394528463d886c3991fb97c3eb5c48906d5d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Nov 2018 22:49:24 +0100 Subject: [PATCH 03/20] add edit_file function + dedicated test for it --- easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 46 +++++++++++++++++++----------------- easybuild/tools/options.py | 1 + test/framework/filetools.py | 12 ++++++++++ 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 72fe9240ed..dca23a14fa 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -134,6 +134,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'download_timeout', 'dump_test_report', 'easyblock', + 'editor', 'extra_modules', 'filter_deps', 'filter_env_vars', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 73a3ed7aec..0f3b63bf2e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -46,6 +46,7 @@ import os import re import shutil +import subprocess import stat import sys import tempfile @@ -59,7 +60,7 @@ # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg from easybuild.tools.config import build_option -from easybuild.tools import run +from easybuild.tools.run import run_cmd try: import requests @@ -371,7 +372,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced if extra_options: cmd = "%s %s" % (cmd, extra_options) - run.run_cmd(cmd, simple=True, force_in_dry_run=forced) + run_cmd(cmd, simple=True, force_in_dry_run=forced) return find_base_dir() @@ -1010,7 +1011,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): _log.debug("Using specified patch level %d for patch %s" % (level, patch_file)) patch_cmd = "patch -b -p%s -i %s" % (level, apatch) - out, ec = run.run_cmd(patch_cmd, simple=False, path=adest, log_ok=False, trace=False) + out, ec = run_cmd(patch_cmd, simple=False, path=adest, log_ok=False, trace=False) if ec: raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out) @@ -1367,7 +1368,7 @@ def move_logs(src_logfile, target_logfile): _log.info("Moved log file %s to %s" % (src_logfile, new_log_path)) if zip_log_cmd: - run.run_cmd("%s %s" % (zip_log_cmd, new_log_path)) + run_cmd("%s %s" % (zip_log_cmd, new_log_path)) _log.info("Zipped log %s using '%s'", new_log_path, zip_log_cmd) except (IOError, OSError), err: @@ -1532,21 +1533,6 @@ def decode_class_name(name): return decode_string(name) -def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): - """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead""" - _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): - """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead""" - _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead""" - _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - def det_size(path): """ Determine total size of given filepath (in bytes). @@ -1724,6 +1710,22 @@ def copy(paths, target_path, force_in_dry_run=False): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) +def edit_file(fp): + """Edit file at specified location.""" + + editor_cmd = build_option('editor') + if which(editor_cmd) is None: + raise EasyBuildError("Editor command '%s' not found", editor_cmd) + + _log.info("Editing %s using '%s'", fp, editor_cmd) + # can't use run_cmd here, just hangs without actually bringing up the editor... + exit_code = subprocess.call([editor_cmd, fp]) + if exit_code: + raise EasyBuildError("Editor '%s' failed with exit code %s while editing", editor_cmd, exit_code, fp) + else: + _log.info("Done editing %s using '%s' (exit code %s)", fp, editor_cmd, exit_code) + + def get_source_tarball_from_git(filename, targetdir, git_config): """ Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it @@ -1780,7 +1782,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) # if a specific commit is asked for, check it out if commit: @@ -1788,11 +1790,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: checkout_cmd.extend(['&&', 'git', 'submodule', 'update']) - run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) # create an archive and delete the git repo directory tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) change_dir(cwd) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 61c4712fcf..7f6cf85749 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -453,6 +453,7 @@ def config_options(self): mk_full_default_path('containerpath')), 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", 'strlist', 'store', None), + 'editor': ("Editor command to use (used for --edit)", 'str', 'store', 'vim'), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 67955f90fd..572ddd1a3a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1797,6 +1797,18 @@ def test_is_sha256_checksum(self): ]: self.assertFalse(ft.is_sha256_checksum(not_a_sha256_checksum)) + def test_edit_file(self): + """Test for edit_file function.""" + + testfile = os.path.join(self.test_prefix, 'test.txt') + ft.write_file(testfile, 'test123') + + init_config(build_options={'editor': 'echo'}) + ft.edit_file(testfile) + + init_config(build_options={'editor': 'nosucheditorcommand'}) + self.assertErrorRegex(EasyBuildError, "Editor command 'nosucheditorcommand' not found", ft.edit_file, testfile) + def suite(): """ returns all the testcases in this module """ From 712303c7cd6901d28e159ee0bfbec3fbc7d73e79 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Nov 2018 22:57:20 +0100 Subject: [PATCH 04/20] add support for --cat, --copy, --edit that collaborate with --new/-n --- easybuild/framework/easyconfig/tools.py | 16 ++++++++-------- easybuild/main.py | 16 ++++++++++++++-- easybuild/tools/options.py | 6 +++++- test/framework/easyconfig.py | 2 +- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 4814bc8d30..ab429987dd 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -703,7 +703,7 @@ def avail_easyblocks(): return easyblocks -def create_new_easyconfig(args): +def create_new_easyconfig(path, args): """Create new easyconfig file based on specified information.""" specs = {} @@ -820,12 +820,12 @@ def create_new_easyconfig(args): except EasyBuildError as err: print_warning("Problem occured when parsing generated easyconfig file: %s" % err) - if ec: - full_ec_ver = det_full_ec_version(specs) - fn = '%s-%s.eb' % (specs['name'], full_ec_ver) - - ec.dump(fn) - print_msg("Easyconfig file %s created based on specified information!" % fn) - else: + if err: print_msg(ec_raw + '\n', prefix=False) raise EasyBuildError("Easyconfig file with raw contents shown above NOT created because of errors: %s", err) + else: + full_ec_ver = det_full_ec_version(specs) + fp = os.path.join(path, '%s-%s.eb' % (specs['name'], full_ec_ver)) + + ec.dump(fp) + return fp diff --git a/easybuild/main.py b/easybuild/main.py index ac91514e53..34aaebf4fa 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -56,7 +56,7 @@ from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, edit_file, read_file, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token from easybuild.tools.github import new_pr, merge_pr, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook @@ -244,7 +244,19 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') elif options.new: - create_new_easyconfig(args) + tmpfp = create_new_easyconfig(eb_tmpdir, args) + + if options.edit: + edit_file(tmpfp) + + fp = os.path.join(options.copy or '', os.path.basename(tmpfp)) + copy_file(tmpfp, fp) + + if options.cat: + print_msg("Contents of easyconfig file %s:\n" % fp) + print_msg(read_file(fp), prefix=False) + + print_msg("Easyconfig file %s created!" % fp) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7f6cf85749..8c0907935d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -665,9 +665,13 @@ def easyconfig_options(self): descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = OrderedDict({ + 'cat': ("Print contents of easyconfig file(s) (works well with --search)", None, 'store_true', False), + 'copy': ("Copy easyconfig file(s) to specified location (works well with --search)", + None, 'store_or_None', '', {'metavar': "PATH"}), + 'edit': ("Edit easyconfig file (using editor specified via --editor)", None, 'store_true', False), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), - 'new': ("Create a new easyconfig file based on specified information", None, 'store_true', False), + 'new': ("Create a new easyconfig file based on specified information", None, 'store_true', False, 'n'), }) self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='') diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 0718483264..54c25fa9d8 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2388,7 +2388,7 @@ def run_eb_new(args): ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') self.assertTrue(os.path.exists(ec_fp)) - expected_stdout = "== Easyconfig file %s created based on specified information!" + expected_stdout = "== Easyconfig file %s created!" self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) ec = EasyConfig(ec_fp) From 84c513d753d22f367e124120e920ab326355fa56 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Nov 2018 23:20:57 +0100 Subject: [PATCH 05/20] add support to search_easyconfigs function to return results rather than print them --- easybuild/tools/robot.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 8e40328b60..9f569879fb 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -279,7 +279,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] - avail_modules = [m for m in avail_modules if not m in being_installed] + avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s' % easyconfigs) @@ -352,7 +352,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: - if not ec in easyconfigs + additional: + if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: @@ -379,8 +379,15 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): return ordered_ecs -def search_easyconfigs(query, short=False, filename_only=False, terse=False): - """Search for easyconfigs, if a query is provided.""" +def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_hits=False): + """ + Search for easyconfigs, if a query is provided. + + :param short: generate short output, by figuring out common prefix among hits + :param filename_only: only return filenames, not file paths + :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing + :param return_hits: return list of search hits, rather than printing them + """ search_path = build_option('robot_path') if not search_path: search_path = [os.getcwd()] @@ -390,11 +397,15 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): ignore_dirs = build_option('ignore_dirs') + # we need long full path results when returns list of hits + if return_hits: + filename_only, short, terse = False, False, True + # note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse, silent=True, filename_only=False) - # filter out archived easyconfigs, these are handled separately + # filter out archived easyconfigs, these are handled separately hits, archived_hits = [], [] for hit in _hits: if EASYCONFIGS_ARCHIVE_DIR in hit.split(os.path.sep): @@ -423,6 +434,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): if not terse: lines.extend(['', "Matching archived easyconfigs:", '']) lines.extend(tmpl % hit for hit in archived_hits) + hits.extend(archived_hits) elif not terse: cnt = len(archived_hits) lines.extend([ @@ -430,4 +442,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt, ]) - print '\n'.join(lines) + if return_hits: + return hits + else: + print '\n'.join(lines) From ecd61f0b3150d3a65c7c53311c37ae423cf13698 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 00:13:23 +0100 Subject: [PATCH 06/20] fix default value for --copy to '.' --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8c0907935d..790115618a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -667,7 +667,7 @@ def easyconfig_options(self): opts = OrderedDict({ 'cat': ("Print contents of easyconfig file(s) (works well with --search)", None, 'store_true', False), 'copy': ("Copy easyconfig file(s) to specified location (works well with --search)", - None, 'store_or_None', '', {'metavar': "PATH"}), + None, 'store_or_None', '.', {'metavar': "PATH"}), 'edit': ("Edit easyconfig file (using editor specified via --editor)", None, 'store_true', False), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), From b832b747cccee3d056bb3030609153111b826e1c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 00:37:31 +0100 Subject: [PATCH 07/20] replace --editor with --editor-command-template, and enhance it to detect whether changes were made --- easybuild/tools/config.py | 2 +- easybuild/tools/filetools.py | 36 +++++++++++++++++++++++++++++------- easybuild/tools/options.py | 2 +- test/framework/filetools.py | 16 ++++++++++++---- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index dca23a14fa..b5995d56ec 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -134,7 +134,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'download_timeout', 'dump_test_report', 'easyblock', - 'editor', + 'editor_command_template', 'extra_modules', 'filter_deps', 'filter_env_vars', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 0f3b63bf2e..e1071f63a4 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1713,17 +1713,39 @@ def copy(paths, target_path, force_in_dry_run=False): def edit_file(fp): """Edit file at specified location.""" - editor_cmd = build_option('editor') - if which(editor_cmd) is None: - raise EasyBuildError("Editor command '%s' not found", editor_cmd) + tmpl_editor_cmd = build_option('editor_command_template') + if not tmpl_editor_cmd: + raise EasyBuildError("No editor command template specified (see --editor-command-template)") + elif tmpl_editor_cmd.count('%s') != 1: + raise EasyBuildError("Editor command template should contain exactly one '%s' as placeholder for filename: '" + + tmpl_editor_cmd + "'") + + # read file before editing so we can detect whether or not any changes were made + txt = read_file(fp) + + # replace placeholder %s with path of file to edit + cmd = [] + for cmd_part in tmpl_editor_cmd.split(' '): + if cmd_part == '%s': + cmd.append(fp) + else: + cmd.append(cmd_part) - _log.info("Editing %s using '%s'", fp, editor_cmd) + print_msg("editing %s... " % fp, newline=False) # can't use run_cmd here, just hangs without actually bringing up the editor... - exit_code = subprocess.call([editor_cmd, fp]) + try: + exit_code = subprocess.call(cmd) + except OSError as err: + raise EasyBuildError("Editor command '%s' failed: %s", ' '.join(cmd), err) if exit_code: - raise EasyBuildError("Editor '%s' failed with exit code %s while editing", editor_cmd, exit_code, fp) + raise EasyBuildError("Editor command '%s' failed with exit code %s", ' '.join(cmd), exit_code) + + if txt == read_file(fp): + done_msg = 'done (no changes)' else: - _log.info("Done editing %s using '%s' (exit code %s)", fp, editor_cmd, exit_code) + done_msg = 'done (changes detected)' + + print_msg(done_msg, prefix=False) def get_source_tarball_from_git(filename, targetdir, git_config): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 790115618a..ff79ec4bff 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -453,7 +453,7 @@ def config_options(self): mk_full_default_path('containerpath')), 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", 'strlist', 'store', None), - 'editor': ("Editor command to use (used for --edit)", 'str', 'store', 'vim'), + 'editor-command-template': ("Editor command template to use (used for --edit)", 'str', 'store', "vim %s"), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 572ddd1a3a..26c230ccd8 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -41,7 +41,6 @@ import urllib2 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner -from urllib2 import URLError import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError @@ -1803,11 +1802,20 @@ def test_edit_file(self): testfile = os.path.join(self.test_prefix, 'test.txt') ft.write_file(testfile, 'test123') - init_config(build_options={'editor': 'echo'}) + for cmd_tmpl in ['true', '%s true %s']: + init_config(build_options={'editor_command_template': cmd_tmpl}) + error_pattern = "Editor command template should contain exactly one '%s' as placeholder for filename" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.edit_file, testfile) + + init_config(build_options={'editor_command_template': 'true %s'}) + self.mock_stdout(True) ft.edit_file(testfile) + stdout = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(stdout.strip(), "== editing %s... done (no changes)" % testfile) - init_config(build_options={'editor': 'nosucheditorcommand'}) - self.assertErrorRegex(EasyBuildError, "Editor command 'nosucheditorcommand' not found", ft.edit_file, testfile) + init_config(build_options={'editor_command_template': 'false %s'}) + self.assertErrorRegex(EasyBuildError, "Editor command 'false .*' failed", ft.edit_file, testfile) def suite(): From 6be0151b65610d2129021504a0f3b567f2370ccb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 00:52:09 +0100 Subject: [PATCH 08/20] make --search aware of --cat, --copy and --edit --- easybuild/main.py | 71 ++++++++++++++++++++++++++++-------- test/framework/easyconfig.py | 2 +- test/framework/filetools.py | 4 ++ 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 34aaebf4fa..96b573e512 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -173,6 +173,60 @@ def run_contrib_style_checks(ecs, check_contrib, check_style): return check_contrib or check_style +def handle_cat_copy_edit(filepaths, print_contents=False, copy=False, edit=False, target_dir=None): + """Handle use of --cat, --copy and --edit.""" + + res = [] + for orig_fp in filepaths: + if copy: + fp = os.path.join(target_dir, os.path.basename(orig_fp)) + copy_file(orig_fp, fp) + res.append(fp) + else: + fp = orig_fp + + if edit: + edit_file(fp) + + if print_contents: + print_msg("Contents of easyconfig file %s:\n" % fp) + print_msg(read_file(fp), prefix=False) + + return res + + +def handle_new(options, tmpdir, args): + """Handle use of --new.""" + tmpfp = create_new_easyconfig(tmpdir, args) + + # use current directory as default location to save generated file, in case no location is specified via --copy + target_dir = options.copy or '.' + res = handle_cat_copy_edit([tmpfp], print_contents=options.cat, copy=True, edit=options.edit, target_dir=target_dir) + + print_msg("easyconfig file %s created!" % res[0]) + + +def handle_search(options, search_query): + """Handle use of --search.""" + + search_action = options.cat or options.copy or options.edit + res = search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, + terse=options.terse, return_hits=search_action) + + if search_action: + # only perform action(s) if there's a single search result, unless --force is used + if len(res) > 1 and not options.force: + raise EasyBuildError("Found %d results, not performing search action(s) without --force", len(res)) + + res = handle_cat_copy_edit(res, print_contents=options.cat, copy=options.copy, edit=options.edit, + target_dir=options.copy or '.') + + if options.copy: + print_msg("copied easyconfig files:") + for path in res: + print_msg("* %s" % path, prefix=False) + + def clean_exit(logfile, tmpdir, testing, silent=False): """Small utility function to perform a clean exit.""" cleanup(logfile, tmpdir, testing, silent=silent) @@ -220,8 +274,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # search for easyconfigs, if a query is specified if search_query: - search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, - terse=options.terse) + handle_search(options, search_query) # GitHub options that warrant a silent cleanup & exit if options.check_github: @@ -244,19 +297,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') elif options.new: - tmpfp = create_new_easyconfig(eb_tmpdir, args) - - if options.edit: - edit_file(tmpfp) - - fp = os.path.join(options.copy or '', os.path.basename(tmpfp)) - copy_file(tmpfp, fp) - - if options.cat: - print_msg("Contents of easyconfig file %s:\n" % fp) - print_msg(read_file(fp), prefix=False) - - print_msg("Easyconfig file %s created!" % fp) + handle_new(options, eb_tmpdir, args) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 54c25fa9d8..027f11dc2f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2388,7 +2388,7 @@ def run_eb_new(args): ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') self.assertTrue(os.path.exists(ec_fp)) - expected_stdout = "== Easyconfig file %s created!" + expected_stdout = "== easyconfig file ./%s created!" self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) ec = EasyConfig(ec_fp) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 26c230ccd8..a32ea3cff0 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1815,7 +1815,11 @@ def test_edit_file(self): self.assertEqual(stdout.strip(), "== editing %s... done (no changes)" % testfile) init_config(build_options={'editor_command_template': 'false %s'}) + self.mock_stdout(True) self.assertErrorRegex(EasyBuildError, "Editor command 'false .*' failed", ft.edit_file, testfile) + stdout = self.get_stdout() + self.mock_stdout(False) + self.assertEqual(stdout.strip(), "== editing %s..." % testfile) def suite(): From 7801f476a5a53763c15625449ad4b08687003eab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 10:49:18 +0100 Subject: [PATCH 09/20] add test for --new combined with --cat, --copy, --edit + smarter handling of argument passed to --copy --- easybuild/main.py | 16 +++-- test/framework/easyconfig.py | 132 ++++++++++++++++++++++++++++------- 2 files changed, 116 insertions(+), 32 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 96b573e512..dfd1df9bdd 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -173,13 +173,19 @@ def run_contrib_style_checks(ecs, check_contrib, check_style): return check_contrib or check_style -def handle_cat_copy_edit(filepaths, print_contents=False, copy=False, edit=False, target_dir=None): +def handle_cat_copy_edit(filepaths, print_contents=False, copy=False, edit=False, target=None): """Handle use of --cat, --copy and --edit.""" res = [] for orig_fp in filepaths: if copy: - fp = os.path.join(target_dir, os.path.basename(orig_fp)) + # if target location is an existing directory, retain filename + # if not, assume last part of specific location is filename + if os.path.isdir(target): + fp = os.path.join(target, os.path.basename(orig_fp)) + else: + fp = target + copy_file(orig_fp, fp) res.append(fp) else: @@ -200,8 +206,8 @@ def handle_new(options, tmpdir, args): tmpfp = create_new_easyconfig(tmpdir, args) # use current directory as default location to save generated file, in case no location is specified via --copy - target_dir = options.copy or '.' - res = handle_cat_copy_edit([tmpfp], print_contents=options.cat, copy=True, edit=options.edit, target_dir=target_dir) + target = options.copy or '.' + res = handle_cat_copy_edit([tmpfp], print_contents=options.cat, copy=True, edit=options.edit, target=target) print_msg("easyconfig file %s created!" % res[0]) @@ -219,7 +225,7 @@ def handle_search(options, search_query): raise EasyBuildError("Found %d results, not performing search action(s) without --force", len(res)) res = handle_cat_copy_edit(res, print_contents=options.cat, copy=options.copy, edit=options.edit, - target_dir=options.copy or '.') + target=options.copy or '.') if options.copy: print_msg("copied easyconfig files:") diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 027f11dc2f..743568d9ec 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -61,8 +61,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates -from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file, remove_file, symlink -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file +from easybuild.tools.filetools import remove_dir, remove_file, symlink, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata @@ -2357,33 +2357,33 @@ def test_deprecated(self): error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n\(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) - def test_new_ec(self): - """Test creating new easyconfigs via 'eb --new'.""" + def run_eb_new(self, args): + """Helper function to run 'eb --new' with specified arguments.""" + change_dir(self.test_prefix) + self.mock_stdout(True) + self.mock_stderr(True) + self.eb_main(['--new'] + args, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) - def run_eb_new(args): - """Helper function to run 'eb --new' with specified arguments.""" - change_dir(self.test_prefix) - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(['--new'] + args, raise_error=True) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + return (stdout, stderr) - return (stdout, stderr) + def test_new_ec(self): + """Test creating new easyconfigs via 'eb --new'.""" # running without any arguments results in a clean error error_pattern = "One or more required parameters are not specified: %s" - self.assertErrorRegex(EasyBuildError, error_pattern % "name, toolchain, version", run_eb_new, []) + self.assertErrorRegex(EasyBuildError, error_pattern % "name, toolchain, version", self.run_eb_new, []) # partial specs also results in a clean error - self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", run_eb_new, ['toy']) - self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", run_eb_new, ['toy', '1.2.3']) - self.assertErrorRegex(EasyBuildError, error_pattern % "version", run_eb_new, ['toy', 'foss/2018a']) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", self.run_eb_new, ['toy']) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", self.run_eb_new, ['toy', '1.2.3']) + self.assertErrorRegex(EasyBuildError, error_pattern % "version", self.run_eb_new, ['toy', 'foss/2018a']) # when just the required specs are specified, a lot of warnings are spit out - (stdout, stderr) = run_eb_new(['toy', '1.2.3', 'foss/2018b']) + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b']) ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') self.assertTrue(os.path.exists(ec_fp)) @@ -2406,14 +2406,14 @@ def run_eb_new(args): self.assertFalse("WARNING: Unhandled argument" in stderr) # warnings are printed for arguments that can't be matched to an easyconfig parameter - (stdout, stderr) = run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) self.assertTrue('WARNING: Unhandled argument: "this_is_a_useless_value"' in stderr) # easyblock names are recognized for easyblock in ['Toolchain', 'EB_toy']: - (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', easyblock]) + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', easyblock]) ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') self.assertTrue(os.path.exists(ec_fp)) @@ -2427,7 +2427,8 @@ def run_eb_new(args): self.assertEqual(ec['easyblock'], easyblock) # check handling of description (first arguments with 3 or more spaces) - (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"]) + args = ['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"] + (stdout, stderr) = self.run_eb_new(args) ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) @@ -2439,7 +2440,8 @@ def run_eb_new(args): self.assertTrue('WARNING: Unhandled argument: "not a description"' in stderr) # check handling of deps/builddeps - (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4']) + args = ['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4'] + (stdout, stderr) = self.run_eb_new(args) ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) @@ -2460,8 +2462,8 @@ def run_eb_new(args): self.assertEqual(ec['builddependencies'][0]['version'], '1.4') # check handling of easyconfig parameters that are specified by =, which get preference - (stdout, stderr) = run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', - 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', + 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) self.assertTrue('WARNING: Unhandled argument: "Toolchain"' in stderr) @@ -2475,7 +2477,83 @@ def run_eb_new(args): error_pattern = "Easyconfig file with raw contents shown above NOT created because of errors: " error_pattern += "'Failed to obtain class for NoSuchEasyBlock easyblock .*'" args = ['bar', '3.4.5', 'foss/2018a', 'easyblock=NoSuchEasyBlock'] - self.assertErrorRegex(EasyBuildError, error_pattern, run_eb_new, args) + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, args) + + def test_new_cat_copy_edit(self): + """Test combining --new with --cat, --copy and --edit.""" + + args = ['toy', '1.2.3', 'foss/2018b'] + ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') + + # contents of easyconfig are printed when using --cat, file still is created into current directory + (stdout, stderr) = self.run_eb_new(args + ['--cat']) + self.assertTrue(os.path.exists(ec_fp)) + + patterns = [ + r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'", + r"^name = 'toy'$", + r"^version = '1.2.3'$", + r"^toolchain = {'name': 'foss', 'version': '2018b'}$", + r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + + # clean up test dir + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # check whether easyconfig file is copied when --copy is used + # --copy without an argument doesn't change anything, since --new always dumps easyconfig in currect dir + (stdout, stderr) = self.run_eb_new(args + ['--copy']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), "== easyconfig file ./toy-1.2.3-foss-2018b.eb created!") + + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # if as argument is passed to --copy, the easyconfig file is created there instead + # (and not in the default location) + copy_args = [ + os.path.join(self.test_prefix, 'test.eb'), # non-existing file in existing directory + os.path.join(self.test_prefix, 'asubdir', 'test.eb'), # non-existing file in non-existing directory + os.path.join(self.test_prefix, 'asubdir'), # existing (sub)directory, no filename specified + ] + for copy_arg in copy_args: + if os.path.isdir(copy_arg): + test_ec = os.path.join(copy_arg, 'toy-1.2.3-foss-2018b.eb') + else: + test_ec = copy_arg + + (stdout, stderr) = self.run_eb_new(args + ['--copy', copy_arg]) + self.assertFalse(os.path.exists(ec_fp)) + self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) + self.assertEqual(stdout.strip(), "== easyconfig file %s created!" % test_ec) + + # combination of --copy and --cat also works + (stdout, stderr) = self.run_eb_new(args + ['--cat', '--copy', copy_arg]) + self.assertFalse(os.path.exists(ec_fp)) + self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) + patterns[0] = r"== Contents of easyconfig file %s:\n\neasyblock = 'ConfigureMake'" % test_ec + patterns[-1] = r"== easyconfig file %s created!" % test_ec + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # throwing --edit in the mix + args.extend(['--edit', '--editor-command-template=echo %s']) + (stdout, stderr) = self.run_eb_new(args + ['--cat']) + self.assertTrue(os.path.exists(ec_fp)) + patterns[0] = r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'" + patterns[-1] = r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!" + patterns.insert(0, r"== editing ./toy-1.2.3-foss-2018b.eb... done \(no changes\)") + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) def suite(): From 398732b2bb4dcc7d5fdecf8a02f5e4db17726267 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 11:05:28 +0100 Subject: [PATCH 10/20] rename --cat to --show, move tests for --new to options.py test module (where they belong) --- easybuild/main.py | 6 +- easybuild/tools/options.py | 2 +- test/framework/easyconfig.py | 201 +---------------------------------- test/framework/options.py | 201 ++++++++++++++++++++++++++++++++++- 4 files changed, 205 insertions(+), 205 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index dfd1df9bdd..90a32f2854 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -207,7 +207,7 @@ def handle_new(options, tmpdir, args): # use current directory as default location to save generated file, in case no location is specified via --copy target = options.copy or '.' - res = handle_cat_copy_edit([tmpfp], print_contents=options.cat, copy=True, edit=options.edit, target=target) + res = handle_cat_copy_edit([tmpfp], print_contents=options.show, copy=True, edit=options.edit, target=target) print_msg("easyconfig file %s created!" % res[0]) @@ -215,7 +215,7 @@ def handle_new(options, tmpdir, args): def handle_search(options, search_query): """Handle use of --search.""" - search_action = options.cat or options.copy or options.edit + search_action = options.show or options.copy or options.edit res = search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse, return_hits=search_action) @@ -224,7 +224,7 @@ def handle_search(options, search_query): if len(res) > 1 and not options.force: raise EasyBuildError("Found %d results, not performing search action(s) without --force", len(res)) - res = handle_cat_copy_edit(res, print_contents=options.cat, copy=options.copy, edit=options.edit, + res = handle_cat_copy_edit(res, print_contents=options.show, copy=options.copy, edit=options.edit, target=options.copy or '.') if options.copy: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ff79ec4bff..efa94fd89f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -665,13 +665,13 @@ def easyconfig_options(self): descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = OrderedDict({ - 'cat': ("Print contents of easyconfig file(s) (works well with --search)", None, 'store_true', False), 'copy': ("Copy easyconfig file(s) to specified location (works well with --search)", None, 'store_or_None', '.', {'metavar': "PATH"}), 'edit': ("Edit easyconfig file (using editor specified via --editor)", None, 'store_true', False), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), 'new': ("Create a new easyconfig file based on specified information", None, 'store_true', False, 'n'), + 'show': ("Print contents of easyconfig file(s) (works well with --search)", None, 'store_true', False), }) self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='') diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 743568d9ec..b267dfc95d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -61,8 +61,7 @@ from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates -from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file -from easybuild.tools.filetools import remove_dir, remove_file, symlink, write_file +from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, read_file, remove_file, symlink, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata @@ -2357,204 +2356,6 @@ def test_deprecated(self): error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n\(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) - def run_eb_new(self, args): - """Helper function to run 'eb --new' with specified arguments.""" - change_dir(self.test_prefix) - self.mock_stdout(True) - self.mock_stderr(True) - self.eb_main(['--new'] + args, raise_error=True) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) - - return (stdout, stderr) - - def test_new_ec(self): - """Test creating new easyconfigs via 'eb --new'.""" - - # running without any arguments results in a clean error - error_pattern = "One or more required parameters are not specified: %s" - self.assertErrorRegex(EasyBuildError, error_pattern % "name, toolchain, version", self.run_eb_new, []) - - # partial specs also results in a clean error - self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", self.run_eb_new, ['toy']) - self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", self.run_eb_new, ['toy', '1.2.3']) - self.assertErrorRegex(EasyBuildError, error_pattern % "version", self.run_eb_new, ['toy', 'foss/2018a']) - - # when just the required specs are specified, a lot of warnings are spit out - (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b']) - - ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') - self.assertTrue(os.path.exists(ec_fp)) - - expected_stdout = "== easyconfig file ./%s created!" - self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) - - ec = EasyConfig(ec_fp) - self.assertEqual(ec.name, 'toy') - self.assertEqual(ec.version, '1.2.3') - self.assertEqual(ec['toolchain'], {'name': 'foss', 'version': '2018b'}) - self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified - self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified - - warn_params = ['description', 'easyblock', 'homepage', 'moduleclass', 'sanity_check_paths', - 'sources', 'source_urls'] - for param in warn_params: - self.assertTrue("WARNING: No value found for '%s' parameter, injected dummy value" % param in stderr) - # no unused arguments - self.assertFalse("WARNING: Unhandled argument" in stderr) - - # warnings are printed for arguments that can't be matched to an easyconfig parameter - (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) - self.assertTrue(os.path.exists(ec_fp)) - self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) - self.assertTrue('WARNING: Unhandled argument: "this_is_a_useless_value"' in stderr) - - # easyblock names are recognized - for easyblock in ['Toolchain', 'EB_toy']: - (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', easyblock]) - - ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') - self.assertTrue(os.path.exists(ec_fp)) - self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) - self.assertFalse("WARNING: Unhandled argument" in stderr) - - ec = EasyConfig(ec_fp) - self.assertEqual(ec.name, 'bar') - self.assertEqual(ec.version, '3.4.5') - self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) - self.assertEqual(ec['easyblock'], easyblock) - - # check handling of description (first arguments with 3 or more spaces) - args = ['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"] - (stdout, stderr) = self.run_eb_new(args) - ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') - self.assertTrue(os.path.exists(ec_fp)) - self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) - ec = EasyConfig(ec_fp) - self.assertEqual(ec.name, 'bar') - self.assertEqual(ec.version, '3.4.5') - self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) - self.assertEqual(ec['description'], "this is a description") - self.assertTrue('WARNING: Unhandled argument: "not a description"' in stderr) - - # check handling of deps/builddeps - args = ['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4'] - (stdout, stderr) = self.run_eb_new(args) - ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') - self.assertTrue(os.path.exists(ec_fp)) - self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) - self.assertFalse("WARNING: Unhandled argument" in stderr) - ec = EasyConfig(ec_fp) - self.assertEqual(ec.name, 'bar') - self.assertEqual(ec.version, '3.4.5') - self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) - self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified - self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified - self.assertEqual(len(ec['dependencies']), 2) - self.assertEqual(ec['dependencies'][0]['name'], 'toy') - self.assertEqual(ec['dependencies'][0]['version'], '0.0') - self.assertEqual(ec['dependencies'][1]['name'], 'GCC') - self.assertEqual(ec['dependencies'][1]['version'], '4.9.2') - self.assertEqual(len(ec['builddependencies']), 1) - self.assertEqual(ec['builddependencies'][0]['name'], 'gzip') - self.assertEqual(ec['builddependencies'][0]['version'], '1.4') - - # check handling of easyconfig parameters that are specified by =, which get preference - (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', - 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) - self.assertTrue(os.path.exists(ec_fp)) - self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) - self.assertTrue('WARNING: Unhandled argument: "Toolchain"' in stderr) - self.assertEqual(ec.name, 'bar') - self.assertEqual(ec.version, '3.4.5') - self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) - self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified - self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified - - # using an unknown easyblock means trouble - error_pattern = "Easyconfig file with raw contents shown above NOT created because of errors: " - error_pattern += "'Failed to obtain class for NoSuchEasyBlock easyblock .*'" - args = ['bar', '3.4.5', 'foss/2018a', 'easyblock=NoSuchEasyBlock'] - self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, args) - - def test_new_cat_copy_edit(self): - """Test combining --new with --cat, --copy and --edit.""" - - args = ['toy', '1.2.3', 'foss/2018b'] - ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') - - # contents of easyconfig are printed when using --cat, file still is created into current directory - (stdout, stderr) = self.run_eb_new(args + ['--cat']) - self.assertTrue(os.path.exists(ec_fp)) - - patterns = [ - r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'", - r"^name = 'toy'$", - r"^version = '1.2.3'$", - r"^toolchain = {'name': 'foss', 'version': '2018b'}$", - r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!", - ] - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) - - # clean up test dir - remove_dir(self.test_prefix) - mkdir(self.test_prefix) - - # check whether easyconfig file is copied when --copy is used - # --copy without an argument doesn't change anything, since --new always dumps easyconfig in currect dir - (stdout, stderr) = self.run_eb_new(args + ['--copy']) - self.assertTrue(os.path.exists(ec_fp)) - self.assertEqual(stdout.strip(), "== easyconfig file ./toy-1.2.3-foss-2018b.eb created!") - - remove_dir(self.test_prefix) - mkdir(self.test_prefix) - - # if as argument is passed to --copy, the easyconfig file is created there instead - # (and not in the default location) - copy_args = [ - os.path.join(self.test_prefix, 'test.eb'), # non-existing file in existing directory - os.path.join(self.test_prefix, 'asubdir', 'test.eb'), # non-existing file in non-existing directory - os.path.join(self.test_prefix, 'asubdir'), # existing (sub)directory, no filename specified - ] - for copy_arg in copy_args: - if os.path.isdir(copy_arg): - test_ec = os.path.join(copy_arg, 'toy-1.2.3-foss-2018b.eb') - else: - test_ec = copy_arg - - (stdout, stderr) = self.run_eb_new(args + ['--copy', copy_arg]) - self.assertFalse(os.path.exists(ec_fp)) - self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) - self.assertEqual(stdout.strip(), "== easyconfig file %s created!" % test_ec) - - # combination of --copy and --cat also works - (stdout, stderr) = self.run_eb_new(args + ['--cat', '--copy', copy_arg]) - self.assertFalse(os.path.exists(ec_fp)) - self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) - patterns[0] = r"== Contents of easyconfig file %s:\n\neasyblock = 'ConfigureMake'" % test_ec - patterns[-1] = r"== easyconfig file %s created!" % test_ec - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) - - remove_dir(self.test_prefix) - mkdir(self.test_prefix) - - # throwing --edit in the mix - args.extend(['--edit', '--editor-command-template=echo %s']) - (stdout, stderr) = self.run_eb_new(args + ['--cat']) - self.assertTrue(os.path.exists(ec_fp)) - patterns[0] = r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'" - patterns[-1] = r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!" - patterns.insert(0, r"== editing ./toy-1.2.3-foss-2018b.eb... done \(no changes\)") - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/options.py b/test/framework/options.py index 5ed8f7a469..af5eecd36c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -49,7 +49,8 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import copy_dir, copy_file, download_file, mkdir, read_file, remove_file, write_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, mkdir +from easybuild.tools.filetools import read_file, remove_dir, remove_file, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, URL_SEPARATOR from easybuild.tools.github import fetch_github_token from easybuild.tools.modules import Lmod @@ -3700,6 +3701,204 @@ def test_enforce_checksums(self): error_pattern = "Missing checksum for toy-0.0.tar.gz" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def run_eb_new(self, args): + """Helper function to run 'eb --new' with specified arguments.""" + change_dir(self.test_prefix) + self.mock_stdout(True) + self.mock_stderr(True) + self.eb_main(['--new'] + args, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + return (stdout, stderr) + + def test_new_ec(self): + """Test creating new easyconfigs via 'eb --new'.""" + + # running without any arguments results in a clean error + error_pattern = "One or more required parameters are not specified: %s" + self.assertErrorRegex(EasyBuildError, error_pattern % "name, toolchain, version", self.run_eb_new, []) + + # partial specs also results in a clean error + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", self.run_eb_new, ['toy']) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", self.run_eb_new, ['toy', '1.2.3']) + self.assertErrorRegex(EasyBuildError, error_pattern % "version", self.run_eb_new, ['toy', 'foss/2018a']) + + # when just the required specs are specified, a lot of warnings are spit out + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b']) + + ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') + self.assertTrue(os.path.exists(ec_fp)) + + expected_stdout = "== easyconfig file ./%s created!" + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'toy') + self.assertEqual(ec.version, '1.2.3') + self.assertEqual(ec['toolchain'], {'name': 'foss', 'version': '2018b'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + + warn_params = ['description', 'easyblock', 'homepage', 'moduleclass', 'sanity_check_paths', + 'sources', 'source_urls'] + for param in warn_params: + self.assertTrue("WARNING: No value found for '%s' parameter, injected dummy value" % param in stderr) + # no unused arguments + self.assertFalse("WARNING: Unhandled argument" in stderr) + + # warnings are printed for arguments that can't be matched to an easyconfig parameter + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertTrue('WARNING: Unhandled argument: "this_is_a_useless_value"' in stderr) + + # easyblock names are recognized + for easyblock in ['Toolchain', 'EB_toy']: + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', easyblock]) + + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertFalse("WARNING: Unhandled argument" in stderr) + + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], easyblock) + + # check handling of description (first arguments with 3 or more spaces) + args = ['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"] + (stdout, stderr) = self.run_eb_new(args) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['description'], "this is a description") + self.assertTrue('WARNING: Unhandled argument: "not a description"' in stderr) + + # check handling of deps/builddeps + args = ['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4'] + (stdout, stderr) = self.run_eb_new(args) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertFalse("WARNING: Unhandled argument" in stderr) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + self.assertEqual(len(ec['dependencies']), 2) + self.assertEqual(ec['dependencies'][0]['name'], 'toy') + self.assertEqual(ec['dependencies'][0]['version'], '0.0') + self.assertEqual(ec['dependencies'][1]['name'], 'GCC') + self.assertEqual(ec['dependencies'][1]['version'], '4.9.2') + self.assertEqual(len(ec['builddependencies']), 1) + self.assertEqual(ec['builddependencies'][0]['name'], 'gzip') + self.assertEqual(ec['builddependencies'][0]['version'], '1.4') + + # check handling of easyconfig parameters that are specified by =, which get preference + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', + 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + self.assertTrue('WARNING: Unhandled argument: "Toolchain"' in stderr) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified + self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + + # using an unknown easyblock means trouble + error_pattern = "Easyconfig file with raw contents shown above NOT created because of errors: " + error_pattern += "'Failed to obtain class for NoSuchEasyBlock easyblock .*'" + args = ['bar', '3.4.5', 'foss/2018a', 'easyblock=NoSuchEasyBlock'] + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, args) + + def test_new_cat_copy_edit(self): + """Test combining --new with --show, --copy and --edit.""" + + args = ['toy', '1.2.3', 'foss/2018b'] + ec_fp = os.path.join(self.test_prefix, 'toy-1.2.3-foss-2018b.eb') + + # contents of easyconfig are printed when using --show, file still is created into current directory + (stdout, stderr) = self.run_eb_new(args + ['--show']) + self.assertTrue(os.path.exists(ec_fp)) + + patterns = [ + r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'", + r"^name = 'toy'$", + r"^version = '1.2.3'$", + r"^toolchain = {'name': 'foss', 'version': '2018b'}$", + r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + + # clean up test dir + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # check whether easyconfig file is copied when --copy is used + # --copy without an argument doesn't change anything, since --new always dumps easyconfig in currect dir + (stdout, stderr) = self.run_eb_new(args + ['--copy']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), "== easyconfig file ./toy-1.2.3-foss-2018b.eb created!") + + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # if as argument is passed to --copy, the easyconfig file is created there instead + # (and not in the default location) + copy_args = [ + os.path.join(self.test_prefix, 'test.eb'), # non-existing file in existing directory + os.path.join(self.test_prefix, 'asubdir', 'test.eb'), # non-existing file in non-existing directory + os.path.join(self.test_prefix, 'asubdir'), # existing (sub)directory, no filename specified + ] + for copy_arg in copy_args: + if os.path.isdir(copy_arg): + test_ec = os.path.join(copy_arg, 'toy-1.2.3-foss-2018b.eb') + else: + test_ec = copy_arg + + (stdout, stderr) = self.run_eb_new(args + ['--copy', copy_arg]) + self.assertFalse(os.path.exists(ec_fp)) + self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) + self.assertEqual(stdout.strip(), "== easyconfig file %s created!" % test_ec) + + # combination of --copy and --show also works + (stdout, stderr) = self.run_eb_new(args + ['--show', '--copy', copy_arg]) + self.assertFalse(os.path.exists(ec_fp)) + self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) + patterns[0] = r"== Contents of easyconfig file %s:\n\neasyblock = 'ConfigureMake'" % test_ec + patterns[-1] = r"== easyconfig file %s created!" % test_ec + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # throwing --edit in the mix + args.extend(['--edit', '--editor-command-template=echo %s']) + (stdout, stderr) = self.run_eb_new(args + ['--show']) + self.assertTrue(os.path.exists(ec_fp)) + patterns[0] = r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'" + patterns[-1] = r"== easyconfig file ./toy-1.2.3-foss-2018b.eb created!" + patterns.insert(0, r"== editing ./toy-1.2.3-foss-2018b.eb... done \(no changes\)") + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' is found in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ From 75092910114294ea6d2f509181397bce1d3ec8d6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 11:34:28 +0100 Subject: [PATCH 11/20] ensure that --copy doesn't blindly overwrite an existing easyconfig file + leverage build_option in main.py to clean up code a bit --- easybuild/main.py | 37 +++++++++++++++++++++---------------- easybuild/tools/config.py | 4 ++++ test/framework/options.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 90a32f2854..e9cff11d31 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -173,9 +173,12 @@ def run_contrib_style_checks(ecs, check_contrib, check_style): return check_contrib or check_style -def handle_cat_copy_edit(filepaths, print_contents=False, copy=False, edit=False, target=None): +def handle_cat_copy_edit(filepaths, target=None, copy=None): """Handle use of --cat, --copy and --edit.""" + if copy is None: + copy = build_option('copy') + res = [] for orig_fp in filepaths: if copy: @@ -186,48 +189,50 @@ def handle_cat_copy_edit(filepaths, print_contents=False, copy=False, edit=False else: fp = target + if os.path.exists(fp) and not build_option('force'): + raise EasyBuildError("Not overwriting existing file %s without --force", fp) + copy_file(orig_fp, fp) res.append(fp) else: fp = orig_fp - if edit: + if build_option('edit'): edit_file(fp) - if print_contents: + if build_option('show'): print_msg("Contents of easyconfig file %s:\n" % fp) print_msg(read_file(fp), prefix=False) return res -def handle_new(options, tmpdir, args): +def handle_new(tmpdir, args): """Handle use of --new.""" tmpfp = create_new_easyconfig(tmpdir, args) # use current directory as default location to save generated file, in case no location is specified via --copy - target = options.copy or '.' - res = handle_cat_copy_edit([tmpfp], print_contents=options.show, copy=True, edit=options.edit, target=target) + res = handle_cat_copy_edit([tmpfp], target=build_option('copy') or '.', copy=True) print_msg("easyconfig file %s created!" % res[0]) -def handle_search(options, search_query): +def handle_search(search_query, search_filename, search_short): """Handle use of --search.""" - search_action = options.show or options.copy or options.edit - res = search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, - terse=options.terse, return_hits=search_action) + copy_path = build_option('copy') + search_action = copy_path or build_option('show') or build_option('edit') + res = search_easyconfigs(search_query, short=search_short, filename_only=search_filename, + terse=build_option('terse'), return_hits=search_action) if search_action: # only perform action(s) if there's a single search result, unless --force is used - if len(res) > 1 and not options.force: + if len(res) > 1 and not build_option('force'): raise EasyBuildError("Found %d results, not performing search action(s) without --force", len(res)) - res = handle_cat_copy_edit(res, print_contents=options.show, copy=options.copy, edit=options.edit, - target=options.copy or '.') + res = handle_cat_copy_edit(res, target=copy_path or '.') - if options.copy: + if copy_path: print_msg("copied easyconfig files:") for path in res: print_msg("* %s" % path, prefix=False) @@ -280,7 +285,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # search for easyconfigs, if a query is specified if search_query: - handle_search(options, search_query) + handle_search(search_query, options.search_filename, options.search_short) # GitHub options that warrant a silent cleanup & exit if options.check_github: @@ -303,7 +308,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') elif options.new: - handle_new(options, eb_tmpdir, args) + handle_new(eb_tmpdir, args) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b5995d56ec..9406b0e954 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -131,6 +131,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'container_image_format', 'container_image_name', 'container_tmpdir', + 'copy', 'download_timeout', 'dump_test_report', 'easyblock', @@ -185,6 +186,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'debug', 'debug_lmod', 'dump_autopep8', + 'edit', 'enforce_checksums', 'extended_dry_run', 'experimental', @@ -208,6 +210,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'set_gid_bit', 'skip_test_cases', 'sticky_bit', + 'terse', 'trace', 'upload_test_report', 'update_modules_tool_cache', @@ -215,6 +218,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'use_f90cache', 'use_existing_modules', 'set_default_module', + 'show', ], True: [ 'cleanup_builddir', diff --git a/test/framework/options.py b/test/framework/options.py index af5eecd36c..de25c0ebcb 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3749,6 +3749,22 @@ def test_new_ec(self): # no unused arguments self.assertFalse("WARNING: Unhandled argument" in stderr) + # --new doesn't blindly overwrite an existing easyconfig file + error_pattern = "Not overwriting existing file ./toy-1.2.3-foss-2018b.eb without --force" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, ['toy', '1.2.3', 'foss/2018b']) + + # works fine with --force + self.assertTrue(os.path.exists(ec_fp)) + (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'moduleclass=bio', '--force']) + self.assertTrue(os.path.exists(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'toy') + self.assertEqual(ec.version, '1.2.3') + self.assertEqual(ec['toolchain'], {'name': 'foss', 'version': '2018b'}) + self.assertEqual(ec['moduleclass'], 'bio') + + remove_file(ec_fp) + # warnings are printed for arguments that can't be matched to an easyconfig parameter (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b', 'this_is_a_useless_value']) self.assertTrue(os.path.exists(ec_fp)) @@ -3770,6 +3786,8 @@ def test_new_ec(self): self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) self.assertEqual(ec['easyblock'], easyblock) + remove_file(ec_fp) + # check handling of description (first arguments with 3 or more spaces) args = ['bar', '3.4.5', 'GCCcore/6.4.0', "not a description", "this is a description"] (stdout, stderr) = self.run_eb_new(args) @@ -3783,6 +3801,8 @@ def test_new_ec(self): self.assertEqual(ec['description'], "this is a description") self.assertTrue('WARNING: Unhandled argument: "not a description"' in stderr) + remove_file(ec_fp) + # check handling of deps/builddeps args = ['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4'] (stdout, stderr) = self.run_eb_new(args) @@ -3805,6 +3825,8 @@ def test_new_ec(self): self.assertEqual(ec['builddependencies'][0]['name'], 'gzip') self.assertEqual(ec['builddependencies'][0]['version'], '1.4') + remove_file(ec_fp) + # check handling of easyconfig parameters that are specified by =, which get preference (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', 'easyblock=EB_toy', 'Toolchain', 'moduleclass=lib', 'configopts="--enable-foo --with=bar=/location/of/bar"']) @@ -3854,6 +3876,16 @@ def test_new_cat_copy_edit(self): self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), "== easyconfig file ./toy-1.2.3-foss-2018b.eb created!") + # --copy doesn't blindly overwrite an existing easyconfig file without --force + error_pattern = "Not overwriting existing file ./toy-1.2.3-foss-2018b.eb without --force" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_eb_new, args + ['--copy']) + + # overwriting works fine with --force + write_file(ec_fp, '') + (stdout, stderr) = self.run_eb_new(args + ['--copy', '--force']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertTrue(read_file(ec_fp) != '') + remove_dir(self.test_prefix) mkdir(self.test_prefix) @@ -3875,6 +3907,8 @@ def test_new_cat_copy_edit(self): self.assertTrue(os.path.exists(test_ec), "%s exists" % test_ec) self.assertEqual(stdout.strip(), "== easyconfig file %s created!" % test_ec) + remove_file(test_ec) + # combination of --copy and --show also works (stdout, stderr) = self.run_eb_new(args + ['--show', '--copy', copy_arg]) self.assertFalse(os.path.exists(ec_fp)) From c3a3da2856ffa3dad534e0bbebe9a46343a40966 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 11:48:37 +0100 Subject: [PATCH 12/20] add --search-action-limit option to control how many search hits are allowed to perform search action --- easybuild/main.py | 6 ++++-- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index e9cff11d31..05215b1677 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -227,8 +227,10 @@ def handle_search(search_query, search_filename, search_short): if search_action: # only perform action(s) if there's a single search result, unless --force is used - if len(res) > 1 and not build_option('force'): - raise EasyBuildError("Found %d results, not performing search action(s) without --force", len(res)) + search_action_limit = build_option('search_action_limit') or 1 + if len(res) > search_action_limit: + err_msg = "Found %d results which is more than search action limit (%d), so not performing search action(s)" + raise EasyBuildError(err_msg, len(res), search_action_limit) res = handle_cat_copy_edit(res, target=copy_path or '.') diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 9406b0e954..82626f0664 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -170,6 +170,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_target_repo', 'rpath_filter', 'regtest_output_dir', + 'search_action_limit', 'skip', 'stop', 'subdir_user_modules', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index efa94fd89f..5f6f6aff60 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -555,6 +555,7 @@ def informative_options(self): None, 'store_true', False), 'search': ("Search for easyconfig files in the robot search path, print full paths", None, 'store', None, {'metavar': 'REGEX'}), + 'search-action-limit': ("Search hit count limit to perform search action(s)", int, 'store', None), 'search-filename': ("Search for easyconfig files in the robot search path, print only filenames", None, 'store', None, {'metavar': 'REGEX'}), 'search-short': ("Search for easyconfig files in the robot search path, print short paths", From 755adda69b03461ad78f6d03662e5a40ca0db0fc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 15:32:39 +0100 Subject: [PATCH 13/20] test interaction between --search and --copy/--edit/--show --- test/framework/options.py | 81 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index de25c0ebcb..73e7cdf8a0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -748,6 +748,85 @@ def test_search_archived(self): ]) self.assertEqual(txt, expected) + def test_search_actions(self): + """Test combo of --search with --copy/--edit/--show.""" + + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + test_gcc_ecs = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.7.2.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', + 'GCC-4.9.2.eb', 'GCC-4.9.3-2.25.eb', 'GCC-4.9.3-2.26.eb'] + + # too many search hits (> 1) for '^gcc-4' search query + error_pattern = r"Found [0-9]* results which is more than search action limit \(1\), " + error_pattern += "so not performing search action\(s\)" + + copy = '--copy=%s' % self.test_prefix + search_actions_to_test = [ + [copy], + ['--edit'], + ['--show'], + [copy, '--show'], + [copy, '--edit'], + ['--edit', '--show'], + [copy, '--edit', '--show'], + ] + for search_actions in search_actions_to_test: + + args = ['--search=^gcc-4', '--editor-command-template=true %s'] + search_actions + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, testing=False, raise_error=True) + + args.append('--search-action-limit=100') + self.mock_stdout(True) + self.eb_main(args, testing=False, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + + ecs_dir = os.path.join(test_ecs, 'g', 'GCC') + + if any(a.startswith('--copy') for a in search_actions): + self.assertTrue("== copied easyconfig files:\n" in stdout) + for ec in test_gcc_ecs: + self.assertTrue("* %s/%s" % (self.test_prefix, ec) in stdout) + self.assertEqual(sorted(f for f in os.listdir(self.test_prefix) if f.startswith('GCC-')), test_gcc_ecs) + ecs_dir = self.test_prefix + else: + self.assertFalse("== copied easyconfig files" in stdout) + + if any(a.startswith('--edit') for a in search_actions): + for ec in test_gcc_ecs: + self.assertTrue("== editing %s/%s... done (no changes)" % (ecs_dir, ec) in stdout) + else: + self.assertFalse("== editing" in stdout) + + if any(a.startswith('--show') for a in search_actions): + for ec in test_gcc_ecs: + self.assertTrue("== Contents of easyconfig file %s/%s:" % (ecs_dir, ec) in stdout) + else: + self.assertFalse("== Contents of easyconfig file" in stdout) + + # clean up and recreate test dir + remove_dir(self.test_prefix) + mkdir(self.test_prefix) + + # test with search query that only returns a single result + args = ['--search=^gcc-4.8.3', '--copy=%s' % self.test_prefix, '--edit', '--show', + '--editor-command-template=true %s'] + self.mock_stdout(True) + self.eb_main(args, testing=False, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + patterns = [ + r"^== editing %s/GCC-4.8.3.eb... done \(no changes\)$" % self.test_prefix, + r"^== Contents of easyconfig file %s/GCC-4.8.3.eb:$" % self.test_prefix, + r'^name = "GCC"$', + r"^version = '4.8.3'$", + r"^toolchain = .*dummy", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + self.assertTrue(stdout.strip().endswith("== copied easyconfig files:\n* %s/GCC-4.8.3.eb" % self.test_prefix)) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3923,7 +4002,7 @@ def test_new_cat_copy_edit(self): mkdir(self.test_prefix) # throwing --edit in the mix - args.extend(['--edit', '--editor-command-template=echo %s']) + args.extend(['--edit', '--editor-command-template=true %s']) (stdout, stderr) = self.run_eb_new(args + ['--show']) self.assertTrue(os.path.exists(ec_fp)) patterns[0] = r"== Contents of easyconfig file \./toy-1.2.3-foss-2018b.eb:\n\neasyblock = 'ConfigureMake'" From e9be0bf5fa9b2e12cab95e9156e9704fbbfb2837 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Nov 2018 17:19:48 +0100 Subject: [PATCH 14/20] flesh out separate function for parsing values as easyconfig parameter values, also handle sanity_check_paths --- easybuild/framework/easyconfig/tools.py | 112 ++++++++++++++++++------ test/framework/easyconfig.py | 39 +++++++++ test/framework/options.py | 3 + 3 files changed, 125 insertions(+), 29 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index ab429987dd..5516eeac95 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -703,14 +703,81 @@ def avail_easyblocks(): return easyblocks +def parse_param_value(string): + """Parse specified string as an easyconfig parameter value.""" + + def split_one(item, sep=','): + """Helper function to split of first part in an item, using given separator.""" + parts = item.split(sep) + return (parts[0], sep.join(parts[1:])) + + param, value = None, None + + # determine list of names of known easyblocks, so we can descriminate an easyblock name + easyblock_names = [e['class'] for e in avail_easyblocks().values()] + + # regular expression to recognise a version + version_regex = re.compile('^[0-9][0-9.-]') + + # first check whether easyconfig parameter name is specfied as '=' + if re.match('^[a-z_]+=', string): + param, string = split_one(string, sep='=') + + # check if the value is most likely a dictionary ':[;:]' + if re.match('^[a-z_]+:', string): + value = {} + for item in string.split(';'): + key, val = split_one(item, sep=':') + value[key] = parse_param_value(val)[1] + + # if we encounter a dictionary with (only) 'files' and/or 'dirs' as key(s), it must be sanity_check_paths + if len(value) <= 2 and ('files' in value or 'dirs' in value): + param = 'sanity_check_paths' + for item_key, item_val in value.items(): + if isinstance(item_val, basestring): + value[item_key] = [item_val] + elif isinstance(item_val, tuple): + value[item_key] = list(item_val) + else: + raise EasyBuildError("Incorrect type of value for sanity_check_paths key '%s': %s", key, value[key]) + + # make sure both files/dirs keys are defined + value.setdefault('files', []) + value.setdefault('dirs', []) + + # ';' is the separator for a list of items + elif ';' in string: + value = [parse_param_value(x)[1] for x in string.split(';')] + + # ',' is the separator for a tuple + elif ',' in string: + value = tuple(parse_param_value(x)[1] for x in string.split(',')) + + # if parameter name is not decided yet, check for a likely match for specific parameters + elif string in easyblock_names and param is None: + param, value = 'easyblock', string + + elif version_regex.match(string) and param is None: + param, value = 'version', string + + elif find_extension(string, raise_error=False) and param is None: + param, value = 'sources', [string] + + # a value with 3 or more spaces should most likely remain a string + elif string.count(' ') >= 3 and param is None: + param, value = 'description', string + + else: + value = string + + return (param, value) + + def create_new_easyconfig(path, args): """Create new easyconfig file based on specified information.""" specs = {} - # regular expression to recognise a version - version_regex = re.compile('^[0-9]') - # try and discriminate between homepage and source URL http_args = [arg for arg in args if arg.startswith('http')] if http_args: @@ -748,43 +815,30 @@ def create_new_easyconfig(path, args): if specs.get('homepage') is None: specs['homepage'] = arg - # determine list of names of known easyblocks, so we can descriminate an easyblock name - easyblock_names = [e['class'] for e in avail_easyblocks().values()] - # iterate over provided arguments, and try to figure out what they specify for arg in args: - # arguments that start with '=' are dealt with first - if re.match('^[a-z_]+=', arg): - key = arg.split('=')[0] - if key in ['builddeps', 'deps']: - deps = [] - for dep in arg.split('=')[-1].split(';'): - deps.append(tuple(dep.split(','))) - specs[key.replace('deps', 'dependencies')] = deps - else: - specs[key] = '='.join(arg.split('=')[1:]) + key, val = parse_param_value(arg) # first argument is assumed to be the software name - elif specs.get('name') is None: + if specs.get('name') is None: specs['name'] = arg - elif version_regex.match(arg) and specs.get('version') is None: - specs['version'] = arg + elif key and specs.get(key) is None: - # toolchain is usually specified as /, e.g. intel/2018a - elif '/' in arg and specs.get('toolchain') is None: - tc_name, tc_ver = arg.split('/') - specs['toolchain'] = {'name': tc_name, 'version': tc_ver} + if key in ['builddeps', 'deps', 'source_urls', 'sources']: + if not isinstance(val, list): + val = [val] - elif arg in easyblock_names and specs.get('easyblock') is None: - specs['easyblock'] = arg + if key in ['builddeps', 'deps']: + key = key.replace('deps', 'dependencies') - elif arg.count(' ') >= 3 and specs.get('description') is None: - specs['description'] = arg + specs[key] = val - elif find_extension(arg, raise_error=False): - specs['sources'] = [arg] + # toolchain is usually specified as /, e.g. intel/2018a + elif isinstance(val, basestring) and '/' in val and specs.get('toolchain') is None: + tc_name, tc_ver = val.split('/') + specs['toolchain'] = {'name': tc_name, 'version': tc_ver} else: print_warning("Unhandled argument: %s" % quote_str(arg)) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b267dfc95d..84532c9ffe 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -56,6 +56,7 @@ from easybuild.framework.easyconfig.style import check_easyconfigs_style from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs +from easybuild.framework.easyconfig.tools import parse_param_value from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes @@ -2356,6 +2357,44 @@ def test_deprecated(self): error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n\(see also" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + def test_parse_param_value(self): + """Tests for parse_param_value function.""" + self.assertEqual(parse_param_value(''), (None, '')) + self.assertEqual(parse_param_value('test123'), (None, 'test123')) + + self.assertEqual(parse_param_value("this is a description"), ('description', "this is a description")) + + self.assertEqual(parse_param_value('1.2.3'), ('version', '1.2.3')) + + # names of source tarballs are recognized + for ext in ['tar.gz', 'gz', 'tar.bz2', 'bz2', 'zip']: + self.assertEqual(parse_param_value('toy-0.0.' + ext), ('sources', ['toy-0.0.' + ext])) + + # known easyblocks are recognized + for easyblock in ['ConfigureMake', 'Toolchain', 'EB_toy']: + self.assertEqual(parse_param_value(easyblock), ('easyblock', easyblock)) + + # list, tuple + self.assertEqual(parse_param_value('one,two,three,four'), (None, ('one', 'two', 'three', 'four'))) + self.assertEqual(parse_param_value('one;two;three;four'), (None, ['one', 'two', 'three', 'four'])) + # list of tuples + self.assertEqual(parse_param_value('one,two;three,four'), (None, [('one', 'two'), ('three', 'four')])) + # dict with string values + self.assertEqual(parse_param_value('one:1;two:2;three:3'), (None, {'one': '1', 'two': '2', 'three': '3'})) + + # specified parameters + self.assertEqual(parse_param_value('moduleclass=lib'), ('moduleclass', 'lib')) + configopts = '--foo --bar --baz --bleh --blah' + self.assertEqual(parse_param_value('configopts=%s' % configopts), ('configopts', configopts)) + + # sanity_check_paths + expected = {'files': ['bin/foo', 'lib/libfoo.a'], 'dirs': ['include']} + self.assertEqual(parse_param_value('files:bin/foo,lib/libfoo.a;dirs:include'), ('sanity_check_paths', expected)) + expected = {'files': [], 'dirs': ['lib']} + self.assertEqual(parse_param_value('dirs:lib'), ('sanity_check_paths', expected)) + expected = {'files': ['bin/foo', 'bin/bar'], 'dirs': []} + self.assertEqual(parse_param_value('files:bin/foo,bin/bar'), ('sanity_check_paths', expected)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/options.py b/test/framework/options.py index 73e7cdf8a0..ac8c6831b2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3805,6 +3805,9 @@ def test_new_ec(self): self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain", self.run_eb_new, ['toy', '1.2.3']) self.assertErrorRegex(EasyBuildError, error_pattern % "version", self.run_eb_new, ['toy', 'foss/2018a']) + # first argument is *always* the software name, even if it matches with a known easyblock (e.g. SCons) + self.assertErrorRegex(EasyBuildError, error_pattern % "toolchain, version", self.run_eb_new, ['ConfigureMake']) + # when just the required specs are specified, a lot of warnings are spit out (stdout, stderr) = self.run_eb_new(['toy', '1.2.3', 'foss/2018b']) From 6b357358f656c069c2b2126dc9fdc307c00a5011 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 18 Nov 2018 11:03:08 +0100 Subject: [PATCH 15/20] take into account empty values in parse_param_value for tuple/list/dict values --- easybuild/framework/easyconfig/tools.py | 8 +++++--- test/framework/easyconfig.py | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 5516eeac95..f60ad6f3b7 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -735,7 +735,9 @@ def split_one(item, sep=','): param = 'sanity_check_paths' for item_key, item_val in value.items(): if isinstance(item_val, basestring): - value[item_key] = [item_val] + value[item_key] = [] + if item_val: + value[item_key].append(item_val) elif isinstance(item_val, tuple): value[item_key] = list(item_val) else: @@ -747,11 +749,11 @@ def split_one(item, sep=','): # ';' is the separator for a list of items elif ';' in string: - value = [parse_param_value(x)[1] for x in string.split(';')] + value = [parse_param_value(x)[1] for x in string.split(';') if x] # ',' is the separator for a tuple elif ',' in string: - value = tuple(parse_param_value(x)[1] for x in string.split(',')) + value = tuple(parse_param_value(x)[1] for x in string.split(',') if x) # if parameter name is not decided yet, check for a likely match for specific parameters elif string in easyblock_names and param is None: diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 84532c9ffe..3232a41c42 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2376,6 +2376,7 @@ def test_parse_param_value(self): # list, tuple self.assertEqual(parse_param_value('one,two,three,four'), (None, ('one', 'two', 'three', 'four'))) + self.assertEqual(parse_param_value('one,two,,four'), (None, ('one', 'two', 'four'))) self.assertEqual(parse_param_value('one;two;three;four'), (None, ['one', 'two', 'three', 'four'])) # list of tuples self.assertEqual(parse_param_value('one,two;three,four'), (None, [('one', 'two'), ('three', 'four')])) @@ -2388,6 +2389,7 @@ def test_parse_param_value(self): self.assertEqual(parse_param_value('configopts=%s' % configopts), ('configopts', configopts)) # sanity_check_paths + self.assertEqual(parse_param_value('files:;dirs:'), ('sanity_check_paths', {'files': [], 'dirs': []})) expected = {'files': ['bin/foo', 'lib/libfoo.a'], 'dirs': ['include']} self.assertEqual(parse_param_value('files:bin/foo,lib/libfoo.a;dirs:include'), ('sanity_check_paths', expected)) expected = {'files': [], 'dirs': ['lib']} From 498e6c1a233c867da6442bdc7a44813db37c4896 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 18 Nov 2018 11:08:29 +0100 Subject: [PATCH 16/20] add tests for sanity_check_paths value in --new --- test/framework/options.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index ac8c6831b2..2d9ca410cd 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3885,10 +3885,35 @@ def test_new_ec(self): remove_file(ec_fp) + # check handling of sanity_check_paths (dict value with files/dir keys) + tests = [ + ('files:', {'files': [], 'dirs': []}), + ('dirs:', {'files': [], 'dirs': []}), + ('files:;dirs:', {'files': [], 'dirs': []}), + ('files:bin/foo', {'files': ['bin/foo'], 'dirs': []}), + ('dirs:include', {'files': [], 'dirs': ['include']}), + ('files:bin/foo,bin/bar', {'files': ['bin/foo', 'bin/bar'], 'dirs': []}), + ('dirs:bin,include,lib', {'files': [], 'dirs': ['bin', 'include', 'lib']}), + ('files:bin/foo;dirs:include', {'files': ['bin/foo'], 'dirs': ['include']}), + ('files:bin/foo,bin/bar;dirs:include,lib', {'files': ['bin/foo', 'bin/bar'], 'dirs': ['include', 'lib']}), + ] + for arg, expected in tests: + args = ['bar', '3.4.5', 'GCCcore/6.4.0', arg] + (stdout, stderr) = self.run_eb_new(args) + ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['sanity_check_paths'], expected) + + remove_file(ec_fp) + # check handling of deps/builddeps args = ['bar', '3.4.5', 'GCCcore/6.4.0', 'deps=toy,0.0;GCC,4.9.2', 'builddeps=gzip,1.4'] (stdout, stderr) = self.run_eb_new(args) - ec_fp = os.path.join(self.test_prefix, 'bar-3.4.5-GCCcore-6.4.0.eb') self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) self.assertFalse("WARNING: Unhandled argument" in stderr) From a110f41983d23964fbcc560f1259921bfe0a1843 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Nov 2018 09:18:47 +0100 Subject: [PATCH 17/20] add logging to parse_param_value, add more tests + minor fixes/tweaks --- easybuild/framework/easyconfig/tools.py | 30 ++++++++++++++++++++----- easybuild/main.py | 2 +- test/framework/easyconfig.py | 19 +++++++++++++--- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index f60ad6f3b7..5bd53f92d3 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -715,25 +715,36 @@ def split_one(item, sep=','): # determine list of names of known easyblocks, so we can descriminate an easyblock name easyblock_names = [e['class'] for e in avail_easyblocks().values()] + _log.debug("List of names for known easyblocks: %s", sorted(easyblock_names)) # regular expression to recognise a version version_regex = re.compile('^[0-9][0-9.-]') - # first check whether easyconfig parameter name is specfied as '=' + # first check whether easyconfig parameter name is specified as '=' if re.match('^[a-z_]+=', string): param, string = split_one(string, sep='=') + _log.info("Found (raw) value for '%s' easyconfig parameter: %s", param, string) # check if the value is most likely a dictionary ':[;:]' - if re.match('^[a-z_]+:', string): + dict_item_sep = ':' + if re.match('^[a-z_]+' + dict_item_sep, string): + _log.info("String value '%s' represents a dictionary value", string) value = {} for item in string.split(';'): - key, val = split_one(item, sep=':') - value[key] = parse_param_value(val)[1] + if dict_item_sep in item: + key, val = split_one(item, sep=dict_item_sep) + # recurse to obtain parsed value for this key + value[key] = parse_param_value(val)[1] + else: + raise EasyBuildError("Wrong format for dictionary item '%s', should be ':= 3 and param is None: @@ -772,6 +787,11 @@ def split_one(item, sep=','): else: value = string + if param: + _log.info("Found value for '%s' easyconfig parameter: %s", param, value) + else: + _log.info("Found value for unknown easyconfig parameter: %s", value) + return (param, value) diff --git a/easybuild/main.py b/easybuild/main.py index 05215b1677..3cb2f8da5f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -234,7 +234,7 @@ def handle_search(search_query, search_filename, search_short): res = handle_cat_copy_edit(res, target=copy_path or '.') - if copy_path: + if res: print_msg("copied easyconfig files:") for path in res: print_msg("* %s" % path, prefix=False) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 3232a41c42..ae752c2cf1 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2368,7 +2368,7 @@ def test_parse_param_value(self): # names of source tarballs are recognized for ext in ['tar.gz', 'gz', 'tar.bz2', 'bz2', 'zip']: - self.assertEqual(parse_param_value('toy-0.0.' + ext), ('sources', ['toy-0.0.' + ext])) + self.assertEqual(parse_param_value('toy-0.0.' + ext), ('sources', 'toy-0.0.' + ext)) # known easyblocks are recognized for easyblock in ['ConfigureMake', 'Toolchain', 'EB_toy']: @@ -2382,13 +2382,26 @@ def test_parse_param_value(self): self.assertEqual(parse_param_value('one,two;three,four'), (None, [('one', 'two'), ('three', 'four')])) # dict with string values self.assertEqual(parse_param_value('one:1;two:2;three:3'), (None, {'one': '1', 'two': '2', 'three': '3'})) + # dict with list values + expected = {'ones': ('1', '1', '1'), 'twos': ('2', '2'), 'threes': ('3',)} + self.assertEqual(parse_param_value('ones:1,1,1;twos:2,2;threes:3,'), (None, expected)) + # dict with a single entry, value can contain additional ':' characters, no problem there + self.assertEqual(parse_param_value('one:1:2:3'), (None, {'one': '1:2:3'})) - # specified parameters + # check error handling for dict with incorrect entry + error_pattern = "Wrong format for dictionary item 'two', should be ': Date: Thu, 22 Nov 2018 09:43:01 +0100 Subject: [PATCH 18/20] refactor to use variables for different separators in parse_param_value --- easybuild/framework/easyconfig/tools.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 5bd53f92d3..580bc6dfc6 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -713,6 +713,11 @@ def split_one(item, sep=','): param, value = None, None + # separators + list_sep = ';' + tuple_sep = ',' + dict_key_val_sep = ':' + # determine list of names of known easyblocks, so we can descriminate an easyblock name easyblock_names = [e['class'] for e in avail_easyblocks().values()] _log.debug("List of names for known easyblocks: %s", sorted(easyblock_names)) @@ -726,13 +731,12 @@ def split_one(item, sep=','): _log.info("Found (raw) value for '%s' easyconfig parameter: %s", param, string) # check if the value is most likely a dictionary ':[;:]' - dict_item_sep = ':' - if re.match('^[a-z_]+' + dict_item_sep, string): + if re.match('^[a-z_]+' + dict_key_val_sep, string): _log.info("String value '%s' represents a dictionary value", string) value = {} - for item in string.split(';'): - if dict_item_sep in item: - key, val = split_one(item, sep=dict_item_sep) + for item in string.split(list_sep): + if dict_key_val_sep in item: + key, val = split_one(item, sep=dict_key_val_sep) # recurse to obtain parsed value for this key value[key] = parse_param_value(val)[1] else: @@ -759,15 +763,15 @@ def split_one(item, sep=','): value.setdefault('dirs', []) # ';' is the separator for a list of items - elif ';' in string: + elif list_sep in string: # recurse for to obtain parsed value for each item in the list - value = [parse_param_value(x)[1] for x in string.split(';') if x] + value = [parse_param_value(x)[1] for x in string.split(list_sep) if x] _log.info("String value '%s' represents a list: %s", string, value) # ',' is the separator for a tuple - elif ',' in string: + elif tuple_sep in string: # recurse for to obtain parsed value for each item in the tuple - value = tuple(parse_param_value(x)[1] for x in string.split(',') if x) + value = tuple(parse_param_value(x)[1] for x in string.split(tuple_sep) if x) _log.info("String value '%s' represents a tuple: %s", string, value) # if parameter name is not decided yet, check for a likely match for specific parameters From 4c17fe6d69e67a16841b91bb643bad9cd11a3f16 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Nov 2018 10:59:26 +0100 Subject: [PATCH 19/20] add test for 'http' arguments to --new --- easybuild/framework/easyconfig/tools.py | 63 +++++++++++++------------ test/framework/options.py | 20 +++++++- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 580bc6dfc6..62c0635f80 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -804,42 +804,43 @@ def create_new_easyconfig(path, args): specs = {} + # handle values that start with 'http' first # try and discriminate between homepage and source URL http_args = [arg for arg in args if arg.startswith('http')] - if http_args: - - # first, check and see if we have a full download URL provided as an argument - for arg in http_args: - maybe_filename = os.path.basename(arg) - ext = find_extension(maybe_filename, raise_error=False) - if ext: - specs['source_urls'] = [os.path.dirname(arg)] - # try to recognise downloading of source tarballs by commit ID - if re.search('^[0-9a-f]+\.', maybe_filename): - specs['sources'] = [{ - 'download_filename': maybe_filename, - 'filename': '%(name)s-%(version)s' + ext, - }] - else: - specs['sources'] = [maybe_filename] - http_args.remove(arg) + + # first, check and see if we have a full download URL provided as an argument + for arg in http_args: + maybe_filename = os.path.basename(arg) + ext = find_extension(maybe_filename, raise_error=False) + if ext: + specs['source_urls'] = [os.path.dirname(arg)] + # try to recognise downloading of source tarballs by commit ID + if re.search('^[0-9a-f]+\.', maybe_filename): + specs['sources'] = [{ + 'download_filename': maybe_filename, + 'filename': '%(name)s-%(version)s' + ext, + }] + else: + specs['sources'] = [maybe_filename] + http_args.remove(arg) + args.remove(arg) + break + + for arg in http_args: + if specs.get('homepage') is None: + # homepage is more like to be of form https://example.com, i.e. top-level domain + # if source_urls is already set, then this URL is likely to be a value for homepage + if '/' not in arg.split('://')[-1] or specs.get('source_urls'): + specs['homepage'] = arg args.remove(arg) - break + # go to next iteration to avoid also using this value for 'source_urls' + continue - for arg in http_args: + if specs.get('source_urls') is None: + specs['source_urls'] = [arg] + args.remove(arg) if specs.get('homepage') is None: - # homepage is more like to be of form https://example.com, i.e. top-level domain - if '/' not in arg.split('://')[-1] or specs.get('source_urls'): - specs['homepage'] = arg - args.remove(arg) - # go to next iteration to avoid also using this value for 'source_urls' - continue - - if specs.get('source_urls') is None: - specs['source_urls'] = [arg] - args.remove(arg) - if specs.get('homepage') is None: - specs['homepage'] = arg + specs['homepage'] = arg # iterate over provided arguments, and try to figure out what they specify for arg in args: diff --git a/test/framework/options.py b/test/framework/options.py index 2d9ca410cd..f28b8488e7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3940,11 +3940,27 @@ def test_new_ec(self): self.assertTrue(os.path.exists(ec_fp)) self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) self.assertTrue('WARNING: Unhandled argument: "Toolchain"' in stderr) + ec = EasyConfig(ec_fp) self.assertEqual(ec.name, 'bar') self.assertEqual(ec.version, '3.4.5') self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) - self.assertEqual(ec['easyblock'], 'ConfigureMake') # this is the default if no easyblock is specified - self.assertEqual(ec['moduleclass'], 'tools') # this is the default if no moduleclass is specified + self.assertEqual(ec['easyblock'], 'EB_toy') + self.assertEqual(ec['moduleclass'], 'lib') + + remove_file(ec_fp) + + # check handling of URLs: source_urls vs homepage + (stdout, stderr) = self.run_eb_new(['bar', '3.4.5', 'GCCcore/6.4.0', + 'https://example.com/files/bar-3.4.5.tar.gz', 'http://example.com/bar']) + self.assertTrue(os.path.exists(ec_fp)) + self.assertEqual(stdout.strip(), expected_stdout % os.path.basename(ec_fp)) + ec = EasyConfig(ec_fp) + self.assertEqual(ec.name, 'bar') + self.assertEqual(ec.version, '3.4.5') + self.assertEqual(ec['toolchain'], {'name': 'GCCcore', 'version': '6.4.0'}) + self.assertEqual(ec['homepage'], 'http://example.com/bar') + self.assertEqual(ec['source_urls'], ['https://example.com/files']) + self.assertEqual(ec['sources'], ['bar-3.4.5.tar.gz']) # using an unknown easyblock means trouble error_pattern = "Easyconfig file with raw contents shown above NOT created because of errors: " From 142155d086765cf741c4e4c5a1e81462c5e6ac41 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Nov 2018 12:27:18 +0100 Subject: [PATCH 20/20] require --experimental for --new --- easybuild/framework/easyconfig/tools.py | 2 ++ easybuild/tools/build_log.py | 2 +- test/framework/options.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 62c0635f80..d0c507585b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -802,6 +802,8 @@ def split_one(item, sep=','): def create_new_easyconfig(path, args): """Create new easyconfig file based on specified information.""" + _log.experimental("Generating easyconfig using 'eb --new'") + specs = {} # handle values that start with 'http' first diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 041cf8a21b..378c153ecf 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -118,7 +118,7 @@ def experimental(self, msg, *args, **kwargs): msg = common_msg + ': ' + msg self.warning(msg, *args, **kwargs) else: - msg = common_msg + " (use --experimental option to enable): " + msg + msg = common_msg + " (use --experimental or -E to enable): " + msg raise EasyBuildError(msg, *args) def deprecated(self, msg, ver, max_ver=None, more_info=None, *args, **kwargs): diff --git a/test/framework/options.py b/test/framework/options.py index 98cbc46abe..290b14b85f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3887,7 +3887,7 @@ def run_eb_new(self, args): change_dir(self.test_prefix) self.mock_stdout(True) self.mock_stderr(True) - self.eb_main(['--new'] + args, raise_error=True) + self.eb_main(['--new', '--experimental'] + args, raise_error=True) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False)