add support for --new, --copy, --edit and --show

easybuild/framework/easyconfig/format/format.py

@@ -62,7 +62,7 @@
     ['name', 'version', 'versionprefix', 'versionsuffix'],
     ['homepage', 'description'],
     ['toolchain', 'toolchainopts'],
-    ['sources', 'source_urls'],
+    ['source_urls', 'sources'],
     ['patches'],
     DEPENDENCY_PARAMETERS,
     ['osdependencies'],

easybuild/framework/easyconfig/tools.py

@@ -50,12 +50,13 @@
 from easybuild.framework.easyconfig.easyconfig import create_paths, get_easyblock_class, process_easyconfig
 from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file
+from easybuild.tools.filetools import find_easyconfigs, find_extension, is_patch_file, read_file, resolve_path
+from easybuild.tools.filetools import which, write_file
 from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo
-from easybuild.tools.modules import modules_tool
+from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.ordereddict import OrderedDict
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
@@ -204,11 +205,12 @@ def mk_node_name(spec):
     # build directed graph
     dgr = digraph()
     dgr.add_nodes(all_nodes)
+    edge_attrs = [('style', 'dotted'), ('color', 'blue'), ('arrowhead', 'diamond')]
     for spec in specs:
         for dep in spec['ec'].all_dependencies:
             dgr.add_edge((spec['module'], dep))
             if dep in spec['ec'].build_dependencies:
-                dgr.add_edge_attributes((spec['module'], dep), attrs=[('style','dotted'), ('color','blue'), ('arrowhead','diamond')])
+                dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
 
     _dep_graph_dump(dgr, filename)
 
@@ -699,3 +701,187 @@ def avail_easyblocks():
                                        easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path)
 
     return easyblocks
+
+
+def parse_param_value(string):
+    """Parse specified string as an easyconfig parameter value."""
+
+    def split_one(item, sep=','):
+        """Helper function to split of first part in an item, using given separator."""
+        parts = item.split(sep)
+        return (parts[0], sep.join(parts[1:]))
+
+    param, value = None, None
+
+    # determine list of names of known easyblocks, so we can descriminate an easyblock name
+    easyblock_names = [e['class'] for e in avail_easyblocks().values()]
+
+    # regular expression to recognise a version
+    version_regex = re.compile('^[0-9][0-9.-]')
+
+    # first check whether easyconfig parameter name is specfied as '<param_name>=<value>'
+    if re.match('^[a-z_]+=', string):
+        param, string = split_one(string, sep='=')
+
+    # check if the value is most likely a dictionary '<key>:<val>[;<key>:<val>]'
+    if re.match('^[a-z_]+:', string):
+        value = {}
+        for item in string.split(';'):
+            key, val = split_one(item, sep=':')
+            value[key] = parse_param_value(val)[1]
+
+        # if we encounter a dictionary with (only) 'files' and/or 'dirs' as key(s), it must be sanity_check_paths
+        if len(value) <= 2 and ('files' in value or 'dirs' in value):
+            param = 'sanity_check_paths'
+            for item_key, item_val in value.items():
+                if isinstance(item_val, basestring):
+                    value[item_key] = []
+                    if item_val:
+                        value[item_key].append(item_val)
+                elif isinstance(item_val, tuple):
+                    value[item_key] = list(item_val)
+                else:
+                    raise EasyBuildError("Incorrect type of value for sanity_check_paths key '%s': %s", key, value[key])
+
+            # make sure both files/dirs keys are defined
+            value.setdefault('files', [])
+            value.setdefault('dirs', [])
+
+    # ';' is the separator for a list of items
+    elif ';' in string:
+        value = [parse_param_value(x)[1] for x in string.split(';') if x]
+
+    # ',' is the separator for a tuple
+    elif ',' in string:
+        value = tuple(parse_param_value(x)[1] for x in string.split(',') if x)
+
+    # if parameter name is not decided yet, check for a likely match for specific parameters
+    elif string in easyblock_names and param is None:
+        param, value = 'easyblock', string
+
+    elif version_regex.match(string) and param is None:
+        param, value = 'version', string
+
+    elif find_extension(string, raise_error=False) and param is None:
+        param, value = 'sources', [string]
+
+    # a value with 3 or more spaces should most likely remain a string
+    elif string.count(' ') >= 3 and param is None:
+        param, value = 'description', string
+
+    else:
+        value = string
+
+    return (param, value)
+
+
+def create_new_easyconfig(path, args):
+    """Create new easyconfig file based on specified information."""
+
+    specs = {}
+
+    # try and discriminate between homepage and source URL
+    http_args = [arg for arg in args if arg.startswith('http')]
+    if http_args:
+
+        # first, check and see if we have a full download URL provided as an argument
+        for arg in http_args:
+            maybe_filename = os.path.basename(arg)
+            ext = find_extension(maybe_filename, raise_error=False)
+            if ext:
+                specs['source_urls'] = [os.path.dirname(arg)]
+                # try to recognise downloading of source tarballs by commit ID
+                if re.search('^[0-9a-f]+\.', maybe_filename):
+                    specs['sources'] = [{
+                        'download_filename': maybe_filename,
+                        'filename': '%(name)s-%(version)s' + ext,
+                    }]
+                else:
+                    specs['sources'] = [maybe_filename]
+                http_args.remove(arg)
+                args.remove(arg)
+                break
+
+        for arg in http_args:
+            if specs.get('homepage') is None:
+                # homepage is more like to be of form https://example.com, i.e. top-level domain
+                if '/' not in arg.split('://')[-1] or specs.get('source_urls'):
+                    specs['homepage'] = arg
+                    args.remove(arg)
+                    # go to next iteration to avoid also using this value for 'source_urls'
+                    continue
+
+            if specs.get('source_urls') is None:
+                specs['source_urls'] = [arg]
+                args.remove(arg)
+                if specs.get('homepage') is None:
+                    specs['homepage'] = arg
+
+    # iterate over provided arguments, and try to figure out what they specify
+    for arg in args:
+
+        key, val = parse_param_value(arg)
+
+        # first argument is assumed to be the software name
+        if specs.get('name') is None:
+            specs['name'] = arg
+
+        elif key and specs.get(key) is None:
+
+            if key in ['builddeps', 'deps', 'source_urls', 'sources']:
+                if not isinstance(val, list):
+                    val = [val]
+
+            if key in ['builddeps', 'deps']:
+                key = key.replace('deps', 'dependencies')
+
+            specs[key] = val
+
+        # toolchain is usually specified as <toolchain_name>/<toolchain_version>, e.g. intel/2018a
+        elif isinstance(val, basestring) and '/' in val and specs.get('toolchain') is None:
+            tc_name, tc_ver = val.split('/')
+            specs['toolchain'] = {'name': tc_name, 'version': tc_ver}
+
+        else:
+            print_warning("Unhandled argument: %s" % quote_str(arg))
+
+    # make sure that at least name, version and toolchain are known
+    missing = [p for p in ['name', 'toolchain', 'version'] if specs.get(p) is None]
+    if missing:
+        raise EasyBuildError("One or more required parameters are not specified: %s", ', '.join(missing))
+
+    # inject educated guesses for some important easyconfig parameters that are not specified
+    educated_guesses = {
+        'description': "This is an example description.",
+        'homepage': 'https://example.com',
+
+        'easyblock': 'ConfigureMake',
+        'moduleclass': 'tools',
+        'sanity_check_paths': {'files': [os.path.join('bin', specs['name'])], 'dirs': []},
+
+        'source_urls': ['%(homepage)s'],
+        'sources': ['%(name)s-%(version)s.tar.gz'],
+    }
+    for key in educated_guesses:
+        if key not in specs:
+            specs[key] = educated_guesses[key]
+            print_warning("No value found for '%s' parameter, injected dummy value: %s" % (key, quote_str(specs[key])))
+
+    # create EasyConfig instance and dump easyconfig file to current directory
+    ec_raw = '\n'.join("%s = %s" % (key, quote_str(specs[key])) for key in specs)
+
+    ec, err = None, None
+    try:
+        ec = EasyConfig(None, rawtxt=ec_raw)
+    except EasyBuildError as err:
+        print_warning("Problem occured when parsing generated easyconfig file: %s" % err)
+
+    if err:
+        print_msg(ec_raw + '\n', prefix=False)
+        raise EasyBuildError("Easyconfig file with raw contents shown above NOT created because of errors: %s", err)
+    else:
+        full_ec_ver = det_full_ec_version(specs)
+        fp = os.path.join(path, '%s-%s.eb' % (specs['name'], full_ec_ver))
+
+        ec.dump(fp)
+        return fp

easybuild/main.py

@@ -49,14 +49,14 @@
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
-from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph
+from easybuild.framework.easyconfig.tools import categorize_files_by_type, create_new_easyconfig, dep_graph
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
-from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
+from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, edit_file, read_file, write_file
 from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token
 from easybuild.tools.github import new_pr, merge_pr, update_pr
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
@@ -173,6 +173,73 @@ def run_contrib_style_checks(ecs, check_contrib, check_style):
     return check_contrib or check_style
 
 
+def handle_cat_copy_edit(filepaths, target=None, copy=None):
+    """Handle use of --cat, --copy and --edit."""
+
+    if copy is None:
+        copy = build_option('copy')
+
+    res = []
+    for orig_fp in filepaths:
+        if copy:
+            # if target location is an existing directory, retain filename
+            # if not, assume last part of specific location is filename
+            if os.path.isdir(target):
+                fp = os.path.join(target, os.path.basename(orig_fp))
+            else:
+                fp = target
+
+            if os.path.exists(fp) and not build_option('force'):
+                raise EasyBuildError("Not overwriting existing file %s without --force", fp)
+
+            copy_file(orig_fp, fp)
+            res.append(fp)
+        else:
+            fp = orig_fp
+
+        if build_option('edit'):
+            edit_file(fp)
+
+        if build_option('show'):
+            print_msg("Contents of easyconfig file %s:\n" % fp)
+            print_msg(read_file(fp), prefix=False)
+
+    return res
+
+
+def handle_new(tmpdir, args):
+    """Handle use of --new."""
+    tmpfp = create_new_easyconfig(tmpdir, args)
+
+    # use current directory as default location to save generated file, in case no location is specified via --copy
+    res = handle_cat_copy_edit([tmpfp], target=build_option('copy') or '.', copy=True)
+
+    print_msg("easyconfig file %s created!" % res[0])
+
+
+def handle_search(search_query, search_filename, search_short):
+    """Handle use of --search."""
+
+    copy_path = build_option('copy')
+    search_action = copy_path or build_option('show') or build_option('edit')
+    res = search_easyconfigs(search_query, short=search_short, filename_only=search_filename,
+                             terse=build_option('terse'), return_hits=search_action)
+
+    if search_action:
+        # only perform action(s) if there's a single search result, unless --force is used
+        search_action_limit = build_option('search_action_limit') or 1
+        if len(res) > search_action_limit:
+            err_msg = "Found %d results which is more than search action limit (%d), so not performing search action(s)"
+            raise EasyBuildError(err_msg, len(res), search_action_limit)
+
+        res = handle_cat_copy_edit(res, target=copy_path or '.')
+
+        if copy_path:
+            print_msg("copied easyconfig files:")
+            for path in res:
+                print_msg("* %s" % path, prefix=False)
+
+
 def clean_exit(logfile, tmpdir, testing, silent=False):
     """Small utility function to perform a clean exit."""
     cleanup(logfile, tmpdir, testing, silent=silent)
@@ -191,7 +258,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     init_session_state = session_state()
 
     eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
-    options, orig_paths = eb_go.options, eb_go.args
+    options, args = eb_go.options, eb_go.args
 
     global _log
     (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings
@@ -220,8 +287,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
 
     # search for easyconfigs, if a query is specified
     if search_query:
-        search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
-                           terse=options.terse)
+        handle_search(search_query, options.search_filename, options.search_short)
 
     # GitHub options that warrant a silent cleanup & exit
     if options.check_github:
@@ -243,13 +309,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     elif options.list_software:
         print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')
 
+    elif options.new:
+        handle_new(eb_tmpdir, args)
+
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
     early_stop_options = [
         options.check_github,
         options.install_github_token,
         options.list_installed_software,
         options.list_software,
         options.merge_pr,
+        options.new,
         options.review_pr,
         options.terse,
         search_query,
@@ -270,14 +340,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         _log.warning("Failed to determine install path for easybuild-easyconfigs package.")
 
     if options.install_latest_eb_release:
-        if orig_paths:
+        if args:
             raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
                                  "other easyconfigs")
         else:
             eb_file = find_easybuild_easyconfig()
-            orig_paths.append(eb_file)
+            args.append(eb_file)
 
-    categorized_paths = categorize_files_by_type(orig_paths)
+    categorized_paths = categorize_files_by_type(args)
 
     # command line options that do not require any easyconfigs to be specified
     new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr