different approach to avoid blind overwriting of existing files with --try-*

.travis.yml

@@ -3,10 +3,10 @@ python: 2.6
 env:
   matrix:
     # purposely specifying slowest builds first, to gain time overall
-    - LMOD_VERSION=6.6.3
-    - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-    - LMOD_VERSION=7.7.16
-    - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+    - LMOD_VERSION=6.5.1
+    - LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+    - LMOD_VERSION=7.8.5
+    - LMOD_VERSION=7.8.5 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules
@@ -102,11 +102,12 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20180916.01 7e7563787a8bab8c30efdbdf95df6ec8ed63230ebcd361fee7b9574cbe9e74ed"
+    - EB_BOOTSTRAP_EXPECTED="20180925.01 d29478d5131fbf560a3806ef2613dc24e653c2857967788aace05107f614913b"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script
     - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap
     # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
     - unset PYTHONPATH
     # simply sanity check on bootstrapped EasyBuild module
-    - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; module load EasyBuild; eb --version
+    - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all
+    - module load EasyBuild; eb --version

RELEASE_NOTES

@@ -3,6 +3,30 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v3.7.1 (October 18th 2018)
+--------------------------
+
+bugfix release
+- various enhancements, including:
+  - generate .modulerc.lua when Lua syntax and Lmod >= 7.8 is used (#2597)
+  - allow --force to use regex if --try-toolchain can not map intelligently (#2605)
+  - add support for disabling modules tool version check (#2610)
+  - add support to ModuleGenerator.modulerc method to also write .modulerc file (#2611)
+  - check whether module file being wrapped exists in same directory as module wrapper when using Lmod 6.x (#2611)
+- various bug fixes, including:
+  - stop relying on 'easy_install' in bootstrap script, use 'python -m easy_install' instead (#2590)
+  - fix templating of values in list_software function (#2591)
+  - fix composing of lib64 fallback paths in sanity check (#2602)
+  - determine file_info for all easyconfigs before any actual copying in copy_easyconfigs function (#2604)
+  - also check for module wrappers in 'ModulesTool.exist' method (#2606)
+  - add trailing newline to module load message if it's not there yet (#2613)
+  - retain all dependencies when determining dependency tree of a toolchain (#2617)
+  - protect exts_lists from templating in dump method (#2619)
+  - making CUDA capability detection more robust (#2621)
+- other changes:
+  - lower required Lmod version to 6.5.1 (#2593)
+
+
 v3.7.0 (September 25th 2018)
 ----------------------------
 

easybuild/framework/easyblock.py

@@ -36,6 +36,7 @@
 :author: Fotis Georgatos (Uni.Lu, NTUA)
 :author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 :author: Maxime Boissonneault (Compute Canada)
+:author: Davide Vanzo (Vanderbilt University)
 """
 
 import copy
@@ -1122,8 +1123,12 @@ def make_module_extra(self, altroot=None, altversion=None):
                                      value, type(value))
             lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path']))
 
-        if self.cfg['modloadmsg']:
-            lines.append(self.module_generator.msg_on_load(self.cfg['modloadmsg']))
+        modloadmsg = self.cfg['modloadmsg']
+        if modloadmsg:
+            # add trailing newline to prevent that shell prompt is 'glued' to module load message
+            if not modloadmsg.endswith('\n'):
+                modloadmsg += '\n'
+            lines.append(self.module_generator.msg_on_load(modloadmsg))
 
         if self.cfg['modtclfooter']:
             if isinstance(self.module_generator, ModuleGeneratorTcl):
@@ -1669,9 +1674,13 @@ def fetch_step(self, skip_checksums=False):
         for mod_symlink_path in mod_symlink_paths:
             pardirs.append(os.path.join(install_path('mod'), mod_symlink_path, mod_subdir))
 
-        self.log.info("Checking dirs that need to be created: %s" % pardirs)
-        for pardir in pardirs:
-            mkdir(pardir, parents=True)
+        # skip directory creation if pre-create-installdir is set to False
+        if build_option('pre_create_installdir'):
+            self.log.info("Checking dirs that need to be created: %s" % pardirs)
+            for pardir in pardirs:
+                mkdir(pardir, parents=True)
+        else:
+            self.log.info("Skipped installation dirs check per user request")
 
     def checksum_step(self):
         """Verify checksum of sources and patches, if a checksum is available."""
@@ -2291,10 +2300,10 @@ def xs2str(xs):
                 # for library files in lib/, also consider fallback to lib64/ equivalent (and vice versa)
                 if not found and build_option('lib64_fallback_sanity_check'):
                     xs_alt = None
-                    if all(x.startswith('lib/') for x in xs):
-                        xs_alt = [os.path.join('lib64', *os.path.split(x)[1:]) for x in xs]
-                    elif all(x.startswith('lib64/') for x in xs):
-                        xs_alt = [os.path.join('lib', *os.path.split(x)[1:]) for x in xs]
+                    if all(x.startswith('lib/') or x == 'lib' for x in xs):
+                        xs_alt = [os.path.join('lib64', *x.split(os.path.sep)[1:]) for x in xs]
+                    elif all(x.startswith('lib64/') or x == 'lib64' for x in xs):
+                        xs_alt = [os.path.join('lib', *x.split(os.path.sep)[1:]) for x in xs]
 
                     if xs_alt:
                         self.log.info("%s not found at %s in %s, consider fallback locations: %s",

easybuild/framework/easyconfig/default.py

@@ -191,6 +191,8 @@
 
     # OTHER easyconfig parameters
     'buildstats': [None, "A list of dicts with build statistics", OTHER],
+    'deprecated': [False, "String specifying reason why this easyconfig file is deprecated "
+                          "and will be archived in the next major release of EasyBuild", OTHER],
 }
 
 

easybuild/framework/easyconfig/easyconfig.py

@@ -72,6 +72,7 @@
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
 from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
 from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars
+from easybuild.tools.version import VERSION
 from easybuild.toolchains.compiler.cuda import Cuda
 
 _log = fancylogger.getLogger('easyconfig.easyconfig', fname=False)
@@ -384,6 +385,9 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
         self.build_specs = build_specs
         self.parse()
 
+        # check whether this easyconfig file is deprecated, and act accordingly if so
+        self.check_deprecated(self.path)
+
         # perform validations
         self.validation = build_option('validate') and validate
         if self.validation:
@@ -442,15 +446,22 @@ def copy(self):
 
         return ec
 
-    def update(self, key, value):
+    def update(self, key, value, allow_duplicate=True):
         """
         Update a string configuration value with a value (i.e. append to it).
         """
         prev_value = self[key]
         if isinstance(prev_value, basestring):
-            self[key] = '%s %s ' % (prev_value, value)
+            if allow_duplicate or value not in prev_value:
+                self[key] = '%s %s ' % (prev_value, value)
         elif isinstance(prev_value, list):
-            self[key] = prev_value + value
+            if allow_duplicate:
+                self[key] = prev_value + value
+            else:
+                for item in value:
+                    # add only those items that aren't already in the list
+                    if item not in prev_value:
+                        self[key] = prev_value + [item]
         else:
             raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key)
 
@@ -534,6 +545,26 @@ def parse(self):
         # indicate that this is a parsed easyconfig
         self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
 
+    def check_deprecated(self, path):
+        """Check whether this easyconfig file is deprecated."""
+
+        depr_msgs = []
+
+        deprecated = self['deprecated']
+        if deprecated:
+            if isinstance(deprecated, basestring):
+                depr_msgs.append("easyconfig file '%s' is marked as deprecated:\n%s\n" % (path, deprecated))
+            else:
+                raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated))
+
+        if self.toolchain.is_deprecated():
+            depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain'])
+
+        if depr_msgs:
+            depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
+            more_info_depr_ec = "(see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)"
+            self.log.deprecated(', '.join(depr_msgs), '%s.0' % depr_maj_ver, more_info=more_info_depr_ec)
+
     def validate(self, check_osdeps=True):
         """
         Validate this easyonfig
@@ -759,9 +790,12 @@ def all_dependencies(self):
 
         return self._all_dependencies
 
-    def dump(self, fp):
+    def dump(self, fp, overwrite=True, backup=False):
         """
         Dump this easyconfig to file, with the given filename.
+
+        :param overwrite: overwrite existing file at specified location without use of --force
+        :param backup: create backup of existing file before overwriting it
         """
         orig_enable_templating = self.enable_templating
 
@@ -797,7 +831,7 @@ def dump(self, fp):
             ectxt = autopep8.fix_code(ectxt, options=autopep8_opts)
             self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt)
 
-        write_file(fp, ectxt.strip())
+        write_file(fp, ectxt, overwrite=overwrite, backup=backup, verbose=backup)
 
         self.enable_templating = orig_enable_templating
 
@@ -1615,16 +1649,19 @@ def clean_up_easyconfigs(paths):
         write_file(path, ectxt, forced=True)
 
 
-def copy_easyconfigs(paths, target_dir):
+def det_file_info(paths, target_dir):
     """
-    Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot.
+    Determine useful information on easyconfig files relative to a target directory,
+    before any actual operation (e.g. copying) is performed
 
-    :param paths: list of paths to copy to git working dir
+    :param paths: list of paths to easyconfig files
     :param target_dir: target directory
-    :return: dict with useful information on copied easyconfig files (corresponding EasyConfig instances, paths, status)
+    :return: dict with useful information on easyconfig files (corresponding EasyConfig instances, paths, status)
+             relative to a target directory
     """
     file_info = {
         'ecs': [],
+        'paths': [],
         'paths_in_repo': [],
         'new': [],
         'new_folder': [],
@@ -1634,6 +1671,7 @@ def copy_easyconfigs(paths, target_dir):
     for path in paths:
         ecs = process_easyconfig(path, validate=False)
         if len(ecs) == 1:
+            file_info['paths'].append(path)
             file_info['ecs'].append(ecs[0]['ec'])
 
             soft_name = file_info['ecs'][-1].name
@@ -1646,14 +1684,27 @@ def copy_easyconfigs(paths, target_dir):
             file_info['new'].append(new_file)
             file_info['new_folder'].append(new_folder)
             file_info['new_file_in_existing_folder'].append(new_file and not new_folder)
-
-            copy_file(path, target_path, force_in_dry_run=True)
-
             file_info['paths_in_repo'].append(target_path)
 
         else:
             raise EasyBuildError("Multiple EasyConfig instances obtained from easyconfig file %s", path)
 
+    return file_info
+
+
+def copy_easyconfigs(paths, target_dir):
+    """
+    Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot.
+
+    :param paths: list of paths to copy to git working dir
+    :param target_dir: target directory
+    :return: dict with useful information on copied easyconfig files (corresponding EasyConfig instances, paths, status)
+    """
+    file_info = det_file_info(paths, target_dir)
+
+    for path, target_path in zip(file_info['paths'], file_info['paths_in_repo']):
+        copy_file(path, target_path, force_in_dry_run=True)
+
     if build_option('cleanup_easyconfigs'):
         clean_up_easyconfigs(file_info['paths_in_repo'])
 

easybuild/framework/easyconfig/format/format.py

@@ -53,8 +53,8 @@
 DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies']
 
 # values for these keys will not be templated in dump()
-EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'homepage', 'name', 'toolchain', 'version'] \
-                                  + DEPENDENCY_PARAMETERS
+EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain',
+                                   'version'] + DEPENDENCY_PARAMETERS
 
 # ordered groups of keys to obtain a nice looking easyconfig file
 GROUPED_PARAMS = [

easybuild/framework/easyconfig/format/one.py

@@ -236,6 +236,8 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
                     # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
                     if key in DEPENDENCY_PARAMETERS:
                         valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
+                    elif key == 'toolchain':
+                        valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
                     else:
                         valstr = quote_py_str(ecfg[key])
 
@@ -287,9 +289,9 @@ def extract_comments(self, rawtxt):
         Inline comments on items of iterable values are also extracted.
         """
         self.comments = {
-            'above' : {},  # comments for a particular parameter definition
-            'header' : [],  # header comment lines
-            'inline' : {},  # inline comments
+            'above': {},  # comments for a particular parameter definition
+            'header': [],  # header comment lines
+            'inline': {},  # inline comments
             'iter': {},  # (inline) comments on elements of iterable values
             'tail': [],
          }
@@ -410,7 +412,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
 
             if 'dependencies' in block:
                 for dep in block['dependencies']:
-                    if not dep in [b['name'] for b in blocks]:
+                    if dep not in [b['name'] for b in blocks]:
                         raise EasyBuildError("Block %s depends on %s, but block was not found.", name, dep)
 
                     dep = [b for b in blocks if b['name'] == dep][0]

easybuild/framework/easyconfig/templates.py

@@ -80,23 +80,23 @@
 # constant templates that can be used in easyconfigs
 TEMPLATE_CONSTANTS = [
     # source url constants
-    ('APACHE_SOURCE', 'http://archive.apache.org/dist/%(namelower)s',
+    ('APACHE_SOURCE', 'https://archive.apache.org/dist/%(namelower)s',
      'apache.org source url'),
-    ('BITBUCKET_SOURCE', 'http://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
+    ('BITBUCKET_SOURCE', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
      'bitbucket.org source url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
-    ('BITBUCKET_DOWNLOADS', 'http://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
+    ('BITBUCKET_DOWNLOADS', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
      'bitbucket.org downloads url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
-    ('CRAN_SOURCE', 'http://cran.r-project.org/src/contrib',
+    ('CRAN_SOURCE', 'https://cran.r-project.org/src/contrib',
      'CRAN (contrib) source url'),
-    ('FTPGNOME_SOURCE', 'http://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
+    ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
      'http download for gnome ftp server'),
     ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive',
      'GitHub source URL (namelower is used if github_account easyconfig parameter is not specified)'),
     ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive',
      'GitHub source URL (lowercase name, namelower is used if github_account easyconfig parameter is not specified)'),
-    ('GNU_SAVANNAH_SOURCE', 'http://download-mirror.savannah.gnu.org/releases/%(namelower)s',
+    ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
      'download.savannah.gnu.org source url'),
-    ('GNU_SOURCE', 'http://ftpmirror.gnu.org/gnu/%(namelower)s',
+    ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
      'gnu.org source url'),
     ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files',
      'googlecode.com source url'),
@@ -106,19 +106,19 @@
      'pypi source url'),  # e.g., Cython, Sphinx
     ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
      'pypi source url (lowercase name)'),  # e.g., Greenlet, PyZMQ
-    ('R_SOURCE', 'http://cran.r-project.org/src/base/R-%(version_major)s',
+    ('R_SOURCE', 'https://cran.r-project.org/src/base/R-%(version_major)s',
      'cran.r-project.org (base) source url'),
-    ('SOURCEFORGE_SOURCE', 'http://download.sourceforge.net/%(namelower)s',
+    ('SOURCEFORGE_SOURCE', 'https://download.sourceforge.net/%(namelower)s',
      'sourceforge.net source url'),
-    ('XORG_DATA_SOURCE', 'http://xorg.freedesktop.org/archive/individual/data/',
+    ('XORG_DATA_SOURCE', 'https://xorg.freedesktop.org/archive/individual/data/',
      'xorg data source url'),
-    ('XORG_LIB_SOURCE', 'http://xorg.freedesktop.org/archive/individual/lib/',
+    ('XORG_LIB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/lib/',
      'xorg lib source url'),
-    ('XORG_PROTO_SOURCE', 'http://xorg.freedesktop.org/archive/individual/proto/',
+    ('XORG_PROTO_SOURCE', 'https://xorg.freedesktop.org/archive/individual/proto/',
      'xorg proto source url'),
-    ('XORG_UTIL_SOURCE', 'http://xorg.freedesktop.org/archive/individual/util/',
+    ('XORG_UTIL_SOURCE', 'https://xorg.freedesktop.org/archive/individual/util/',
      'xorg util source url'),
-    ('XORG_XCB_SOURCE', 'http://xorg.freedesktop.org/archive/individual/xcb/',
+    ('XORG_XCB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/xcb/',
      'xorg xcb source url'),
 
     # TODO, not urgent, yet nice to have: