diff --git a/.travis.yml b/.travis.yml
index e75b9dc9ea..125b435b4d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,10 +3,10 @@ python: 2.6
env:
matrix:
# purposely specifying slowest builds first, to gain time overall
- - LMOD_VERSION=6.6.3
- - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
- - LMOD_VERSION=7.7.16
- - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+ - LMOD_VERSION=6.5.1
+ - LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+ - LMOD_VERSION=7.8.5
+ - LMOD_VERSION=7.8.5 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
- ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
- ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
- ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules
@@ -102,11 +102,12 @@ script:
- EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
- EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
- EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
- - EB_BOOTSTRAP_EXPECTED="20180916.01 7e7563787a8bab8c30efdbdf95df6ec8ed63230ebcd361fee7b9574cbe9e74ed"
+ - EB_BOOTSTRAP_EXPECTED="20180925.01 d29478d5131fbf560a3806ef2613dc24e653c2857967788aace05107f614913b"
- test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
# test bootstrap script
- python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap
# unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
- unset PYTHONPATH
# simply sanity check on bootstrapped EasyBuild module
- - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; module load EasyBuild; eb --version
+ - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all
+ - module load EasyBuild; eb --version
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index d432727995..080c267830 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -3,6 +3,30 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
+v3.7.1 (October 18th 2018)
+--------------------------
+
+bugfix release
+- various enhancements, including:
+ - generate .modulerc.lua when Lua syntax and Lmod >= 7.8 is used (#2597)
+ - allow --force to use regex if --try-toolchain can not map intelligently (#2605)
+ - add support for disabling modules tool version check (#2610)
+ - add support to ModuleGenerator.modulerc method to also write .modulerc file (#2611)
+ - check whether module file being wrapped exists in same directory as module wrapper when using Lmod 6.x (#2611)
+- various bug fixes, including:
+ - stop relying on 'easy_install' in bootstrap script, use 'python -m easy_install' instead (#2590)
+ - fix templating of values in list_software function (#2591)
+ - fix composing of lib64 fallback paths in sanity check (#2602)
+ - determine file_info for all easyconfigs before any actual copying in copy_easyconfigs function (#2604)
+ - also check for module wrappers in 'ModulesTool.exist' method (#2606)
+ - add trailing newline to module load message if it's not there yet (#2613)
+ - retain all dependencies when determining dependency tree of a toolchain (#2617)
+ - protect exts_lists from templating in dump method (#2619)
+ - making CUDA capability detection more robust (#2621)
+- other changes:
+ - lower required Lmod version to 6.5.1 (#2593)
+
+
v3.7.0 (September 25th 2018)
----------------------------
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 27987fb9a5..a305d046e5 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -36,6 +36,7 @@
:author: Fotis Georgatos (Uni.Lu, NTUA)
:author: Damian Alvarez (Forschungszentrum Juelich GmbH)
:author: Maxime Boissonneault (Compute Canada)
+:author: Davide Vanzo (Vanderbilt University)
"""
import copy
@@ -1122,8 +1123,12 @@ def make_module_extra(self, altroot=None, altversion=None):
value, type(value))
lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path']))
- if self.cfg['modloadmsg']:
- lines.append(self.module_generator.msg_on_load(self.cfg['modloadmsg']))
+ modloadmsg = self.cfg['modloadmsg']
+ if modloadmsg:
+ # add trailing newline to prevent that shell prompt is 'glued' to module load message
+ if not modloadmsg.endswith('\n'):
+ modloadmsg += '\n'
+ lines.append(self.module_generator.msg_on_load(modloadmsg))
if self.cfg['modtclfooter']:
if isinstance(self.module_generator, ModuleGeneratorTcl):
@@ -1669,9 +1674,13 @@ def fetch_step(self, skip_checksums=False):
for mod_symlink_path in mod_symlink_paths:
pardirs.append(os.path.join(install_path('mod'), mod_symlink_path, mod_subdir))
- self.log.info("Checking dirs that need to be created: %s" % pardirs)
- for pardir in pardirs:
- mkdir(pardir, parents=True)
+ # skip directory creation if pre-create-installdir is set to False
+ if build_option('pre_create_installdir'):
+ self.log.info("Checking dirs that need to be created: %s" % pardirs)
+ for pardir in pardirs:
+ mkdir(pardir, parents=True)
+ else:
+ self.log.info("Skipped installation dirs check per user request")
def checksum_step(self):
"""Verify checksum of sources and patches, if a checksum is available."""
@@ -2291,10 +2300,10 @@ def xs2str(xs):
# for library files in lib/, also consider fallback to lib64/ equivalent (and vice versa)
if not found and build_option('lib64_fallback_sanity_check'):
xs_alt = None
- if all(x.startswith('lib/') for x in xs):
- xs_alt = [os.path.join('lib64', *os.path.split(x)[1:]) for x in xs]
- elif all(x.startswith('lib64/') for x in xs):
- xs_alt = [os.path.join('lib', *os.path.split(x)[1:]) for x in xs]
+ if all(x.startswith('lib/') or x == 'lib' for x in xs):
+ xs_alt = [os.path.join('lib64', *x.split(os.path.sep)[1:]) for x in xs]
+ elif all(x.startswith('lib64/') or x == 'lib64' for x in xs):
+ xs_alt = [os.path.join('lib', *x.split(os.path.sep)[1:]) for x in xs]
if xs_alt:
self.log.info("%s not found at %s in %s, consider fallback locations: %s",
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index fc02703501..b0adea255d 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -191,6 +191,8 @@
# OTHER easyconfig parameters
'buildstats': [None, "A list of dicts with build statistics", OTHER],
+ 'deprecated': [False, "String specifying reason why this easyconfig file is deprecated "
+ "and will be archived in the next major release of EasyBuild", OTHER],
}
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 3d041bdfe8..2c692a217f 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -72,6 +72,7 @@
from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars
+from easybuild.tools.version import VERSION
from easybuild.toolchains.compiler.cuda import Cuda
_log = fancylogger.getLogger('easyconfig.easyconfig', fname=False)
@@ -384,6 +385,9 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
self.build_specs = build_specs
self.parse()
+ # check whether this easyconfig file is deprecated, and act accordingly if so
+ self.check_deprecated(self.path)
+
# perform validations
self.validation = build_option('validate') and validate
if self.validation:
@@ -442,15 +446,22 @@ def copy(self):
return ec
- def update(self, key, value):
+ def update(self, key, value, allow_duplicate=True):
"""
Update a string configuration value with a value (i.e. append to it).
"""
prev_value = self[key]
if isinstance(prev_value, basestring):
- self[key] = '%s %s ' % (prev_value, value)
+ if allow_duplicate or value not in prev_value:
+ self[key] = '%s %s ' % (prev_value, value)
elif isinstance(prev_value, list):
- self[key] = prev_value + value
+ if allow_duplicate:
+ self[key] = prev_value + value
+ else:
+ for item in value:
+ # add only those items that aren't already in the list
+ if item not in prev_value:
+ self[key] = prev_value + [item]
else:
raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key)
@@ -534,6 +545,26 @@ def parse(self):
# indicate that this is a parsed easyconfig
self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
+ def check_deprecated(self, path):
+ """Check whether this easyconfig file is deprecated."""
+
+ depr_msgs = []
+
+ deprecated = self['deprecated']
+ if deprecated:
+ if isinstance(deprecated, basestring):
+ depr_msgs.append("easyconfig file '%s' is marked as deprecated:\n%s\n" % (path, deprecated))
+ else:
+ raise EasyBuildError("Wrong type for value of 'deprecated' easyconfig parameter: %s", type(deprecated))
+
+ if self.toolchain.is_deprecated():
+ depr_msgs.append("toolchain '%(name)s/%(version)s' is marked as deprecated" % self['toolchain'])
+
+ if depr_msgs:
+ depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
+ more_info_depr_ec = "(see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)"
+ self.log.deprecated(', '.join(depr_msgs), '%s.0' % depr_maj_ver, more_info=more_info_depr_ec)
+
def validate(self, check_osdeps=True):
"""
Validate this easyonfig
@@ -759,9 +790,12 @@ def all_dependencies(self):
return self._all_dependencies
- def dump(self, fp):
+ def dump(self, fp, overwrite=True, backup=False):
"""
Dump this easyconfig to file, with the given filename.
+
+ :param overwrite: overwrite existing file at specified location without use of --force
+ :param backup: create backup of existing file before overwriting it
"""
orig_enable_templating = self.enable_templating
@@ -797,7 +831,7 @@ def dump(self, fp):
ectxt = autopep8.fix_code(ectxt, options=autopep8_opts)
self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt)
- write_file(fp, ectxt.strip())
+ write_file(fp, ectxt, overwrite=overwrite, backup=backup, verbose=backup)
self.enable_templating = orig_enable_templating
@@ -1615,16 +1649,19 @@ def clean_up_easyconfigs(paths):
write_file(path, ectxt, forced=True)
-def copy_easyconfigs(paths, target_dir):
+def det_file_info(paths, target_dir):
"""
- Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot.
+ Determine useful information on easyconfig files relative to a target directory,
+ before any actual operation (e.g. copying) is performed
- :param paths: list of paths to copy to git working dir
+ :param paths: list of paths to easyconfig files
:param target_dir: target directory
- :return: dict with useful information on copied easyconfig files (corresponding EasyConfig instances, paths, status)
+ :return: dict with useful information on easyconfig files (corresponding EasyConfig instances, paths, status)
+ relative to a target directory
"""
file_info = {
'ecs': [],
+ 'paths': [],
'paths_in_repo': [],
'new': [],
'new_folder': [],
@@ -1634,6 +1671,7 @@ def copy_easyconfigs(paths, target_dir):
for path in paths:
ecs = process_easyconfig(path, validate=False)
if len(ecs) == 1:
+ file_info['paths'].append(path)
file_info['ecs'].append(ecs[0]['ec'])
soft_name = file_info['ecs'][-1].name
@@ -1646,14 +1684,27 @@ def copy_easyconfigs(paths, target_dir):
file_info['new'].append(new_file)
file_info['new_folder'].append(new_folder)
file_info['new_file_in_existing_folder'].append(new_file and not new_folder)
-
- copy_file(path, target_path, force_in_dry_run=True)
-
file_info['paths_in_repo'].append(target_path)
else:
raise EasyBuildError("Multiple EasyConfig instances obtained from easyconfig file %s", path)
+ return file_info
+
+
+def copy_easyconfigs(paths, target_dir):
+ """
+ Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot.
+
+ :param paths: list of paths to copy to git working dir
+ :param target_dir: target directory
+ :return: dict with useful information on copied easyconfig files (corresponding EasyConfig instances, paths, status)
+ """
+ file_info = det_file_info(paths, target_dir)
+
+ for path, target_path in zip(file_info['paths'], file_info['paths_in_repo']):
+ copy_file(path, target_path, force_in_dry_run=True)
+
if build_option('cleanup_easyconfigs'):
clean_up_easyconfigs(file_info['paths_in_repo'])
diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py
index 0d4b68fc30..56e82e0e26 100644
--- a/easybuild/framework/easyconfig/format/format.py
+++ b/easybuild/framework/easyconfig/format/format.py
@@ -53,8 +53,8 @@
DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies']
# values for these keys will not be templated in dump()
-EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'homepage', 'name', 'toolchain', 'version'] \
- + DEPENDENCY_PARAMETERS
+EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain',
+ 'version'] + DEPENDENCY_PARAMETERS
# ordered groups of keys to obtain a nice looking easyconfig file
GROUPED_PARAMS = [
diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py
index c1342e1bb4..7e505deb4a 100644
--- a/easybuild/framework/easyconfig/format/one.py
+++ b/easybuild/framework/easyconfig/format/one.py
@@ -236,6 +236,8 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
if key in DEPENDENCY_PARAMETERS:
valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
+ elif key == 'toolchain':
+ valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
else:
valstr = quote_py_str(ecfg[key])
@@ -287,9 +289,9 @@ def extract_comments(self, rawtxt):
Inline comments on items of iterable values are also extracted.
"""
self.comments = {
- 'above' : {}, # comments for a particular parameter definition
- 'header' : [], # header comment lines
- 'inline' : {}, # inline comments
+ 'above': {}, # comments for a particular parameter definition
+ 'header': [], # header comment lines
+ 'inline': {}, # inline comments
'iter': {}, # (inline) comments on elements of iterable values
'tail': [],
}
@@ -410,7 +412,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
if 'dependencies' in block:
for dep in block['dependencies']:
- if not dep in [b['name'] for b in blocks]:
+ if dep not in [b['name'] for b in blocks]:
raise EasyBuildError("Block %s depends on %s, but block was not found.", name, dep)
dep = [b for b in blocks if b['name'] == dep][0]
diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py
index 963129f743..0d0819ea95 100644
--- a/easybuild/framework/easyconfig/templates.py
+++ b/easybuild/framework/easyconfig/templates.py
@@ -80,23 +80,23 @@
# constant templates that can be used in easyconfigs
TEMPLATE_CONSTANTS = [
# source url constants
- ('APACHE_SOURCE', 'http://archive.apache.org/dist/%(namelower)s',
+ ('APACHE_SOURCE', 'https://archive.apache.org/dist/%(namelower)s',
'apache.org source url'),
- ('BITBUCKET_SOURCE', 'http://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
+ ('BITBUCKET_SOURCE', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
'bitbucket.org source url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
- ('BITBUCKET_DOWNLOADS', 'http://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
+ ('BITBUCKET_DOWNLOADS', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
'bitbucket.org downloads url (namelower is used if bitbucket_account easyconfig parameter is not specified)'),
- ('CRAN_SOURCE', 'http://cran.r-project.org/src/contrib',
+ ('CRAN_SOURCE', 'https://cran.r-project.org/src/contrib',
'CRAN (contrib) source url'),
- ('FTPGNOME_SOURCE', 'http://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
+ ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
'http download for gnome ftp server'),
('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive',
'GitHub source URL (namelower is used if github_account easyconfig parameter is not specified)'),
('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive',
'GitHub source URL (lowercase name, namelower is used if github_account easyconfig parameter is not specified)'),
- ('GNU_SAVANNAH_SOURCE', 'http://download-mirror.savannah.gnu.org/releases/%(namelower)s',
+ ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
'download.savannah.gnu.org source url'),
- ('GNU_SOURCE', 'http://ftpmirror.gnu.org/gnu/%(namelower)s',
+ ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
'gnu.org source url'),
('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files',
'googlecode.com source url'),
@@ -106,19 +106,19 @@
'pypi source url'), # e.g., Cython, Sphinx
('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ
- ('R_SOURCE', 'http://cran.r-project.org/src/base/R-%(version_major)s',
+ ('R_SOURCE', 'https://cran.r-project.org/src/base/R-%(version_major)s',
'cran.r-project.org (base) source url'),
- ('SOURCEFORGE_SOURCE', 'http://download.sourceforge.net/%(namelower)s',
+ ('SOURCEFORGE_SOURCE', 'https://download.sourceforge.net/%(namelower)s',
'sourceforge.net source url'),
- ('XORG_DATA_SOURCE', 'http://xorg.freedesktop.org/archive/individual/data/',
+ ('XORG_DATA_SOURCE', 'https://xorg.freedesktop.org/archive/individual/data/',
'xorg data source url'),
- ('XORG_LIB_SOURCE', 'http://xorg.freedesktop.org/archive/individual/lib/',
+ ('XORG_LIB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/lib/',
'xorg lib source url'),
- ('XORG_PROTO_SOURCE', 'http://xorg.freedesktop.org/archive/individual/proto/',
+ ('XORG_PROTO_SOURCE', 'https://xorg.freedesktop.org/archive/individual/proto/',
'xorg proto source url'),
- ('XORG_UTIL_SOURCE', 'http://xorg.freedesktop.org/archive/individual/util/',
+ ('XORG_UTIL_SOURCE', 'https://xorg.freedesktop.org/archive/individual/util/',
'xorg util source url'),
- ('XORG_XCB_SOURCE', 'http://xorg.freedesktop.org/archive/individual/xcb/',
+ ('XORG_XCB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/xcb/',
'xorg xcb source url'),
# TODO, not urgent, yet nice to have:
diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py
index 3a83e7294d..fc28b4bf7e 100644
--- a/easybuild/framework/easyconfig/tweak.py
+++ b/easybuild/framework/easyconfig/tweak.py
@@ -120,8 +120,28 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
else:
target_toolchain['version'] = source_toolchain['version']
- src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool)
+ try:
+ src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool)
+ except EasyBuildError as err:
+ # make sure exception was raised by match_minimum_tc_specs because toolchain mapping could not be done
+ if "No possible mapping from source toolchain" in err.msg:
+
+ if build_option('force'):
+ warning_msg = "Combining --try-toolchain with --force for toolchains with unequal capabilities:"
+ warning_msg += " disabling recursion and not changing (sub)toolchains for dependencies."
+ print_warning(warning_msg)
+ revert_to_regex = True
+ modifying_toolchains = False
+ else:
+ error_msg = err.msg + '\n'
+ error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. "
+ error_msg += "(If you know what you are doing, you can use --force to proceed anyway.)"
+ raise EasyBuildError(error_msg, target_toolchain['name'], source_toolchain['name'])
+ else:
+ # simply re-raise the exception if something else went wrong
+ raise err
+ if not revert_to_regex:
_log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs)
orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)
@@ -338,14 +358,7 @@ def __repr__(self):
_log.debug("Generated file name for tweaked easyconfig file: %s", tweaked_ec)
# write out tweaked easyconfig file
- if os.path.exists(tweaked_ec):
- if build_option('force'):
- print_warning("Overwriting existing file at %s with tweaked easyconfig file (due to --force)", tweaked_ec)
- else:
- raise EasyBuildError("A file already exists at %s where tweaked easyconfig file would be written",
- tweaked_ec)
-
- write_file(tweaked_ec, ectxt)
+ write_file(tweaked_ec, ectxt, backup=True, overwrite=False, verbose=True)
_log.info("Tweaked easyconfig file written to %s", tweaked_ec)
return tweaked_ec
@@ -470,17 +483,17 @@ def select_or_generate_ec(fp, paths, specs):
# TOOLCHAIN NAME
# we can't rely on set, because we also need to be able to obtain a list of unique lists
- def unique(l):
+ def unique(lst):
"""Retain unique elements in a sorted list."""
- l = sorted(l)
- if len(l) > 1:
- l2 = [l[0]]
- for x in l:
- if not x == l2[-1]:
- l2.append(x)
- return l2
+ lst = sorted(lst)
+ if len(lst) > 1:
+ res = [lst[0]]
+ for x in lst:
+ if not x == res[-1]:
+ res.append(x)
+ return res
else:
- return l
+ return lst
# determine list of unique toolchain names
tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files])
@@ -750,7 +763,7 @@ def get_dep_tree_of_toolchain(toolchain_spec, modtool):
toolchain_spec['name'], toolchain_spec['version'])
ec = process_easyconfig(path, validate=False)
- return [dep['ec'] for dep in resolve_dependencies(ec, modtool)]
+ return [dep['ec'] for dep in resolve_dependencies(ec, modtool, retain_all_deps=True)]
def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool):
@@ -843,7 +856,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
else:
raise EasyBuildError("A file already exists at %s where tweaked easyconfig file would be written",
tweaked_spec)
- parsed_ec['ec'].dump(tweaked_spec)
+ parsed_ec['ec'].dump(tweaked_spec, overwrite=False, backup=True)
_log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec)
return tweaked_spec
diff --git a/easybuild/main.py b/easybuild/main.py
index 38444a5ea5..4de63d1166 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -39,20 +39,17 @@
import os
import stat
import sys
-import tempfile
import traceback
# IMPORTANT this has to be the first easybuild import as it customises the logging
# expect missing log output when this not the case!
-from easybuild.tools.build_log import EasyBuildError, init_logging, print_error, print_msg, print_warning, stop_logging
+from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, stop_logging
-import easybuild.tools.config as config
-import easybuild.tools.options as eboptions
-from easybuild.framework.easyblock import EasyBlock, build_and_install_one, inject_checksums
+from easybuild.framework.easyblock import build_and_install_one, inject_checksums
from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
from easybuild.framework.easyconfig.easyconfig import verify_easyconfig_filename
from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
-from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, dep_graph
+from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph
from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
@@ -64,27 +61,16 @@
from easybuild.tools.github import new_pr, merge_pr, update_pr
from easybuild.tools.hooks import START, END, load_hooks, run_hook
from easybuild.tools.modules import modules_tool
-from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color
-from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs
+from easybuild.tools.options import set_up_configuration, use_color
+from easybuild.tools.robot import check_conflicts, dry_run, resolve_dependencies, search_easyconfigs
from easybuild.tools.package.utilities import check_pkg_support
from easybuild.tools.parallelbuild import submit_jobs
from easybuild.tools.repository.repository import init_repository
from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
-from easybuild.tools.version import this_is_easybuild
_log = None
-def log_start(eb_command_line, eb_tmpdir):
- """Log startup info."""
- _log.info(this_is_easybuild())
-
- # log used command line
- _log.info("Command line: %s" % (' '.join(eb_command_line)))
-
- _log.info("Using %s as temporary directory" % eb_tmpdir)
-
-
def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=False):
"""Find easyconfigs by build specifications."""
generated, ec_file = obtain_ec_for(build_specs, robot_path, None)
@@ -187,23 +173,6 @@ def run_contrib_style_checks(ecs, check_contrib, check_style):
return check_contrib or check_style
-def check_root_usage(allow_use_as_root=False):
- """
- Check whether we are running as root, and act accordingly
-
- :param allow_use_as_root: allow use of EasyBuild as root (but do print a warning when doing so)
- """
- if os.getuid() == 0:
- if allow_use_as_root:
- msg = "Using EasyBuild as root is NOT recommended, please proceed with care!\n"
- msg += "(this is only allowed because EasyBuild was configured with "
- msg += "--allow-use-as-root-and-accept-consequences)"
- print_warning(msg)
- else:
- raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
- "so let's end this here.")
-
-
def clean_exit(logfile, tmpdir, testing, silent=False):
"""Small utility function to perform a clean exit."""
cleanup(logfile, tmpdir, testing, silent=silent)
@@ -221,63 +190,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
- # initialise options
- eb_go = eboptions.parse_options(args=args)
- options = eb_go.options
- orig_paths = eb_go.args
-
- # set umask (as early as possible)
- if options.umask is not None:
- new_umask = int(options.umask, 8)
- old_umask = os.umask(new_umask)
+ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
+ options, orig_paths = eb_go.options, eb_go.args
- # set by option parsers via set_tmpdir
- eb_tmpdir = tempfile.gettempdir()
-
- search_query = options.search or options.search_filename or options.search_short
-
- # initialise logging for main
global _log
- _log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
- silent=(testing or options.terse or search_query), colorize=options.color)
-
- # disallow running EasyBuild as root (by default)
- check_root_usage(allow_use_as_root=options.allow_use_as_root_and_accept_consequences)
-
- # log startup info
- eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
- log_start(eb_cmd_line, eb_tmpdir)
-
- if options.umask is not None:
- _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
-
- # process software build specifications (if any), i.e.
- # software name/version, toolchain name/version, extra patches, ...
- (try_to_generate, build_specs) = process_software_build_specs(options)
-
- # determine robot path
- # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
- tweaked_ecs = try_to_generate and build_specs
- tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
- auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
- robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
- _log.debug("Full robot path: %s" % robot_path)
-
- # configure & initialize build options
- config_options_dict = eb_go.get_options_by_section('config')
- build_options = {
- 'build_specs': build_specs,
- 'command_line': eb_cmd_line,
- 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
- 'pr_path': pr_path,
- 'robot_path': robot_path,
- 'silent': testing,
- 'try_to_generate': try_to_generate,
- 'valid_stops': [x[0] for x in EasyBlock.get_steps()],
- }
- # initialise the EasyBuild configuration & build options
- config.init(options, config_options_dict)
- config.init_build_options(build_options=build_options, cmdline_options=options)
+ (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
@@ -527,5 +444,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
if __name__ == "__main__":
try:
main()
- except EasyBuildError, e:
- print_error(e.msg)
+ except EasyBuildError as err:
+ print_error(err.msg)
+ except KeyboardInterrupt:
+ print_error("Cancelled by user (keyboard interrupt)")
diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py
index 8d497ef6d5..f2705f74f2 100644
--- a/easybuild/scripts/bootstrap_eb.py
+++ b/easybuild/scripts/bootstrap_eb.py
@@ -54,7 +54,7 @@
from hashlib import md5
-EB_BOOTSTRAP_VERSION = '20180916.01'
+EB_BOOTSTRAP_VERSION = '20180925.01'
# argparse preferrred, optparse deprecated >=2.7
HAVE_ARGPARSE = False
@@ -381,26 +381,16 @@ def check_easy_install_cmd():
easy_install_regex = re.compile('^(setuptools|distribute) %s' % setuptools.__version__)
debug("Pattern for 'easy_install --version': %s" % easy_install_regex.pattern)
- for path in os.getenv('PATH', '').split(os.pathsep):
- easy_install = os.path.join(path, 'easy_install')
- debug("Checking %s..." % easy_install)
- res = False
- if os.path.exists(easy_install):
- cmd = "PYTHONPATH='%s' %s %s --version" % (os.getenv('PYTHONPATH', ''), sys.executable, easy_install)
- os.system("%s > %s 2>&1" % (cmd, outfile))
- outtxt = open(outfile).read().strip()
- debug("Output of '%s':\n%s" % (cmd, outtxt))
- res = bool(easy_install_regex.match(outtxt))
- debug("Result for %s: %s" % (easy_install, res))
- else:
- debug("%s does not exist" % easy_install)
-
- if res:
- debug("Found right 'easy_install' command in %s" % path)
- curr_path = os.environ.get('PATH', '').split(os.pathsep)
- os.environ['PATH'] = os.pathsep.join([path] + curr_path)
- debug("$PATH: %s" % os.environ['PATH'])
- return
+ pythonpath = os.getenv('PYTHONPATH', '')
+ cmd = "PYTHONPATH='%s' %s -m easy_install --version" % (pythonpath, sys.executable)
+ os.system("%s > %s 2>&1" % (cmd, outfile))
+ outtxt = open(outfile).read().strip()
+ debug("Output of '%s':\n%s" % (cmd, outtxt))
+ res = bool(easy_install_regex.match(outtxt))
+ debug("Result: %s" % res)
+ if res:
+ debug("Found right 'easy_install' command")
+ return
error("Failed to find right 'easy_install' command!")
@@ -641,15 +631,15 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
preinstallopts = ''
if distribute_egg_dir is not None:
- # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
+ # inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
- # this is required for the easy_install from stage 1 to work
+ # this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
- preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable
+ preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
@@ -658,7 +648,7 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
- preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
+ preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
@@ -704,8 +694,10 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
- handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
+ ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
+ handle.write(ebfile_txt)
handle.close()
+ debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh
old mode 100644
new mode 100755
index b5cd7ddd28..b5ea2eb3d1
--- a/easybuild/scripts/install-EasyBuild-develop.sh
+++ b/easybuild/scripts/install-EasyBuild-develop.sh
@@ -34,13 +34,13 @@ github_clone_branch()
cd "${REPO}"
git remote add "github_hpcugent" "git@github.com:hpcugent/${REPO}.git"
git fetch github_hpcugent
- git branch --set-upstream "${BRANCH}" "github_hpcugent/${BRANCH}"
+ git branch --set-upstream-to "github_hpcugent/${BRANCH}" "${BRANCH}"
else
echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..."
cd "${REPO}"
git remote add "github_easybuilders" "git@github.com:easybuilders/${REPO}.git"
git fetch github_easybuilders
- git branch --set-upstream "${BRANCH}" "github_easybuilders/${BRANCH}"
+ git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}"
fi
}
diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py
index 338789be69..8c56cdd507 100644
--- a/easybuild/toolchains/compiler/pgi.py
+++ b/easybuild/toolchains/compiler/pgi.py
@@ -26,7 +26,7 @@
# along with EasyBuild. If not, see .
##
"""
-Support for PGI compilers (pgcc, pgc++, pgfortran) as toolchain compilers.
+Support for PGI compilers (pgcc, pgc++, pgf90/pgfortran) as toolchain compilers.
:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada)
"""
@@ -84,7 +84,7 @@ class Pgi(Compiler):
COMPILER_CXX = None
COMPILER_F77 = 'pgf77'
- COMPILER_F90 = 'pgfortran'
+ COMPILER_F90 = 'pgf90'
COMPILER_FC = 'pgfortran'
LINKER_TOGGLE_STATIC_DYNAMIC = {
diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py
index 8979eb2840..041cf8a21b 100644
--- a/easybuild/tools/build_log.py
+++ b/easybuild/tools/build_log.py
@@ -41,7 +41,7 @@
from vsc.utils import fancylogger
from vsc.utils.exceptions import LoggedException
-from easybuild.tools.version import VERSION
+from easybuild.tools.version import VERSION, this_is_easybuild
# EasyBuild message prefix
@@ -121,7 +121,7 @@ def experimental(self, msg, *args, **kwargs):
msg = common_msg + " (use --experimental option to enable): " + msg
raise EasyBuildError(msg, *args)
- def deprecated(self, msg, ver, max_ver=None, *args, **kwargs):
+ def deprecated(self, msg, ver, max_ver=None, more_info=None, *args, **kwargs):
"""
Print deprecation warning or raise an exception, depending on specified version(s)
@@ -129,12 +129,13 @@ def deprecated(self, msg, ver, max_ver=None, *args, **kwargs):
:param ver: if max_ver is None: threshold for EasyBuild version to determine warning vs exception
else: version to check against max_ver to determine warning vs exception
:param max_ver: version threshold for warning vs exception (compared to 'ver')
+ :param more_info: additional message with instructions where to get more information
"""
# provide log_callback function that both logs a warning and prints to stderr
def log_callback_warning_and_print(msg):
"""Log warning message, and also print it to stderr."""
self.warning(msg)
- sys.stderr.write(msg + '\n')
+ sys.stderr.write('\nWARNING: ' + msg + '\n\n')
kwargs['log_callback'] = log_callback_warning_and_print
@@ -142,7 +143,10 @@ def log_callback_warning_and_print(msg):
kwargs['exception'] = EasyBuildError
if max_ver is None:
- msg += "; see %s for more information" % DEPRECATED_DOC_URL
+ if more_info:
+ msg += more_info
+ else:
+ msg += "; see %s for more information" % DEPRECATED_DOC_URL
fancylogger.FancyLogger.deprecated(self, msg, str(CURRENT_VERSION), ver, *args, **kwargs)
else:
fancylogger.FancyLogger.deprecated(self, msg, ver, max_ver, *args, **kwargs)
@@ -204,6 +208,16 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.
return log, logfile
+def log_start(log, eb_command_line, eb_tmpdir):
+ """Log startup info."""
+ log.info(this_is_easybuild())
+
+ # log used command line
+ log.info("Command line: %s", ' '.join(eb_command_line))
+
+ log.info("Using %s as temporary directory", eb_tmpdir)
+
+
def stop_logging(logfile, logtostdout=False):
"""Stop logging."""
if logtostdout:
@@ -212,13 +226,6 @@ def stop_logging(logfile, logtostdout=False):
fancylogger.logToFile(logfile, enable=False)
-def get_log(name=None):
- """
- (NO LONGER SUPPORTED!) Generate logger object
- """
- log.nosupport("Use of get_log function", '2.0')
-
-
def print_msg(msg, log=None, silent=False, prefix=True, newline=True, stderr=False):
"""
Print a message.
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index d9b6e22eaa..72fe9240ed 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -105,6 +105,9 @@
FORCE_DOWNLOAD_CHOICES = [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES]
DEFAULT_FORCE_DOWNLOAD = FORCE_DOWNLOAD_SOURCES
+JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error'
+JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run'
+
DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04'
DOCKER_BASE_IMAGE_CENTOS = 'centos:7'
@@ -146,6 +149,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'ignore_dirs',
'job_backend_config',
'job_cores',
+ 'job_deps_type',
'job_max_jobs',
'job_max_walltime',
'job_output_dir',
@@ -217,6 +221,8 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'cleanup_tmpdir',
'extended_dry_run_ignore_errors',
'mpi_tests',
+ 'modules_tool_version_check',
+ 'pre_create_installdir',
],
WARN: [
'check_ebroot_env_vars',
@@ -437,13 +443,17 @@ def init_build_options(build_options=None, cmdline_options=None):
def build_option(key, **kwargs):
"""Obtain value specified build option."""
+
build_options = BuildOptions()
if key in build_options:
return build_options[key]
elif 'default' in kwargs:
return kwargs['default']
else:
- raise EasyBuildError("Undefined build option: %s", key)
+ error_msg = "Undefined build option: '%s'. " % key
+ error_msg += "Make sure you have set up the EasyBuild configuration using set_up_configuration() "
+ error_msg += "(from easybuild.tools.options) in case you're not using EasyBuild via the 'eb' CLI."
+ raise EasyBuildError(error_msg)
def build_path():
diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py
index 8bdb2338da..1b6aad11ff 100644
--- a/easybuild/tools/docs.py
+++ b/easybuild/tools/docs.py
@@ -559,20 +559,23 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False
else:
toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])
- versionsuffix = ec.get('versionsuffix', '')
-
- # make sure versionsuffix gets properly templated
- if versionsuffix and isinstance(ec, dict):
- template_values = template_constant_dict(ec)
- versionsuffix = versionsuffix % template_values
-
- software[ec['name']].append({
- 'description': ec['description'],
- 'homepage': ec['homepage'],
- 'toolchain': toolchain,
- 'version': ec['version'],
- 'versionsuffix': versionsuffix,
- })
+ keys = ['description', 'homepage', 'version', 'versionsuffix']
+
+ info = {'toolchain': toolchain}
+ for key in keys:
+ info[key] = ec.get(key, '')
+
+ # make sure values like homepage & versionsuffix get properly templated
+ if isinstance(ec, dict):
+ template_values = template_constant_dict(ec, skip_lower=False)
+ for key in keys:
+ if '%(' in info[key]:
+ try:
+ info[key] = info[key] % template_values
+ except (KeyError, TypeError, ValueError) as err:
+ _log.debug("Ignoring failure to resolve templates: %s", err)
+
+ software[ec['name']].append(info)
if only_installed:
software[ec['name']][-1].update({'mod_name': ec.full_mod_name})
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index b2fb9186d0..499a7ff459 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -193,7 +193,7 @@ def read_file(path, log_error=True):
return txt
-def write_file(path, txt, append=False, forced=False, backup=False):
+def write_file(path, txt, append=False, forced=False, backup=False, overwrite=True, verbose=False):
"""
Write given contents to file at given path;
overwrites current file contents without backup by default!
@@ -203,15 +203,26 @@ def write_file(path, txt, append=False, forced=False, backup=False):
:param append: append to existing file rather than overwrite
:param forced: force actually writing file in (extended) dry run mode
:param backup: back up existing file before overwriting or modifying it
+ :param overwrite: don't require --force to overwrite an existing file
+ :param verbose: be verbose, i.e. inform where backup file was created
"""
# early exit in 'dry run' mode
if not forced and build_option('extended_dry_run'):
dry_run_msg("file written: %s" % path, silent=build_option('silent'))
return
- if backup and os.path.exists(path):
- backup = back_up_file(path)
- _log.info("Existing file %s backed up to %s", path, backup)
+ if os.path.exists(path):
+ if not append:
+ if overwrite or build_option('force'):
+ _log.info("Overwriting existing file %s", path)
+ else:
+ raise EasyBuildError("File exists, not overwriting it without --force: %s", path)
+
+ if backup:
+ backed_up_fp = back_up_file(path)
+ _log.info("Existing file %s backed up to %s", path, backed_up_fp)
+ if verbose:
+ print_msg("Backup of %s created at %s" % (path, backed_up_fp))
# note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
try:
@@ -956,7 +967,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None):
apatch_root, apatch_file = os.path.split(apatch)
apatch_name, apatch_extension = os.path.splitext(apatch_file)
# Supports only bz2, gz and xz. zip can be archives which are not supported.
- if apatch_extension in ['.gz','.bz2','.xz']:
+ if apatch_extension in ['.gz', '.bz2', '.xz']:
# split again to get the second extension
apatch_subname, apatch_subextension = os.path.splitext(apatch_name)
if apatch_subextension == ".patch":
@@ -1054,12 +1065,17 @@ def convert_name(name, upper=False):
def adjust_permissions(name, permissionBits, add=True, onlyfiles=False, onlydirs=False, recursive=True,
- group_id=None, relative=True, ignore_errors=False, skip_symlinks=True):
+ group_id=None, relative=True, ignore_errors=False, skip_symlinks=None):
"""
Add or remove (if add is False) permissionBits from all files (if onlydirs is False)
and directories (if onlyfiles is False) in path
"""
+ if skip_symlinks is not None:
+ depr_msg = "Use of 'skip_symlinks' argument for 'adjust_permissions' is deprecated "
+ depr_msg += "(symlinks are never followed anymore)"
+ _log.deprecated(depr_msg, '4.0')
+
name = os.path.abspath(name)
if recursive:
@@ -1068,14 +1084,7 @@ def adjust_permissions(name, permissionBits, add=True, onlyfiles=False, onlydirs
for root, dirs, files in os.walk(name):
paths = []
if not onlydirs:
- if skip_symlinks:
- for path in files:
- if os.path.islink(os.path.join(root, path)):
- _log.debug("Not adjusting permissions for symlink %s", path)
- else:
- paths.append(path)
- else:
- paths.extend(files)
+ paths.extend(files)
if not onlyfiles:
# os.walk skips symlinked dirs by default, i.e., no special handling needed here
paths.extend(dirs)
@@ -1092,26 +1101,30 @@ def adjust_permissions(name, permissionBits, add=True, onlyfiles=False, onlydirs
for path in allpaths:
try:
- if relative:
+ # don't change permissions if path is a symlink, since we're not checking where the symlink points to
+ # this is done because of security concerns (symlink may point out of installation directory)
+ # (note: os.lchmod is not supported on Linux)
+ if not os.path.islink(path):
+ if relative:
- # relative permissions (add or remove)
- perms = os.stat(path)[stat.ST_MODE]
+ # relative permissions (add or remove)
+ perms = os.lstat(path)[stat.ST_MODE]
- if add:
- os.chmod(path, perms | permissionBits)
- else:
- os.chmod(path, perms & ~permissionBits)
+ if add:
+ os.chmod(path, perms | permissionBits)
+ else:
+ os.chmod(path, perms & ~permissionBits)
- else:
- # hard permissions bits (not relative)
- os.chmod(path, permissionBits)
+ else:
+ # hard permissions bits (not relative)
+ os.chmod(path, permissionBits)
if group_id:
# only change the group id if it the current gid is different from what we want
- cur_gid = os.stat(path).st_gid
+ cur_gid = os.lstat(path).st_gid
if not cur_gid == group_id:
_log.debug("Changing group id of %s to %s" % (path, group_id))
- os.chown(path, -1, group_id)
+ os.lchown(path, -1, group_id)
else:
_log.debug("Group id of %s is already OK (%s)" % (path, group_id))
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
index 674d23095e..c6be3e2c7c 100644
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -46,12 +46,10 @@
from vsc.utils.missing import nub
from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, process_easyconfig
-from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION
-from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION
from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.filetools import apply_patch, copy_dir, det_patched_files, download_file, extract_file
-from easybuild.tools.filetools import mkdir, read_file, which, write_file
+from easybuild.tools.filetools import mkdir, read_file, symlink, which, write_file
from easybuild.tools.systemtools import UNKNOWN, get_tool_version
from easybuild.tools.utilities import only_if_module_is_available
@@ -368,33 +366,33 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
# make sure path exists, create it if necessary
mkdir(path, parents=True)
- _log.debug("Fetching easyconfigs from PR #%s into %s" % (pr, path))
- pr_url = lambda g: g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr]
+ github_account = build_option('pr_target_account')
+ github_repo = GITHUB_EASYCONFIGS_REPO
+
+ def pr_url(gh):
+ """Utility function to fetch data for a specific PR."""
+ return gh.repos[github_account][github_repo].pulls[pr]
+
+ _log.debug("Fetching easyconfigs from %s/%s PR #%s into %s", github_account, github_repo, pr, path)
status, pr_data = github_api_get_request(pr_url, github_user)
if status != HTTP_STATUS_OK:
raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
- pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data)
-
- # if PR is open and mergable, download develop and patch
- stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN
- closed = pr_data['state'] == GITHUB_STATE_CLOSED and not pr_data['merged']
+ pr, github_account, github_repo, status, pr_data)
- # 'clean' on successful (or missing) test, 'unstable' on failed tests or merge conflict
- if not stable:
- _log.warning("Mergeable state for PR #%d is not '%s': %s.",
- pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state'])
+ pr_merged = pr_data['merged']
+ pr_closed = pr_data['state'] == GITHUB_STATE_CLOSED and not pr_merged
- if (stable or pr_data['merged']) and not closed:
- # whether merged or not, download develop
- final_path = download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='develop', github_user=github_user)
+ pr_target_branch = pr_data['base']['ref']
+ _log.info("Target branch for PR #%s: %s", pr, pr_target_branch)
- else:
- final_path = path
+ # download target branch of PR so we can try and apply the PR patch on top of it
+ repo_target_branch = download_repo(repo=github_repo, account=github_account, branch=pr_target_branch,
+ github_user=github_user)
# determine list of changed files via diff
diff_fn = os.path.basename(pr_data['diff_url'])
- diff_filepath = os.path.join(final_path, diff_fn)
+ diff_filepath = os.path.join(path, diff_fn)
download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True)
diff_txt = read_file(diff_filepath)
@@ -402,53 +400,74 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
_log.debug("List of patched files: %s" % patched_files)
for key, val in sorted(pr_data.items()):
- _log.debug("\n%s:\n\n%s\n" % (key, val))
+ _log.debug("\n%s:\n\n%s\n", key, val)
# obtain last commit
# get all commits, increase to (max of) 100 per page
if pr_data['commits'] > GITHUB_MAX_PER_PAGE:
- raise EasyBuildError("PR #%s contains more than %s commits, can't obtain last commit", pr, GITHUB_MAX_PER_PAGE)
- status, commits_data = github_api_get_request(lambda g: pr_url(g).commits, github_user,
+ raise EasyBuildError("PR #%s contains more than %s commits, can't obtain last commit",
+ pr, GITHUB_MAX_PER_PAGE)
+
+ status, commits_data = github_api_get_request(lambda gh: pr_url(gh).commits, github_user,
per_page=GITHUB_MAX_PER_PAGE)
if status != HTTP_STATUS_OK:
raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
- pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, commits_data)
+ pr, github_account, github_repo, status, commits_data)
last_commit = commits_data[-1]
- _log.debug("Commits: %s, last commit: %s" % (commits_data, last_commit['sha']))
-
- if not(pr_data['merged']):
- if not stable or closed:
- state_msg = "unstable (pending/failed tests or merge conflict)" if not stable else "closed (but not merged)"
- print "\n*** WARNING: Using easyconfigs from %s PR #%s ***\n" % (state_msg, pr)
- # obtain most recent version of patched files
- for patched_file in patched_files:
- # path to patch file, incl. subdir it is in
- fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:])
- sha = last_commit['sha']
- full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file])
- _log.info("Downloading %s from %s" % (fn, full_url))
- download_file(fn, full_url, path=os.path.join(path, fn), forced=True)
- else:
- apply_patch(diff_filepath, final_path, level=1)
+ _log.debug("Commits: %s, last commit: %s", commits_data, last_commit['sha'])
+
+ final_path = None
+ # try to apply PR patch on top of target branch, unless the PR is closed or already merged
+ if pr_merged:
+ _log.info("PR is already merged, so using current version of PR target branch")
+ final_path = repo_target_branch
+
+ elif not pr_closed:
+ try:
+ _log.debug("Trying to apply PR patch %s to %s...", diff_filepath, repo_target_branch)
+ apply_patch(diff_filepath, repo_target_branch, level=1)
+ _log.info("Using %s which included PR patch to test PR #%s", repo_target_branch, pr)
+ final_path = repo_target_branch
+
+ except EasyBuildError as err:
+ _log.warning("Ignoring problem that occured when applying PR patch: %s", err)
+
+ if final_path is None:
+
+ if pr_closed:
+ print_warning("Using easyconfigs from closed PR #%s" % pr)
+
+ # obtain most recent version of patched files
+ for patched_file in patched_files:
+ # path to patch file, incl. subdir it is in
+ fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:])
+ sha = last_commit['sha']
+ full_url = URL_SEPARATOR.join([GITHUB_RAW, github_account, github_repo, sha, patched_file])
+ _log.info("Downloading %s from %s", fn, full_url)
+ download_file(fn, full_url, path=os.path.join(path, fn), forced=True)
+
+ final_path = path
+
+ # symlink directories into expected place if they're not there yet
if final_path != path:
dirpath = os.path.join(final_path, 'easybuild', 'easyconfigs')
for eb_dir in os.listdir(dirpath):
- os.symlink(os.path.join(dirpath, eb_dir), os.path.join(path, os.path.basename(eb_dir)))
+ symlink(os.path.join(dirpath, eb_dir), os.path.join(path, os.path.basename(eb_dir)))
# sanity check: make sure all patched files are downloaded
ec_files = []
for patched_file in [f for f in patched_files if not f.startswith('test/')]:
fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:])
- if os.path.exists(os.path.join(path, fn)):
- ec_files.append(os.path.join(path, fn))
+ full_path = os.path.join(path, fn)
+ if os.path.exists(full_path):
+ ec_files.append(full_path)
else:
- raise EasyBuildError("Couldn't find path to patched file %s", os.path.join(path, fn))
+ raise EasyBuildError("Couldn't find path to patched file %s", full_path)
return ec_files
-
def create_gist(txt, fn, descr=None, github_user=None):
"""Create a gist with the provided text."""
if descr is None:
diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py
index 31b7d8181a..f6e67f6050 100644
--- a/easybuild/tools/job/gc3pie.py
+++ b/easybuild/tools/job/gc3pie.py
@@ -31,14 +31,13 @@
"""
from distutils.version import LooseVersion
from time import gmtime, strftime
-import re
import time
from pkg_resources import get_distribution, DistributionNotFound
from vsc.utils import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.config import build_option
+from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.utilities import only_if_module_is_available
@@ -50,7 +49,6 @@
import gc3libs
import gc3libs.exceptions
from gc3libs import Application, Run, create_engine
- from gc3libs.core import Engine
from gc3libs.quantity import hours as hr
from gc3libs.workflow import AbortOnError, DependentTaskCollection
@@ -63,7 +61,7 @@
gc3libs.UNIGNORE_ALL_ERRORS = True
# note: order of class inheritance is important!
- class _BuildTaskCollection(AbortOnError, DependentTaskCollection):
+ class AbortingDependentTaskCollection(AbortOnError, DependentTaskCollection):
"""
A `DependentTaskCollection`:class: that aborts execution upon error.
@@ -125,9 +123,22 @@ def init(self):
self.config_files.append(cfgfile)
self.output_dir = build_option('job_output_dir')
- self.jobs = _BuildTaskCollection(output_dir=self.output_dir)
self.job_cnt = 0
+ job_deps_type = build_option('job_deps_type')
+ if job_deps_type is None:
+ job_deps_type = JOB_DEPS_TYPE_ABORT_ON_ERROR
+ self.log.info("Using default job dependency type: %s", job_deps_type)
+ else:
+ self.log.info("Using specified job dependency type: %s", job_deps_type)
+
+ if job_deps_type == JOB_DEPS_TYPE_ALWAYS_RUN:
+ self.jobs = DependentTaskCollection(output_dir=self.output_dir)
+ elif job_deps_type == JOB_DEPS_TYPE_ABORT_ON_ERROR:
+ self.jobs = AbortingDependentTaskCollection(output_dir=self.output_dir)
+ else:
+ raise EasyBuildError("Unknown job dependency type specified: %s", job_deps_type)
+
# after polling for job status, sleep for this time duration
# before polling again (in seconds)
self.poll_interval = build_option('job_polling_interval')
@@ -152,8 +163,8 @@ def make_job(self, script, name, env_vars=None, hours=None, cores=None):
- *cores* depends on which cluster the job is being run.
"""
named_args = {
- 'jobname': name, # job name in GC3Pie
- 'name': name, # job name in EasyBuild
+ 'jobname': name, # job name in GC3Pie
+ 'name': name, # job name in EasyBuild
}
# environment
diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py
index 14225825af..ed0acb7e1a 100644
--- a/easybuild/tools/job/pbs_python.py
+++ b/easybuild/tools/job/pbs_python.py
@@ -36,7 +36,7 @@
from vsc.utils import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.config import build_option
+from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.utilities import only_if_module_is_available
@@ -244,6 +244,22 @@ def __init__(self, server, script, name, env_vars=None,
# list of holds that are placed on this job
self.holds = []
+ job_deps_type = build_option('job_deps_type')
+
+ # mark job dependencies with 'afterany' by default to retain backward compatibility for pbs_python job backend
+ if job_deps_type is None:
+ job_deps_type = JOB_DEPS_TYPE_ALWAYS_RUN
+ self.log.info("Using default job dependency type: %s", job_deps_type)
+ else:
+ self.log.info("Using specified job dependency type: %s", job_deps_type)
+
+ if job_deps_type == JOB_DEPS_TYPE_ABORT_ON_ERROR:
+ self.job_deps_type = 'afterok'
+ elif job_deps_type == JOB_DEPS_TYPE_ALWAYS_RUN:
+ self.job_deps_type = 'afterany'
+ else:
+ raise EasyBuildError("Unknown job dependency type specified: %s", job_deps_type)
+
def __str__(self):
"""Return the job ID as a string."""
return (str(self.jobid) if self.jobid is not None
@@ -288,7 +304,7 @@ def _submit(self):
if self.deps:
deps_attributes = pbs.new_attropl(1)
deps_attributes[0].name = pbs.ATTR_depend
- deps_attributes[0].value = ",".join(["afterany:%s" % dep.jobid for dep in self.deps])
+ deps_attributes[0].value = ','.join([self.job_deps_type + ':' + dep.jobid for dep in self.deps])
pbs_attributes.extend(deps_attributes)
self.log.debug("Job deps attributes: %s" % deps_attributes[0].value)
diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py
new file mode 100644
index 0000000000..b5891de0c9
--- /dev/null
+++ b/easybuild/tools/job/slurm.py
@@ -0,0 +1,179 @@
+##
+# Copyright 2018-2018 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Support for using Slurm as a backend for --job
+
+:author: Kenneth Hoste (Ghent University)
+"""
+import re
+from distutils.version import LooseVersion
+from vsc.utils import fancylogger
+
+from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
+from easybuild.tools.job.backend import JobBackend
+from easybuild.tools.filetools import which
+from easybuild.tools.run import run_cmd
+
+
+_log = fancylogger.getLogger('slurm', fname=False)
+
+
+class Slurm(JobBackend):
+ """
+ Manage SLURM server communication and create `SlurmJob` objects.
+ """
+
+ REQ_VERSION = '17'
+
+ def __init__(self, *args, **kwargs):
+ """Constructor."""
+
+ # early check for required commands
+ for cmd in ['sbatch', 'scontrol']:
+ path = which(cmd)
+ if path is None:
+ raise EasyBuildError("Required command '%s' not found", cmd)
+
+ super(Slurm, self).__init__(*args, **kwargs)
+
+ job_deps_type = build_option('job_deps_type')
+ if job_deps_type is None:
+ job_deps_type = JOB_DEPS_TYPE_ABORT_ON_ERROR
+ self.log.info("Using default job dependency type: %s", job_deps_type)
+ else:
+ self.log.info("Using specified job dependency type: %s", job_deps_type)
+
+ if job_deps_type == JOB_DEPS_TYPE_ABORT_ON_ERROR:
+ self.job_deps_type = 'afterok'
+ elif job_deps_type == JOB_DEPS_TYPE_ALWAYS_RUN:
+ self.job_deps_type = 'afterany'
+ else:
+ raise EasyBuildError("Unknown job dependency type specified: %s", job_deps_type)
+
+ def _check_version(self):
+ """Check whether version of Slurm complies with required version."""
+ (out, _) = run_cmd("sbatch --version", trace=False)
+ slurm_ver = out.strip().split(' ')[-1]
+ self.log.info("Found Slurm version %s", slurm_ver)
+
+ if LooseVersion(slurm_ver) < LooseVersion(self.REQ_VERSION):
+ raise EasyBuildError("Found Slurm version %s, but version %s or more recent is required",
+ slurm_ver, self.REQ_VERSION)
+
+ def init(self):
+ """
+ Initialise the PySlurm job backend.
+ """
+ self._submitted = []
+
+ def queue(self, job, dependencies=frozenset()):
+ """
+ Add a job to the queue.
+
+ :param dependencies: jobs on which this job depends.
+ """
+ submit_cmd = 'sbatch'
+
+ if dependencies:
+ job.job_specs['dependency'] = self.job_deps_type + ':' + ':'.join(str(d.jobid) for d in dependencies)
+ # make sure job that has invalid dependencies doesn't remain queued indefinitely
+ submit_cmd += " --kill-on-invalid-dep=yes"
+
+ # submit job with hold in place
+ job.job_specs['hold'] = True
+
+ self.log.info("Submitting job with following specs: %s", job.job_specs)
+ for key in sorted(job.job_specs):
+ if key in ['hold']:
+ if job.job_specs[key]:
+ submit_cmd += " --%s" % key
+ else:
+ submit_cmd += ' --%s "%s"' % (key, job.job_specs[key])
+
+ (out, _) = run_cmd(submit_cmd, trace=False)
+
+ jobid_regex = re.compile("^Submitted batch job (?P[0-9]+)")
+
+ res = jobid_regex.search(out)
+ if res:
+ job.jobid = res.group('jobid')
+ self.log.info("Job submitted, got job ID %s", job.jobid)
+ else:
+ raise EasyBuildError("Failed to determine job ID from output of submission command: %s", out)
+
+ self._submitted.append(job)
+
+ def complete(self):
+ """
+ Complete a bulk job submission.
+
+ Release all user holds on submitted jobs, and disconnect from server.
+ """
+ job_ids = []
+ for job in self._submitted:
+ if job.job_specs['hold']:
+ self.log.info("releasing user hold on job %s" % job.jobid)
+ job_ids.append(job.jobid)
+ run_cmd("scontrol release %s" % ' '.join(job_ids), trace=False)
+
+ submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted])
+ print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log)
+
+ def make_job(self, script, name, env_vars=None, hours=None, cores=None):
+ """Create and return a job dict with the given parameters."""
+ return SlurmJob(script, name, env_vars=env_vars, hours=hours, cores=cores)
+
+
+class SlurmJob(object):
+ """Job class for SLURM jobs."""
+
+ def __init__(self, script, name, env_vars=None, hours=None, cores=None):
+ """Create a new Job to be submitted to SLURM."""
+ self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
+
+ self.jobid = None
+ self.script = script
+ self.name = name
+
+ self.job_specs = {'wrap': self.script, 'job-name': self.name}
+
+ if env_vars:
+ self.job_specs['export'] = ','.join(sorted(env_vars.keys()))
+
+ max_walltime = build_option('job_max_walltime')
+ if hours is None:
+ hours = max_walltime
+ if hours > max_walltime:
+ self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, max_walltime))
+ hours = max_walltime
+ self.job_specs['time'] = hours * 60
+
+ if cores:
+ self.job_specs['nodes'] = 1
+ self.job_specs['ntasks'] = cores
+ self.job_specs['ntasks-per-node'] = cores
+ else:
+ self.log.warn("Number of cores to request not specified, falling back to whatever Slurm does by default")
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index fcf5baa035..e575d9642b 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -46,7 +46,7 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_module_syntax, install_path
from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file
-from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, modules_tool
+from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool
from easybuild.tools.utilities import quote_str
@@ -106,7 +106,7 @@ def dependencies_for(mod_name, modtool, depth=sys.maxint):
# add dependencies of dependency modules only if they're not there yet
for moddepdeps in moddeps:
for dep in moddepdeps:
- if not dep in mods:
+ if dep not in mods:
mods.append(dep)
return mods
@@ -123,6 +123,7 @@ class ModuleGenerator(object):
MODULE_FILE_EXTENSION = None
MODULE_SHEBANG = None
+ DOT_MODULERC = '.modulerc'
# a single level of indentation
INDENTATION = ' ' * 4
@@ -133,6 +134,8 @@ def __init__(self, application, fake=False):
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.fake_mod_path = tempfile.mkdtemp()
+ self.modules_tool = modules_tool()
+
def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
"""
Generate append-path statements for the given list of paths.
@@ -198,52 +201,109 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
"""
return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
- def modulerc(self, module_version=None):
+ def _modulerc_check_module_version(self, module_version):
"""
- Generate contents of .modulerc file, in Tcl syntax (compatible with all module tools, incl. Lmod)
+ Check value type & contents of specified module-version spec.
:param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys)
+ :return: True if spec is OK
"""
- modulerc = [ModuleGeneratorTcl.MODULE_SHEBANG]
-
+ res = False
if module_version:
if isinstance(module_version, dict):
expected_keys = ['modname', 'sym_version', 'version']
if sorted(module_version.keys()) == expected_keys:
+ res = True
+ else:
+ raise EasyBuildError("Incorrect module_version spec, expected keys: %s", expected_keys)
+ else:
+ raise EasyBuildError("Incorrect module_version value type: %s", type(module_version))
- module_version_statement = "module-version %(modname)s %(sym_version)s"
+ return res
- # for Environment Modules we need to guard the module-version statement,
- # to avoid "Duplicate version symbol" warning messages where EasyBuild trips over,
- # which occur because the .modulerc is parsed twice
- # "module-info version " returns its argument if that argument is not a symbolic version (yet),
- # and returns the corresponding real version in case the argument is an existing symbolic version
- # cfr. https://sourceforge.net/p/modules/mailman/message/33399425/
- if modules_tool().__class__ == EnvironmentModulesC:
+ def _write_modulerc_file(self, modulerc_path, modulerc_txt, wrapped_mod_name=None):
+ """
+ Write modulerc file with specified contents.
- modname, sym_version, version = [module_version[key] for key in expected_keys]
+ :param modulerc_path: location of .modulerc file to write
+ :param modulerc_txt: contents of .modulerc file
+ :param wrapped_mod_name: name of module file for which a wrapper is defined in the .modulerc file (if any)
+ """
+ if os.path.exists(modulerc_path) and not (build_option('force') or build_option('rebuild')):
+ raise EasyBuildError("Found existing .modulerc at %s, not overwriting without --force or --rebuild",
+ modulerc_path)
- # determine module name with symbolic version
- if version in modname:
- # take a copy so we don't modify original value
- module_version = copy.copy(module_version)
- module_version['sym_modname'] = modname.replace(version, sym_version)
- else:
- raise EasyBuildError("Version '%s' does not appear in module name '%s'", version, modname)
+ # Lmod 6.x requires that module being wrapped is in same location as .modulerc file...
+ if wrapped_mod_name is not None:
+ if isinstance(self.modules_tool, Lmod) and LooseVersion(self.modules_tool.version) < LooseVersion('7.0'):
+ mod_dir = os.path.dirname(modulerc_path)
- module_version_statement = '\n'.join([
- 'if {"%(sym_modname)s" eq [module-info version %(sym_modname)s]} {',
- ' ' * 4 + module_version_statement,
- "}",
- ])
+ # need to consider existing module file in both Tcl (no extension) & Lua (.lua extension) syntax...
+ wrapped_mod_fp = os.path.join(mod_dir, os.path.basename(wrapped_mod_name))
+ wrapped_mod_exists = os.path.exists(wrapped_mod_fp)
+ if not wrapped_mod_exists and self.MODULE_FILE_EXTENSION:
+ wrapped_mod_exists = os.path.exists(wrapped_mod_fp + self.MODULE_FILE_EXTENSION)
- modulerc.append(module_version_statement % module_version)
- else:
- raise EasyBuildError("Incorrect module_version spec, expected keys: %s", expected_keys)
- else:
- raise EasyBuildError("Incorrect module_version value type: %s", type(module_version))
+ if not wrapped_mod_exists:
+ error_msg = "Expected module file %s not found; " % wrapped_mod_fp
+ error_msg += "Lmod 6.x requires that .modulerc and wrapped module file are in same directory!"
+ raise EasyBuildError(error_msg)
+
+ write_file(modulerc_path, modulerc_txt, backup=True)
+
+ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
+ """
+ Generate contents of .modulerc file, in Tcl syntax (compatible with all module tools, incl. Lmod).
+ If 'filepath' is specified, the .modulerc file will be written as well.
+
+ :param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys)
+ :param filepath: location where .modulerc file should be written to
+ :param modulerc_txt: contents of .modulerc to use
+ :return: contents of .modulerc file
+ """
+ if modulerc_txt is None:
+
+ self.log.info("Generating .modulerc contents in Tcl syntax (args: module_version: %s", module_version)
+ modulerc = [ModuleGeneratorTcl.MODULE_SHEBANG]
+
+ if self._modulerc_check_module_version(module_version):
+
+ module_version_statement = "module-version %(modname)s %(sym_version)s"
- return '\n'.join(modulerc)
+ # for Environment Modules we need to guard the module-version statement,
+ # to avoid "Duplicate version symbol" warning messages where EasyBuild trips over,
+ # which occur because the .modulerc is parsed twice
+ # "module-info version " returns its argument if that argument is not a symbolic version (yet),
+ # and returns the corresponding real version in case the argument is an existing symbolic version
+ # cfr. https://sourceforge.net/p/modules/mailman/message/33399425/
+ if self.modules_tool.__class__ == EnvironmentModulesC:
+
+ keys = ['modname', 'sym_version', 'version']
+ modname, sym_version, version = [module_version[key] for key in keys]
+
+ # determine module name with symbolic version
+ if version in modname:
+ # take a copy so we don't modify original value
+ module_version = copy.copy(module_version)
+ module_version['sym_modname'] = modname.replace(version, sym_version)
+ else:
+ raise EasyBuildError("Version '%s' does not appear in module name '%s'", version, modname)
+
+ module_version_statement = '\n'.join([
+ 'if {"%(sym_modname)s" eq [module-info version %(sym_modname)s]} {',
+ ' ' * 4 + module_version_statement,
+ "}",
+ ])
+
+ modulerc.append(module_version_statement % module_version)
+
+ modulerc_txt = '\n'.join(modulerc)
+
+ if filepath:
+ self.log.info("Writing %s with contents:\n%s", filepath, modulerc_txt)
+ self._write_modulerc_file(filepath, modulerc_txt, wrapped_mod_name=module_version['modname'])
+
+ return modulerc_txt
# From this point on just not implemented methods
@@ -261,7 +321,7 @@ def comment(self, msg):
"""Return given string formatted as a comment."""
raise NotImplementedError
- def conditional_statement(self, condition, body, negative=False, else_body=None):
+ def conditional_statement(self, condition, body, negative=False, else_body=None, indent=True):
"""
Return formatted conditional statement, with given condition and body.
@@ -269,6 +329,7 @@ def conditional_statement(self, condition, body, negative=False, else_body=None)
:param body: (multiline) string with if body (in correct syntax, without indentation)
:param negative: boolean indicating whether the condition should be negated
:param else_body: optional body for 'else' part
+ :param indent: indent if/else body
"""
raise NotImplementedError
@@ -507,7 +568,7 @@ def comment(self, msg):
"""Return string containing given message as a comment."""
return "# %s\n" % msg
- def conditional_statement(self, condition, body, negative=False, else_body=None):
+ def conditional_statement(self, condition, body, negative=False, else_body=None, indent=True):
"""
Return formatted conditional statement, with given condition and body.
@@ -515,6 +576,7 @@ def conditional_statement(self, condition, body, negative=False, else_body=None)
:param body: (multiline) string with if body (in correct syntax, without indentation)
:param negative: boolean indicating whether the condition should be negated
:param else_body: optional body for 'else' part
+ :param indent: indent if/else body
"""
if negative:
lines = ["if { ![ %s ] } {" % condition]
@@ -522,14 +584,18 @@ def conditional_statement(self, condition, body, negative=False, else_body=None)
lines = ["if { [ %s ] } {" % condition]
for line in body.split('\n'):
- lines.append(self.INDENTATION + line)
+ if indent:
+ line = self.INDENTATION + line
+ lines.append(line)
if else_body is None:
lines.extend(['}', ''])
else:
lines.append('} else {')
for line in else_body.split('\n'):
- lines.append(self.INDENTATION + line)
+ if indent:
+ line = self.INDENTATION + line
+ lines.append(line)
lines.extend(['}', ''])
return '\n'.join(lines)
@@ -594,7 +660,7 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload
load_template = self.LOAD_TEMPLATE
# Lmod 7.6.1+ supports depends-on which does this most nicely:
if build_option('mod_depends_on') or depends_on:
- if not modules_tool().supports_depends_on:
+ if not self.modules_tool.supports_depends_on:
raise EasyBuildError("depends-on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
body.append(load_template)
@@ -614,9 +680,9 @@ def msg_on_load(self, msg):
Add a message that should be printed when loading the module.
"""
# escape any (non-escaped) characters with special meaning by prefixing them with a backslash
- msg = re.sub(r'((?= LooseVersion('7.7.38'):
+ self.DOT_MODULERC = '.modulerc.lua'
+
def check_group(self, group, error_msg=None):
"""
Generate a check of the software group and the current user, and refuse to load the module if the user don't
@@ -792,10 +866,10 @@ def check_group(self, group, error_msg=None):
:param group: string with the group name
:param error_msg: error message to print for users outside that group
"""
- lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND')
+ lmod_version = self.modules_tool.version
min_lmod_version = '6.0.8'
- if lmod_version != 'NOT_FOUND' and LooseVersion(lmod_version) >= LooseVersion(min_lmod_version):
+ if LooseVersion(lmod_version) >= LooseVersion(min_lmod_version):
if error_msg is None:
error_msg = "You are not part of '%s' group of users that have access to this software; " % group
error_msg += "Please consult with user support how to become a member of this group"
@@ -821,7 +895,7 @@ def comment(self, msg):
"""Return string containing given message as a comment."""
return "-- %s\n" % msg
- def conditional_statement(self, condition, body, negative=False, else_body=None):
+ def conditional_statement(self, condition, body, negative=False, else_body=None, indent=True):
"""
Return formatted conditional statement, with given condition and body.
@@ -829,6 +903,7 @@ def conditional_statement(self, condition, body, negative=False, else_body=None)
:param body: (multiline) string with if body (in correct syntax, without indentation)
:param negative: boolean indicating whether the condition should be negated
:param else_body: optional body for 'else' part
+ :param indent: indent if/else body
"""
if negative:
lines = ["if not %s then" % condition]
@@ -836,14 +911,18 @@ def conditional_statement(self, condition, body, negative=False, else_body=None)
lines = ["if %s then" % condition]
for line in body.split('\n'):
- lines.append(self.INDENTATION + line)
+ if indent:
+ line = self.INDENTATION + line
+ lines.append(line)
if else_body is None:
lines.extend(['end', ''])
else:
lines.append('else')
for line in else_body.split('\n'):
- lines.append(self.INDENTATION + line)
+ if indent:
+ line = self.INDENTATION + line
+ lines.append(line)
lines.extend(['end', ''])
return '\n'.join(lines)
@@ -906,7 +985,7 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload
load_template = self.LOAD_TEMPLATE
# Lmod 7.6+ supports depends_on which does this most nicely:
if build_option('mod_depends_on') or depends_on:
- if not modules_tool().supports_depends_on:
+ if not self.modules_tool.supports_depends_on:
raise EasyBuildError("depends_on statements in generated module are not supported by modules tool")
load_template = self.LOAD_TEMPLATE_DEPENDS_ON
@@ -934,7 +1013,37 @@ def msg_on_load(self, msg):
"""
# take into account possible newlines in messages by using [==...==] (requires Lmod 5.8)
stmt = 'io.stderr:write(%s%s%s)' % (self.START_STR, self.check_str(msg), self.END_STR)
- return '\n' + self.conditional_statement('mode() == "load"', stmt)
+ return '\n' + self.conditional_statement('mode() == "load"', stmt, indent=False)
+
+ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
+ """
+ Generate contents of .modulerc(.lua) file, in Lua syntax (but only if Lmod is recent enough, i.e. >= 7.7.38)
+
+ :param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys)
+ :param filepath: location where .modulerc file should be written to
+ :param modulerc_txt: contents of .modulerc to use
+ :return: contents of .modulerc file
+ """
+ if modulerc_txt is None:
+ lmod_ver = self.modules_tool.version
+ min_ver = '7.7.38'
+
+ if LooseVersion(lmod_ver) >= LooseVersion(min_ver):
+ self.log.info("Found Lmod v%s >= v%s, so will generate .modulerc.lua in Lua syntax", lmod_ver, min_ver)
+
+ modulerc = []
+
+ if self._modulerc_check_module_version(module_version):
+ module_version_statement = 'module_version("%(modname)s", "%(sym_version)s")'
+ modulerc.append(module_version_statement % module_version)
+
+ modulerc_txt = '\n'.join(modulerc)
+
+ else:
+ self.log.info("Lmod v%s < v%s, need to stick to Tcl syntax for .modulerc", lmod_ver, min_ver)
+
+ return super(ModuleGeneratorLua, self).modulerc(module_version=module_version, filepath=filepath,
+ modulerc_txt=modulerc_txt)
def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
"""
diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py
index 7550b79dde..301b46ff26 100644
--- a/easybuild/tools/modules.py
+++ b/easybuild/tools/modules.py
@@ -239,21 +239,26 @@ def set_and_check_version(self):
if self.REQ_VERSION is None and self.MAX_VERSION is None:
self.log.debug("No version requirement defined.")
- if self.REQ_VERSION is not None:
- self.log.debug("Required minimum version defined.")
- if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION):
- raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s",
- self.__class__.__name__, self.REQ_VERSION, self.version)
- else:
- self.log.debug('Version %s matches requirement >= %s', self.version, self.REQ_VERSION)
+ elif build_option('modules_tool_version_check'):
+ self.log.debug("Checking whether modules tool version '%s' meets requirements", self.version)
+
+ if self.REQ_VERSION is not None:
+ self.log.debug("Required minimum version defined.")
+ if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION):
+ raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s",
+ self.__class__.__name__, self.REQ_VERSION, self.version)
+ else:
+ self.log.debug('Version %s matches requirement >= %s', self.version, self.REQ_VERSION)
- if self.MAX_VERSION is not None:
- self.log.debug("Maximum allowed version defined.")
- if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION):
- raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s",
- self.__class__.__name__, self.MAX_VERSION, self.version)
- else:
- self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION)
+ if self.MAX_VERSION is not None:
+ self.log.debug("Maximum allowed version defined.")
+ if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION):
+ raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s",
+ self.__class__.__name__, self.MAX_VERSION, self.version)
+ else:
+ self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION)
+ else:
+ self.log.debug("Skipping modules tool version '%s' requirements check", self.version)
MODULE_VERSION_CACHE[self.COMMAND] = self.version
@@ -458,6 +463,51 @@ def available(self, mod_name=None, extra_args=None):
return ans
+ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_regex_template=None):
+ """
+ Determine whether a module wrapper with specified name exists.
+ Only .modulerc file in Tcl syntax is considered here.
+ """
+ if mod_wrapper_regex_template is None:
+ mod_wrapper_regex_template = "^[ ]*module-version (?P[^ ]*) %s$"
+
+ wrapped_mod = None
+
+ mod_dir = os.path.dirname(mod_name)
+ wrapper_regex = re.compile(mod_wrapper_regex_template % os.path.basename(mod_name), re.M)
+ for mod_path in curr_module_paths():
+ modulerc_cand = os.path.join(mod_path, mod_dir, modulerc_fn)
+ if os.path.exists(modulerc_cand):
+ self.log.debug("Found %s that may define %s as a wrapper for a module file", modulerc_cand, mod_name)
+ res = wrapper_regex.search(read_file(modulerc_cand))
+ if res:
+ wrapped_mod = res.group('wrapped_mod')
+ self.log.debug("Confirmed that %s is a module wrapper for %s", mod_name, wrapped_mod)
+ break
+
+ mod_dir = os.path.dirname(mod_name)
+ if wrapped_mod is not None and not wrapped_mod.startswith(mod_dir):
+ # module wrapper uses 'short' module name of module being wrapped,
+ # so we need to correct it in case a hierarchical module naming scheme is used...
+ # e.g. 'Java/1.8.0_181' should become 'Core/Java/1.8.0_181' for wrapper 'Core/Java/1.8'
+ self.log.debug("Full module name prefix mismatch between module wrapper '%s' and wrapped module '%s'",
+ mod_name, wrapped_mod)
+
+ mod_name_parts = mod_name.split(os.path.sep)
+ wrapped_mod_subdir = ''
+ while not os.path.join(wrapped_mod_subdir, wrapped_mod).startswith(mod_dir) and mod_name_parts:
+ wrapped_mod_subdir = os.path.join(wrapped_mod_subdir, mod_name_parts.pop(0))
+
+ full_wrapped_mod_name = os.path.join(wrapped_mod_subdir, wrapped_mod)
+ if full_wrapped_mod_name.startswith(mod_dir):
+ self.log.debug("Full module name for wrapped module %s: %s", wrapped_mod, full_wrapped_mod_name)
+ wrapped_mod = full_wrapped_mod_name
+ else:
+ raise EasyBuildError("Failed to determine full module name for module wrapped by %s: %s | %s",
+ mod_name, wrapped_mod_subdir, wrapped_mod)
+
+ return wrapped_mod
+
def exist(self, mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*:\s*$', skip_avail=False):
"""
Check if modules with specified names exists.
@@ -491,11 +541,24 @@ def mod_exists_via_show(mod_name):
for (mod_name, visible) in mod_names:
if visible:
# module name may be partial, so also check via 'module show' as fallback
- mods_exist.append(mod_name in avail_mod_names or mod_exists_via_show(mod_name))
+ mod_exists = mod_name in avail_mod_names or mod_exists_via_show(mod_name)
else:
# hidden modules are not visible in 'avail', need to use 'show' instead
self.log.debug("checking whether hidden module %s exists via 'show'..." % mod_name)
- mods_exist.append(mod_exists_via_show(mod_name))
+ mod_exists = mod_exists_via_show(mod_name)
+
+ # if no module file was found, check whether specified module name can be a 'wrapper' module...
+ if not mod_exists:
+ self.log.debug("Module %s not found via module avail/show, checking whether it is a wrapper", mod_name)
+ wrapped_mod = self.module_wrapper_exists(mod_name)
+ if wrapped_mod is not None:
+ # module wrapper only really exists if the wrapped module file is also available
+ mod_exists = wrapped_mod in avail_mod_names or mod_exists_via_show(wrapped_mod)
+ self.log.debug("Result for existence check of wrapped module %s: %s", wrapped_mod, mod_exists)
+
+ self.log.debug("Result for existence check of %s module: %s", mod_name, mod_exists)
+
+ mods_exist.append(mod_exists)
return mods_exist
@@ -1107,7 +1170,7 @@ class Lmod(ModulesTool):
"""Interface to Lmod."""
COMMAND = 'lmod'
COMMAND_ENVIRONMENT = 'LMOD_CMD'
- REQ_VERSION = '6.6.3'
+ REQ_VERSION = '6.5.1'
REQ_VERSION_DEPENDS_ON = '7.6.1'
VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s"
USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
@@ -1241,6 +1304,25 @@ def prepend_module_path(self, path, set_mod_paths=True, priority=None):
if set_mod_paths:
self.set_mod_paths()
+ def module_wrapper_exists(self, mod_name):
+ """
+ Determine whether a module wrapper with specified name exists.
+ First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax).
+ """
+ res = None
+
+ # first consider .modulerc.lua with Lmod 7.8 (or newer)
+ if StrictVersion(self.version) >= StrictVersion('7.8'):
+ mod_wrapper_regex_template = '^module_version\("(?P.*)", "%s"\)$'
+ res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua',
+ mod_wrapper_regex_template=mod_wrapper_regex_template)
+
+ # fall back to checking for .modulerc in Tcl syntax
+ if res is None:
+ res = super(Lmod, self).module_wrapper_exists(mod_name)
+
+ return res
+
def exist(self, mod_names, skip_avail=False):
"""
Check if modules with specified names exists.
@@ -1407,8 +1489,9 @@ def __init__(self, *args, **kwargs):
class NoModulesTool(ModulesTool):
"""Class that mock the module behaviour, used for operation not requiring modules. Eg. tests, fetch only"""
+
def __init__(self, *args, **kwargs):
- pass
+ self.version = None
def exist(self, mod_names, *args, **kwargs):
"""No modules, so nothing exists"""
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 85cdab8a32..4df30e7baa 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -53,17 +53,19 @@
from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
from easybuild.framework.easyconfig.easyconfig import HAVE_AUTOPEP8
from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
-from easybuild.framework.easyconfig.tools import get_paths_for
+from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for
from easybuild.tools import build_log, run # build_log should always stay there, to ensure EasyBuildLog
-from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, raise_easybuilderror
+from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError
+from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror
from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE
from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_JOB_BACKEND
from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS, DEFAULT_MNS
from easybuild.tools.config import DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES
from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE
-from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS
-from easybuild.tools.config import ERROR, IGNORE, FORCE_DOWNLOAD_CHOICES, LOADED_MODULES_ACTIONS, WARN
-from easybuild.tools.config import get_pretend_installpath, mk_full_default_path
+from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS, ERROR
+from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR
+from easybuild.tools.config import JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS, WARN
+from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
from easybuild.tools.configobj import ConfigObj, ConfigObjError
from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST
from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
@@ -82,12 +84,14 @@
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes
from easybuild.tools.modules import Lmod
from easybuild.tools.ordereddict import OrderedDict
+from easybuild.tools.robot import det_robot_path
from easybuild.tools.run import run_cmd
from easybuild.tools.package.utilities import avail_package_naming_schemes
from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
from easybuild.tools.repository.repository import avail_repositories
from easybuild.tools.version import this_is_easybuild
+
try:
from humanfriendly.terminal import terminal_supports_colors
except ImportError:
@@ -391,6 +395,7 @@ def override_options(self):
'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False),
'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS),
None, 'store_true', False),
+ 'modules-tool-version-check': ("Check version of modules tool being used", None, 'store_true', True),
'mpi-cmd-template': ("Template for MPI commands (template keys: %(nr_ranks)s, %(cmd)s)",
None, 'store', None),
'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True),
@@ -399,6 +404,8 @@ def override_options(self):
'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_TXT, FORMAT_RST]),
'parallel': ("Specify (maximum) level of parallellism used during build procedure",
'int', 'store', None),
+ 'pre-create-installdir': ("Create installation directory before submitting build jobs",
+ None, 'store_true', True),
'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
None, 'store_true', False, 'p'),
'read-only-installdir': ("Set read-only permissions on installation directory after installation",
@@ -670,6 +677,8 @@ def job_options(self):
opts = OrderedDict({
'backend-config': ("Configuration file for job backend", None, 'store', None),
'cores': ("Number of cores to request per job", 'int', 'store', None),
+ 'deps-type': ("Type of dependency to set between jobs (default depends on job backend)",
+ 'choice', 'store', None, [JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN]),
'max-jobs': ("Maximum number of concurrent jobs (queued and running, 0 = unlimited)", 'int', 'store', 0),
'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24),
'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()),
@@ -1148,6 +1157,94 @@ def parse_options(args=None, with_include=True):
return eb_go
+def check_root_usage(allow_use_as_root=False):
+ """
+ Check whether we are running as root, and act accordingly
+
+ :param allow_use_as_root: allow use of EasyBuild as root (but do print a warning when doing so)
+ """
+ if os.getuid() == 0:
+ if allow_use_as_root:
+ msg = "Using EasyBuild as root is NOT recommended, please proceed with care!\n"
+ msg += "(this is only allowed because EasyBuild was configured with "
+ msg += "--allow-use-as-root-and-accept-consequences)"
+ print_warning(msg)
+ else:
+ raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
+ "so let's end this here.")
+
+
+def set_up_configuration(args=None, logfile=None, testing=False, silent=False):
+ """
+ Set up EasyBuild configuration, by parsing configuration settings & initialising build options.
+
+ :param args: command line arguments to take into account when parsing the EasyBuild configuration settings
+ :param logfile: log file to use
+ :param testing: enable testing mode
+ :param silent: stay silent (no printing)
+ """
+
+ # parse EasyBuild configuration settings
+ eb_go = parse_options(args=args)
+ options = eb_go.options
+
+ # tmpdir is set by option parser via set_tmpdir function
+ tmpdir = tempfile.gettempdir()
+
+ # set umask (as early as possible)
+ if options.umask is not None:
+ new_umask = int(options.umask, 8)
+ old_umask = os.umask(new_umask)
+
+ search_query = options.search or options.search_filename or options.search_short
+
+ # initialise logging for main
+ log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
+ silent=(testing or options.terse or search_query or silent),
+ colorize=options.color)
+
+ # log startup info (must be done after setting up logger)
+ eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
+ log_start(log, eb_cmd_line, tmpdir)
+
+ # can't log umask setting before logger is set up...
+ if options.umask is not None:
+ log.info("umask set to '%s' (used to be '%s')", oct(new_umask), oct(old_umask))
+
+ # disallow running EasyBuild as root (by default)
+ check_root_usage(allow_use_as_root=options.allow_use_as_root_and_accept_consequences)
+
+ # process software build specifications (if any), i.e.
+ # software name/version, toolchain name/version, extra patches, ...
+ (try_to_generate, build_specs) = process_software_build_specs(options)
+
+ # determine robot path
+ # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
+ tweaked_ecs = try_to_generate and build_specs
+ tweaked_ecs_paths, pr_path = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
+ auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
+ robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
+ log.debug("Full robot path: %s" % robot_path)
+
+ # configure & initialize build options
+ config_options_dict = eb_go.get_options_by_section('config')
+ build_options = {
+ 'build_specs': build_specs,
+ 'command_line': eb_cmd_line,
+ 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
+ 'pr_path': pr_path,
+ 'robot_path': robot_path,
+ 'silent': testing,
+ 'try_to_generate': try_to_generate,
+ 'valid_stops': [x[0] for x in EasyBlock.get_steps()],
+ }
+ # initialise the EasyBuild configuration & build options
+ init(options, config_options_dict)
+ init_build_options(build_options=build_options, cmdline_options=options)
+
+ return eb_go, (build_specs, log, logfile, robot_path, search_query, tmpdir, try_to_generate, tweaked_ecs_paths)
+
+
def process_software_build_specs(options):
"""
Create a dictionary with specified software build options.
diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py
index 1a5018a3e8..a9c6151aff 100644
--- a/easybuild/tools/parallelbuild.py
+++ b/easybuild/tools/parallelbuild.py
@@ -155,18 +155,6 @@ def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-bui
returns the job
"""
- # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
- easybuild_vars = {}
- for name in os.environ:
- if name.startswith("EASYBUILD"):
- easybuild_vars[name] = os.environ[name]
-
- for env_var in ["PYTHONPATH", "MODULEPATH"]:
- if env_var in os.environ:
- easybuild_vars[env_var] = os.environ[env_var]
-
- _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
-
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
@@ -194,7 +182,7 @@ def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-bui
if build_option('job_cores'):
extra['cores'] = build_option('job_cores')
- job = job_backend.make_job(command, name, easybuild_vars, **extra)
+ job = job_backend.make_job(command, name, **extra)
job.module = easyconfig['ec'].full_mod_name
return job
diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py
index 91bec43645..19adb0342c 100644
--- a/easybuild/tools/toolchain/toolchain.py
+++ b/easybuild/tools/toolchain/toolchain.py
@@ -670,6 +670,10 @@ def compilers(self):
return (c_comps, fortran_comps)
+ def is_deprecated(self):
+ """Return whether or not this toolchain is deprecated."""
+ return False
+
def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index 0c137edc4d..af2161324e 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -43,7 +43,7 @@
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
-VERSION = LooseVersion('3.7.0')
+VERSION = LooseVersion('3.7.2.dev0')
UNKNOWN = 'UNKNOWN'
diff --git a/test/framework/build_log.py b/test/framework/build_log.py
index 759d79b4d6..eefb09abeb 100644
--- a/test/framework/build_log.py
+++ b/test/framework/build_log.py
@@ -31,9 +31,8 @@
import re
import sys
import tempfile
-from distutils.version import LooseVersion
from datetime import datetime, timedelta
-from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
+from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
from unittest import TextTestRunner
from vsc.utils.fancylogger import getLogger, getRootLoggerName, logToFile, setLogFormat
@@ -115,11 +114,11 @@ def test_easybuildlog(self):
self.mock_stderr(False)
more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information"
- expected_stderr = '\n'.join([
- "Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
- "Deprecated functionality, will no longer work in v2.0: onemorewarning",
- "Deprecated functionality, will no longer work in v2.0: lastwarning",
- ]) + '\n'
+ expected_stderr = '\n\n'.join([
+ "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
+ "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning",
+ "\nWARNING: Deprecated functionality, will no longer work in v2.0: lastwarning",
+ ]) + '\n\n'
self.assertEqual(stderr, expected_stderr)
try:
@@ -172,6 +171,21 @@ def test_easybuildlog(self):
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
+ write_file(tmplog, '')
+ logToFile(tmplog, enable=True)
+
+ # also test use of 'more_info' named argument for log.deprecated
+ self.mock_stderr(True)
+ log.deprecated("\nthis is just a test\n", newer_ver, more_info="(see URLGOESHERE for more information)")
+ self.mock_stderr(False)
+ logtxt = read_file(tmplog)
+ expected_logtxt = '\n'.join([
+ "[WARNING] :: Deprecated functionality, will no longer work in v10000001: ",
+ "this is just a test",
+ "(see URLGOESHERE for more information)",
+ ])
+ self.assertTrue(logtxt.strip().endswith(expected_logtxt))
+
def test_log_levels(self):
"""Test whether log levels are respected"""
fd, tmplog = tempfile.mkstemp()
@@ -274,5 +288,6 @@ def suite():
""" returns all the testcases in this module """
return TestLoaderFiltered().loadTestsFromTestCase(BuildLogTest, sys.argv[1:])
+
if __name__ == '__main__':
TextTestRunner(verbosity=1).run(suite())
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 30fbb9e8e2..f138e164e1 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1452,7 +1452,8 @@ def test_stale_module_caches(self):
module_version_spec = {'modname': 'one/1.0.2', 'sym_version': '1.0', 'version': '1.0.2'}
modulerc_txt = modgen.modulerc(module_version=module_version_spec)
one_moddir = os.path.join(self.test_installpath, 'modules', 'all', 'one')
- write_file(os.path.join(one_moddir, '.modulerc'), modulerc_txt)
+
+ write_file(os.path.join(one_moddir, modgen.DOT_MODULERC), modulerc_txt)
# check again, this just grabs the cached results for 'avail one/1.0' & 'show one/1.0'
self.assertFalse(self.modtool.exist(['one/1.0'])[0])
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index 9a34fc4767..0447b20c2d 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -53,6 +53,7 @@
from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3
from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str
+from easybuild.framework.easyconfig.style import check_easyconfigs_style
from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph
from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs
from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one
@@ -344,7 +345,7 @@ def test_exts_list(self):
' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail
' "checksums": [',
# SHA256 checksum for source (gzip-1.4.eb)
- ' "6f281b6d7a3965476324a23b9d80232bd4ffe3967da85e4b7c01d9d81d649a09",',
+ ' "f0235f93773e40a9120e8970e438023d46bbf205d44828beffb60905a8644156",',
# SHA256 checksum for 'patch' (toy-0.0.eb)
' "a79ba0ef5dceb5b8829268247feae8932bed2034c6628ff1d92c84bf45e9a546",',
' ],',
@@ -366,7 +367,7 @@ def test_exts_list(self):
self.assertEqual(exts_sources[1]['name'], 'ext2')
self.assertEqual(exts_sources[1]['version'], '2.0')
self.assertEqual(exts_sources[1]['options'], {
- 'checksums': ['6f281b6d7a3965476324a23b9d80232bd4ffe3967da85e4b7c01d9d81d649a09',
+ 'checksums': ['f0235f93773e40a9120e8970e438023d46bbf205d44828beffb60905a8644156',
'a79ba0ef5dceb5b8829268247feae8932bed2034c6628ff1d92c84bf45e9a546'],
'patches': ['toy-0.0.eb'],
'source_tmpl': 'gzip-1.4.eb',
@@ -1381,6 +1382,18 @@ def test_update(self):
toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch'
self.assertEqual(ec['patches'], [toy_patch_fn, ('toy-extra.txt', 'toy-0.0'), 'foo.patch', 'bar.patch'])
+ # for unallowed duplicates
+ ec.update('configopts', 'SOME_VALUE')
+ configopts_tmp = ec['configopts']
+ ec.update('configopts', 'SOME_VALUE', allow_duplicate=False)
+ self.assertEquals(ec['configopts'], configopts_tmp)
+
+ # for unallowed duplicates when a list is used
+ ec.update('patches', ['foo2.patch', 'bar2.patch'])
+ patches_tmp = copy.deepcopy(ec['patches'])
+ ec.update('patches', ['foo2.patch', 'bar2.patch'], allow_duplicate=False)
+ self.assertEquals(ec['patches'], patches_tmp)
+
def test_hide_hidden_deps(self):
"""Test use of --hide-deps on hiddendependencies."""
test_dir = os.path.dirname(os.path.abspath(__file__))
@@ -1430,38 +1443,53 @@ def test_quote_str(self):
def test_dump(self):
"""Test EasyConfig's dump() method."""
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+ build_options = {
+ 'valid_module_classes': module_classes(),
+ 'check_osdeps': False,
+ }
+ init_config(build_options=build_options)
ecfiles = [
't/toy/toy-0.0.eb',
'g/goolf/goolf-1.4.10.eb',
's/ScaLAPACK/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb',
'g/gzip/gzip-1.4-GCC-4.6.3.eb',
+ 'p/Python/Python-2.7.10-ictce-4.1.13.eb',
]
for ecfile in ecfiles:
test_ec = os.path.join(self.test_prefix, 'test.eb')
ec = EasyConfig(os.path.join(test_ecs_dir, ecfile))
+ ec.enable_templating = False
ecdict = ec.asdict()
ec.dump(test_ec)
# dict representation of EasyConfig instance should not change after dump
self.assertEqual(ecdict, ec.asdict())
ectxt = read_file(test_ec)
- patterns = [r"^name = ['\"]", r"^version = ['0-9\.]", r'^description = ["\']']
+ patterns = [
+ r"^name = ['\"]",
+ r"^version = ['0-9\.]",
+ r'^description = ["\']',
+ r"^toolchain = {'name': .*, 'version': .*}",
+ ]
for pattern in patterns:
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(ectxt), "Pattern '%s' found in: %s" % (regex.pattern, ectxt))
# parse result again
dumped_ec = EasyConfig(test_ec)
+ dumped_ec.enable_templating = False
# check that selected parameters still have the same value
params = [
'name',
'toolchain',
'dependencies', # checking this is important w.r.t. filtered hidden dependencies being restored in dump
+ 'exts_list', # exts_lists (in Python easyconfig) use another layer of templating so shouldn't change
]
for param in params:
- self.assertEqual(ec[param], dumped_ec[param])
+ if param in ec:
+ self.assertEqual(ec[param], dumped_ec[param])
def test_dump_autopep8(self):
"""Test dump() with autopep8 usage enabled (only if autopep8 is available)."""
@@ -1486,7 +1514,7 @@ def test_dump_extra(self):
"homepage = 'http://foo.com/'",
'description = "foo description"',
'',
- "toolchain = {'version': 'dummy', 'name': 'dummy'}",
+ "toolchain = {'name': 'dummy', 'version': 'dummy'}",
'',
"dependencies = [",
" ('GCC', '4.6.4', '-test'),",
@@ -1496,6 +1524,7 @@ def test_dump_extra(self):
"]",
'',
"foo_extra1 = 'foobar'",
+ '',
])
handle, testec = tempfile.mkstemp(prefix=self.test_prefix, suffix='.eb')
@@ -1506,7 +1535,10 @@ def test_dump_extra(self):
ectxt = read_file(testec)
self.assertEqual(rawtxt, ectxt)
- dumped_ec = EasyConfig(testec)
+ # check parsing of dumped easyconfig
+ EasyConfig(testec)
+
+ check_easyconfigs_style([testec])
def test_dump_template(self):
""" Test EasyConfig's dump() method for files containing templates"""
@@ -1573,7 +1605,9 @@ def test_dump_template(self):
self.assertTrue(regex.search(ectxt), "Pattern '%s' found in: %s" % (regex.pattern, ectxt))
# reparsing the dumped easyconfig file should work
- ecbis = EasyConfig(testec)
+ EasyConfig(testec)
+
+ check_easyconfigs_style([testec])
def test_dump_comments(self):
""" Test dump() method for files containing comments """
@@ -1631,7 +1665,9 @@ def test_dump_comments(self):
self.assertTrue(ectxt.endswith("# trailing comment"))
# reparsing the dumped easyconfig file should work
- ecbis = EasyConfig(testec)
+ EasyConfig(testec)
+
+ check_easyconfigs_style([testec])
def test_to_template_str(self):
""" Test for to_template_str method """
@@ -1851,7 +1887,7 @@ def test_copy_easyconfigs(self):
# passing an empty list of paths is fine
res = copy_easyconfigs([], target_dir)
self.assertEqual(res, {'ecs': [], 'new': [], 'new_file_in_existing_folder': [],
- 'new_folder': [], 'paths_in_repo': []})
+ 'new_folder': [], 'paths': [], 'paths_in_repo': []})
self.assertEqual(os.listdir(ecs_target_dir), [])
# copy test easyconfigs, purposely under a different name
@@ -1868,7 +1904,7 @@ def test_copy_easyconfigs(self):
res = copy_easyconfigs(ecs_to_copy, target_dir)
self.assertEqual(sorted(res.keys()), ['ecs', 'new', 'new_file_in_existing_folder',
- 'new_folder', 'paths_in_repo'])
+ 'new_folder', 'paths', 'paths_in_repo'])
self.assertEqual(len(res['ecs']), len(test_ecs))
self.assertTrue(all(isinstance(ec, EasyConfig) for ec in res['ecs']))
self.assertTrue(all(res['new']))
@@ -1900,7 +1936,7 @@ def test_copy_easyconfigs(self):
# copy single easyconfig with buildstats included for running further tests
res = copy_easyconfigs([toy_ec], target_dir)
- self.assertEqual([len(x) for x in res.values()], [1, 1, 1, 1, 1])
+ self.assertEqual([len(x) for x in res.values()], [1, 1, 1, 1, 1, 1])
self.assertEqual(res['ecs'][0].full_mod_name, 'toy/0.0')
# toy-0.0.eb was already copied into target_dir, so should not be marked as new anymore
@@ -2311,6 +2347,16 @@ def test_check_sha256_checksums(self):
# multiple checksums listed for source tarball, while exactly one (SHA256) checksum is expected
self.assertTrue(res[1].startswith("Non-SHA256 checksum found for toy-0.0.tar.gz: "))
+ def test_deprecated(self):
+ """Test use of 'deprecated' easyconfig parameter."""
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ toy_ec_txt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, toy_ec_txt + "\ndeprecated = 'this is just a test'")
+
+ error_pattern = r"easyconfig file '.*/test.eb' is marked as deprecated:\nthis is just a test\n\(see also"
+ self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb
index 1ae4242e80..fdaac2ae20 100644
--- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb
+++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb
@@ -24,7 +24,7 @@ toolchain = {'name': 'dummy', 'version': 'dummy'}
sources = [SOURCE_TAR_GZ]
# download location for source files
-source_urls = ['http://ftpmirror.gnu.org/gzip']
+source_urls = ['https://ftpmirror.gnu.org/gzip']
# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb
index cf97f23ae0..c1ac02b7d9 100644
--- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb
+++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb
@@ -14,8 +14,8 @@ easyblock = 'ConfigureMake'
name = 'gzip'
version = '1.4'
-homepage = "http://www.gzip.org/"
-description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"
+homepage = "http://www.%(namelower)s.org/"
+description = "%(name)s (GNU zip) is a popular data compression program as a replacement for compress"
# test toolchain specification
toolchain = {'name':'dummy','version':'dummy'}
@@ -24,7 +24,7 @@ toolchain = {'name':'dummy','version':'dummy'}
sources = [SOURCE_TAR_GZ]
# download location for source files
-source_urls = ['http://ftpmirror.gnu.org/gnu/gzip']
+source_urls = ['https://ftpmirror.gnu.org/gnu/gzip']
# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-goolf-1.4.10.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-goolf-1.4.10.eb
index 978e0c5e81..2a42eaf88f 100644
--- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-goolf-1.4.10.eb
+++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-goolf-1.4.10.eb
@@ -20,7 +20,7 @@ description = "gzip (GNU zip) is a popular data compression program as a replace
toolchain = {'name': 'goolf', 'version': '1.4.10'}
# eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz
-source_urls = ['http://ftpmirror.gnu.org/gnu/gzip']
+source_urls = ['https://ftpmirror.gnu.org/gnu/gzip']
sources = [SOURCE_TAR_GZ]
# make sure the gzip, gunzip and compress binaries are available after installation
diff --git a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-ictce-4.1.13.eb b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-ictce-4.1.13.eb
index c73162550d..d578178841 100644
--- a/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-ictce-4.1.13.eb
+++ b/test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-ictce-4.1.13.eb
@@ -14,13 +14,13 @@ easyblock = 'ConfigureMake'
name = 'gzip'
version = '1.5'
-homepage = "http://www.gzip.org/"
-description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"
+homepage = "http://www.%(name)s.org/"
+description = "%(namelower)s (GNU zip) is a popular data compression program as a replacement for compress"
toolchain = {'name': 'ictce', 'version': '4.1.13'}
# eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz
-source_urls = ['http://ftpmirror.gnu.org/gnu/gzip']
+source_urls = ['https://ftpmirror.gnu.org/gnu/gzip']
sources = [SOURCE_TAR_GZ]
# make sure the gzip, gunzip and compress binaries are available after installation
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 5020b99a1f..58a69f1739 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -547,6 +547,34 @@ def test_read_write_file(self):
self.assertEqual(ft.read_file(backup1), txt + txt2)
self.assertEqual(ft.read_file(backup2), 'foo')
+ # tese use of 'verbose' to make write_file print location of backed up file
+ self.mock_stdout(True)
+ ft.write_file(fp, 'foo', backup=True, verbose=True)
+ stdout = self.get_stdout()
+ self.mock_stdout(False)
+ regex = re.compile("^== Backup of .*/test.txt created at .*/test.txt.bak_[0-9]*")
+ self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+
+ # by default, write_file will just blindly overwrite an already existing file
+ self.assertTrue(os.path.exists(fp))
+ ft.write_file(fp, 'blah')
+ self.assertEqual(ft.read_file(fp), 'blah')
+
+ # blind overwriting can be disabled via 'overwrite'
+ error_pattern = "File exists, not overwriting it without --force: %s" % fp
+ self.assertErrorRegex(EasyBuildError, error_pattern, ft.write_file, fp, 'blah', overwrite=False)
+ self.assertErrorRegex(EasyBuildError, error_pattern, ft.write_file, fp, 'blah', overwrite=False, backup=True)
+
+ # use of --force ensuring that file gets written regardless of whether or not it exists already
+ build_options = {'force': True}
+ init_config(build_options=build_options)
+
+ ft.write_file(fp, 'overwrittenbyforce', overwrite=False)
+ self.assertEqual(ft.read_file(fp), 'overwrittenbyforce')
+
+ ft.write_file(fp, 'overwrittenbyforcewithbackup', overwrite=False, backup=True)
+ self.assertEqual(ft.read_file(fp), 'overwrittenbyforcewithbackup')
+
# also test behaviour of write_file under --dry-run
build_options = {
'extended_dry_run': True,
@@ -568,7 +596,6 @@ def test_read_write_file(self):
self.assertTrue(os.path.exists(foo))
self.assertEqual(ft.read_file(foo), 'bar')
-
def test_det_patched_files(self):
"""Test det_patched_files function."""
toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch'
@@ -944,10 +971,6 @@ def test_adjust_permissions(self):
for bit in [stat.S_IXGRP, stat.S_IWOTH, stat.S_IXOTH]:
self.assertFalse(perms & bit)
- # broken symlinks are trouble if symlinks are not skipped
- self.assertErrorRegex(EasyBuildError, "No such file or directory", ft.adjust_permissions, self.test_prefix,
- stat.S_IXUSR, skip_symlinks=False)
-
# restore original umask
os.umask(orig_umask)
@@ -961,20 +984,19 @@ def test_adjust_permissions_max_fail_ratio(self):
ft.write_file(test_files[-1], '')
ft.symlink(test_files[-1], os.path.join(testdir, 'symlink%s' % idx))
- # by default, 50% of failures are allowed (to be robust against broken symlinks)
+ # by default, 50% of failures are allowed (to be robust against failures to change permissions)
perms = stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR
- # one file remove, 1 dir + 2 files + 3 symlinks (of which 1 broken) left => 1/6 (16%) fail ratio is OK
+ ft.adjust_permissions(testdir, perms, recursive=True, ignore_errors=True)
+
+ # introducing a broken symlinks doesn't cause problems
ft.remove_file(test_files[0])
- ft.adjust_permissions(testdir, perms, recursive=True, skip_symlinks=False, ignore_errors=True)
- # 2 files removed, 1 dir + 1 file + 3 symlinks (of which 2 broken) left => 2/5 (40%) fail ratio is OK
+ ft.adjust_permissions(testdir, perms, recursive=True, ignore_errors=True)
+
+ # multiple/all broken symlinks is no problem either, since symlinks are never followed
ft.remove_file(test_files[1])
- ft.adjust_permissions(testdir, perms, recursive=True, skip_symlinks=False, ignore_errors=True)
- # 3 files removed, 1 dir + 3 broken symlinks => 75% fail ratio is too high, so error is raised
ft.remove_file(test_files[2])
- error_pattern = r"75.00% of permissions/owner operations failed \(more than 50.00%\), something must be wrong"
- self.assertErrorRegex(EasyBuildError, error_pattern, ft.adjust_permissions, testdir, perms,
- recursive=True, skip_symlinks=False, ignore_errors=True)
+ ft.adjust_permissions(testdir, perms, recursive=True, ignore_errors=True)
# reconfigure EasyBuild to allow even higher fail ratio (80%)
build_options = {
@@ -983,7 +1005,7 @@ def test_adjust_permissions_max_fail_ratio(self):
init_config(build_options=build_options)
# 75% < 80%, so OK
- ft.adjust_permissions(testdir, perms, recursive=True, skip_symlinks=False, ignore_errors=True)
+ ft.adjust_permissions(testdir, perms, recursive=True, ignore_errors=True)
# reconfigure to allow less failures (10%)
build_options = {
@@ -991,21 +1013,12 @@ def test_adjust_permissions_max_fail_ratio(self):
}
init_config(build_options=build_options)
- # way too many failures with 3 broken symlinks
- error_pattern = r"75.00% of permissions/owner operations failed \(more than 10.00%\), something must be wrong"
- self.assertErrorRegex(EasyBuildError, error_pattern, ft.adjust_permissions, testdir, perms,
- recursive=True, skip_symlinks=False, ignore_errors=True)
+ ft.adjust_permissions(testdir, perms, recursive=True, ignore_errors=True)
- # one broken symlink is still too much with max fail ratio of 10%
ft.write_file(test_files[0], '')
ft.write_file(test_files[1], '')
- error_pattern = r"16.67% of permissions/owner operations failed \(more than 10.00%\), something must be wrong"
- self.assertErrorRegex(EasyBuildError, error_pattern, ft.adjust_permissions, testdir, perms,
- recursive=True, skip_symlinks=False, ignore_errors=True)
-
- # all files restored, no more broken symlinks, so OK
ft.write_file(test_files[2], '')
- ft.adjust_permissions(testdir, perms, recursive=True, skip_symlinks=False, ignore_errors=True)
+ ft.adjust_permissions(testdir, perms, recursive=True, ignore_errors=True)
def test_apply_regex_substitutions(self):
"""Test apply_regex_substitutions function."""
diff --git a/test/framework/github.py b/test/framework/github.py
index 3591947f83..e9a8f22379 100644
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -120,6 +120,10 @@ def test_fetch_easyconfigs_from_pr(self):
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
+ init_config(build_options={
+ 'pr_target_account': gh.GITHUB_EB_MAIN,
+ })
+
# PR for rename of ffmpeg to FFmpeg,
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files
all_ecs_pr2481 = [
diff --git a/test/framework/lib.py b/test/framework/lib.py
new file mode 100644
index 0000000000..11e84b664e
--- /dev/null
+++ b/test/framework/lib.py
@@ -0,0 +1,130 @@
+# #
+# Copyright 2018-2018 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+# #
+"""
+Unit tests for using EasyBuild as a library.
+
+@author: Kenneth Hoste (Ghent University)
+"""
+import os
+import shutil
+import sys
+import tempfile
+from unittest import TextTestRunner
+
+from test.framework.utilities import TestLoaderFiltered
+
+# deliberately *not* using EnhancedTestCase from test.framework.utilities to avoid automatic configuration via setUp
+from vsc.utils.testing import EnhancedTestCase
+
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import BuildOptions
+from easybuild.tools.options import set_up_configuration
+from easybuild.tools.filetools import mkdir
+from easybuild.tools.modules import modules_tool
+from easybuild.tools.run import run_cmd
+
+
+class EasyBuildLibTest(EnhancedTestCase):
+ """Test cases for using EasyBuild as a library."""
+
+ def setUp(self):
+ """Prepare for running test."""
+ super(EasyBuildLibTest, self).setUp()
+
+ # make sure BuildOptions instance is re-created
+ del BuildOptions._instances[BuildOptions]
+
+ self.tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ """Cleanup after running test."""
+ super(EasyBuildLibTest, self).tearDown()
+
+ shutil.rmtree(self.tmpdir)
+
+ def configure(self):
+ """Utility function to set up EasyBuild configuration."""
+
+ # wipe BuildOption singleton instance, so it gets re-created when set_up_configuration is called
+ del BuildOptions._instances[BuildOptions]
+
+ self.assertFalse(BuildOptions in BuildOptions._instances)
+ set_up_configuration(silent=True)
+ self.assertTrue(BuildOptions in BuildOptions._instances)
+
+ def test_run_cmd(self):
+ """Test use of run_cmd function in the context of using EasyBuild framework as a library."""
+
+ error_pattern = "Undefined build option: .*"
+ error_pattern += " Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)"
+ self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd, "echo hello")
+
+ self.configure()
+
+ # run_cmd works fine if set_up_configuration was called first
+ (out, ec) = run_cmd("echo hello")
+ self.assertEqual(ec, 0)
+ self.assertEqual(out, 'hello\n')
+
+ def test_mkdir(self):
+ """Test use of run_cmd function in the context of using EasyBuild framework as a library."""
+
+ test_dir = os.path.join(self.tmpdir, 'test123')
+
+ error_pattern = "Undefined build option: .*"
+ error_pattern += " Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)"
+ self.assertErrorRegex(EasyBuildError, error_pattern, mkdir, test_dir)
+
+ self.configure()
+
+ # mkdir works fine if set_up_configuration was called first
+ self.assertFalse(os.path.exists(test_dir))
+ mkdir(test_dir)
+ self.assertTrue(os.path.exists(test_dir))
+
+ def test_modules_tool(self):
+ """Test use of modules_tool function in the context of using EasyBuild framework as a library."""
+
+ error_pattern = "Undefined build option: .*"
+ error_pattern += " Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)"
+ self.assertErrorRegex(EasyBuildError, error_pattern, modules_tool)
+
+ self.configure()
+
+ test_mods_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
+
+ modtool = modules_tool()
+ modtool.use(test_mods_path)
+ self.assertTrue('GCC/4.7.2' in modtool.available())
+ modtool.load(['GCC/4.7.2'])
+ self.assertEqual(modtool.list(), [{'default': None, 'mod_name': 'GCC/4.7.2'}])
+
+
+def suite():
+ return TestLoaderFiltered().loadTestsFromTestCase(EasyBuildLibTest, sys.argv[1:])
+
+
+if __name__ == '__main__':
+ TextTestRunner(verbosity=1).run(suite())
diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index 287f766eea..bf52f5b56d 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -32,6 +32,7 @@
import os
import sys
import tempfile
+from distutils.version import LooseVersion
from unittest import TextTestRunner, TestSuite
from vsc.utils.fancylogger import setLogLevelDebug, logToScreen
from vsc.utils.missing import nub
@@ -323,8 +324,26 @@ def test_modulerc(self):
error_pattern = "Incorrect module_version spec, expected keys"
self.assertErrorRegex(EasyBuildError, error_pattern, self.modgen.modulerc, arg)
- modulerc = self.modgen.modulerc({'modname': 'test/1.2.3.4.5', 'sym_version': '1.2.3', 'version': '1.2.3.4.5'})
+ mod_ver_spec = {'modname': 'test/1.2.3.4.5', 'sym_version': '1.2.3', 'version': '1.2.3.4.5'}
+ modulerc_path = os.path.join(self.test_prefix, 'test', self.modgen.DOT_MODULERC)
+ # with Lmod 6.x, both .modulerc and wrapper module must be in the same location
+ if isinstance(self.modtool, Lmod) and LooseVersion(self.modtool.version) < LooseVersion('7.0'):
+ error = "Expected module file .* not found; "
+ error += "Lmod 6.x requires that .modulerc and wrapped module file are in same directory"
+ self.assertErrorRegex(EasyBuildError, error, self.modgen.modulerc, mod_ver_spec, filepath=modulerc_path)
+
+ # if the wrapped module file is in place, everything should be fine
+ write_file(os.path.join(self.test_prefix, 'test', '1.2.3.4.5'), '#%Module')
+ modulerc = self.modgen.modulerc(mod_ver_spec, filepath=modulerc_path)
+
+ # first, check raw contents of generated .modulerc file
+ expected = '\n'.join([
+ '#%Module',
+ "module-version test/1.2.3.4.5 1.2.3",
+ ])
+
+ # two exceptions: EnvironmentModulesC, or Lmod 7.8 (or newer) and Lua syntax
if self.modtool.__class__ == EnvironmentModulesC:
expected = '\n'.join([
'#%Module',
@@ -332,16 +351,12 @@ def test_modulerc(self):
' module-version test/1.2.3.4.5 1.2.3',
'}',
])
- else:
- expected = '\n'.join([
- '#%Module',
- "module-version test/1.2.3.4.5 1.2.3",
- ])
+ elif self.MODULE_GENERATOR_CLASS == ModuleGeneratorLua:
+ if isinstance(self.modtool, Lmod) and LooseVersion(self.modtool.version) >= LooseVersion('7.8'):
+ expected = 'module_version("test/1.2.3.4.5", "1.2.3")'
self.assertEqual(modulerc, expected)
-
- write_file(os.path.join(self.test_prefix, 'test', '1.2.3.4.5'), '#%Module')
- write_file(os.path.join(self.test_prefix, 'test', '.modulerc'), modulerc)
+ self.assertEqual(read_file(modulerc_path), expected)
self.modtool.use(self.test_prefix)
@@ -355,6 +370,23 @@ def test_modulerc(self):
self.assertEqual(len(res), 1)
self.assertEqual(res[0]['mod_name'], 'test/1.2.3.4.5')
+ # overwriting existing .modulerc requires --force or --rebuild
+ error_msg = "Found existing .modulerc at .*, not overwriting without --force or --rebuild"
+ self.assertErrorRegex(EasyBuildError, error_msg, self.modgen.modulerc, mod_ver_spec, filepath=modulerc_path)
+
+ init_config(build_options={'force': True})
+ modulerc = self.modgen.modulerc(mod_ver_spec, filepath=modulerc_path)
+ self.assertEqual(modulerc, expected)
+ self.assertEqual(read_file(modulerc_path), expected)
+
+ init_config(build_options={})
+ self.assertErrorRegex(EasyBuildError, error_msg, self.modgen.modulerc, mod_ver_spec, filepath=modulerc_path)
+
+ init_config(build_options={'rebuild': True})
+ modulerc = self.modgen.modulerc(mod_ver_spec, filepath=modulerc_path)
+ self.assertEqual(modulerc, expected)
+ self.assertEqual(read_file(modulerc_path), expected)
+
def test_unload(self):
"""Test unload part in generated module file."""
@@ -597,7 +629,10 @@ def test_alias(self):
def test_conditional_statement(self):
"""Test formatting of conditional statements."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- simple_cond = self.modgen.conditional_statement("is-loaded foo", "module load bar")
+ cond = "is-loaded foo"
+ load = "module load bar"
+
+ simple_cond = self.modgen.conditional_statement(cond, load)
expected = '\n'.join([
"if { [ is-loaded foo ] } {",
" module load bar",
@@ -606,7 +641,7 @@ def test_conditional_statement(self):
])
self.assertEqual(simple_cond, expected)
- neg_cond = self.modgen.conditional_statement("is-loaded foo", "module load bar", negative=True)
+ neg_cond = self.modgen.conditional_statement(cond, load, negative=True)
expected = '\n'.join([
"if { ![ is-loaded foo ] } {",
" module load bar",
@@ -615,7 +650,7 @@ def test_conditional_statement(self):
])
self.assertEqual(neg_cond, expected)
- if_else_cond = self.modgen.conditional_statement("is-loaded foo", "module load bar", else_body='puts "foo"')
+ if_else_cond = self.modgen.conditional_statement(cond, load, else_body='puts "foo"')
expected = '\n'.join([
"if { [ is-loaded foo ] } {",
" module load bar",
@@ -626,8 +661,22 @@ def test_conditional_statement(self):
])
self.assertEqual(if_else_cond, expected)
+ if_else_cond = self.modgen.conditional_statement(cond, load, else_body='puts "foo"', indent=False)
+ expected = '\n'.join([
+ "if { [ is-loaded foo ] } {",
+ "module load bar",
+ "} else {",
+ 'puts "foo"',
+ '}',
+ '',
+ ])
+ self.assertEqual(if_else_cond, expected)
+
elif self.MODULE_GENERATOR_CLASS == ModuleGeneratorLua:
- simple_cond = self.modgen.conditional_statement('isloaded("foo")', 'load("bar")')
+ cond = 'isloaded("foo")'
+ load = 'load("bar")'
+
+ simple_cond = self.modgen.conditional_statement(cond, load)
expected = '\n'.join([
'if isloaded("foo") then',
' load("bar")',
@@ -636,7 +685,7 @@ def test_conditional_statement(self):
])
self.assertEqual(simple_cond, expected)
- neg_cond = self.modgen.conditional_statement('isloaded("foo")', 'load("bar")', negative=True)
+ neg_cond = self.modgen.conditional_statement(cond, load, negative=True)
expected = '\n'.join([
'if not isloaded("foo") then',
' load("bar")',
@@ -645,7 +694,7 @@ def test_conditional_statement(self):
])
self.assertEqual(neg_cond, expected)
- if_else_cond = self.modgen.conditional_statement('isloaded("foo")', 'load("bar")', else_body='load("bleh")')
+ if_else_cond = self.modgen.conditional_statement(cond, load, else_body='load("bleh")')
expected = '\n'.join([
'if isloaded("foo") then',
' load("bar")',
@@ -655,34 +704,46 @@ def test_conditional_statement(self):
'',
])
self.assertEqual(if_else_cond, expected)
+
+ if_else_cond = self.modgen.conditional_statement(cond, load, else_body='load("bleh")', indent=False)
+ expected = '\n'.join([
+ 'if isloaded("foo") then',
+ 'load("bar")',
+ 'else',
+ 'load("bleh")',
+ 'end',
+ '',
+ ])
+ self.assertEqual(if_else_cond, expected)
+
else:
self.assertTrue(False, "Unknown module syntax")
def test_load_msg(self):
"""Test including a load message in the module file."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
- expected = "\nif { [ module-info mode load ] } {\n puts stderr \"test\"\n}\n"
+ expected = "\nif { [ module-info mode load ] } {\nputs stderr \"test\"\n}\n"
self.assertEqual(expected, self.modgen.msg_on_load('test'))
tcl_load_msg = '\n'.join([
'',
"if { [ module-info mode load ] } {",
- " puts stderr \"test \\$test \\$test",
- " test \\$foo \\$bar\"",
+ "puts stderr \"test \\$test \\$test",
+ "test \\$foo \\$bar\"",
"}",
'',
])
self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar'))
else:
- expected = '\nif mode() == "load" then\n io.stderr:write([==[test]==])\nend\n'
+ expected = '\nif mode() == "load" then\nio.stderr:write([==[test]==])\nend\n'
self.assertEqual(expected, self.modgen.msg_on_load('test'))
lua_load_msg = '\n'.join([
'',
'if mode() == "load" then',
- ' io.stderr:write([==[test $test \\$test',
- ' test $foo \\$bar]==])',
+ 'io.stderr:write([==[test $test \\$test',
+ 'test $foo \\$bar]==])',
'end',
'',
])
diff --git a/test/framework/modules.py b/test/framework/modules.py
index d8943faca4..ad3e3fc90f 100644
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -42,8 +42,8 @@
import easybuild.tools.modules as mod
from easybuild.framework.easyblock import EasyBlock
from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import copy_file, copy_dir, mkdir, read_file, write_file
-from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesTcl, Lmod, NoModulesTool
+from easybuild.tools.filetools import copy_file, copy_dir, mkdir, read_file, remove_file, write_file
+from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool
from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version
from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches
from easybuild.tools.run import run_cmd
@@ -139,8 +139,8 @@ def test_exists(self):
self.assertEqual(self.modtool.exist(['OpenMPI/1.6.4'], skip_avail=True), [False])
# exists works on hidden modules in Lua syntax (only with Lmod)
+ test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
if isinstance(self.modtool, Lmod):
- test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
# make sure only the .lua module file is there, otherwise this test doesn't work as intended
self.assertTrue(os.path.exists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua')))
self.assertFalse(os.path.exists(os.path.join(test_modules_path, 'bzip2', '.1.0.6')))
@@ -155,6 +155,69 @@ def test_exists(self):
self.assertEqual(self.modtool.exist(mod_names), [True, False, True, False, True, True, True])
self.assertEqual(self.modtool.exist(mod_names, skip_avail=True), [True, False, True, False, True, True, True])
+ # verify whether checking for existence of a module wrapper works
+ self.modtool.unuse(test_modules_path)
+ self.modtool.use(self.test_prefix)
+
+ java_mod_dir = os.path.join(self.test_prefix, 'Java')
+ write_file(os.path.join(java_mod_dir, '1.8.0_181'), '#%Module')
+
+ if self.modtool.__class__ == EnvironmentModulesC:
+ modulerc_tcl_txt = '\n'.join([
+ '#%Module',
+ 'if {"Java/1.8" eq [module-info version Java/1.8]} {',
+ ' module-version Java/1.8.0_181 1.8',
+ '}',
+ ])
+ else:
+ modulerc_tcl_txt = '\n'.join([
+ '#%Module',
+ 'module-version Java/1.8.0_181 1.8',
+ ])
+
+ write_file(os.path.join(java_mod_dir, '.modulerc'), modulerc_tcl_txt)
+
+ avail_mods = self.modtool.available()
+ self.assertTrue('Java/1.8.0_181' in avail_mods)
+ if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'):
+ self.assertTrue('Java/1.8' in avail_mods)
+ self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True])
+ self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181')
+
+ reset_module_caches()
+
+ # what if we're in an HMNS setting...
+ mkdir(os.path.join(self.test_prefix, 'Core'))
+ shutil.move(java_mod_dir, os.path.join(self.test_prefix, 'Core', 'Java'))
+
+ self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available())
+ self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True])
+ self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True])
+ self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181')
+
+ # also check with .modulerc.lua for Lmod 7.8 or newer
+ if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'):
+ shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir)
+ reset_module_caches()
+
+ remove_file(os.path.join(java_mod_dir, '.modulerc'))
+ write_file(os.path.join(java_mod_dir, '.modulerc.lua'), 'module_version("Java/1.8.0_181", "1.8")')
+
+ avail_mods = self.modtool.available()
+ self.assertTrue('Java/1.8.0_181' in avail_mods)
+ self.assertTrue('Java/1.8' in avail_mods)
+ self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True])
+ self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181')
+
+ reset_module_caches()
+
+ # back to HMNS setup
+ shutil.move(java_mod_dir, os.path.join(self.test_prefix, 'Core', 'Java'))
+ self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available())
+ self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True])
+ self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True])
+ self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181')
+
def test_load(self):
""" test if we load one module it is in the loaded_modules """
self.init_testmods()
diff --git a/test/framework/options.py b/test/framework/options.py
index aed8e8b8b5..cdfed6b74d 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -1343,7 +1343,7 @@ def test_deprecated(self):
except easybuild.tools.build_log.EasyBuildError, err:
self.assertTrue(False, 'Deprecated logging should work')
- stderr_regex = re.compile("^Deprecated functionality, will no longer work in")
+ stderr_regex = re.compile("^\nWARNING: Deprecated functionality, will no longer work in")
self.assertTrue(stderr_regex.search(stderr), "Pattern '%s' found in: %s" % (stderr_regex.pattern, stderr))
# force it to current version, which should result in deprecation
diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py
index 78d43da20b..2fdb45c5bd 100644
--- a/test/framework/parallelbuild.py
+++ b/test/framework/parallelbuild.py
@@ -33,11 +33,11 @@
import sys
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
-from vsc.utils.fancylogger import setLogLevelDebug, logToScreen
+import easybuild.tools.job.slurm as slurm
from easybuild.framework.easyconfig.tools import process_easyconfig
from easybuild.tools import config
-from easybuild.tools.filetools import adjust_permissions, mkdir, which, write_file
+from easybuild.tools.filetools import adjust_permissions, mkdir, remove_dir, which, write_file
from easybuild.tools.job import pbs_python
from easybuild.tools.job.pbs_python import PbsPython
from easybuild.tools.options import parse_options
@@ -63,6 +63,21 @@
time_cmd = %(time)s
"""
+
+MOCKED_SBATCH = """#!/bin/bash
+if [[ $1 == '--version' ]]; then
+ echo "slurm 17.0"
+else
+ echo "Submitted batch job $RANDOM"
+ echo "(submission args: $@)"
+fi
+"""
+
+MOCKED_SCONTROL = """#!/bin/bash
+ echo "(scontrol args: $@)"
+"""
+
+
def mock(*args, **kwargs):
"""Function used for mocking several functions imported in parallelbuild module."""
return 1
@@ -160,6 +175,31 @@ def test_build_easyconfigs_in_parallel_pbs_python(self):
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
+ # also test use of --pre-create-installdir
+ ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool)
+
+ # installation directory doesn't exist yet before submission
+ toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
+ self.assertFalse(os.path.exists(toy_installdir))
+
+ jobs = submit_jobs(ordered_ecs, '', testing=False)
+ self.assertEqual(len(jobs), 1)
+
+ # software install dir is created (by default) as part of job submission process (fetch_step is run)
+ self.assertTrue(os.path.exists(toy_installdir))
+ remove_dir(toy_installdir)
+ remove_dir(os.path.dirname(toy_installdir))
+ self.assertFalse(os.path.exists(toy_installdir))
+
+ # installation directory does *not* get created when --pre-create-installdir is used
+ build_options['pre_create_installdir'] = False
+ init_config(args=['--job-backend=PbsPython'], build_options=build_options)
+
+ jobs = submit_jobs(ordered_ecs, '', testing=False)
+ self.assertEqual(len(jobs), 1)
+ self.assertFalse(os.path.exists(toy_installdir))
+
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
@@ -171,7 +211,7 @@ def test_build_easyconfigs_in_parallel_pbs_python(self):
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
- import gc3libs
+ import gc3libs # noqa (ignore unused import)
except ImportError:
print "GC3Pie not available, skipping test"
return
@@ -207,7 +247,7 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
'valid_module_classes': config.module_classes(),
'validate': False,
}
- options = init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
+ init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
@@ -215,7 +255,7 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
- jobs = build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
+ build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0'))
self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
@@ -260,12 +300,59 @@ def test_submit_jobs(self):
regex = re.compile(regex)
self.assertFalse(regex.search(cmd), "Pattern '%s' *not* found in: %s" % (regex.pattern, cmd))
+ def test_build_easyconfigs_in_parallel_slurm(self):
+ """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""
+
+ # install mocked versions of 'sbatch' and 'scontrol' commands
+ sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch')
+ write_file(sbatch, MOCKED_SBATCH)
+ adjust_permissions(sbatch, stat.S_IXUSR, add=True)
+
+ scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol')
+ write_file(scontrol, MOCKED_SCONTROL)
+ adjust_permissions(scontrol, stat.S_IXUSR, add=True)
+
+ os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')])
+
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-goolf-1.4.10.eb')
+
+ build_options = {
+ 'external_modules_metadata': {},
+ 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
+ 'valid_module_classes': config.module_classes(),
+ 'validate': False,
+ 'job_cores': 3,
+ 'job_max_walltime': 5,
+ }
+ init_config(args=['--job-backend=Slurm'], build_options=build_options)
+
+ easyconfigs = process_easyconfig(test_ec)
+ ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
+ self.mock_stdout(True)
+ jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
+ self.mock_stdout(False)
+
+ self.assertEqual(len(jobs), 8)
+
+ expected = {
+ 'job-name': 'gzip-1.5-goolf-1.4.10',
+ 'wrap': "echo '%s'" % test_ec,
+ 'nodes': 1,
+ 'ntasks': 3,
+ 'time': 300, # 60*5 (unit is minutes)
+ 'dependency': 'afterok:%s' % jobs[-2].jobid,
+ 'hold': True,
+ }
+ for key, val in expected.items():
+ self.assertTrue(key in jobs[-1].job_specs)
+ self.assertEqual(jobs[-1].job_specs[key], expected[key])
+
def suite():
""" returns all the testcases in this module """
return TestLoaderFiltered().loadTestsFromTestCase(ParallelBuildTest, sys.argv[1:])
+
if __name__ == '__main__':
- #logToScreen(enable=True)
- #setLogLevelDebug()
TextTestRunner(verbosity=1).run(suite())
diff --git a/test/framework/suite.py b/test/framework/suite.py
index 3e9687ff81..8d170b336e 100755
--- a/test/framework/suite.py
+++ b/test/framework/suite.py
@@ -59,6 +59,7 @@
import test.framework.github as g
import test.framework.hooks as h
import test.framework.include as i
+import test.framework.lib as lib
import test.framework.license as lic
import test.framework.module_generator as mg
import test.framework.modules as m
@@ -118,7 +119,7 @@
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, sc,
- tw, p, i, pkg, d, env, et, y, st, h, ct]
+ tw, p, i, pkg, d, env, et, y, st, h, ct, lib]
SUITE = unittest.TestSuite([x.suite() for x in tests])
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 69579a3867..dc2f26fbbd 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -950,7 +950,7 @@ def test_pgi_toolchain(self):
self.assertEqual(tc.get_variable('CC'), 'pgcc')
self.assertEqual(tc.get_variable('CXX'), 'pgCC')
self.assertEqual(tc.get_variable('F77'), 'pgf77')
- self.assertEqual(tc.get_variable('F90'), 'pgfortran')
+ self.assertEqual(tc.get_variable('F90'), 'pgf90')
self.assertEqual(tc.get_variable('FC'), 'pgfortran')
self.modtool.purge()
@@ -961,7 +961,7 @@ def test_pgi_toolchain(self):
self.assertEqual(tc.get_variable('CC'), 'pgcc')
self.assertEqual(tc.get_variable('CXX'), 'pgc++')
self.assertEqual(tc.get_variable('F77'), 'pgf77')
- self.assertEqual(tc.get_variable('F90'), 'pgfortran')
+ self.assertEqual(tc.get_variable('F90'), 'pgf90')
self.assertEqual(tc.get_variable('FC'), 'pgfortran')
def test_pgi_imkl(self):
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 9b7e59be5c..1111570803 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -215,8 +215,8 @@ def test_toy_tweaked(self):
shutil.copy2(os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb'), ec_file)
modloadmsg = 'THANKS FOR LOADING ME\\nI AM %(name)s v%(version)s'
- modloadmsg_regex_tcl = 'THANKS.*\n\s*I AM toy v0.0"'
- modloadmsg_regex_lua = '\[==\[THANKS.*\n\s*I AM toy v0.0\]==\]'
+ modloadmsg_regex_tcl = 'THANKS.*\n\s*I AM toy v0.0\n\s*"'
+ modloadmsg_regex_lua = '\[==\[THANKS.*\n\s*I AM toy v0.0\n\s*\]==\]'
# tweak easyconfig by appending to it
ec_extra = '\n'.join([
@@ -269,6 +269,23 @@ def test_toy_tweaked(self):
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
+ # newline between "I AM toy v0.0" (modloadmsg) and "oh hai!" (mod*footer) is added automatically
+ expected = "\nTHANKS FOR LOADING ME\nI AM toy v0.0\n"
+
+ # with module files in Tcl syntax, a newline is added automatically
+ if get_module_syntax() == 'Tcl':
+ expected += "\n"
+
+ expected += "oh hai!"
+
+ # setting $LMOD_QUIET results in suppression of printed message with Lmod & module files in Tcl syntax
+ if 'LMOD_QUIET' in os.environ:
+ del os.environ['LMOD_QUIET']
+
+ self.modtool.use(os.path.join(self.test_installpath, 'modules', 'all'))
+ out = self.modtool.run_module('load', 'toy/0.0-tweaked', return_output=True)
+ self.assertTrue(out.strip().endswith(expected))
+
def test_toy_buggy_easyblock(self):
"""Test build using a buggy/broken easyblock, make sure a traceback is reported."""
ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
@@ -1070,12 +1087,14 @@ def test_toy_module_fulltxt(self):
toy_mod_txt = read_file(toy_module)
modloadmsg_tcl = [
- r' puts stderr "THANKS FOR LOADING ME',
- r' I AM toy v0.0"',
+ r'puts stderr "THANKS FOR LOADING ME',
+ r'I AM toy v0.0',
+ '"',
]
modloadmsg_lua = [
- r' io.stderr:write\(\[==\[THANKS FOR LOADING ME',
- r' I AM toy v0.0\]==\]\)',
+ r'io.stderr:write\(\[==\[THANKS FOR LOADING ME',
+ r'I AM toy v0.0',
+ '\]==\]\)',
]
help_txt = '\n'.join([
@@ -1624,6 +1643,17 @@ def test_sanity_check_paths_lib64(self):
# all is fine is lib64 fallback check is enabled (which it is by default)
self.test_toy_build(ec_file=test_ec, raise_error=True)
+ # also check with 'lib' in sanity check dirs (special case)
+ ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy'],", ectxt)
+ ectxt = re.sub("\s*'dirs'.*", "'dirs': ['lib'],", ectxt)
+ write_file(test_ec, ectxt)
+
+ error_pattern = r"Sanity check failed: no \(non-empty\) directory found at 'lib' in "
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
+ extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False)
+
+ self.test_toy_build(ec_file=test_ec, raise_error=True)
+
# also check other way around (lib64 -> lib)
ectxt = read_file(ec_file)
ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', 'lib64/libtoy.a'],", ectxt)
@@ -1637,6 +1667,26 @@ def test_sanity_check_paths_lib64(self):
# sanity check passes when lib64 fallback is enabled (by default), since lib/libtoy.a is also considered
self.test_toy_build(ec_file=test_ec, raise_error=True)
+ # also check with 'lib64' in sanity check dirs (special case)
+ ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy'],", ectxt)
+ ectxt = re.sub("\s*'dirs'.*", "'dirs': ['lib64'],", ectxt)
+ write_file(test_ec, ectxt)
+
+ error_pattern = r"Sanity check failed: no \(non-empty\) directory found at 'lib64' in "
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
+ extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False)
+
+ self.test_toy_build(ec_file=test_ec, raise_error=True)
+
+ # check whether fallback works for files that's more than 1 subdir deep
+ ectxt = read_file(ec_file)
+ ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', 'lib/test/libtoy.a'],", ectxt)
+ postinstallcmd = "mkdir -p %(installdir)s/lib64/test && "
+ postinstallcmd += "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/test/libtoy.a"
+ ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt)
+ write_file(test_ec, ectxt)
+ self.test_toy_build(ec_file=test_ec, raise_error=True)
+
def test_toy_dumped_easyconfig(self):
""" Test dumping of file in eb_filerepo in both .eb and .yeb format """
filename = 'toy-0.0'
diff --git a/test/framework/tweak.py b/test/framework/tweak.py
index 427fdf2484..9334187b7f 100644
--- a/test/framework/tweak.py
+++ b/test/framework/tweak.py
@@ -223,6 +223,7 @@ def test_dep_tree_of_toolchain(self):
toolchain_spec = {'name': 'goolf', 'version': '1.4.10'}
list_of_deps = get_dep_tree_of_toolchain(toolchain_spec, self.modtool)
expected_deps = [
+ ['GCC', '4.7.2'],
['OpenBLAS', '0.2.6'],
['hwloc', '1.6.2'],
['OpenMPI', '1.6.4'],