diff --git a/.travis.yml b/.travis.yml index 44e97f45a0..2313f2a366 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ env: - LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - LMOD_VERSION=6.6.3 - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - - LMOD_VERSION=7.7.16 - - LMOD_VERSION=7.7.16 TEST_EASYBUILD_MODULE_SYNTAX=Tcl + - LMOD_VERSION=7.8.5 + - LMOD_VERSION=7.8.5 TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl - ENV_MOD_VERSION=4.0.0 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index fcf5baa035..46d516cb6e 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -46,7 +46,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file -from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, modules_tool +from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool from easybuild.tools.utilities import quote_str @@ -106,7 +106,7 @@ def dependencies_for(mod_name, modtool, depth=sys.maxint): # add dependencies of dependency modules only if they're not there yet for moddepdeps in moddeps: for dep in moddepdeps: - if not dep in mods: + if dep not in mods: mods.append(dep) return mods @@ -123,6 +123,7 @@ class ModuleGenerator(object): MODULE_FILE_EXTENSION = None MODULE_SHEBANG = None + DOT_MODULERC = '.modulerc' # a single level of indentation INDENTATION = ' ' * 4 @@ -133,6 +134,8 @@ def __init__(self, application, fake=False): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.fake_mod_path = tempfile.mkdtemp() + self.modules_tool = modules_tool() + def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): """ Generate append-path statements for the given list of paths. @@ -198,51 +201,65 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): """ return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) - def modulerc(self, module_version=None): + def _modulerc_check_module_version(self, module_version): """ - Generate contents of .modulerc file, in Tcl syntax (compatible with all module tools, incl. Lmod) + Check value type & contents of specified module-version spec. :param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys) + :return: True if spec is OK """ - modulerc = [ModuleGeneratorTcl.MODULE_SHEBANG] - + res = False if module_version: if isinstance(module_version, dict): expected_keys = ['modname', 'sym_version', 'version'] if sorted(module_version.keys()) == expected_keys: - - module_version_statement = "module-version %(modname)s %(sym_version)s" - - # for Environment Modules we need to guard the module-version statement, - # to avoid "Duplicate version symbol" warning messages where EasyBuild trips over, - # which occur because the .modulerc is parsed twice - # "module-info version " returns its argument if that argument is not a symbolic version (yet), - # and returns the corresponding real version in case the argument is an existing symbolic version - # cfr. https://sourceforge.net/p/modules/mailman/message/33399425/ - if modules_tool().__class__ == EnvironmentModulesC: - - modname, sym_version, version = [module_version[key] for key in expected_keys] - - # determine module name with symbolic version - if version in modname: - # take a copy so we don't modify original value - module_version = copy.copy(module_version) - module_version['sym_modname'] = modname.replace(version, sym_version) - else: - raise EasyBuildError("Version '%s' does not appear in module name '%s'", version, modname) - - module_version_statement = '\n'.join([ - 'if {"%(sym_modname)s" eq [module-info version %(sym_modname)s]} {', - ' ' * 4 + module_version_statement, - "}", - ]) - - modulerc.append(module_version_statement % module_version) + res = True else: raise EasyBuildError("Incorrect module_version spec, expected keys: %s", expected_keys) else: raise EasyBuildError("Incorrect module_version value type: %s", type(module_version)) + return res + + def modulerc(self, module_version=None): + """ + Generate contents of .modulerc file, in Tcl syntax (compatible with all module tools, incl. Lmod) + + :param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys) + """ + self.log.info("Generating .modulerc contents in Tcl syntax (args: module_version: %s", module_version) + modulerc = [ModuleGeneratorTcl.MODULE_SHEBANG] + + if self._modulerc_check_module_version(module_version): + + module_version_statement = "module-version %(modname)s %(sym_version)s" + + # for Environment Modules we need to guard the module-version statement, + # to avoid "Duplicate version symbol" warning messages where EasyBuild trips over, + # which occur because the .modulerc is parsed twice + # "module-info version " returns its argument if that argument is not a symbolic version (yet), + # and returns the corresponding real version in case the argument is an existing symbolic version + # cfr. https://sourceforge.net/p/modules/mailman/message/33399425/ + if self.modules_tool.__class__ == EnvironmentModulesC: + + modname, sym_version, version = [module_version[key] for key in sorted(module_version.keys())] + + # determine module name with symbolic version + if version in modname: + # take a copy so we don't modify original value + module_version = copy.copy(module_version) + module_version['sym_modname'] = modname.replace(version, sym_version) + else: + raise EasyBuildError("Version '%s' does not appear in module name '%s'", version, modname) + + module_version_statement = '\n'.join([ + 'if {"%(sym_modname)s" eq [module-info version %(sym_modname)s]} {', + ' ' * 4 + module_version_statement, + "}", + ]) + + modulerc.append(module_version_statement % module_version) + return '\n'.join(modulerc) # From this point on just not implemented methods @@ -594,7 +611,7 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload load_template = self.LOAD_TEMPLATE # Lmod 7.6.1+ supports depends-on which does this most nicely: if build_option('mod_depends_on') or depends_on: - if not modules_tool().supports_depends_on: + if not self.modules_tool.supports_depends_on: raise EasyBuildError("depends-on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON body.append(load_template) @@ -614,7 +631,7 @@ def msg_on_load(self, msg): Add a message that should be printed when loading the module. """ # escape any (non-escaped) characters with special meaning by prefixing them with a backslash - msg = re.sub(r'((?= LooseVersion('7.8'): + self.DOT_MODULERC = '.modulerc.lua' + def check_group(self, group, error_msg=None): """ Generate a check of the software group and the current user, and refuse to load the module if the user don't @@ -792,10 +818,10 @@ def check_group(self, group, error_msg=None): :param group: string with the group name :param error_msg: error message to print for users outside that group """ - lmod_version = os.environ.get('LMOD_VERSION', 'NOT_FOUND') + lmod_version = self.modules_tool.version min_lmod_version = '6.0.8' - if lmod_version != 'NOT_FOUND' and LooseVersion(lmod_version) >= LooseVersion(min_lmod_version): + if LooseVersion(lmod_version) >= LooseVersion(min_lmod_version): if error_msg is None: error_msg = "You are not part of '%s' group of users that have access to this software; " % group error_msg += "Please consult with user support how to become a member of this group" @@ -906,7 +932,7 @@ def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload load_template = self.LOAD_TEMPLATE # Lmod 7.6+ supports depends_on which does this most nicely: if build_option('mod_depends_on') or depends_on: - if not modules_tool().supports_depends_on: + if not self.modules_tool.supports_depends_on: raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") load_template = self.LOAD_TEMPLATE_DEPENDS_ON @@ -936,6 +962,32 @@ def msg_on_load(self, msg): stmt = 'io.stderr:write(%s%s%s)' % (self.START_STR, self.check_str(msg), self.END_STR) return '\n' + self.conditional_statement('mode() == "load"', stmt) + def modulerc(self, module_version=None): + """ + Generate contents of .modulerc file, in Lua syntax (but only if Lmod is recent enough, i.e. >= 7.8) + + :param module_version: specs for module-version statement (dict with 'modname', 'sym_version' & 'version' keys) + """ + lmod_ver = self.modules_tool.version + min_ver = '7.8' + + if LooseVersion(lmod_ver) >= LooseVersion(min_ver): + self.log.info("Found Lmod v%s >= v%s, so will generate .modulerc.lua in Lua syntax", lmod_ver, min_ver) + + modulerc = [] + + if self._modulerc_check_module_version(module_version): + module_version_statement = 'module_version("%(modname)s", "%(sym_version)s")' + modulerc.append(module_version_statement % module_version) + + modulerc = '\n'.join(modulerc) + + else: + self.log.info("Lmod v%s < v%s, need to stick to Tcl syntax for .modulerc", lmod_ver, min_ver) + modulerc = super(ModuleGeneratorLua, self).modulerc(module_version=module_version) + + return modulerc + def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True): """ Generate prepend_path or append_path statements for the given list of paths diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 03a3df26c9..a9291c5bfc 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1407,8 +1407,9 @@ def __init__(self, *args, **kwargs): class NoModulesTool(ModulesTool): """Class that mock the module behaviour, used for operation not requiring modules. Eg. tests, fetch only""" + def __init__(self, *args, **kwargs): - pass + self.version = None def exist(self, mod_names, *args, **kwargs): """No modules, so nothing exists""" diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 30fbb9e8e2..f138e164e1 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1452,7 +1452,8 @@ def test_stale_module_caches(self): module_version_spec = {'modname': 'one/1.0.2', 'sym_version': '1.0', 'version': '1.0.2'} modulerc_txt = modgen.modulerc(module_version=module_version_spec) one_moddir = os.path.join(self.test_installpath, 'modules', 'all', 'one') - write_file(os.path.join(one_moddir, '.modulerc'), modulerc_txt) + + write_file(os.path.join(one_moddir, modgen.DOT_MODULERC), modulerc_txt) # check again, this just grabs the cached results for 'avail one/1.0' & 'show one/1.0' self.assertFalse(self.modtool.exist(['one/1.0'])[0]) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 287f766eea..5eb62bc2db 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -32,6 +32,7 @@ import os import sys import tempfile +from distutils.version import LooseVersion from unittest import TextTestRunner, TestSuite from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from vsc.utils.missing import nub @@ -325,6 +326,12 @@ def test_modulerc(self): modulerc = self.modgen.modulerc({'modname': 'test/1.2.3.4.5', 'sym_version': '1.2.3', 'version': '1.2.3.4.5'}) + expected = '\n'.join([ + '#%Module', + "module-version test/1.2.3.4.5 1.2.3", + ]) + + # two exceptions: EnvironmentModulesC, or Lmod 7.8 (or newer) and Lua syntax if self.modtool.__class__ == EnvironmentModulesC: expected = '\n'.join([ '#%Module', @@ -332,16 +339,14 @@ def test_modulerc(self): ' module-version test/1.2.3.4.5 1.2.3', '}', ]) - else: - expected = '\n'.join([ - '#%Module', - "module-version test/1.2.3.4.5 1.2.3", - ]) + elif self.MODULE_GENERATOR_CLASS == ModuleGeneratorLua: + if isinstance(self.modtool, Lmod) and LooseVersion(self.modtool.version) >= LooseVersion('7.8'): + expected = 'module_version("test/1.2.3.4.5", "1.2.3")' self.assertEqual(modulerc, expected) write_file(os.path.join(self.test_prefix, 'test', '1.2.3.4.5'), '#%Module') - write_file(os.path.join(self.test_prefix, 'test', '.modulerc'), modulerc) + write_file(os.path.join(self.test_prefix, 'test', self.modgen.DOT_MODULERC), modulerc) self.modtool.use(self.test_prefix)