diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a87aa44874..dc1c049c13 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -937,6 +937,7 @@ def make_devel_module(self, create_in_builddir=False): # capture all the EBDEVEL vars # these should be all the dependencies and we should load them recursive_unload = self.cfg['recursive_module_unload'] + depends_on = self.cfg['module_depends_on'] for key in os.environ: # legacy support if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX): @@ -944,7 +945,8 @@ def make_devel_module(self, create_in_builddir=False): path = os.environ[key] if os.path.isfile(path): mod_name = path.rsplit(os.path.sep, 1)[-1] - load_statement = self.module_generator.load_module(mod_name, recursive_unload=recursive_unload) + load_statement = self.module_generator.load_module(mod_name, recursive_unload=recursive_unload, + depends_on=depends_on) load_lines.append(load_statement) elif key.startswith('SOFTDEVEL'): self.log.nosupport("Environment variable SOFTDEVEL* being relied on", '2.0') @@ -1042,12 +1044,15 @@ def make_module_dep(self, unload_info=None): self.log.debug("List of retained deps to load in generated module: %s", deps) # include load statements for retained dependencies + recursive_unload = self.cfg['recursive_module_unload'] + depends_on = self.cfg['module_depends_on'] loads = [] for dep in deps: unload_modules = [] if dep in unload_info: unload_modules.append(unload_info[dep]) - loads.append(self.module_generator.load_module(dep, recursive_unload=self.cfg['recursive_module_unload'], + loads.append(self.module_generator.load_module(dep, recursive_unload=recursive_unload, + depends_on=depends_on, unload_modules=unload_modules)) # Force unloading any other modules diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index d45c3c81fc..7f68136361 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -171,6 +171,8 @@ 'moduleclass': ['base', 'Module class to be used for this software', MODULES], 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], + 'module_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' + '(implies recursive unloading of modules).', MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], # MODULES documentation easyconfig parameters diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f73da0351c..28b471a64f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -236,6 +236,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): False: [ 'dry_run', 'recursive_mod_unload', + 'mod_depends_on', 'retain_all_deps', 'silent', 'try_to_generate', @@ -386,6 +387,7 @@ def init_build_options(build_options=None, cmdline_options=None): 'check_osdeps': not cmdline_options.ignore_osdeps, 'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short, 'recursive_mod_unload': cmdline_options.recursive_module_unload, + 'mod_depends_on': cmdline_options.module_depends_on, 'retain_all_deps': retain_all_deps, 'validate': not cmdline_options.force, 'valid_module_classes': module_classes(), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e4715a4b51..375fe7e160 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -403,8 +403,9 @@ class ModuleGeneratorTcl(ModuleGenerator): MODULE_SHEBANG = '#%Module' CHARS_TO_ESCAPE = ['$'] - LOAD_REGEX = r"^\s*module\s+load\s+(\S+)" + LOAD_REGEX = r"^\s*module\s+(?:load|depends-on)\s+(\S+)" LOAD_TEMPLATE = "module load %(mod_name)s" + LOAD_TEMPLATE_DEPENDS_ON = "depends-on %(mod_name)s" def check_group(self, group, error_msg=None): """ @@ -494,7 +495,7 @@ def getenv_cmd(self, envvar): """ return '$env(%s)' % envvar - def load_module(self, mod_name, recursive_unload=False, unload_modules=None): + def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None): """ Generate load statement for specified module. @@ -505,9 +506,15 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): body = [] if unload_modules: body.extend([self.unload_module(m).strip() for m in unload_modules]) - body.append(self.LOAD_TEMPLATE) - - if build_option('recursive_mod_unload') or recursive_unload: + load_template = self.LOAD_TEMPLATE + # Lmod 7.6.1+ supports depends-on which does this most nicely: + if build_option('mod_depends_on') or depends_on: + if not modules_tool().supports_depends_on: + raise EasyBuildError("depends-on statements in generated module are not supported by modules tool") + load_template = self.LOAD_TEMPLATE_DEPENDS_ON + body.append(load_template) + + if build_option('recursive_mod_unload') or recursive_unload or load_template == self.LOAD_TEMPLATE_DEPENDS_ON: # not wrapping the 'module load' with an is-loaded guard ensures recursive unloading; # when "module unload" is called on the module in which the dependency "module load" is present, # it will get translated to "module unload" @@ -668,8 +675,9 @@ class ModuleGeneratorLua(ModuleGenerator): MODULE_SHEBANG = '' # no 'shebang' in Lua module files CHARS_TO_ESCAPE = [] - LOAD_REGEX = r'^\s*load\("(\S+)"' + LOAD_REGEX = r'^\s*(?:load|depends_on)\("(\S+)"' LOAD_TEMPLATE = 'load("%(mod_name)s")' + LOAD_TEMPLATE_DEPENDS_ON = 'depends_on("%(mod_name)s")' PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")' UPDATE_PATH_TEMPLATE = '%s_path("%s", %s)' @@ -784,7 +792,7 @@ def getenv_cmd(self, envvar): """ return 'os.getenv("%s")' % envvar - def load_module(self, mod_name, recursive_unload=False, unload_modules=None): + def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None): """ Generate load statement for specified module. @@ -795,19 +803,29 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None): body = [] if unload_modules: body.extend([self.unload_module(m).strip() for m in unload_modules]) - body.append(self.LOAD_TEMPLATE) - - if build_option('recursive_mod_unload') or recursive_unload: - # wrapping the 'module load' with an 'is-loaded or mode == unload' - # guard ensures recursive unloading while avoiding load storms, - # when "module unload" is called on the module in which the - # depedency "module load" is present, it will get translated - # to "module unload" - # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html - load_guard = 'isloaded("%(mod_name)s") or mode() == "unload"' + + load_template = self.LOAD_TEMPLATE + # Lmod 7.6+ supports depends_on which does this most nicely: + if build_option('mod_depends_on') or depends_on: + if not modules_tool().supports_depends_on: + raise EasyBuildError("depends_on statements in generated module are not supported by modules tool") + load_template = self.LOAD_TEMPLATE_DEPENDS_ON + + body.append(load_template) + if load_template == self.LOAD_TEMPLATE_DEPENDS_ON: + load_statement = body + [''] else: - load_guard = 'isloaded("%(mod_name)s")' - load_statement = [self.conditional_statement(load_guard, '\n'.join(body), negative=True)] + if build_option('recursive_mod_unload') or recursive_unload: + # wrapping the 'module load' with an 'is-loaded or mode == unload' + # guard ensures recursive unloading while avoiding load storms, + # when "module unload" is called on the module in which the + # depedency "module load" is present, it will get translated + # to "module unload" + # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html + load_guard = 'isloaded("%(mod_name)s") or mode() == "unload"' + else: + load_guard = 'isloaded("%(mod_name)s")' + load_statement = [self.conditional_statement(load_guard, '\n'.join(body), negative=True)] return '\n'.join([''] + load_statement) % {'mod_name': mod_name} diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 62ae8db77c..6ee00cfbf3 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -197,6 +197,7 @@ def __init__(self, mod_paths=None, testing=False): self.check_module_path() self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch')) self.set_and_check_version() + self.supports_depends_on = False def buildstats(self): """Return tuple with data to be included in buildstats""" @@ -1101,6 +1102,7 @@ class Lmod(ModulesTool): COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' REQ_VERSION = '5.8' + REQ_VERSION_DEPENDS_ON = '7.6.1' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') @@ -1116,6 +1118,7 @@ def __init__(self, *args, **kwargs): setvar('LMOD_REDIRECT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) + self.supports_depends_on = StrictVersion(self.version) >= StrictVersion(self.REQ_VERSION_DEPENDS_ON) def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 85d0620314..b696ae5b44 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -453,6 +453,9 @@ def config_options(self): # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}), + 'module-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules " + "(implies recursive unloading of modules).", + None, 'store_true', False), 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index d9b9bfe0d5..1220946d62 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -258,6 +258,22 @@ def test_load(self): init_config(build_options={'recursive_mod_unload': True}) self.assertEqual(expected, self.modgen.load_module("mod_name")) + + # Lmod 7.6+ depends-on + if self.modtool.supports_depends_on: + expected = '\n'.join([ + '', + "depends-on mod_name", + '', + ]) + self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True)) + init_config(build_options={'mod_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name")) + else: + expected = "depends-on statements in generated module are not supported by modules tool" + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True) + init_config(build_options={'mod_depends_on': 'True'}) + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name") else: # default: guarded module load (which implies no recursive unloading) expected = '\n'.join([ @@ -283,6 +299,22 @@ def test_load(self): init_config(build_options={'recursive_mod_unload': True}) self.assertEqual(expected, self.modgen.load_module("mod_name")) + # Lmod 7.6+ depends_on + if self.modtool.supports_depends_on: + expected = '\n'.join([ + '', + 'depends_on("mod_name")', + '', + ]) + self.assertEqual(expected, self.modgen.load_module("mod_name", depends_on=True)) + init_config(build_options={'mod_depends_on': 'True'}) + self.assertEqual(expected, self.modgen.load_module("mod_name")) + else: + expected = "depends_on statements in generated module are not supported by modules tool" + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name", depends_on=True) + init_config(build_options={'mod_depends_on': 'True'}) + self.assertErrorRegex(EasyBuildError, expected, self.modgen.load_module, "mod_name") + def test_unload(self): """Test unload part in generated module file.""" diff --git a/test/framework/options.py b/test/framework/options.py index 4a6b6e9b92..6847aff8b7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1156,23 +1156,27 @@ def test_recursive_module_unload(self): eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb') # check log message with --skip for existing module - args = [ - eb_file, - '--sourcepath=%s' % self.test_sourcepath, - '--buildpath=%s' % self.test_buildpath, - '--installpath=%s' % self.test_installpath, - '--debug', - '--force', - '--recursive-module-unload', - ] - self.eb_main(args, do_build=True, verbose=True) - - toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') - if get_module_syntax() == 'Lua': - toy_module += '.lua' - toy_module_txt = read_file(toy_module) - is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) - self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) + lastargs = ['--recursive-module-unload'] + if self.modtool.supports_depends_on: + lastargs.append('--module-depends-on') + for lastarg in lastargs: + args = [ + eb_file, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + '--force', + lastarg, + ] + self.eb_main(args, do_build=True, verbose=True) + + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + if get_module_syntax() == 'Lua': + toy_module += '.lua' + toy_module_txt = read_file(toy_module) + is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) + self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) def test_tmpdir(self): """Test setting temporary directory to use by EasyBuild."""