support depends_on "load" statements in generated modules via --module-depends-on and module_depends_on easyconfig parameter

easybuild/framework/easyblock.py

@@ -937,6 +937,8 @@ def make_devel_module(self, create_in_builddir=False):
         # capture all the EBDEVEL vars
         # these should be all the dependencies and we should load them
         recursive_unload = self.cfg['recursive_module_unload']
+        if self.cfg['recursive_module_unload_depends_on']:
+            recursive_unload = 'depends_on'
         for key in os.environ:
             # legacy support
             if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX):
@@ -1042,12 +1044,15 @@ def make_module_dep(self, unload_info=None):
         self.log.debug("List of retained deps to load in generated module: %s", deps)
 
         # include load statements for retained dependencies
+        recursive_unload = self.cfg['recursive_module_unload']
+        if self.cfg['recursive_module_unload_depends_on']:
+            recursive_unload = 'depends_on'
         loads = []
         for dep in deps:
             unload_modules = []
             if dep in unload_info:
                 unload_modules.append(unload_info[dep])
-            loads.append(self.module_generator.load_module(dep, recursive_unload=self.cfg['recursive_module_unload'],
+            loads.append(self.module_generator.load_module(dep, recursive_unload=recursive_unload,
                                                            unload_modules=unload_modules))
 
         # Force unloading any other modules

easybuild/framework/easyconfig/default.py

@@ -172,6 +172,7 @@
     'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES],
     'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
     'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES],
+    'recursive_module_unload_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module', MODULES],
 
     # MODULES documentation easyconfig parameters
     #    (docurls is part of MANDATORY)

easybuild/tools/config.py

@@ -236,6 +236,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     False: [
         'dry_run',
         'recursive_mod_unload',
+        'recursive_mod_unload_depends_on',
         'retain_all_deps',
         'silent',
         'try_to_generate',
@@ -386,6 +387,7 @@ def init_build_options(build_options=None, cmdline_options=None):
             'check_osdeps': not cmdline_options.ignore_osdeps,
             'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short,
             'recursive_mod_unload': cmdline_options.recursive_module_unload,
+            'recursive_mod_unload_depends_on': cmdline_options.recursive_module_unload_depends_on,
             'retain_all_deps': retain_all_deps,
             'validate': not cmdline_options.force,
             'valid_module_classes': module_classes(),

easybuild/tools/module_generator.py

@@ -403,8 +403,9 @@ class ModuleGeneratorTcl(ModuleGenerator):
     MODULE_SHEBANG = '#%Module'
     CHARS_TO_ESCAPE = ['$']
 
-    LOAD_REGEX = r"^\s*module\s+load\s+(\S+)"
+    LOAD_REGEX = r"^\s*module\s+(?:load|depends-on)\s+(\S+)"
     LOAD_TEMPLATE = "module load %(mod_name)s"
+    LOAD_TEMPLATE_DEPENDS_ON = "depends-on %(mod_name)s"
 
     def check_group(self, group, error_msg=None):
         """
@@ -505,9 +506,15 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None):
         body = []
         if unload_modules:
             body.extend([self.unload_module(m).strip() for m in unload_modules])
-        body.append(self.LOAD_TEMPLATE)
-
-        if build_option('recursive_mod_unload') or recursive_unload:
+        load_template = self.LOAD_TEMPLATE
+        # Lmod 7.6.1+ supports depends-on which does this most nicely:
+        if (build_option('recursive_mod_unload_depends_on') or
+            recursive_unload == 'depends_on') and modules_tool().has_depends_on:
+            load_template = self.LOAD_TEMPLATE_DEPENDS_ON
+        body.append(load_template)
+
+        if (build_option('recursive_mod_unload') or recursive_unload or
+            load_template == self.LOAD_TEMPLATE_DEPENDS_ON):
             # not wrapping the 'module load' with an is-loaded guard ensures recursive unloading;
             # when "module unload" is called on the module in which the dependency "module load" is present,
             # it will get translated to "module unload"
@@ -668,8 +675,9 @@ class ModuleGeneratorLua(ModuleGenerator):
     MODULE_SHEBANG = ''  # no 'shebang' in Lua module files
     CHARS_TO_ESCAPE = []
 
-    LOAD_REGEX = r'^\s*load\("(\S+)"'
+    LOAD_REGEX = r'^\s*(?:load|depends_on)\("(\S+)"'
     LOAD_TEMPLATE = 'load("%(mod_name)s")'
+    LOAD_TEMPLATE_DEPENDS_ON = 'depends_on("%(mod_name)s")'
 
     PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")'
     UPDATE_PATH_TEMPLATE = '%s_path("%s", %s)'
@@ -795,19 +803,28 @@ def load_module(self, mod_name, recursive_unload=False, unload_modules=None):
         body = []
         if unload_modules:
             body.extend([self.unload_module(m).strip() for m in unload_modules])
-        body.append(self.LOAD_TEMPLATE)
-
-        if build_option('recursive_mod_unload') or recursive_unload:
-            # wrapping the 'module load' with an 'is-loaded or mode == unload'
-            # guard ensures recursive unloading while avoiding load storms,
-            # when "module unload" is called on the module in which the
-            # depedency "module load" is present, it will get translated
-            # to "module unload"
-            # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html
-            load_guard = 'isloaded("%(mod_name)s") or mode() == "unload"'
+
+        load_template = self.LOAD_TEMPLATE
+        # Lmod 7.6+ supports depends_on which does this most nicely:
+        if (build_option('recursive_mod_unload_depends_on') or
+            recursive_unload == 'depends_on') and modules_tool().has_depends_on:
+            load_template = self.LOAD_TEMPLATE_DEPENDS_ON
+
+        body.append(load_template)
+        if load_template == self.LOAD_TEMPLATE_DEPENDS_ON:
+            load_statement = body + ['']
         else:
-            load_guard = 'isloaded("%(mod_name)s")'
-        load_statement = [self.conditional_statement(load_guard, '\n'.join(body), negative=True)]
+            if build_option('recursive_mod_unload') or recursive_unload:
+                # wrapping the 'module load' with an 'is-loaded or mode == unload'
+                # guard ensures recursive unloading while avoiding load storms,
+                # when "module unload" is called on the module in which the
+                # depedency "module load" is present, it will get translated
+                # to "module unload"
+                # see also http://lmod.readthedocs.io/en/latest/210_load_storms.html
+                load_guard = 'isloaded("%(mod_name)s") or mode() == "unload"'
+            else:
+                load_guard = 'isloaded("%(mod_name)s")'
+            load_statement = [self.conditional_statement(load_guard, '\n'.join(body), negative=True)]
 
         return '\n'.join([''] + load_statement) % {'mod_name': mod_name}
 

easybuild/tools/modules.py

@@ -209,8 +209,11 @@ def modules(self):
 
     def set_and_check_version(self):
         """Get the module version, and check any requirements"""
+        lmod_depends_on_min = '7.6.1'
         if self.COMMAND in MODULE_VERSION_CACHE:
             self.version = MODULE_VERSION_CACHE[self.COMMAND]
+            self.has_depends_on = (isinstance(self, Lmod) and
+                                   StrictVersion(self.version) >= StrictVersion(lmod_depends_on_min))
             self.log.debug("Found cached version for %s: %s", self.COMMAND, self.version)
             return
 
@@ -259,6 +262,8 @@ def set_and_check_version(self):
                 self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION)
 
         MODULE_VERSION_CACHE[self.COMMAND] = self.version
+        self.has_depends_on = (isinstance(self, Lmod) and
+                               StrictVersion(self.version) >= StrictVersion(lmod_depends_on_min))
 
     def check_cmd_avail(self):
         """Check whether modules tool command is available."""

easybuild/tools/options.py

@@ -474,6 +474,8 @@ def config_options(self):
                        None, 'store', None),
             'recursive-module-unload': ("Enable generating of modules that unload recursively.",
                                         None, 'store_true', False),
+            'recursive-module-unload-depends-on': ("Use depends_on (Lmod 7.6.1+) for dependencies in all generated modules.",
+                                        None, 'store_true', False),
             'repository': ("Repository type, using repositorypath",
                            'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())),
             'repositorypath': (("Repository path, used by repository "

test/framework/module_generator.py

@@ -258,6 +258,17 @@ def test_load(self):
 
             init_config(build_options={'recursive_mod_unload': True})
             self.assertEqual(expected, self.modgen.load_module("mod_name"))
+
+            # Lmod 7.6+ depends-on
+            if self.modtool.has_depends_on:
+                expected = '\n'.join([
+                    '',
+                    "depends-on mod_name",
+                    '',
+                ])
+                self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload="depends_on"))
+                init_config(build_options={'recursive_mod_unload_depends_on': 'True'})
+                self.assertEqual(expected, self.modgen.load_module("mod_name"))
         else:
             # default: guarded module load (which implies no recursive unloading)
             expected = '\n'.join([
@@ -283,6 +294,17 @@ def test_load(self):
             init_config(build_options={'recursive_mod_unload': True})
             self.assertEqual(expected, self.modgen.load_module("mod_name"))
 
+            # Lmod 7.6+ depends_on
+            if self.modtool.has_depends_on:
+                expected = '\n'.join([
+                    '',
+                    'depends_on("mod_name")',
+                    '',
+                ])
+                self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload="depends_on"))
+                init_config(build_options={'recursive_mod_unload_depends_on': 'True'})
+                self.assertEqual(expected, self.modgen.load_module("mod_name"))
+
     def test_unload(self):
         """Test unload part in generated module file."""
 

test/framework/options.py

@@ -1156,23 +1156,27 @@ def test_recursive_module_unload(self):
         eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')
 
         # check log message with --skip for existing module
-        args = [
-            eb_file,
-            '--sourcepath=%s' % self.test_sourcepath,
-            '--buildpath=%s' % self.test_buildpath,
-            '--installpath=%s' % self.test_installpath,
-            '--debug',
-            '--force',
-            '--recursive-module-unload',
-        ]
-        self.eb_main(args, do_build=True, verbose=True)
-
-        toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps')
-        if get_module_syntax() == 'Lua':
-            toy_module += '.lua'
-        toy_module_txt = read_file(toy_module)
-        is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M)
-        self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt)
+        lastargs = ['--recursive-module-unload']
+        if self.modtool.has_depends_on:
+            lastargs.append(lastargs[0]+'-depends-on')
+        for lastarg in lastargs:
+            args = [
+                eb_file,
+                '--sourcepath=%s' % self.test_sourcepath,
+                '--buildpath=%s' % self.test_buildpath,
+                '--installpath=%s' % self.test_installpath,
+                '--debug',
+                '--force',
+                lastarg,
+            ]
+            self.eb_main(args, do_build=True, verbose=True)
+
+            toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps')
+            if get_module_syntax() == 'Lua':
+                toy_module += '.lua'
+            toy_module_txt = read_file(toy_module)
+            is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M)
+            self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt)
 
     def test_tmpdir(self):
         """Test setting temporary directory to use by EasyBuild."""