From 2c65cee0fce0ad3ec998d0d3ccb48889e840b8b9 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Sat, 21 Oct 2017 13:59:24 +0200 Subject: [PATCH 01/26] Add a architecture env variable to the user subdir module path. On systems with multiple architectures sharing a single home directory for the users there need to be a way to install architecture specific builds when using the subdir-user-modules extension. --- easybuild/framework/easyblock.py | 7 ++++--- easybuild/tools/config.py | 1 + easybuild/tools/module_generator.py | 24 +++++++++++++++++++++--- easybuild/tools/options.py | 1 + 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 806061fe84..79315fe6b9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1179,12 +1179,13 @@ def make_module_extend_modpath(self): if user_modpath: # If a mod_path_suffix is being used, we should respect it mod_path_suffix = build_option('suffix_modules_path') - user_modpath = os.path.join(user_modpath, mod_path_suffix) + # Use arch env if required + arch_env_name = build_option('subdir_arch_env') user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) - user_modpath_exts = [os.path.join(user_modpath, e) for e in user_modpath_exts] self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True) + guarded=True, user_modpath=user_modpath, mod_path_suffix=mod_path_suffix, + arch_env_name=arch_env_name) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f73da0351c..356fdec512 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -147,6 +147,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'skip', 'stop', 'subdir_user_modules', + 'subdir_arch_env', 'test_report_env_filter', 'testoutput', 'umask', diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e4715a4b51..2be1950a6d 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -307,12 +307,15 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath """ raise NotImplementedError - def use(self, paths, prefix=None, guarded=False): + def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None, arch_env_name=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists + :param user_modpath: optional user subdir + :param mod_path_suffix: optional path suffix + :param arch_env_name: optional name of environment variable to use """ raise NotImplementedError @@ -636,12 +639,15 @@ def unload_module(self, mod_name): """ return '\n'.join(['', "module unload %s" % mod_name]) - def use(self, paths, prefix=None, guarded=False): + def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None, arch_env_name=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists + :param user_modpath: optional user subdir + :param mod_path_suffix: optional path suffix + :param arch_env_name: optional name of environment variable to use """ use_statements = [] for path in paths: @@ -936,20 +942,32 @@ def unload_module(self, mod_name): """ return '\n'.join(['', 'unload("%s")' % mod_name]) - def use(self, paths, prefix=None, guarded=False): + def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None, arch_env_name=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists + :param user_modpath: optional user subdir path + :param arch_env_name: optional name of environment variable to use, only used with user_modpath + :param mod_path_suffix: optional path suffix, only used with user_modpath """ use_statements = [] + if user_modpath: + user_modpath = quote_str(user_modpath) + if arch_env_name: + user_modpath = 'pathJoin(%s, os.getenv("%s"))' % (user_modpath, arch_env_name) + if mod_path_suffix: + user_modpath = 'pathJoin(%s, %s)' % (user_modpath, quote_str(mod_path_suffix)) for path in paths: quoted_path = quote_str(path) + if user_modpath: + quoted_path = 'pathJoin(%s, %s)' % (user_modpath, quoted_path) if prefix: full_path = 'pathJoin(%s, %s)' % (prefix, quoted_path) else: full_path = quoted_path + prepend_modulepath = self.UPDATE_PATH_TEMPLATE % ('prepend', 'MODULEPATH', full_path) if guarded: cond_statement = self.conditional_statement('isDir(%s)' % full_path, prepend_modulepath) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 85d0620314..a7c8257b12 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -486,6 +486,7 @@ def config_options(self): 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None), + 'subdir-arch-env': ("Name of environment variable to add to user-specific module path", None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage From 0a6e4fea65470f6b23f933a96e16d7827ab95b4b Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 7 Feb 2018 08:35:39 +0100 Subject: [PATCH 02/26] Use {RUNTIME_ENV::SOME_ENV_VAR} instead of subdir_arch_env. Change from using a specific subdir_arch_env to expanding "{RUNTIME_ENV::SOME_ENV_VAR}" into module runtime equivalences of os.getenv("SOME_ENV_VAR") --- easybuild/framework/easyblock.py | 5 +---- easybuild/tools/config.py | 1 - easybuild/tools/module_generator.py | 33 +++++++++++++++++++++-------- easybuild/tools/options.py | 1 - 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 79315fe6b9..eaf350734e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1179,13 +1179,10 @@ def make_module_extend_modpath(self): if user_modpath: # If a mod_path_suffix is being used, we should respect it mod_path_suffix = build_option('suffix_modules_path') - # Use arch env if required - arch_env_name = build_option('subdir_arch_env') user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True, user_modpath=user_modpath, mod_path_suffix=mod_path_suffix, - arch_env_name=arch_env_name) + guarded=True, user_modpath=user_modpath, mod_path_suffix=mod_path_suffix) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 356fdec512..f73da0351c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -147,7 +147,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'skip', 'stop', 'subdir_user_modules', - 'subdir_arch_env', 'test_report_env_filter', 'testoutput', 'umask', diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 2be1950a6d..69d332f45f 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -307,7 +307,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath """ raise NotImplementedError - def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None, arch_env_name=None): + def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted @@ -315,7 +315,6 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suf :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: optional user subdir :param mod_path_suffix: optional path suffix - :param arch_env_name: optional name of environment variable to use """ raise NotImplementedError @@ -639,7 +638,7 @@ def unload_module(self, mod_name): """ return '\n'.join(['', "module unload %s" % mod_name]) - def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None, arch_env_name=None): + def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted @@ -647,7 +646,6 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suf :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: optional user subdir :param mod_path_suffix: optional path suffix - :param arch_env_name: optional name of environment variable to use """ use_statements = [] for path in paths: @@ -942,21 +940,38 @@ def unload_module(self, mod_name): """ return '\n'.join(['', 'unload("%s")' % mod_name]) - def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None, arch_env_name=None): + def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: optional user subdir path - :param arch_env_name: optional name of environment variable to use, only used with user_modpath :param mod_path_suffix: optional path suffix, only used with user_modpath """ use_statements = [] if user_modpath: - user_modpath = quote_str(user_modpath) - if arch_env_name: - user_modpath = 'pathJoin(%s, os.getenv("%s"))' % (user_modpath, arch_env_name) + # Check for occurenses of {RUNTIME_ENV::SOME_ENV_VAR} + # SOME_ENV_VAR will be expanded at module load time. + runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') + sub_paths = [] + expanded_user_modpath = '' + for sub_path in re.split(os.path.sep, user_modpath): + matched_re = runtime_env_re.match(sub_path) + if matched_re: + path = os.path.join(*sub_paths) + sub_paths = [] + if expanded_user_modpath: + expanded_user_modpath = 'pathJoin(%s, %s, os.getenv(%s))' % (expanded_user_modpath, quote_str(path), quote_str(matched_re.group(1))) + else: + expanded_user_modpath = 'pathJoin(%s, os.getenv(%s))' % (quote_str(path), quote_str(matched_re.group(1))) + else: + sub_paths.append(sub_path) + if sub_paths: + path = os.path.join(*sub_paths) + expanded_user_modpath = 'pathJoin(%s, %s)' % (expanded_user_modpath, quote_str(path)) + + user_modpath = expanded_user_modpath if mod_path_suffix: user_modpath = 'pathJoin(%s, %s)' % (user_modpath, quote_str(mod_path_suffix)) for path in paths: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a7c8257b12..85d0620314 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -486,7 +486,6 @@ def config_options(self): 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None), - 'subdir-arch-env': ("Name of environment variable to add to user-specific module path", None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage From 07e6b3f0068ae8adcd2b09a36bfdf083ed4d1742 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 7 Feb 2018 12:53:21 +0100 Subject: [PATCH 03/26] Simplify the code for handling {RUNTIME_ENV::SOME_ENV_VAR} in user_modpath. --- easybuild/tools/module_generator.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 69d332f45f..b386232986 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -955,25 +955,22 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suf # SOME_ENV_VAR will be expanded at module load time. runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') sub_paths = [] - expanded_user_modpath = '' + expanded_user_modpath = [] for sub_path in re.split(os.path.sep, user_modpath): matched_re = runtime_env_re.match(sub_path) if matched_re: - path = os.path.join(*sub_paths) - sub_paths = [] - if expanded_user_modpath: - expanded_user_modpath = 'pathJoin(%s, %s, os.getenv(%s))' % (expanded_user_modpath, quote_str(path), quote_str(matched_re.group(1))) - else: - expanded_user_modpath = 'pathJoin(%s, os.getenv(%s))' % (quote_str(path), quote_str(matched_re.group(1))) + if sub_paths: + path = quote_str(os.path.join(*sub_paths)) + expanded_user_modpath.extend([path]) + sub_paths = [] + expanded_user_modpath.extend(['os.getenv(%s)' % quote_str(matched_re.group(1))]) else: sub_paths.append(sub_path) if sub_paths: - path = os.path.join(*sub_paths) - expanded_user_modpath = 'pathJoin(%s, %s)' % (expanded_user_modpath, quote_str(path)) - - user_modpath = expanded_user_modpath + expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) if mod_path_suffix: - user_modpath = 'pathJoin(%s, %s)' % (user_modpath, quote_str(mod_path_suffix)) + expanded_user_modpath.extend([quote_str(mod_path_suffix)]) + user_modpath = ', '.join(expanded_user_modpath) for path in paths: quoted_path = quote_str(path) if user_modpath: From 7b92aafe933a4490c819c6191013bf385376de71 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Tue, 3 Jul 2018 21:11:33 +0200 Subject: [PATCH 04/26] Change tabs to spaces. --- easybuild/tools/module_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 8ee6f9b5d3..25d125a80f 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -977,11 +977,11 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suf for sub_path in re.split(os.path.sep, user_modpath): matched_re = runtime_env_re.match(sub_path) if matched_re: - if sub_paths: - path = quote_str(os.path.join(*sub_paths)) + if sub_paths: + path = quote_str(os.path.join(*sub_paths)) expanded_user_modpath.extend([path]) - sub_paths = [] - expanded_user_modpath.extend(['os.getenv(%s)' % quote_str(matched_re.group(1))]) + sub_paths = [] + expanded_user_modpath.extend(['os.getenv(%s)' % quote_str(matched_re.group(1))]) else: sub_paths.append(sub_path) if sub_paths: From 4d2369be9683063555e8c7ca463a8dce0f3a3140 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Tue, 3 Jul 2018 21:13:53 +0200 Subject: [PATCH 05/26] Increase indentation for visual indent as per houndci-bot. --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 927504a0f8..b4c720d016 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1191,7 +1191,7 @@ def make_module_extend_modpath(self): user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True, user_modpath=user_modpath, mod_path_suffix=mod_path_suffix) + guarded=True, user_modpath=user_modpath, mod_path_suffix=mod_path_suffix) else: self.log.debug("Not including module path extensions, as specified.") return txt From e95c96596f2626abcdf0afb05386429ac1763e3b Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Tue, 3 Jul 2018 22:57:52 +0200 Subject: [PATCH 06/26] Adjust test for Lua modpath extension to match new code in module_generator.use --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index bda482b05c..5be92d27e7 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -283,9 +283,9 @@ def test_make_module_extend_modpath(self): home = r'os.getenv\("HOME"\)' regexs.extend([ # extension for user modules is guarded - r'if isDir\(pathJoin\(%s, "%s/funky/Compiler/pi/3.14"\)\) then' % (home, usermodsdir), + r'if isDir\(pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/funky/Compiler/pi/3.14"\)\)' % (home, usermodsdir), + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\)' % (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From fb88ba5e2eb78bf8af47c4017102e30c87c2b0ac Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Tue, 3 Jul 2018 23:04:06 +0200 Subject: [PATCH 07/26] Adjust line length as per houndci-bot. --- test/framework/easyblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5be92d27e7..e3c2eb3cd7 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -285,7 +285,8 @@ def test_make_module_extend_modpath(self): # extension for user modules is guarded r'if isDir\(pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\)' % (home, usermodsdir), + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\)' % + (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From a17c9e0f6e5fae85501175c5cf2e08d17b658652 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Tue, 3 Jul 2018 23:08:17 +0200 Subject: [PATCH 08/26] Increase indentation more since houndci is still complaining. --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e3c2eb3cd7..d307b60618 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -286,7 +286,7 @@ def test_make_module_extend_modpath(self): r'if isDir\(pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\)' % - (home, usermodsdir), + (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From 04a16209d13c92bc97913f46d94ef0baaf6955b4 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Tue, 3 Jul 2018 23:09:51 +0200 Subject: [PATCH 09/26] Clumsily got tabs into the code, changed to spaces. --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d307b60618..70f2fac170 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -286,7 +286,7 @@ def test_make_module_extend_modpath(self): r'if isDir\(pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\)' % - (home, usermodsdir), + (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From 6f6789c3eb32dee66b3498cdc64e6b851345c990 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 06:33:24 +0200 Subject: [PATCH 10/26] Forgot the extra pathJoin (in the test regex) that the module generator adds when using subdir-user-modules. --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 70f2fac170..b25c721ae3 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -283,9 +283,9 @@ def test_make_module_extend_modpath(self): home = r'os.getenv\("HOME"\)' regexs.extend([ # extension for user modules is guarded - r'if isDir\(pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\) then' % (home, usermodsdir), + r'if isDir\(pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s", "funky", "Compiler/pi/3.14"\)\)' % + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, pathJoin("%s", "funky", "Compiler/pi/3.14"\)\)\)' % (home, usermodsdir), ]) else: From b6e491b77816efec7ab2e4fcadd9d24fd1c1083a Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 07:50:19 +0200 Subject: [PATCH 11/26] Missing \ protection for ( causing misbalanced () --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index b25c721ae3..806cba86c2 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -285,7 +285,7 @@ def test_make_module_extend_modpath(self): # extension for user modules is guarded r'if isDir\(pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, pathJoin("%s", "funky", "Compiler/pi/3.14"\)\)\)' % + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\)' % (home, usermodsdir), ]) else: From bece4934a171b4129b6b335b004e8af72179f905 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 08:28:58 +0200 Subject: [PATCH 12/26] Add Tcl support for {RUNTIME_ENV::SOME_ENV_VAR} in subdir-user-modules. --- easybuild/tools/module_generator.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 25d125a80f..e4f7d9a4d6 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -655,8 +655,31 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suf :param mod_path_suffix: optional path suffix """ use_statements = [] + if user_modpath: + # Check for occurenses of {RUNTIME_ENV::SOME_ENV_VAR} + # SOME_ENV_VAR will be expanded at module load time. + runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') + sub_paths = [] + expanded_user_modpath = [] + for sub_path in re.split(os.path.sep, user_modpath): + matched_re = runtime_env_re.match(sub_path) + if matched_re: + if sub_paths: + path = quote_str(os.path.join(*sub_paths)) + expanded_user_modpath.extend([path]) + sub_paths = [] + expanded_user_modpath.extend(['$::env(%s)' % quote_str(matched_re.group(1))]) + else: + sub_paths.append(sub_path) + if sub_paths: + expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) + if mod_path_suffix: + expanded_user_modpath.extend([quote_str(mod_path_suffix)]) + user_modpath = ' '.join(expanded_user_modpath) for path in paths: quoted_path = quote_str(path) + if user_modpath: + quoted_path = '[ file join %s %s' ] % (user_modpath, quoted_path) if prefix: full_path = '[ file join %s %s ]' % (prefix, quoted_path) else: From f38846a3fa23bcc2c7628e73a251b1fc5c478401 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 08:30:36 +0200 Subject: [PATCH 13/26] Fix "'" position typo. --- easybuild/tools/module_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e4f7d9a4d6..07fff6966e 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -679,7 +679,7 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suf for path in paths: quoted_path = quote_str(path) if user_modpath: - quoted_path = '[ file join %s %s' ] % (user_modpath, quoted_path) + quoted_path = '[ file join %s %s ]' % (user_modpath, quoted_path) if prefix: full_path = '[ file join %s %s ]' % (prefix, quoted_path) else: From 9b8fd37dcf1b9034bbe27fdcced9e809f62c090b Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 09:03:36 +0200 Subject: [PATCH 14/26] Update test for Tcl style of the "{RUNTIME_ENV::SOME_ENV_VAR} in subdir-user-modules" changes. --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 806cba86c2..a061dc0126 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -274,9 +274,9 @@ def test_make_module_extend_modpath(self): home = r'\$env\(HOME\)' regexs.extend([ # extension for user modules is guarded - r'if { \[ file isdirectory \[ file join %s "%s/funky/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir), + r'if { \[ file isdirectory \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \] \] } {$' % (home, usermodsdir), # no per-moduleclass extension for user modules - r'^\s+module use \[ file join %s "%s/funky/Compiler/pi/3.14"\ ]$' % (home, usermodsdir), + r'^\s+module use \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \]$' % (home, usermodsdir), ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] From 3fdb798af0dbe38b6f71d3f761cb785410256a82 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 09:06:45 +0200 Subject: [PATCH 15/26] Fix long line and increase indentation again since houndci is complaining. --- test/framework/easyblock.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index a061dc0126..2cad8b87ae 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -274,7 +274,8 @@ def test_make_module_extend_modpath(self): home = r'\$env\(HOME\)' regexs.extend([ # extension for user modules is guarded - r'if { \[ file isdirectory \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \] \] } {$' % (home, usermodsdir), + r'if { \[ file isdirectory \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \] \] } {$' % + (home, usermodsdir), # no per-moduleclass extension for user modules r'^\s+module use \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \]$' % (home, usermodsdir), ]) @@ -286,7 +287,7 @@ def test_make_module_extend_modpath(self): r'if isDir\(pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\)' % - (home, usermodsdir), + (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From 4c5579fa47ef168fb8c3600057cca9859a2b6af5 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 4 Jul 2018 09:08:45 +0200 Subject: [PATCH 16/26] One more too long line, and increase indentation alot this time. --- test/framework/easyblock.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 2cad8b87ae..e71e5e6cf4 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -275,9 +275,10 @@ def test_make_module_extend_modpath(self): regexs.extend([ # extension for user modules is guarded r'if { \[ file isdirectory \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \] \] } {$' % - (home, usermodsdir), + (home, usermodsdir), # no per-moduleclass extension for user modules - r'^\s+module use \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \]$' % (home, usermodsdir), + r'^\s+module use \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \]$' % + (home, usermodsdir), ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] @@ -287,7 +288,7 @@ def test_make_module_extend_modpath(self): r'if isDir\(pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\)' % - (home, usermodsdir), + (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From a7202dcde2bc84d0dc5b1e91fbc2a40525e9e2f8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Jul 2018 20:49:01 +0200 Subject: [PATCH 17/26] minor cleanup in test_make_module_extend_modpath --- test/framework/easyblock.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e71e5e6cf4..43e484581d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -272,23 +272,22 @@ def test_make_module_extend_modpath(self): if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] home = r'\$env\(HOME\)' + fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded - r'if { \[ file isdirectory \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \] \] } {$' % - (home, usermodsdir), + r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (home, fj_usermodsdir), # no per-moduleclass extension for user modules - r'^\s+module use \[ file join %s \[ file join "%s" "funky" "Compiler/pi/3.14" \] \]$' % - (home, usermodsdir), + r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] home = r'os.getenv\("HOME"\)' + pj_usermodsdir = 'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded - r'if isDir\(pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\) then' % (home, usermodsdir), + r'if isDir\(pathJoin\(%s, %s\)\) then' % (home, pj_usermodsdir), # no per-moduleclass extension for user modules - r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, pathJoin\("%s", "funky", "Compiler/pi/3.14"\)\)\)' % - (home, usermodsdir), + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) From 018508bf0ab13a8b0cde25fab3066c8b9978188a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Jul 2018 20:50:05 +0200 Subject: [PATCH 18/26] flesh out duplicate code in implementation of use methods in module_generator.py (& leverage existing getenv_cmd method) --- easybuild/framework/easyblock.py | 4 +- easybuild/tools/module_generator.py | 92 +++++++++++++---------------- 2 files changed, 42 insertions(+), 54 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b4c720d016..746339725c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1186,12 +1186,10 @@ def make_module_extend_modpath(self): # add user-specific module path; use statement will be guarded so no need to create the directories user_modpath = build_option('subdir_user_modules') if user_modpath: - # If a mod_path_suffix is being used, we should respect it - mod_path_suffix = build_option('suffix_modules_path') user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True, user_modpath=user_modpath, mod_path_suffix=mod_path_suffix) + guarded=True, user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 07fff6966e..4d513665be 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -307,14 +307,46 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath """ raise NotImplementedError - def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None): + def det_user_modpath(self, user_modpath): + """ + Determine user-specific modules subdirectory, to be used in 'use' statements + (cfr. implementations of use() method). + """ + if user_modpath: + # Check for occurences of {RUNTIME_ENV::SOME_ENV_VAR} + # SOME_ENV_VAR will be expanded at module load time. + runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') + sub_paths = [] + expanded_user_modpath = [] + for sub_path in re.split(os.path.sep, user_modpath): + matched_re = runtime_env_re.match(sub_path) + if matched_re: + if sub_paths: + path = quote_str(os.path.join(*sub_paths)) + expanded_user_modpath.extend([path]) + sub_paths = [] + expanded_user_modpath.extend([self.getenv_cmd(matched_re.group(1))]) + else: + sub_paths.append(sub_path) + if sub_paths: + expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) + + # if a mod_path_suffix is being used, we should respect it + mod_path_suffix = build_option('suffix_modules_path') + if mod_path_suffix: + expanded_user_modpath.extend([quote_str(mod_path_suffix)]) + + user_modpath = ', '.join(expanded_user_modpath) + + return user_modpath + + def use(self, paths, prefix=None, guarded=False, user_modpath=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists - :param user_modpath: optional user subdir - :param mod_path_suffix: optional path suffix + :param user_modpath: user-specific modules subdirectory to include in use statements """ raise NotImplementedError @@ -645,37 +677,16 @@ def unload_module(self, mod_name): """ return '\n'.join(['', "module unload %s" % mod_name]) - def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None): + def use(self, paths, prefix=None, guarded=False, user_modpath=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists - :param user_modpath: optional user subdir - :param mod_path_suffix: optional path suffix + :param user_modpath: user-specific modules subdirectory to include in use statements """ + user_modpath = self.det_user_modpath(user_modpath) use_statements = [] - if user_modpath: - # Check for occurenses of {RUNTIME_ENV::SOME_ENV_VAR} - # SOME_ENV_VAR will be expanded at module load time. - runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') - sub_paths = [] - expanded_user_modpath = [] - for sub_path in re.split(os.path.sep, user_modpath): - matched_re = runtime_env_re.match(sub_path) - if matched_re: - if sub_paths: - path = quote_str(os.path.join(*sub_paths)) - expanded_user_modpath.extend([path]) - sub_paths = [] - expanded_user_modpath.extend(['$::env(%s)' % quote_str(matched_re.group(1))]) - else: - sub_paths.append(sub_path) - if sub_paths: - expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) - if mod_path_suffix: - expanded_user_modpath.extend([quote_str(mod_path_suffix)]) - user_modpath = ' '.join(expanded_user_modpath) for path in paths: quoted_path = quote_str(path) if user_modpath: @@ -981,37 +992,16 @@ def unload_module(self, mod_name): """ return '\n'.join(['', 'unload("%s")' % mod_name]) - def use(self, paths, prefix=None, guarded=False, user_modpath=None, mod_path_suffix=None): + def use(self, paths, prefix=None, guarded=False, user_modpath=None): """ Generate module use statements for given list of module paths. :param paths: list of module path extensions to generate use statements for; paths will be quoted :param prefix: optional path prefix; not quoted, i.e., can be a statement :param guarded: use statements will be guarded to only apply if path exists - :param user_modpath: optional user subdir path - :param mod_path_suffix: optional path suffix, only used with user_modpath + :param user_modpath: user-specific modules subdirectory to include in use statements """ + user_modpath = self.det_user_modpath(user_modpath) use_statements = [] - if user_modpath: - # Check for occurenses of {RUNTIME_ENV::SOME_ENV_VAR} - # SOME_ENV_VAR will be expanded at module load time. - runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') - sub_paths = [] - expanded_user_modpath = [] - for sub_path in re.split(os.path.sep, user_modpath): - matched_re = runtime_env_re.match(sub_path) - if matched_re: - if sub_paths: - path = quote_str(os.path.join(*sub_paths)) - expanded_user_modpath.extend([path]) - sub_paths = [] - expanded_user_modpath.extend(['os.getenv(%s)' % quote_str(matched_re.group(1))]) - else: - sub_paths.append(sub_path) - if sub_paths: - expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) - if mod_path_suffix: - expanded_user_modpath.extend([quote_str(mod_path_suffix)]) - user_modpath = ', '.join(expanded_user_modpath) for path in paths: quoted_path = quote_str(path) if user_modpath: From 98f4498b730d4a9c6f293951b2e8e803b58ba3d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Jul 2018 21:30:08 +0200 Subject: [PATCH 19/26] add test for test_det_user_modpath method --- test/framework/module_generator.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 1220946d62..b63940d9da 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -471,6 +471,22 @@ def test_prepend_paths(self): "which only expects relative paths." % self.modgen.app.installdir, self.modgen.prepend_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir]) + def test_det_user_modpath(self): + """Test for generic det_user_modpath method.""" + # None by default + self.assertEqual(self.modgen.det_user_modpath(None), None) + + self.assertEqual(self.modgen.det_user_modpath('my/own/modules'), '"my/own/modules", "all"') + + # result is affected by --suffix-modules-path + # {RUNTIME_ENV::FOO} gets translated into Tcl/Lua syntax for resolving $FOO at runtime + init_config(build_options={'suffix_modules_path': ''}) + user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules' + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", $env(TEST123), "modules"') + else: + self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"') + def test_use(self): """Test generating module use statements.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: From 73c51805ec3494a93722a379376635707e33477c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Jul 2018 21:37:46 +0200 Subject: [PATCH 20/26] use $::env rather than to access environment variables in Tcl syntax --- easybuild/tools/module_generator.py | 2 +- test/framework/module_generator.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 4d513665be..5b7df1cfd1 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -527,7 +527,7 @@ def getenv_cmd(self, envvar): """ Return module-syntax specific code to get value of specific environment variable. """ - return '$env(%s)' % envvar + return '$::env(%s)' % envvar def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None): """ diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index b63940d9da..54b75c8b82 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -483,7 +483,7 @@ def test_det_user_modpath(self): init_config(build_options={'suffix_modules_path': ''}) user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules' if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", $env(TEST123), "modules"') + self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", $::env(TEST123), "modules"') else: self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"') @@ -534,8 +534,8 @@ def test_env(self): def test_getenv_cmd(self): """Test getting value of environment variable.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - self.assertEqual('$env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) - self.assertEqual('$env(HOME)', self.modgen.getenv_cmd('HOME')) + self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) + self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) else: self.assertEqual('os.getenv("HOSTNAME")', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('os.getenv("HOME")', self.modgen.getenv_cmd('HOME')) From 81a59a56d2407f41a0d94c26418598857c6e6a8d Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 5 Jul 2018 22:18:53 +0200 Subject: [PATCH 21/26] Tcl and Lua differs in argument separator for their respective file join/pathjoin, so add a join_str to det_user_modpath. --- easybuild/tools/module_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 5b7df1cfd1..df09440490 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -307,7 +307,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath """ raise NotImplementedError - def det_user_modpath(self, user_modpath): + def det_user_modpath(self, user_modpath, join_str): """ Determine user-specific modules subdirectory, to be used in 'use' statements (cfr. implementations of use() method). @@ -336,7 +336,7 @@ def det_user_modpath(self, user_modpath): if mod_path_suffix: expanded_user_modpath.extend([quote_str(mod_path_suffix)]) - user_modpath = ', '.join(expanded_user_modpath) + user_modpath = join_str.join(expanded_user_modpath) return user_modpath @@ -685,7 +685,7 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None): :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: user-specific modules subdirectory to include in use statements """ - user_modpath = self.det_user_modpath(user_modpath) + user_modpath = self.det_user_modpath(user_modpath, join_str=' ') use_statements = [] for path in paths: quoted_path = quote_str(path) @@ -1000,7 +1000,7 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None): :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: user-specific modules subdirectory to include in use statements """ - user_modpath = self.det_user_modpath(user_modpath) + user_modpath = self.det_user_modpath(user_modpath, join_str=', ') use_statements = [] for path in paths: quoted_path = quote_str(path) From ece102f6155d9c70a06a1a7123689f8200a526ff Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 6 Jul 2018 07:42:16 +0200 Subject: [PATCH 22/26] Use self.SYNTAX to determine string to use in join of expanded_user_modpath. Removes the need for an extra argument to det_user_modpath. --- easybuild/tools/module_generator.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index df09440490..94d4721def 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -307,11 +307,17 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath """ raise NotImplementedError - def det_user_modpath(self, user_modpath, join_str): + def det_user_modpath(self, user_modpath): """ Determine user-specific modules subdirectory, to be used in 'use' statements (cfr. implementations of use() method). """ + + if self.SYNTAX == 'Tcl': + join_str = ' ' + else: + join_str = ', ' + if user_modpath: # Check for occurences of {RUNTIME_ENV::SOME_ENV_VAR} # SOME_ENV_VAR will be expanded at module load time. @@ -685,7 +691,7 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None): :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: user-specific modules subdirectory to include in use statements """ - user_modpath = self.det_user_modpath(user_modpath, join_str=' ') + user_modpath = self.det_user_modpath(user_modpath) use_statements = [] for path in paths: quoted_path = quote_str(path) @@ -1000,7 +1006,7 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None): :param guarded: use statements will be guarded to only apply if path exists :param user_modpath: user-specific modules subdirectory to include in use statements """ - user_modpath = self.det_user_modpath(user_modpath, join_str=', ') + user_modpath = self.det_user_modpath(user_modpath) use_statements = [] for path in paths: quoted_path = quote_str(path) From 674b52f5887e4c6308db00823a175beccdb90e19 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 6 Jul 2018 07:43:43 +0200 Subject: [PATCH 23/26] Fix incorrect Tcl syntax in expected strings from det_user_modpath. --- test/framework/module_generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 54b75c8b82..88ff06d98b 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -476,14 +476,17 @@ def test_det_user_modpath(self): # None by default self.assertEqual(self.modgen.det_user_modpath(None), None) - self.assertEqual(self.modgen.det_user_modpath('my/own/modules'), '"my/own/modules", "all"') + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + self.assertEqual(self.modgen.det_user_modpath('my/own/modules'), '"my/own/modules" "all"') + else: + self.assertEqual(self.modgen.det_user_modpath('my/own/modules'), '"my/own/modules", "all"') # result is affected by --suffix-modules-path # {RUNTIME_ENV::FOO} gets translated into Tcl/Lua syntax for resolving $FOO at runtime init_config(build_options={'suffix_modules_path': ''}) user_modpath = 'my/{RUNTIME_ENV::TEST123}/modules' if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: - self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", $::env(TEST123), "modules"') + self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my" $::env(TEST123) "modules"') else: self.assertEqual(self.modgen.det_user_modpath(user_modpath), '"my", os.getenv("TEST123"), "modules"') From 1d205c552e0d0250de2aab120bd8720942e4be35 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 6 Jul 2018 07:48:18 +0200 Subject: [PATCH 24/26] Fix tabs to spaces --- easybuild/tools/module_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 94d4721def..3621f274e5 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -313,10 +313,10 @@ def det_user_modpath(self, user_modpath): (cfr. implementations of use() method). """ - if self.SYNTAX == 'Tcl': - join_str = ' ' - else: - join_str = ', ' + if self.SYNTAX == 'Tcl': + join_str = ' ' + else: + join_str = ', ' if user_modpath: # Check for occurences of {RUNTIME_ENV::SOME_ENV_VAR} From 7c147c51ec9c5649b165da739584faea588ac8ad Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 6 Jul 2018 08:27:42 +0200 Subject: [PATCH 25/26] flesh out common part of det_user_modpath into _det_user_modpath_common method --- easybuild/tools/module_generator.py | 85 +++++++++++++++++------------ 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 3621f274e5..ed8c43702c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -307,44 +307,41 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath """ raise NotImplementedError + def _det_user_modpath_common(self, user_modpath): + """ + Helper function for det_user_modpath. + """ + # Check for occurences of {RUNTIME_ENV::SOME_ENV_VAR} + # SOME_ENV_VAR will be expanded at module load time. + runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') + sub_paths = [] + expanded_user_modpath = [] + for sub_path in re.split(os.path.sep, user_modpath): + matched_re = runtime_env_re.match(sub_path) + if matched_re: + if sub_paths: + path = quote_str(os.path.join(*sub_paths)) + expanded_user_modpath.extend([path]) + sub_paths = [] + expanded_user_modpath.extend([self.getenv_cmd(matched_re.group(1))]) + else: + sub_paths.append(sub_path) + if sub_paths: + expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) + + # if a mod_path_suffix is being used, we should respect it + mod_path_suffix = build_option('suffix_modules_path') + if mod_path_suffix: + expanded_user_modpath.extend([quote_str(mod_path_suffix)]) + + return expanded_user_modpath + def det_user_modpath(self, user_modpath): """ Determine user-specific modules subdirectory, to be used in 'use' statements - (cfr. implementations of use() method). + (cfr. implementation of use() method). """ - - if self.SYNTAX == 'Tcl': - join_str = ' ' - else: - join_str = ', ' - - if user_modpath: - # Check for occurences of {RUNTIME_ENV::SOME_ENV_VAR} - # SOME_ENV_VAR will be expanded at module load time. - runtime_env_re = re.compile(r'{RUNTIME_ENV::(\w+)}') - sub_paths = [] - expanded_user_modpath = [] - for sub_path in re.split(os.path.sep, user_modpath): - matched_re = runtime_env_re.match(sub_path) - if matched_re: - if sub_paths: - path = quote_str(os.path.join(*sub_paths)) - expanded_user_modpath.extend([path]) - sub_paths = [] - expanded_user_modpath.extend([self.getenv_cmd(matched_re.group(1))]) - else: - sub_paths.append(sub_path) - if sub_paths: - expanded_user_modpath.extend([quote_str(os.path.join(*sub_paths))]) - - # if a mod_path_suffix is being used, we should respect it - mod_path_suffix = build_option('suffix_modules_path') - if mod_path_suffix: - expanded_user_modpath.extend([quote_str(mod_path_suffix)]) - - user_modpath = join_str.join(expanded_user_modpath) - - return user_modpath + raise NotImplementedError def use(self, paths, prefix=None, guarded=False, user_modpath=None): """ @@ -683,6 +680,16 @@ def unload_module(self, mod_name): """ return '\n'.join(['', "module unload %s" % mod_name]) + def det_user_modpath(self, user_modpath): + """ + Determine user-specific modules subdirectory, to be used in 'use' statements + (cfr. implementation of use() method). + """ + if user_modpath: + user_modpath = ' '.join(self._det_user_modpath_common(user_modpath)) + + return user_modpath + def use(self, paths, prefix=None, guarded=False, user_modpath=None): """ Generate module use statements for given list of module paths. @@ -998,6 +1005,16 @@ def unload_module(self, mod_name): """ return '\n'.join(['', 'unload("%s")' % mod_name]) + def det_user_modpath(self, user_modpath): + """ + Determine user-specific modules subdirectory, to be used in 'use' statements + (cfr. implementations of use() method). + """ + if user_modpath: + user_modpath = ', '.join(self._det_user_modpath_common(user_modpath)) + + return user_modpath + def use(self, paths, prefix=None, guarded=False, user_modpath=None): """ Generate module use statements for given list of module paths. From 4dc5f5698866d3627a5416cb7b31ee0d44594e68 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 6 Jul 2018 09:10:03 +0200 Subject: [PATCH 26/26] fix check for $env(HOME) in test_make_module_extend_modpath, should be $::env(HOME) --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 43e484581d..bcf69e1bab 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -271,7 +271,7 @@ def test_make_module_extend_modpath(self): txt = eb.make_module_extend_modpath() if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\$env\(HOME\)' + home = r'\$::env\(HOME\)' fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded