diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f4d9d8562b..0f6f0cc16d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1896,11 +1896,12 @@ def patch_step(self, beginpath=None): if not apply_patch(patch['path'], src, copy=copy_patch, level=level): raise EasyBuildError("Applying patch %s failed", patch['name']) - def prepare_step(self, start_dir=True): + def prepare_step(self, start_dir=True, load_tc_deps_modules=True): """ Pre-configure step. Set's up the builddir just before starting configure :param start_dir: guess start directory based on unpacked sources + :param load_tc_deps_modules: load modules for toolchain and dependencies in build environment """ if self.dry_run: self.dry_run_msg("Defining build environment, based on toolchain (options) and specified dependencies...\n") @@ -1941,7 +1942,8 @@ def prepare_step(self, start_dir=True): # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, - rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) + loadmod=load_tc_deps_modules, rpath_filter_dirs=self.rpath_filter_dirs, + rpath_include_dirs=self.rpath_include_dirs) # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index ce6d5f3f29..160f40c78c 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1080,6 +1080,29 @@ class EnvironmentModulesC(ModulesTool): MAX_VERSION = '3.99' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' + def run_module(self, *args, **kwargs): + """ + Run module command, tweak output that is exec'ed if necessary. + """ + if isinstance(args[0], (list, tuple,)): + args = args[0] + + # some versions of Cray's environment modules tool (3.2.10.x) include a "source */init/bash" command + # in the output of some "modulecmd python load" calls, which is not a valid Python command, + # which must be stripped out to avoid "invalid syntax" errors when evaluating the output + def tweak_stdout(txt): + """Tweak stdout before it's exec'ed as Python code.""" + source_regex = re.compile("^source .*$", re.M) + return source_regex.sub('', txt) + + tweak_stdout_fn = None + # for 'active' module (sub)commands that yield changes in environment, we need to tweak stdout before exec'ing + if args[0] in ['load', 'purge', 'swap', 'unload', 'use', 'unuse']: + tweak_stdout_fn = tweak_stdout + kwargs.update({'tweak_stdout': tweak_stdout_fn}) + + return super(EnvironmentModulesC, self).run_module(*args, **kwargs) + def update(self): """Update after new modules were added.""" pass diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index c6f005c400..5b9b2a1613 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1514,6 +1514,39 @@ def test_prepare_step(self): self.assertEqual(len(self.modtool.list()), 1) self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/6.4.0-2.28') + def test_prepare_step_load_tc_deps_modules(self): + """Test disabling loading of toolchain + dependencies in build environment.""" + + init_config(build_options={'robot_path': os.environ['EASYBUILD_ROBOT_PATHS']}) + + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + ompi_ec_file = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb') + ec = process_easyconfig(ompi_ec_file, validate=False)[0] + + mkdir(os.path.join(self.test_buildpath, 'OpenMPI', '2.1.2', 'GCC-6.4.0-2.28'), parents=True) + eb = EasyBlock(ec['ec']) + eb.silent = True + + # $EBROOTGCC and $EBROOTHWLOC must be set to set up build environment + os.environ['EBROOTGCC'] = self.test_prefix + os.environ['EBROOTHWLOC'] = self.test_prefix + + # loaded of modules for toolchain + dependencies can be disabled via load_tc_deps_modules=False + eb.prepare_step(load_tc_deps_modules=False) + self.assertEqual(self.modtool.list(), []) + + del os.environ['EBROOTGCC'] + del os.environ['EBROOTHWLOC'] + + # modules for toolchain + dependencies are still loaded by default + eb.prepare_step() + loaded_modules = self.modtool.list() + self.assertEqual(len(loaded_modules), 2) + self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28') + self.assertTrue(os.environ['EBROOTGCC']) + self.assertEqual(loaded_modules[1]['mod_name'], 'hwloc/1.11.8-GCC-6.4.0-2.28') + self.assertTrue(os.environ['EBROOTHWLOC']) + def test_prepare_step_hmns(self): """ Check whether loading of already existing dependencies during prepare step works when HierarchicalMNS is used. diff --git a/test/framework/modules.py b/test/framework/modules.py index 8e080e2d06..d6298b598c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -34,6 +34,7 @@ import re import tempfile import shutil +import stat import sys from distutils.version import StrictVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -42,7 +43,7 @@ import easybuild.tools.modules as mod from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import copy_file, copy_dir, mkdir, read_file, remove_file, write_file +from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir, read_file, remove_file, write_file from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches @@ -1057,6 +1058,7 @@ def check_loaded_modules(): self.assertErrorRegex(EasyBuildError, error_msg, init_config, args=['--detect-loaded-modules=sdvbfdgh']) def test_NoModulesTool(self): + """Test use of NoModulesTool class.""" nmt = NoModulesTool(testing=True) self.assertEqual(len(nmt.available()), 0) self.assertEqual(len(nmt.available(mod_names='foo')), 0) @@ -1064,6 +1066,33 @@ def test_NoModulesTool(self): self.assertEqual(nmt.exist(['foo', 'bar']), [False, False]) self.assertEqual(nmt.exist(['foo', 'bar'], r'^\s*\S*/%s.*:\s*$', skip_avail=False), [False, False]) + def test_modulecmd_strip_source(self): + """Test stripping of 'source' command in output of 'modulecmd python load'.""" + + init_config(build_options={'allow_modules_tool_mismatch': True}) + + # install dummy modulecmd command that always produces a 'source command' in its output + modulecmd = os.path.join(self.test_prefix, 'modulecmd') + modulecmd_txt = '\n'.join([ + '#!/bin/bash', + # if last argument (${!#})) is --version, print version + 'if [ x"${!#}" == "x--version" ]; then', + ' echo 3.2.10', + # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command + 'else', + ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', + " echo \"os.environ['TEST123'] = 'test123'\"", + 'fi', + ]) + write_file(modulecmd, modulecmd_txt) + adjust_permissions(modulecmd, stat.S_IXUSR, add=True) + + os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) + + modtool = EnvironmentModulesC() + modtool.run_module('load', 'test123') + self.assertEqual(os.getenv('TEST123'), 'test123') + def suite(): """ returns all the testcases in this module """