From e2e684395aefee395cd513b7f37568d40a4024e4 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 28 Nov 2018 01:29:54 +0000 Subject: [PATCH 1/7] Add dependencies in the prepare step instead of check_readiness. Using the prepare step we will be able to iterate over multiple build-dependencies such as multiple Python versions. This also removes the dependencies field from the Toolchain class which handles them completely functionally now. The EasyBlock class now has self.dependencies to remember them for outputting in modulefiles. --- easybuild/framework/easyblock.py | 20 ++++----- easybuild/tools/toolchain/toolchain.py | 62 ++++++++++++++++---------- test/framework/easyconfig.py | 3 +- test/framework/toolchain.py | 9 ++-- 4 files changed, 51 insertions(+), 43 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8425125808..e3e952ce14 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -185,6 +185,9 @@ def __init__(self, ec): # indicates whether build should be performed in installation dir self.build_in_installdir = self.cfg['buildininstalldir'] + # list of dependencies to load in generated module + build dependencies + self.dependencies = [] + # list of locations to include in RPATH filter used by toolchain self.rpath_filter_dirs = [] @@ -1011,8 +1014,8 @@ def make_module_dep(self, unload_info=None): # include load/unload statements for dependencies deps = [] - self.log.debug("List of deps considered to load in generated module: %s", self.toolchain.dependencies) - for dep in self.toolchain.dependencies: + self.log.debug("List of deps considered to load in generated module: %s", self.dependencies) + for dep in self.dependencies: if dep['build_only']: self.log.debug("Skipping build dependency %s", dep) else: @@ -1537,13 +1540,6 @@ def check_readiness_step(self): if len(loadedmods) > 0: self.log.warning("Loaded modules detected: %s" % loadedmods) - # do all dependencies have a toolchain version? - self.toolchain.add_dependencies(self.cfg.dependencies()) - if not len(self.cfg.dependencies()) == len(self.toolchain.dependencies): - self.log.debug("dep %s (%s)" % (len(self.cfg.dependencies()), self.cfg.dependencies())) - self.log.debug("tc.dep %s (%s)" % (len(self.toolchain.dependencies), self.toolchain.dependencies)) - raise EasyBuildError('Not all dependencies have a matching toolchain version') - # check if the application is not loaded at the moment (root, env_var) = get_software_root(self.name, with_env_var=True) if root: @@ -1832,8 +1828,10 @@ def prepare_step(self, start_dir=True): self.rpath_include_dirs.append('$ORIGIN/../lib64') # prepare toolchain: load toolchain module and dependencies, set up build environment - self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, - rpath_include_dirs=self.rpath_include_dirs) + deps = self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, + rpath_filter_dirs=self.rpath_filter_dirs, + rpath_include_dirs=self.rpath_include_dirs) + self.dependencies = deps # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 19adb0342c..d612d5fd39 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -120,7 +120,6 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde """ self.base_init() - self.dependencies = [] self.toolchain_dep_mods = [] self.cached_compilers = set() @@ -385,9 +384,9 @@ def get_dependency_version(self, dependency): raise EasyBuildError("No toolchain version for dependency name %s (suffix %s) found", dependency['name'], toolchain_suffix) - def add_dependencies(self, dependencies): - """ Verify if the given dependencies exist and add them """ - self.log.debug("add_dependencies: adding toolchain dependencies %s", dependencies) + def _check_dependencies(self, dependencies): + """ Verify if the given dependencies exist and return them """ + self.log.debug("_check_dependencies: adding toolchain dependencies %s", dependencies) # use *full* module name to check existence of dependencies, since the modules may not be available in the # current $MODULEPATH without loading the prior dependencies in a module hierarchy @@ -397,23 +396,26 @@ def add_dependencies(self, dependencies): dep_mod_names = [dep['full_mod_name'] for dep in dependencies] # check whether modules exist - self.log.debug("add_dependencies: MODULEPATH: %s", os.environ['MODULEPATH']) + self.log.debug("_check_dependencies: MODULEPATH: %s", os.environ['MODULEPATH']) if self.dry_run: deps_exist = [True] * len(dep_mod_names) else: deps_exist = self.modules_tool.exist(dep_mod_names) missing_dep_mods = [] + deps = [] for dep, dep_mod_name, dep_exists in zip(dependencies, dep_mod_names, deps_exist): if dep_exists: - self.dependencies.append(dep) - self.log.devel("add_dependencies: added toolchain dependency %s", str(dep)) + deps.append(dep) + self.log.devel("_check_dependencies: added toolchain dependency %s", str(dep)) else: missing_dep_mods.append(dep_mod_name) if missing_dep_mods: raise EasyBuildError("Missing modules for one or more dependencies: %s", ', '.join(missing_dep_mods)) + return deps + def is_required(self, name): """Determine whether this is a required toolchain element.""" # default: assume every element is required @@ -513,9 +515,9 @@ def _load_toolchain_module(self, silent=False): # append toolchain module to list of modules self.modules.append(tc_mod) - def _load_dependencies_modules(self, silent=False): + def _load_dependencies_modules(self, dependencies, silent=False): """Load modules for dependencies, and handle special cases like external modules.""" - dep_mods = [dep['short_mod_name'] for dep in self.dependencies] + dep_mods = [dep['short_mod_name'] for dep in dependencies] if self.dry_run: dry_run_msg("\nLoading modules for dependencies...\n", silent=silent) @@ -523,7 +525,7 @@ def _load_dependencies_modules(self, silent=False): mods_exist = self.modules_tool.exist(dep_mods) # load available modules for dependencies, simulate load for others - for dep, dep_mod_exists in zip(self.dependencies, mods_exist): + for dep, dep_mod_exists in zip(dependencies, mods_exist): mod_name = dep['short_mod_name'] if dep_mod_exists: self.modules_tool.load([mod_name]) @@ -539,8 +541,8 @@ def _load_dependencies_modules(self, silent=False): self.log.debug("Loading modules for dependencies: %s", dep_mods) self.modules_tool.load(dep_mods) - if self.dependencies: - build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']] + if dependencies: + build_dep_mods = [dep['short_mod_name'] for dep in dependencies if dep['build_only']] if build_dep_mods: trace_msg("loading modules for build dependencies:") for dep_mod in build_dep_mods: @@ -548,7 +550,7 @@ def _load_dependencies_modules(self, silent=False): else: trace_msg("(no build dependencies specified)") - run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']] + run_dep_mods = [dep['short_mod_name'] for dep in dependencies if not dep['build_only']] if run_dep_mods: trace_msg("loading modules for (runtime) dependencies:") for dep_mod in run_dep_mods: @@ -560,7 +562,7 @@ def _load_dependencies_modules(self, silent=False): self.modules.extend(dep_mods) # define $EBROOT* and $EBVERSION* for external modules, if metadata is available - for dep in [d for d in self.dependencies if d['external_module']]: + for dep in [d for d in dependencies if d['external_module']]: mod_name = dep['full_mod_name'] metadata = dep['external_module_metadata'] self.log.debug("Metadata for external module %s: %s", mod_name, metadata) @@ -573,7 +575,7 @@ def _load_dependencies_modules(self, silent=False): for name, version in zip(names, versions): self._simulated_load_dependency_module(name, version, metadata, verbose=True) - def _load_modules(self, silent=False): + def _load_modules(self, dependencies, silent=False): """Load modules for toolchain and dependencies.""" if self.modules_tool is None: raise EasyBuildError("No modules tool defined in Toolchain instance.") @@ -588,12 +590,12 @@ def _load_modules(self, silent=False): dry_run_msg("(no modules are loaded for a dummy-dummy toolchain)", silent=silent) else: self.log.info('prepare: toolchain dummy mode and loading dependencies') - self._load_dependencies_modules(silent=silent) + self._load_dependencies_modules(dependencies, silent=silent) else: # load the toolchain and dependencies modules self.log.debug("Loading toolchain module and dependencies...") self._load_toolchain_module(silent=silent) - self._load_dependencies_modules(silent=silent) + self._load_dependencies_modules(dependencies, silent=silent) # include list of loaded modules in dry run output if self.dry_run: @@ -674,12 +676,14 @@ def is_deprecated(self): """Return whether or not this toolchain is deprecated.""" return False - def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=None, rpath_include_dirs=None): + def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, + rpath_filter_dirs=None, rpath_include_dirs=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment + :param deps: list of dependencies :param onlymod: boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). @@ -688,8 +692,18 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) :param rpath_include_dirs: extra directories to include in RPATH """ + + # do all dependencies have a toolchain version? + if deps is None: + deps = [] + tcdeps = self._check_dependencies(deps) + if not len(deps) == len(tcdeps): + self.log.debug("dep %s (%s)" % (len(deps), deps)) + self.log.debug("tc.dep %s (%s)" % (len(tcdeps), tcdeps)) + raise EasyBuildError('Not all dependencies have a matching toolchain version') + if loadmod: - self._load_modules(silent=silent) + self._load_modules(deps, silent=silent) if self.name != DUMMY_TOOLCHAIN_NAME: @@ -710,7 +724,7 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No self.log.debug("prepare: set additional variables onlymod=%s", onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars - self._add_dependency_variables() + self._add_dependency_variables(deps=tcdeps) self.generate_vars() self._setenv_variables(onlymod, verbose=not silent) @@ -726,6 +740,8 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No else: self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option") + return tcdeps + def comp_cache_compilers(self, cache_tool): """ Determine list of relevant compilers for specified compiler caching tool. @@ -860,7 +876,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None else: self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd) - def _add_dependency_variables(self, names=None, cpp=None, ld=None): + def _add_dependency_variables(self, names=None, cpp=None, ld=None, deps=None): """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies names should be a list of strings containing the name of the dependency """ @@ -878,9 +894,7 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None): if p not in ld_paths: ld_paths.append(p) - if not names: - deps = self.dependencies - else: + if names: deps = [{'name': name} for name in names if name is not None] # collect software install prefixes for dependencies diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 45b33d3685..3e1be05479 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1345,8 +1345,7 @@ def test_external_dependencies(self): os.environ['PI_PREFIX'] = '/test/prefix/PI' os.environ['TEST_INC'] = '/test/prefix/test/include' ec.toolchain.dry_run = True - ec.toolchain.add_dependencies(ec.dependencies()) - ec.toolchain.prepare(silent=True) + ec.toolchain.prepare(deps=ec.dependencies(), silent=True) self.assertEqual(os.environ.get('EBROOTBAR'), '/foo/bar') self.assertEqual(os.environ.get('EBROOTFOO'), '/foo/bar') diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index a9c85e2b52..5ff0dd7f0d 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -796,8 +796,7 @@ def test_prepare_deps(self): 'build_only': False, }, ] - tc.add_dependencies(deps) - tc.prepare() + tc.prepare(deps=deps) mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4'] self.assertTrue([m['mod_name'] for m in self.modtool.list()], mods) @@ -825,8 +824,7 @@ def test_prepare_deps_external(self): } ] tc = self.get_toolchain('GCC', version='4.6.4') - tc.add_dependencies(deps) - tc.prepare() + tc.prepare(deps=deps) mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4', 'toy/0.0'] self.assertTrue([m['mod_name'] for m in self.modtool.list()], mods) self.assertTrue(os.environ['EBROOTTOY'].endswith('software/toy/0.0')) @@ -846,9 +844,8 @@ def test_prepare_deps_external(self): 'build_only': False, } tc = self.get_toolchain('GCC', version='4.6.4') - tc.add_dependencies(deps) os.environ['FOOBAR_PREFIX'] = '/foo/bar' - tc.prepare() + tc.prepare(deps=deps) mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4', 'toy/0.0'] self.assertTrue([m['mod_name'] for m in self.modtool.list()], mods) self.assertEqual(os.environ['EBROOTTOY'], '/foo/bar') From c23b1beddaf2030bff9e5ea778b8a30b0527182c Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 28 Nov 2018 02:16:06 +0000 Subject: [PATCH 2/7] prepare wrappers must return a result now (dependencies) also use tcdeps instead of deps where appropriate for _load_modules() --- easybuild/toolchains/compiler/craype.py | 4 +++- easybuild/toolchains/mpi/openmpi.py | 4 +++- easybuild/tools/toolchain/toolchain.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index 4b36275cd1..c1ded82525 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -122,12 +122,14 @@ def _set_optimal_architecture(self): def prepare(self, *args, **kwargs): """Prepare to use this toolchain; define $CRAYPE_LINK_TYPE if 'dynamic' toolchain option is enabled.""" - super(CrayPECompiler, self).prepare(*args, **kwargs) + res = super(CrayPECompiler, self).prepare(*args, **kwargs) if self.options['dynamic'] or self.options['shared']: self.log.debug("Enabling building of shared libs/dynamically linked executables via $CRAYPE_LINK_TYPE") env.setvar('CRAYPE_LINK_TYPE', 'dynamic') + return res + class CrayPEGCC(CrayPECompiler): """Support for using the Cray GNU compiler wrappers.""" diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index a9434a1f57..e19812c08c 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -72,7 +72,7 @@ def prepare(self, *args, **kwargs): """ Prepare for using OpenMPI library in toolchain environment """ - super(OpenMPI, self).prepare(*args, **kwargs) + res = super(OpenMPI, self).prepare(*args, **kwargs) # OpenMPI 2.x trips if path specified in $TMPDIR is too long # see https://www.open-mpi.org/faq/?category=osx#startup-errors-with-open-mpi-2.0.x @@ -86,6 +86,8 @@ def prepare(self, *args, **kwargs): self.log.warning(warn_msg) print_warning(warn_msg) + return res + def _set_mpi_compiler_variables(self): """Define MPI wrapper commands (depends on OpenMPI version) and add OMPI_* variables to set.""" ompi_ver = self.get_software_version(self.MPI_MODULE_NAME)[0] diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index d612d5fd39..adf23307c0 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -703,7 +703,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, raise EasyBuildError('Not all dependencies have a matching toolchain version') if loadmod: - self._load_modules(deps, silent=silent) + self._load_modules(tcdeps, silent=silent) if self.name != DUMMY_TOOLCHAIN_NAME: From 8a50726880eb8f4935c4310feacd94236e9c97a6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 28 Nov 2018 02:49:38 +0000 Subject: [PATCH 3/7] Add prepare_step() calls to make_module_dep/step() tests. Now that prepare_step() takes care of dependencies they now no longer fail. --- test/framework/easyblock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index f6fbd13db9..323669352e 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -473,6 +473,7 @@ def test_make_module_dep(self): eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() + eb.prepare_step() if get_module_syntax() == 'Tcl': tc_load = '\n'.join([ @@ -715,6 +716,7 @@ def test_make_module_step(self): eb = EasyBlock(ec) eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() + eb.prepare_step() modpath = os.path.join(eb.make_module_step(), name, version) if get_module_syntax() == 'Lua': From 7f0aebaf9fc37b8d1d1e1f9bf5bcd58fd0225eb0 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 28 Nov 2018 03:18:39 +0000 Subject: [PATCH 4/7] Tests that call prepare_step also need to call make_builddir. --- test/framework/easyblock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 323669352e..1259da2912 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -473,6 +473,7 @@ def test_make_module_dep(self): eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() + eb.make_builddir() eb.prepare_step() if get_module_syntax() == 'Tcl': @@ -716,6 +717,7 @@ def test_make_module_step(self): eb = EasyBlock(ec) eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() + eb.make_builddir() eb.prepare_step() modpath = os.path.join(eb.make_module_step(), name, version) From c0df892dbabdef11d73adcb1cd78884cdbbda0bc Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 29 Nov 2018 14:07:55 +0000 Subject: [PATCH 5/7] Keep self.toolchain.dependencies after all. This avoids needing to worry about prepare returning a value, and easyblocks relying on it. --- easybuild/framework/easyblock.py | 13 ++++--------- easybuild/toolchains/compiler/craype.py | 4 +--- easybuild/toolchains/mpi/openmpi.py | 4 +--- easybuild/tools/toolchain/toolchain.py | 3 ++- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e3e952ce14..e2bcb78264 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -185,9 +185,6 @@ def __init__(self, ec): # indicates whether build should be performed in installation dir self.build_in_installdir = self.cfg['buildininstalldir'] - # list of dependencies to load in generated module + build dependencies - self.dependencies = [] - # list of locations to include in RPATH filter used by toolchain self.rpath_filter_dirs = [] @@ -1014,8 +1011,8 @@ def make_module_dep(self, unload_info=None): # include load/unload statements for dependencies deps = [] - self.log.debug("List of deps considered to load in generated module: %s", self.dependencies) - for dep in self.dependencies: + self.log.debug("List of deps considered to load in generated module: %s", self.toolchain.dependencies) + for dep in self.toolchain.dependencies: if dep['build_only']: self.log.debug("Skipping build dependency %s", dep) else: @@ -1828,10 +1825,8 @@ def prepare_step(self, start_dir=True): self.rpath_include_dirs.append('$ORIGIN/../lib64') # prepare toolchain: load toolchain module and dependencies, set up build environment - deps = self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, - rpath_filter_dirs=self.rpath_filter_dirs, - rpath_include_dirs=self.rpath_include_dirs) - self.dependencies = deps + self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, + rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index c1ded82525..4b36275cd1 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -122,14 +122,12 @@ def _set_optimal_architecture(self): def prepare(self, *args, **kwargs): """Prepare to use this toolchain; define $CRAYPE_LINK_TYPE if 'dynamic' toolchain option is enabled.""" - res = super(CrayPECompiler, self).prepare(*args, **kwargs) + super(CrayPECompiler, self).prepare(*args, **kwargs) if self.options['dynamic'] or self.options['shared']: self.log.debug("Enabling building of shared libs/dynamically linked executables via $CRAYPE_LINK_TYPE") env.setvar('CRAYPE_LINK_TYPE', 'dynamic') - return res - class CrayPEGCC(CrayPECompiler): """Support for using the Cray GNU compiler wrappers.""" diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index e19812c08c..a9434a1f57 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -72,7 +72,7 @@ def prepare(self, *args, **kwargs): """ Prepare for using OpenMPI library in toolchain environment """ - res = super(OpenMPI, self).prepare(*args, **kwargs) + super(OpenMPI, self).prepare(*args, **kwargs) # OpenMPI 2.x trips if path specified in $TMPDIR is too long # see https://www.open-mpi.org/faq/?category=osx#startup-errors-with-open-mpi-2.0.x @@ -86,8 +86,6 @@ def prepare(self, *args, **kwargs): self.log.warning(warn_msg) print_warning(warn_msg) - return res - def _set_mpi_compiler_variables(self): """Define MPI wrapper commands (depends on OpenMPI version) and add OMPI_* variables to set.""" ompi_ver = self.get_software_version(self.MPI_MODULE_NAME)[0] diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index adf23307c0..3e6992dd05 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -120,6 +120,7 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde """ self.base_init() + self.dependencies = [] self.toolchain_dep_mods = [] self.cached_compilers = set() @@ -740,7 +741,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, else: self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option") - return tcdeps + self.dependencies = tcdeps def comp_cache_compilers(self, cache_tool): """ From b7e418ae7f46ca67d8144f9ed5a921be675dda72 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 7 Dec 2018 15:20:57 +0000 Subject: [PATCH 6/7] Keep using self.dependencies as was done before, where possible. --- easybuild/tools/toolchain/toolchain.py | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 3e6992dd05..6e46d54918 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -516,9 +516,9 @@ def _load_toolchain_module(self, silent=False): # append toolchain module to list of modules self.modules.append(tc_mod) - def _load_dependencies_modules(self, dependencies, silent=False): + def _load_dependencies_modules(self, silent=False): """Load modules for dependencies, and handle special cases like external modules.""" - dep_mods = [dep['short_mod_name'] for dep in dependencies] + dep_mods = [dep['short_mod_name'] for dep in self.dependencies] if self.dry_run: dry_run_msg("\nLoading modules for dependencies...\n", silent=silent) @@ -526,7 +526,7 @@ def _load_dependencies_modules(self, dependencies, silent=False): mods_exist = self.modules_tool.exist(dep_mods) # load available modules for dependencies, simulate load for others - for dep, dep_mod_exists in zip(dependencies, mods_exist): + for dep, dep_mod_exists in zip(self.dependencies, mods_exist): mod_name = dep['short_mod_name'] if dep_mod_exists: self.modules_tool.load([mod_name]) @@ -542,8 +542,8 @@ def _load_dependencies_modules(self, dependencies, silent=False): self.log.debug("Loading modules for dependencies: %s", dep_mods) self.modules_tool.load(dep_mods) - if dependencies: - build_dep_mods = [dep['short_mod_name'] for dep in dependencies if dep['build_only']] + if self.dependencies: + build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']] if build_dep_mods: trace_msg("loading modules for build dependencies:") for dep_mod in build_dep_mods: @@ -551,7 +551,7 @@ def _load_dependencies_modules(self, dependencies, silent=False): else: trace_msg("(no build dependencies specified)") - run_dep_mods = [dep['short_mod_name'] for dep in dependencies if not dep['build_only']] + run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']] if run_dep_mods: trace_msg("loading modules for (runtime) dependencies:") for dep_mod in run_dep_mods: @@ -563,7 +563,7 @@ def _load_dependencies_modules(self, dependencies, silent=False): self.modules.extend(dep_mods) # define $EBROOT* and $EBVERSION* for external modules, if metadata is available - for dep in [d for d in dependencies if d['external_module']]: + for dep in [d for d in self.dependencies if d['external_module']]: mod_name = dep['full_mod_name'] metadata = dep['external_module_metadata'] self.log.debug("Metadata for external module %s: %s", mod_name, metadata) @@ -576,7 +576,7 @@ def _load_dependencies_modules(self, dependencies, silent=False): for name, version in zip(names, versions): self._simulated_load_dependency_module(name, version, metadata, verbose=True) - def _load_modules(self, dependencies, silent=False): + def _load_modules(self, silent=False): """Load modules for toolchain and dependencies.""" if self.modules_tool is None: raise EasyBuildError("No modules tool defined in Toolchain instance.") @@ -591,12 +591,12 @@ def _load_modules(self, dependencies, silent=False): dry_run_msg("(no modules are loaded for a dummy-dummy toolchain)", silent=silent) else: self.log.info('prepare: toolchain dummy mode and loading dependencies') - self._load_dependencies_modules(dependencies, silent=silent) + self._load_dependencies_modules(silent=silent) else: # load the toolchain and dependencies modules self.log.debug("Loading toolchain module and dependencies...") self._load_toolchain_module(silent=silent) - self._load_dependencies_modules(dependencies, silent=silent) + self._load_dependencies_modules(silent=silent) # include list of loaded modules in dry run output if self.dry_run: @@ -697,14 +697,14 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, # do all dependencies have a toolchain version? if deps is None: deps = [] - tcdeps = self._check_dependencies(deps) - if not len(deps) == len(tcdeps): + self.dependencies = self._check_dependencies(deps) + if not len(deps) == len(self.dependencies): self.log.debug("dep %s (%s)" % (len(deps), deps)) - self.log.debug("tc.dep %s (%s)" % (len(tcdeps), tcdeps)) + self.log.debug("tc.dep %s (%s)" % (len(self.dependencies), self.dependencies)) raise EasyBuildError('Not all dependencies have a matching toolchain version') if loadmod: - self._load_modules(tcdeps, silent=silent) + self._load_modules(silent=silent) if self.name != DUMMY_TOOLCHAIN_NAME: @@ -725,7 +725,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, self.log.debug("prepare: set additional variables onlymod=%s", onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars - self._add_dependency_variables(deps=tcdeps) + self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod, verbose=not silent) @@ -741,8 +741,6 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, else: self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option") - self.dependencies = tcdeps - def comp_cache_compilers(self, cache_tool): """ Determine list of relevant compilers for specified compiler caching tool. @@ -877,7 +875,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None else: self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd) - def _add_dependency_variables(self, names=None, cpp=None, ld=None, deps=None): + def _add_dependency_variables(self, names=None, cpp=None, ld=None): """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies names should be a list of strings containing the name of the dependency """ @@ -895,7 +893,9 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None, deps=None): if p not in ld_paths: ld_paths.append(p) - if names: + if not names: + deps = self.dependencies + else: deps = [{'name': name} for name in names if name is not None] # collect software install prefixes for dependencies From 4c628ea57531f4610f3c50099f3fada5d78ccb30 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Dec 2018 16:51:39 +0100 Subject: [PATCH 7/7] add back Toolchain.add_dependencies as deprecated method --- easybuild/tools/toolchain/toolchain.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 6e46d54918..fd036e7e65 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -417,6 +417,16 @@ def _check_dependencies(self, dependencies): return deps + def add_dependencies(self, dependencies): + """ + [DEPRECATED] Verify if the given dependencies exist, and return them. + + This method is deprecated. + You should pass the dependencies to the 'prepare' method instead, via the 'deps' named argument. + """ + self.log.deprecated("use of 'Toolchain.add_dependencies' method", '4.0') + self.dependencies = self._check_dependencies(dependencies) + def is_required(self, name): """Determine whether this is a required toolchain element.""" # default: assume every element is required