diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c85104d687..f4e60cdc8b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1294,6 +1294,9 @@ def make_module_dep(self, unload_info=None): if tc_mod in excluded_deps and tc_dep_mods: excluded_deps.extend(tc_dep_mods) + # Compute Canada specific: exclude gcccore from toolchain deps, we treat it as build dependency + excluded_deps.extend([d for d in tc_dep_mods if d.startswith('gcccore/')]) + self.log.debug("List of excluded deps: %s", excluded_deps) # expand toolchain into toolchain components if desired @@ -1619,7 +1622,7 @@ def make_module_req_guess(self): 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in lib_paths + ['share']], 'ACLOCAL_PATH': [os.path.join('share', 'aclocal')], 'CLASSPATH': ['*.jar'], - 'XDG_DATA_DIRS': ['share'], + #'XDG_DATA_DIRS': ['share'], #disable for now 'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths], 'CMAKE_PREFIX_PATH': [''], 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above @@ -3961,7 +3964,7 @@ def run_step(self, step, step_methods): raise StopException(step) @staticmethod - def get_steps(run_test_cases=True, iteration_count=1): + def get_steps(run_test_cases=True, iteration_count=1, multi_deps_extensions_only=False): """Return a list of all steps to be performed.""" def get_step(tag, descr, substeps, skippable, initial=True): @@ -3976,11 +3979,24 @@ def get_step(tag, descr, substeps, skippable, initial=True): (True, lambda x: x.reset_env), (True, lambda x: x.handle_iterate_opts), ] + # if multi_deps is used only for extensions, don't make the build directory each iteration + if multi_deps_extensions_only: + ready_substeps_iterates = [ + (False, lambda x: x.check_readiness_step), + (True, lambda x: x.reset_env), + (True, lambda x: x.handle_iterate_opts), + ] + else: + ready_substeps_iterates = ready_substeps def ready_step_spec(initial): """Return ready step specified.""" - return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False, - initial=initial) + if initial: + return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False, + initial=initial) + else: + return get_step(READY_STEP, "creating build dir, resetting environment", + ready_substeps_iterates, False, initial=initial) source_substeps = [ (False, lambda x: x.checksum_step), @@ -4027,17 +4043,24 @@ def install_step_spec(initial): # part 2: iterated part, from 2nd iteration onwards # repeat core procedure again depending on specified iteration count # not all parts of all steps need to be rerun (see e.g., ready, prepare) - steps_part2 = [ - ready_step_spec(False), - source_step_spec(False), - patch_step_spec, - prepare_step_spec, - configure_step_spec, - build_step_spec, - test_step_spec, - install_step_spec(False), - extensions_step_spec, - ] * (iteration_count - 1) + if multi_deps_extensions_only: + steps_part2 = [ + ready_step_spec(False), + prepare_step_spec, + extensions_step_spec, + ] * (iteration_count - 1) + else: + steps_part2 = [ + ready_step_spec(False), + source_step_spec(False), + patch_step_spec, + prepare_step_spec, + configure_step_spec, + build_step_spec, + test_step_spec, + install_step_spec(False), + extensions_step_spec, + ] * (iteration_count - 1) # part 3: post-iteration part steps_part3 = [ (POSTITER_STEP, 'restore after iterating', [lambda x: x.post_iter_step], False), @@ -4068,7 +4091,8 @@ def run_all_steps(self, run_test_cases): if self.cfg['stop'] and self.cfg['stop'] == 'cfg': return True - steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) + steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt(), + multi_deps_extensions_only=self.cfg["multi_deps_extensions_only"]) # figure out how many steps will actually be run (not be skipped) step_cnt = 0 diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index b37c3660f0..5720906238 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -167,6 +167,7 @@ 'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES], 'multi_deps': [{}, "Dict of lists of dependency versions over which to iterate", DEPENDENCIES], 'multi_deps_load_default': [True, "Load module for first version listed in multi_deps by default", DEPENDENCIES], + 'multi_deps_extensions_only': [False, "Use multi_deps dependencies only to install extensions", DEPENDENCIES], 'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES], 'moddependpaths': [None, "Absolute path(s) to prepend to MODULEPATH before loading dependencies", DEPENDENCIES], diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 41c688355b..ba30f1f19e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -385,6 +385,7 @@ def __repr__(self): try: # obtain temporary filename fd, tmpfn = tempfile.mkstemp() + os.chmod(tmpfn, 0o644) os.close(fd) # write easyconfig to temporary file diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index c87f1b21d1..9788e9b4ed 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -31,6 +31,7 @@ """ from easybuild.tools.toolchain.compiler import Compiler +from easybuild.tools.modules import get_software_root TC_CONSTANT_CUDA = "CUDA" @@ -77,7 +78,9 @@ def __init__(self, *args, **kwargs): def _set_compiler_vars(self): """Set the compiler variables""" # append lib dir paths to LDFLAGS (only if the paths are actually there) - root = self.get_software_root('CUDA')[0] + root = get_software_root('CUDAcore') + if not root: + root = self.get_software_root('CUDA')[0] self.variables.append_subdirs("LDFLAGS", root, subdirs=["lib64", "lib"]) super(Cuda, self)._set_compiler_vars() diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 43b450b3de..3e9c567622 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -36,6 +36,7 @@ from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP TC_CONSTANT_FLEXIBLAS = 'FlexiBLAS' @@ -78,6 +79,12 @@ class FlexiBLAS(LinAlg): LAPACK_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS + def _set_blas_variables(self): + """Fix the BLAS_LIB for Intel""" + if self.COMPILER_FAMILY == TC_CONSTANT_INTELCOMP: + self.BLAS_LIB = ['flexiblas_intel'] + super(FlexiBLAS, self)._set_blas_variables() + def banned_linked_shared_libs(self): """ List of shared libraries (names, file names, paths) which are diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 215b8d38aa..9fa15fa863 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -214,6 +214,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger. # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-', dir=tmp_logdir) + os.chmod(logfile, 0o644) os.close(fd) fancylogger.logToFile(logfile, max_bytes=0) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index aaf97d3194..316ff76d03 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -514,6 +514,12 @@ def set_alias(self, key, value): """ raise NotImplementedError + def set_shell_function(self, key, bashStr, cshStr): + """ + Generate set_shell_function statement in modulefile for the given key/value pair. + """ + raise NotImplementedError + def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None): """ Set generated module as default module @@ -1005,6 +1011,12 @@ def set_alias(self, key, value): # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles return 'set-alias\t%s\t\t%s\n' % (key, quote_str(value, tcl=True)) + def set_shell_function(self, key, bashStr, cshStr): + """ + Generate set_shell_function statement in modulefile for the given key/value pair. + """ + raise NotImplementedError + def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None): """ Create a .version file inside the package module folder in order to set the default version for TMod @@ -1190,6 +1202,8 @@ def check_group(self, group, error_msg=None): error_msg = 'LmodError("' + error_msg + '")' res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True) + if 'EASYBUILD_CONFIGFILES' in os.environ: + res = '' # disable for Compute Canada: else: warn_msg = "Can't generate robust check in Lua modules for users belonging to group %s. " warn_msg += "Lmod version not recent enough (%s), should be >= %s" @@ -1478,6 +1492,13 @@ def set_alias(self, key, value): # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles return 'set_alias("%s", %s)\n' % (key, quote_str(value)) + def set_shell_function(self, key, bashStr, cshStr): + """ + Generate set_shell_function statement in modulefile for the given key/value pair. + """ + # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles + return 'set_shell_function("%s", %s, %s)\n' % (key, quote_str(bashStr), quote_str(cshStr)) + def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None): """ Create a symlink named 'default' inside the package's module folder in order to set the default module version diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 8e8f3e919e..5220008ba9 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -69,8 +69,8 @@ def det_full_ec_version(ec): type(versionprefix).__name__, versionprefix, ec) versionsuffix = ec.get('versionsuffix', '') - if not isinstance(versionsuffix, string_type): - raise EasyBuildError("versionsuffix value should be a string, found '%s': %s (full spec: %s)", + if not isinstance(versionsuffix, string_type) and versionsuffix is not None: + raise EasyBuildError("versionsuffix value should be a string or None, found '%s': %s (full spec: %s)", type(versionsuffix).__name__, versionsuffix, ec) ecver = ''.join([x for x in [versionprefix, ecver, versionsuffix] if x]) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 761cdc262f..85a325ae94 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1874,6 +1874,7 @@ def set_tmpdir(tmpdir=None, raise_error=False): raise EasyBuildError("Failed to create path to temporary directory %s: %s", current_tmpdir, err) _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir) + os.chmod(current_tmpdir, 0o755) for var in ['TMPDIR', 'TEMP', 'TMP']: env.setvar(var, current_tmpdir, verbose=False) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..50b9c44831 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -439,7 +439,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency - verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) + #verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: if ec not in easyconfigs + additional: diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index b7ce694ff7..4b1075a4d9 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -92,7 +92,7 @@ def set_variables(self): # TODO is link order fully preserved with this order ? self._set_blas_variables() self._set_lapack_variables() - if getattr(self, 'MPI_MODULE_NAME', None): + if getattr(self, 'MPI_MODULE_NAME', None) and self.SCALAPACK_LIB is not None: self._set_blacs_variables() self._set_scalapack_variables()