Eliminate duplicate multideps from modules

easybuild/framework/easyblock.py

@@ -1294,6 +1294,9 @@ def make_module_dep(self, unload_info=None):
         if tc_mod in excluded_deps and tc_dep_mods:
             excluded_deps.extend(tc_dep_mods)
 
+        # Compute Canada specific: exclude gcccore from toolchain deps, we treat it as build dependency
+        excluded_deps.extend([d for d in tc_dep_mods if d.startswith('gcccore/')])
+
         self.log.debug("List of excluded deps: %s", excluded_deps)
 
         # expand toolchain into toolchain components if desired
@@ -1619,7 +1622,7 @@ def make_module_req_guess(self):
             'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in lib_paths + ['share']],
             'ACLOCAL_PATH': [os.path.join('share', 'aclocal')],
             'CLASSPATH': ['*.jar'],
-            'XDG_DATA_DIRS': ['share'],
+            #'XDG_DATA_DIRS': ['share'], #disable for now
             'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths],
             'CMAKE_PREFIX_PATH': [''],
             'CMAKE_LIBRARY_PATH': ['lib64'],  # lib and lib32 are searched through the above
@@ -3961,7 +3964,7 @@ def run_step(self, step, step_methods):
             raise StopException(step)
 
     @staticmethod
-    def get_steps(run_test_cases=True, iteration_count=1):
+    def get_steps(run_test_cases=True, iteration_count=1, multi_deps_extensions_only=False):
         """Return a list of all steps to be performed."""
 
         def get_step(tag, descr, substeps, skippable, initial=True):
@@ -3976,11 +3979,24 @@ def get_step(tag, descr, substeps, skippable, initial=True):
             (True, lambda x: x.reset_env),
             (True, lambda x: x.handle_iterate_opts),
         ]
+        # if multi_deps is used only for extensions, don't make the build directory each iteration
+        if multi_deps_extensions_only:
+            ready_substeps_iterates = [
+                (False, lambda x: x.check_readiness_step),
+                (True, lambda x: x.reset_env),
+                (True, lambda x: x.handle_iterate_opts),
+            ]
+        else:
+            ready_substeps_iterates = ready_substeps
 
         def ready_step_spec(initial):
             """Return ready step specified."""
-            return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False,
-                            initial=initial)
+            if initial:
+                return get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False,
+                                initial=initial)
+            else:
+                return get_step(READY_STEP, "creating build dir, resetting environment",
+                                ready_substeps_iterates, False, initial=initial)
 
         source_substeps = [
             (False, lambda x: x.checksum_step),
@@ -4027,17 +4043,24 @@ def install_step_spec(initial):
         # part 2: iterated part, from 2nd iteration onwards
         # repeat core procedure again depending on specified iteration count
         # not all parts of all steps need to be rerun (see e.g., ready, prepare)
-        steps_part2 = [
-            ready_step_spec(False),
-            source_step_spec(False),
-            patch_step_spec,
-            prepare_step_spec,
-            configure_step_spec,
-            build_step_spec,
-            test_step_spec,
-            install_step_spec(False),
-            extensions_step_spec,
-        ] * (iteration_count - 1)
+        if multi_deps_extensions_only:
+            steps_part2 = [
+                ready_step_spec(False),
+                prepare_step_spec,
+                extensions_step_spec,
+            ] * (iteration_count - 1)
+        else:
+            steps_part2 = [
+                ready_step_spec(False),
+                source_step_spec(False),
+                patch_step_spec,
+                prepare_step_spec,
+                configure_step_spec,
+                build_step_spec,
+                test_step_spec,
+                install_step_spec(False),
+                extensions_step_spec,
+            ] * (iteration_count - 1)
         # part 3: post-iteration part
         steps_part3 = [
             (POSTITER_STEP, 'restore after iterating', [lambda x: x.post_iter_step], False),
@@ -4068,7 +4091,8 @@ def run_all_steps(self, run_test_cases):
         if self.cfg['stop'] and self.cfg['stop'] == 'cfg':
             return True
 
-        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
+        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt(),
+                               multi_deps_extensions_only=self.cfg["multi_deps_extensions_only"])
 
         # figure out how many steps will actually be run (not be skipped)
         step_cnt = 0

easybuild/framework/easyconfig/default.py

@@ -167,6 +167,7 @@
     'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES],
     'multi_deps': [{}, "Dict of lists of dependency versions over which to iterate", DEPENDENCIES],
     'multi_deps_load_default': [True, "Load module for first version listed in multi_deps by default", DEPENDENCIES],
+    'multi_deps_extensions_only': [False, "Use multi_deps dependencies only to install extensions", DEPENDENCIES],
     'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES],
     'moddependpaths': [None, "Absolute path(s) to prepend to MODULEPATH before loading dependencies", DEPENDENCIES],
 

easybuild/framework/easyconfig/tweak.py

@@ -385,6 +385,7 @@ def __repr__(self):
         try:
             # obtain temporary filename
             fd, tmpfn = tempfile.mkstemp()
+            os.chmod(tmpfn, 0o644)
             os.close(fd)
 
             # write easyconfig to temporary file

easybuild/toolchains/compiler/cuda.py

@@ -31,6 +31,7 @@
 """
 
 from easybuild.tools.toolchain.compiler import Compiler
+from easybuild.tools.modules import get_software_root
 
 
 TC_CONSTANT_CUDA = "CUDA"
@@ -77,7 +78,9 @@ def __init__(self, *args, **kwargs):
     def _set_compiler_vars(self):
         """Set the compiler variables"""
         # append lib dir paths to LDFLAGS (only if the paths are actually there)
-        root = self.get_software_root('CUDA')[0]
+        root = get_software_root('CUDAcore')
+        if not root:
+            root = self.get_software_root('CUDA')[0]
         self.variables.append_subdirs("LDFLAGS", root, subdirs=["lib64", "lib"])
         super(Cuda, self)._set_compiler_vars()
 

easybuild/toolchains/linalg/flexiblas.py

@@ -36,6 +36,7 @@
 
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
+from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP
 
 
 TC_CONSTANT_FLEXIBLAS = 'FlexiBLAS'
@@ -78,6 +79,12 @@ class FlexiBLAS(LinAlg):
     LAPACK_INCLUDE_DIR = [os.path.join('include', 'flexiblas')]
     LAPACK_FAMILY = TC_CONSTANT_FLEXIBLAS
 
+    def _set_blas_variables(self):
+        """Fix the BLAS_LIB for Intel"""
+        if self.COMPILER_FAMILY == TC_CONSTANT_INTELCOMP:
+            self.BLAS_LIB = ['flexiblas_intel']
+        super(FlexiBLAS, self)._set_blas_variables()
+
     def banned_linked_shared_libs(self):
         """
         List of shared libraries (names, file names, paths) which are

easybuild/tools/build_log.py

@@ -214,6 +214,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.
 
             # mkstemp returns (fd,filename), fd is from os.open, not regular open!
             fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-', dir=tmp_logdir)
+            os.chmod(logfile, 0o644)
             os.close(fd)
 
         fancylogger.logToFile(logfile, max_bytes=0)

easybuild/tools/module_generator.py

@@ -514,6 +514,12 @@ def set_alias(self, key, value):
         """
         raise NotImplementedError
 
+    def set_shell_function(self, key, bashStr, cshStr):
+        """
+        Generate set_shell_function statement in modulefile for the given key/value pair.
+        """
+        raise NotImplementedError
+
     def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None):
         """
         Set generated module as default module
@@ -1005,6 +1011,12 @@ def set_alias(self, key, value):
         # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
         return 'set-alias\t%s\t\t%s\n' % (key, quote_str(value, tcl=True))
 
+    def set_shell_function(self, key, bashStr, cshStr):
+        """
+        Generate set_shell_function statement in modulefile for the given key/value pair.
+        """
+        raise NotImplementedError
+
     def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None):
         """
         Create a .version file inside the package module folder in order to set the default version for TMod
@@ -1190,6 +1202,8 @@ def check_group(self, group, error_msg=None):
 
             error_msg = 'LmodError("' + error_msg + '")'
             res = self.conditional_statement('userInGroup("%s")' % group, error_msg, negative=True)
+            if 'EASYBUILD_CONFIGFILES' in os.environ:
+                res = '' # disable for Compute Canada:
         else:
             warn_msg = "Can't generate robust check in Lua modules for users belonging to group %s. "
             warn_msg += "Lmod version not recent enough (%s), should be >= %s"
@@ -1478,6 +1492,13 @@ def set_alias(self, key, value):
         # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
         return 'set_alias("%s", %s)\n' % (key, quote_str(value))
 
+    def set_shell_function(self, key, bashStr, cshStr):
+        """
+        Generate set_shell_function statement in modulefile for the given key/value pair.
+        """
+        # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
+        return 'set_shell_function("%s", %s, %s)\n' % (key, quote_str(bashStr), quote_str(cshStr))
+
     def set_as_default(self, module_dir_path, module_version, mod_symlink_paths=None):
         """
         Create a symlink named 'default' inside the package's module folder in order to set the default module version

easybuild/tools/module_naming_scheme/utilities.py

@@ -69,8 +69,8 @@ def det_full_ec_version(ec):
                              type(versionprefix).__name__, versionprefix, ec)
 
     versionsuffix = ec.get('versionsuffix', '')
-    if not isinstance(versionsuffix, string_type):
-        raise EasyBuildError("versionsuffix value should be a string, found '%s': %s (full spec: %s)",
+    if not isinstance(versionsuffix, string_type) and versionsuffix is not None:
+        raise EasyBuildError("versionsuffix value should be a string or None, found '%s': %s (full spec: %s)",
                              type(versionsuffix).__name__, versionsuffix, ec)
 
     ecver = ''.join([x for x in [versionprefix, ecver, versionsuffix] if x])

easybuild/tools/options.py

@@ -1874,6 +1874,7 @@ def set_tmpdir(tmpdir=None, raise_error=False):
             raise EasyBuildError("Failed to create path to temporary directory %s: %s", current_tmpdir, err)
 
     _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir)
+    os.chmod(current_tmpdir, 0o755)
 
     for var in ['TMPDIR', 'TEMP', 'TMP']:
         env.setvar(var, current_tmpdir, verbose=False)

easybuild/tools/robot.py

@@ -439,7 +439,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro
                         processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
 
                         # ensure that selected easyconfig provides required dependency
-                        verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs)
+                        #verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs)
 
                         for ec in processed_ecs:
                             if ec not in easyconfigs + additional:

easybuild/tools/toolchain/linalg.py

@@ -92,7 +92,7 @@ def set_variables(self):
         # TODO is link order fully preserved with this order ?
         self._set_blas_variables()
         self._set_lapack_variables()
-        if getattr(self, 'MPI_MODULE_NAME', None):
+        if getattr(self, 'MPI_MODULE_NAME', None) and self.SCALAPACK_LIB is not None:
             self._set_blacs_variables()
             self._set_scalapack_variables()