From 732a4a28a673f97b5700d6a6246531da3102659d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 13 Jan 2025 11:32:49 +0100 Subject: [PATCH 1/4] replace make_module_req_guess with module_load_environment in ICC easyblock --- easybuild/easyblocks/i/icc.py | 154 ++++++++++++++++------------------ 1 file changed, 70 insertions(+), 84 deletions(-) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 167a6646f2d..55bdfb4d407 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -66,9 +66,7 @@ def __init__(self, *args, **kwargs): """Constructor, initialize class variables.""" super(EB_icc, self).__init__(*args, **kwargs) - self.debuggerpath = None - - self.comp_libs_subdir = None + self.comp_libs_subdir = "" # need to make sure version is an actual version # required because of support in SystemCompiler generic easyblock to specify 'system' as version, @@ -92,6 +90,68 @@ def __init__(self, *args, **kwargs): f"Using {self.cfg['components']} instead." ) + # define list of subdirectories in search paths of module load environment + # some of these paths only apply to certain versions, but that doesn't really matter + # existence of paths is checked by module generator before 'prepend-paths' statements are included + + # new directory layout since Intel Parallel Studio XE 2016 + # https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016 + # in recent Intel compiler distributions, the actual binaries are + # in deeper directories, and symlinked in top-level directories + # however, not all binaries are symlinked (e.g. mcpcom is not) + # we only need to include the deeper directories (same as compilervars.sh) + + self.module_load_environment.PATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + 'bin/intel64', + 'ipp/bin/intel64', + 'mpi/intel64/bin', + 'tbb/bin/emt64', + 'tbb/bin/intel64', + )] + # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' + self.module_load_environment.LD_LIBRARY_PATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + 'lib', + 'compiler/lib/intel64', + 'debugger/ipt/intel64/lib', + 'ipp/lib/intel64', + 'mkl/lib/intel64', + 'mpi/intel64', + f"tbb/lib/intel64/{get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64'))}", + )] + # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex + # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 + self.module_load_environment.CPATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + 'daal/include', + 'ipp/include', + 'mkl/include', + 'tbb/include', + )] + self.module_load_environment.CLASSPATH = [os.path.join(self.comp_libs_subdir, 'daal/lib/daal.jar')] + self.module_load_environment.DAALROOT = [os.path.join(self.comp_libs_subdir, 'daal')] + self.module_load_environment.IPPROOT = [os.path.join(self.comp_libs_subdir, 'ipp')] + self.module_load_environment.MANPATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + 'debugger/gdb/intel64/share/man', + 'man/common', + 'man/en_US', + 'share/man', + )] + self.module_load_environment.TBBROOT = [os.path.join(self.comp_libs_subdir, 'tbb')] + + # Debugger requires INTEL_PYTHONHOME, which only allows for a single value + self.debuggerpath = f"debugger_{self.version.split('.')[0]}" + self.module_load_environment.LD_LIBRARY_PATH.extend([ + os.path.join(self.debuggerpath, 'libipt/intel64/lib'), + 'daal/lib/intel64_lin', # out of self.comp_libs_subdir to not break libipt library loading for gdb + ]) + self.module_load_environment.PATH.append( + os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin') + ) + + # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs + self.module_load_environment.LD_LIBRARY_PATH.append(os.path.join(self.comp_libs_subdir, 'lib/intel64')) + + self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH + def sanity_check_step(self): """Custom sanity check paths for icc.""" @@ -118,98 +178,24 @@ def sanity_check_step(self): super(EB_icc, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - def make_module_req_guess(self): + def make_module_step(self, *args, **kwargs): """ - Additional paths to consider for prepend-paths statements in module file + Additional paths for module load environment that are conditional """ - prefix = None - - guesses = super(EB_icc, self).make_module_req_guess() - - # guesses per environment variables - # some of these paths only apply to certain versions, but that doesn't really matter - # existence of paths is checked by module generator before 'prepend-paths' statements are included - guesses.update({ - 'CLASSPATH': ['daal/lib/daal.jar'], - # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex - # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 - 'CPATH': ['daal/include', 'ipp/include', 'mkl/include', 'tbb/include'], - 'DAALROOT': ['daal'], - 'IPPROOT': ['ipp'], - 'LD_LIBRARY_PATH': ['lib'], - 'MANPATH': ['debugger/gdb/intel64/share/man', 'man/common', 'man/en_US', 'share/man'], - 'PATH': [], - 'TBBROOT': ['tbb'], - }) - - # 64-bit toolkit - guesses['PATH'].extend([ - 'bin/intel64', - 'debugger/gdb/intel64/bin', - 'ipp/bin/intel64', - 'mpi/intel64/bin', - 'tbb/bin/emt64', - 'tbb/bin/intel64', - ]) - - # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' - guesses['LD_LIBRARY_PATH'].extend([ - 'compiler/lib/intel64', - 'debugger/ipt/intel64/lib', - 'ipp/lib/intel64', - 'mkl/lib/intel64', - 'mpi/intel64', - 'tbb/lib/intel64/%s' % get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64')), - ]) - - # new directory layout since Intel Parallel Studio XE 2016 - # https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016 - prefix = self.comp_libs_subdir - # Debugger requires INTEL_PYTHONHOME, which only allows for a single value - self.debuggerpath = 'debugger_%s' % self.version.split('.')[0] - - guesses['LD_LIBRARY_PATH'].extend([ - os.path.join(self.debuggerpath, 'libipt/intel64/lib'), - 'daal/lib/intel64_lin', - ]) - - # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs - guesses['LD_LIBRARY_PATH'].append('lib/intel64') - - guesses['LIBRARY_PATH'] = guesses['LD_LIBRARY_PATH'] - - # set debugger path - if self.debuggerpath: - guesses['PATH'].append(os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin')) - - # in recent Intel compiler distributions, the actual binaries are - # in deeper directories, and symlinked in top-level directories - # however, not all binaries are symlinked (e.g. mcpcom is not) - # we only need to include the deeper directories (same as compilervars.sh) - if prefix and os.path.isdir(os.path.join(self.installdir, prefix)): - for key, subdirs in guesses.items(): - guesses[key] = [os.path.join(prefix, subdir) for subdir in subdirs] - - # The for loop above breaks libipt library loading for gdb - this fixes that - guesses['LD_LIBRARY_PATH'].append('daal/lib/intel64_lin') - if self.debuggerpath: - guesses['LD_LIBRARY_PATH'].append(os.path.join(self.debuggerpath, 'libipt/intel64/lib')) - # only set $IDB_HOME if idb exists idb_home_subdir = 'bin/intel64' if os.path.isfile(os.path.join(self.installdir, idb_home_subdir, 'idb')): - guesses['IDB_HOME'] = [idb_home_subdir] + self.module_load_environment.IDB_HOME = [idb_home_subdir] - return guesses + return super().make_module_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): """Additional custom variables for icc: $INTEL_PYTHONHOME.""" txt = super(EB_icc, self).make_module_extra(*args, **kwargs) - if self.debuggerpath: - intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64') - if os.path.isdir(intel_pythonhome): - txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome) + intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64') + if os.path.isdir(intel_pythonhome): + txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome) # on Debian/Ubuntu, /usr/include/x86_64-linux-gnu needs to be included in $CPATH for icc res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False) From 696c3e9dfe8a5658d01392740bc889eec694f050 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 13 Jan 2025 13:23:38 +0100 Subject: [PATCH 2/4] replace make_module_req_guess with module_load_environment in ifort easyblock --- easybuild/easyblocks/i/ifort.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index d2d66adf952..adaf47dbfb7 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -58,6 +58,10 @@ def __init__(self, *args, **kwargs): f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0." ) + # define list of subdirectories in search paths of module load environment + # add additional paths to those of ICC only needed for separate ifort installations + self.module_load_environment.CPATH.append(os.path.join(self.comp_libs_subdir, 'compiler/include')) + def sanity_check_step(self): """Custom sanity check paths for ifort.""" shlib_ext = get_shared_lib_ext() @@ -84,12 +88,3 @@ def sanity_check_step(self): IntelBase.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) - def make_module_req_guess(self): - """ - Additional paths to consider for prepend-paths statements in module file - """ - guesses = super(EB_ifort, self).make_module_req_guess() - # This enables the creation of fortran 2008 bindings in MPI - guesses['CPATH'].append('include') - - return guesses From 2bec777637c007a14b34def7b93c8b48df875874 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 13 Jan 2025 13:23:53 +0100 Subject: [PATCH 3/4] replace make_module_req_guess with module_load_environment in iccifort easyblock --- easybuild/easyblocks/i/iccifort.py | 36 +++++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/easybuild/easyblocks/i/iccifort.py b/easybuild/easyblocks/i/iccifort.py index 1072de00d7b..4b385a37a02 100644 --- a/easybuild/easyblocks/i/iccifort.py +++ b/easybuild/easyblocks/i/iccifort.py @@ -41,6 +41,26 @@ class EB_iccifort(EB_ifort, EB_icc): Class that can be used to install iccifort """ + def __init__(self, *args, **kwargs): + """Constructor, initialize class variables.""" + super().__init__(*args, **kwargs) + + # Exclude 'compiler/include' for CPATH, including it causes problems, e.g. with complex.h and std::complex + # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 + self.module_load_environment.CPATH.remove(os.path.join(self.comp_libs_subdir, 'compiler/include')) + + # remove entries from LIBRARY_PATH that icc and co already know about at compile time + # only do this for iccifort merged installations so that icc can still find ifort + # libraries and vice versa for split installations + if self.comp_libs_subdir: + excluded_library_paths = [os.path.join(self.comp_libs_subdir, path) for path in ( + 'compiler/lib/intel64', + 'lib', + 'lib/intel64', + )] + for excluded_path in excluded_library_paths: + self.module_load_environment.LIBRARY_PATH.remove(excluded_path) + def sanity_check_step(self): """Custom sanity check paths for iccifort.""" EB_icc.sanity_check_step(self) @@ -56,19 +76,3 @@ def make_module_extra(self): txt += self.module_generator.set_environment('EBVERSIONIFORT', self.version) return txt - - def make_module_req_guess(self): - # Use EB_icc because its make_module_req_guess deliberately omits 'include' for CPATH: - # including it causes problems, e.g. with complex.h and std::complex - # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 - # whereas EB_ifort adds 'include' but that's only needed if icc and ifort are separate - guesses = EB_icc.make_module_req_guess(self) - - # remove entries from LIBRARY_PATH that icc and co already know about at compile time - # only do this for iccifort merged installations so that icc can still find ifort - # libraries and vice versa for split installations - if self.comp_libs_subdir is not None: - compiler_library_paths = [os.path.join(self.comp_libs_subdir, p) - for p in ('lib', 'compiler/lib/intel64', 'lib/ia32', 'lib/intel64')] - guesses['LIBRARY_PATH'] = [p for p in guesses['LIBRARY_PATH'] if p not in compiler_library_paths] - return guesses From 3df7f2eb89f954f7d8845a36188ca97a98ddcc39 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Feb 2025 14:01:21 +0100 Subject: [PATCH 4/4] remove empty line at end of ifort.py --- easybuild/easyblocks/i/ifort.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index 414d16b3ed3..87c986d5dd6 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -87,4 +87,3 @@ def sanity_check_step(self): custom_commands = ["which ifort"] IntelBase.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) -