Skip to content

adopt module_load_environment: icc, ifort, iccifort #3550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 5 commits into from
Feb 8, 2025
Merged

adopt module_load_environment: icc, ifort, iccifort #3550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
732a4a2
replace make_module_req_guess with module_load_environment in ICC eas…
lexming Jan 13, 2025
696c3e9
replace make_module_req_guess with module_load_environment in ifort e…
lexming Jan 13, 2025
2bec777
replace make_module_req_guess with module_load_environment in iccifor…
lexming Jan 13, 2025
5266247
Merge branch '5.0.x' into modload-iccifort
boegel Feb 5, 2025
3df7f2e
remove empty line at end of ifort.py
boegel Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 70 additions & 84 deletions easybuild/easyblocks/i/icc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,7 @@ def __init__(self, *args, **kwargs):
"""Constructor, initialize class variables."""
super(EB_icc, self).__init__(*args, **kwargs)

self.debuggerpath = None

self.comp_libs_subdir = None
self.comp_libs_subdir = ""

# need to make sure version is an actual version
# required because of support in SystemCompiler generic easyblock to specify 'system' as version,
Expand All @@ -92,6 +90,68 @@ def __init__(self, *args, **kwargs):
f"Using {self.cfg['components']} instead."
)

# define list of subdirectories in search paths of module load environment
# some of these paths only apply to certain versions, but that doesn't really matter
# existence of paths is checked by module generator before 'prepend-paths' statements are included

# new directory layout since Intel Parallel Studio XE 2016
# https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016
# in recent Intel compiler distributions, the actual binaries are
# in deeper directories, and symlinked in top-level directories
# however, not all binaries are symlinked (e.g. mcpcom is not)
# we only need to include the deeper directories (same as compilervars.sh)

self.module_load_environment.PATH = [os.path.join(self.comp_libs_subdir, path) for path in (
'bin/intel64',
'ipp/bin/intel64',
'mpi/intel64/bin',
'tbb/bin/emt64',
'tbb/bin/intel64',
)]
# in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH'
self.module_load_environment.LD_LIBRARY_PATH = [os.path.join(self.comp_libs_subdir, path) for path in (
'lib',
'compiler/lib/intel64',
'debugger/ipt/intel64/lib',
'ipp/lib/intel64',
'mkl/lib/intel64',
'mpi/intel64',
f"tbb/lib/intel64/{get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64'))}",
)]
# 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex
# cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378
self.module_load_environment.CPATH = [os.path.join(self.comp_libs_subdir, path) for path in (
'daal/include',
'ipp/include',
'mkl/include',
'tbb/include',
)]
self.module_load_environment.CLASSPATH = [os.path.join(self.comp_libs_subdir, 'daal/lib/daal.jar')]
self.module_load_environment.DAALROOT = [os.path.join(self.comp_libs_subdir, 'daal')]
self.module_load_environment.IPPROOT = [os.path.join(self.comp_libs_subdir, 'ipp')]
self.module_load_environment.MANPATH = [os.path.join(self.comp_libs_subdir, path) for path in (
'debugger/gdb/intel64/share/man',
'man/common',
'man/en_US',
'share/man',
)]
self.module_load_environment.TBBROOT = [os.path.join(self.comp_libs_subdir, 'tbb')]

# Debugger requires INTEL_PYTHONHOME, which only allows for a single value
self.debuggerpath = f"debugger_{self.version.split('.')[0]}"
self.module_load_environment.LD_LIBRARY_PATH.extend([
os.path.join(self.debuggerpath, 'libipt/intel64/lib'),
'daal/lib/intel64_lin', # out of self.comp_libs_subdir to not break libipt library loading for gdb
])
self.module_load_environment.PATH.append(
os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin')
)

# 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs
self.module_load_environment.LD_LIBRARY_PATH.append(os.path.join(self.comp_libs_subdir, 'lib/intel64'))

self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH

def sanity_check_step(self):
"""Custom sanity check paths for icc."""

Expand All @@ -118,98 +178,24 @@ def sanity_check_step(self):

super(EB_icc, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)

def make_module_req_guess(self):
def make_module_step(self, *args, **kwargs):
"""
Additional paths to consider for prepend-paths statements in module file
Additional paths for module load environment that are conditional
"""
prefix = None

guesses = super(EB_icc, self).make_module_req_guess()

# guesses per environment variables
# some of these paths only apply to certain versions, but that doesn't really matter
# existence of paths is checked by module generator before 'prepend-paths' statements are included
guesses.update({
'CLASSPATH': ['daal/lib/daal.jar'],
# 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex
# cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378
'CPATH': ['daal/include', 'ipp/include', 'mkl/include', 'tbb/include'],
'DAALROOT': ['daal'],
'IPPROOT': ['ipp'],
'LD_LIBRARY_PATH': ['lib'],
'MANPATH': ['debugger/gdb/intel64/share/man', 'man/common', 'man/en_US', 'share/man'],
'PATH': [],
'TBBROOT': ['tbb'],
})

# 64-bit toolkit
guesses['PATH'].extend([
'bin/intel64',
'debugger/gdb/intel64/bin',
'ipp/bin/intel64',
'mpi/intel64/bin',
'tbb/bin/emt64',
'tbb/bin/intel64',
])

# in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH'
guesses['LD_LIBRARY_PATH'].extend([
'compiler/lib/intel64',
'debugger/ipt/intel64/lib',
'ipp/lib/intel64',
'mkl/lib/intel64',
'mpi/intel64',
'tbb/lib/intel64/%s' % get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64')),
])

# new directory layout since Intel Parallel Studio XE 2016
# https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016
prefix = self.comp_libs_subdir
# Debugger requires INTEL_PYTHONHOME, which only allows for a single value
self.debuggerpath = 'debugger_%s' % self.version.split('.')[0]

guesses['LD_LIBRARY_PATH'].extend([
os.path.join(self.debuggerpath, 'libipt/intel64/lib'),
'daal/lib/intel64_lin',
])

# 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs
guesses['LD_LIBRARY_PATH'].append('lib/intel64')

guesses['LIBRARY_PATH'] = guesses['LD_LIBRARY_PATH']

# set debugger path
if self.debuggerpath:
guesses['PATH'].append(os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin'))

# in recent Intel compiler distributions, the actual binaries are
# in deeper directories, and symlinked in top-level directories
# however, not all binaries are symlinked (e.g. mcpcom is not)
# we only need to include the deeper directories (same as compilervars.sh)
if prefix and os.path.isdir(os.path.join(self.installdir, prefix)):
for key, subdirs in guesses.items():
guesses[key] = [os.path.join(prefix, subdir) for subdir in subdirs]

# The for loop above breaks libipt library loading for gdb - this fixes that
guesses['LD_LIBRARY_PATH'].append('daal/lib/intel64_lin')
if self.debuggerpath:
guesses['LD_LIBRARY_PATH'].append(os.path.join(self.debuggerpath, 'libipt/intel64/lib'))

# only set $IDB_HOME if idb exists
idb_home_subdir = 'bin/intel64'
if os.path.isfile(os.path.join(self.installdir, idb_home_subdir, 'idb')):
guesses['IDB_HOME'] = [idb_home_subdir]
self.module_load_environment.IDB_HOME = [idb_home_subdir]

return guesses
return super().make_module_step(*args, **kwargs)

def make_module_extra(self, *args, **kwargs):
"""Additional custom variables for icc: $INTEL_PYTHONHOME."""
txt = super(EB_icc, self).make_module_extra(*args, **kwargs)

if self.debuggerpath:
intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64')
if os.path.isdir(intel_pythonhome):
txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome)
intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64')
if os.path.isdir(intel_pythonhome):
txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome)

# on Debian/Ubuntu, /usr/include/x86_64-linux-gnu needs to be included in $CPATH for icc
res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False)
Expand Down
36 changes: 20 additions & 16 deletions easybuild/easyblocks/i/iccifort.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,26 @@ class EB_iccifort(EB_ifort, EB_icc):
Class that can be used to install iccifort
"""

def __init__(self, *args, **kwargs):
"""Constructor, initialize class variables."""
super().__init__(*args, **kwargs)

# Exclude 'compiler/include' for CPATH, including it causes problems, e.g. with complex.h and std::complex
# cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378
self.module_load_environment.CPATH.remove(os.path.join(self.comp_libs_subdir, 'compiler/include'))

# remove entries from LIBRARY_PATH that icc and co already know about at compile time
# only do this for iccifort merged installations so that icc can still find ifort
# libraries and vice versa for split installations
if self.comp_libs_subdir:
excluded_library_paths = [os.path.join(self.comp_libs_subdir, path) for path in (
'compiler/lib/intel64',
'lib',
'lib/intel64',
)]
for excluded_path in excluded_library_paths:
self.module_load_environment.LIBRARY_PATH.remove(excluded_path)

def sanity_check_step(self):
"""Custom sanity check paths for iccifort."""
EB_icc.sanity_check_step(self)
Expand All @@ -56,19 +76,3 @@ def make_module_extra(self):
txt += self.module_generator.set_environment('EBVERSIONIFORT', self.version)

return txt

def make_module_req_guess(self):
# Use EB_icc because its make_module_req_guess deliberately omits 'include' for CPATH:
# including it causes problems, e.g. with complex.h and std::complex
# cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378
# whereas EB_ifort adds 'include' but that's only needed if icc and ifort are separate
guesses = EB_icc.make_module_req_guess(self)

# remove entries from LIBRARY_PATH that icc and co already know about at compile time
# only do this for iccifort merged installations so that icc can still find ifort
# libraries and vice versa for split installations
if self.comp_libs_subdir is not None:
compiler_library_paths = [os.path.join(self.comp_libs_subdir, p)
for p in ('lib', 'compiler/lib/intel64', 'lib/ia32', 'lib/intel64')]
guesses['LIBRARY_PATH'] = [p for p in guesses['LIBRARY_PATH'] if p not in compiler_library_paths]
return guesses
14 changes: 4 additions & 10 deletions easybuild/easyblocks/i/ifort.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ def __init__(self, *args, **kwargs):
f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0."
)

# define list of subdirectories in search paths of module load environment
# add additional paths to those of ICC only needed for separate ifort installations
self.module_load_environment.CPATH.append(os.path.join(self.comp_libs_subdir, 'compiler/include'))

def sanity_check_step(self):
"""Custom sanity check paths for ifort."""
shlib_ext = get_shared_lib_ext()
Expand All @@ -83,13 +87,3 @@ def sanity_check_step(self):
custom_commands = ["which ifort"]

IntelBase.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands)

def make_module_req_guess(self):
"""
Additional paths to consider for prepend-paths statements in module file
"""
guesses = super(EB_ifort, self).make_module_req_guess()
# This enables the creation of fortran 2008 bindings in MPI
guesses['CPATH'].append('include')

return guesses