diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 04addf4f536..e0ce3f11513 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -44,6 +44,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file from easybuild.tools.run import run_cmd from vsc.utils import fancylogger @@ -81,6 +82,9 @@ LICENSE_FILE_NAME = 'ACTIVATION_LICENSE_FILE' # since icc/ifort v2013_sp1, impi v4.1.1, imkl v11.1 LICENSE_FILE_NAME_2012 = 'PSET_LICENSE_FILE' # previous license file parameter used in older versions +COMP_ALL = 'ALL' +COMP_DEFAULTS = 'DEFAULTS' + class IntelBase(EasyBlock): """ @@ -100,6 +104,8 @@ def __init__(self, *args, **kwargs): common_tmp_dir = os.path.dirname(tempfile.gettempdir()) # common tmp directory, same across nodes self.home_subdir_local = os.path.join(common_tmp_dir, os.getenv('USER'), 'easybuild_intel') + self.install_components = None + @staticmethod def extra_options(extra_vars=None): extra_vars = EasyBlock.extra_options(extra_vars) @@ -111,10 +117,38 @@ def extra_options(extra_vars=None): # disables TMP_PATH env and command line option 'usetmppath': [False, "Use temporary path for installation", CUSTOM], 'm32': [False, "Enable 32-bit toolchain", CUSTOM], + 'components': [None, "List of components to install", CUSTOM], }) return extra_vars + def parse_components_list(self): + """parse the regex in the components extra_options and select the matching components + from the mediaconfig.xml file in the install dir""" + + mediaconfigpath = os.path.join(self.cfg['start_dir'], 'pset', 'mediaconfig.xml') + if not os.path.isfile(mediaconfigpath): + raise EasyBuildError("Could not find %s to find list of components." % mediaconfigpath) + + mediaconfig = read_file(mediaconfigpath) + available_components = re.findall("(?P[^<]+)", mediaconfig, re.M) + self.log.debug("Intel components found: %s" % available_components) + self.log.debug("Using regex list: %s" % self.cfg['components']) + + if COMP_ALL in self.cfg['components'] or COMP_DEFAULTS in self.cfg['components']: + if len(self.cfg['components']) == 1: + self.install_components = self.cfg['components'] + else: + raise EasyBuildError("If you specify %s as components, you cannot specify anything else: %s", + ' or '.join([COMP_ALL, COMP_DEFAULTS]), self.cfg['components']) + else: + self.install_components = [] + for comp_regex in self.cfg['components']: + comps = [comp for comp in available_components if re.match(comp_regex, comp)] + self.install_components.extend(comps) + + self.log.debug("Components to install: %s" % self.install_components) + def clean_home_subdir(self): """Remove contents of (local) 'intel' directory home subdir, where stuff is cached.""" @@ -254,6 +288,12 @@ def configure_step(self): # clean home directory self.clean_home_subdir() + # determine list of components, based on 'components' easyconfig parameter (if specified) + if self.cfg['components']: + self.parse_components_list() + else: + self.log.debug("No components specified") + def build_step(self): """Binary installation files, so no building.""" pass @@ -277,7 +317,8 @@ def install_step(self, silent_cfg_names_map=None, silent_cfg_extras=None): if lic_activation in lic_file_server_activations: lic_file_entry = "%(license_file_name)s=%(license_file)s" elif not self.cfg['license_activation'] in other_activations: - raise EasyBuildError("Unknown type of activation specified: %s (known :%s)", lic_activation, ACTIVATION_TYPES) + raise EasyBuildError("Unknown type of activation specified: %s (known :%s)", + lic_activation, ACTIVATION_TYPES) silent = '\n'.join([ "%(activation_name)s=%(activation)s", @@ -298,6 +339,16 @@ def install_step(self, silent_cfg_names_map=None, silent_cfg_extras=None): 'install_mode_name': silent_cfg_names_map.get('install_mode_name', INSTALL_MODE_NAME_2015), } + if self.install_components is not None: + if len(self.install_components) == 1 and self.install_components[0] in [COMP_ALL, COMP_DEFAULTS]: + # no quotes should be used for ALL or DEFAULTS + silent += 'COMPONENTS=%s\n' % self.install_components[0] + elif self.install_components: + # a list of components is specified (needs quotes) + silent += 'COMPONENTS="' + ';'.join(self.install_components) + '"\n' + else: + raise EasyBuildError("Empty list of matching components obtained via %s", self.cfg['components']) + if silent_cfg_extras is not None: if isinstance(silent_cfg_extras, dict): silent += '\n'.join("%s=%s" % (key, value) for (key, value) in silent_cfg_extras.iteritems()) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 6f863c56021..a6a5cc96846 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -30,13 +30,17 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) +@author: Ward Poelmans (Ghent University) +@author: Fokko Masselink """ import os import re from distutils.version import LooseVersion -from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 +from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, COMP_ALL +from easybuild.easyblocks.generic.intelbase import LICENSE_FILE_NAME_2012 +from easybuild.easyblocks.t.tbb import get_tbb_gccprefix from easybuild.tools.run import run_cmd @@ -58,6 +62,18 @@ class EB_icc(IntelBase): - will fail for all older versions (due to newer silent installer) """ + def __init__(self, *args, **kwargs): + """Constructor, initialize class variables.""" + super(EB_icc, self).__init__(*args, **kwargs) + + self.debuggerpath = None + + if LooseVersion(self.version) >= LooseVersion('2016') and self.cfg['components'] is None: + # we need to use 'ALL' by default, using 'DEFAULTS' results in key things not being installed (e.g. bin/icc) + self.cfg['components'] = [COMP_ALL] + self.log.debug("Nothing specified for components, but required for version 2016, using %s instead", + self.cfg['components']) + def install_step(self): """ Actual installation @@ -79,79 +95,128 @@ def install_step(self): def sanity_check_step(self): """Custom sanity check paths for icc.""" - binprefix = "bin/intel64" - libprefix = "lib/intel64/lib" - if LooseVersion(self.version) >= LooseVersion("2011"): - if LooseVersion(self.version) <= LooseVersion("2011.3.174"): - binprefix = "bin" - elif LooseVersion(self.version) >= LooseVersion("2013_sp1"): - binprefix = "bin" - libprefix = "lib/intel64/lib" + binprefix = 'bin/intel64' + libprefix = 'lib/intel64' + if LooseVersion(self.version) >= LooseVersion('2011'): + if LooseVersion(self.version) <= LooseVersion('2011.3.174'): + binprefix = 'bin' + elif LooseVersion(self.version) >= LooseVersion('2013_sp1'): + binprefix = 'bin' else: - libprefix = "compiler/lib/intel64/lib" + libprefix = 'compiler/lib/intel64' - binfiles = ["icc", "icpc"] - if LooseVersion(self.version) < LooseVersion("2014"): - binfiles += ["idb"] + binfiles = ['icc', 'icpc'] + if LooseVersion(self.version) < LooseVersion('2014'): + binfiles += ['idb'] + + binaries = [os.path.join(binprefix, f) for f in binfiles] + libraries = [os.path.join(libprefix, 'lib%s' % l) for l in ['iomp5.a', 'iomp5.so']] + sanity_check_files = binaries + libraries + if LooseVersion(self.version) > LooseVersion('2015'): + sanity_check_files.append('include/omp.h') custom_paths = { - 'files': ["%s/%s" % (binprefix, x) for x in binfiles] + - ["%s%s" % (libprefix, x) for x in ["iomp5.a", "iomp5.so"]], + 'files': sanity_check_files, 'dirs': [], } super(EB_icc, self).sanity_check_step(custom_paths=custom_paths) def make_module_req_guess(self): - """Customize paths to check and add in environment. """ + Additional paths to consider for prepend-paths statements in module file + """ + prefix = None + + # guesses per environment variables + # some of these paths only apply to certain versions, but that doesn't really matter + # existence of paths is checked by module generator before 'prepend-paths' statements are included + guesses = { + 'CLASSPATH': ['daal/lib/daal.jar'], + # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex + # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 + 'CPATH': ['daal/include', 'ipp/include', 'mkl/include', 'tbb/include'], + 'DAALROOT': ['daal'], + 'IDB_HOME': ['bin/intel64'], + 'IPPROOT': ['ipp'], + 'LD_LIBRARY_PATH': ['lib'], + 'LIBRARY_PATH': ['lib'], + 'MANPATH': ['debugger/gdb/intel64/share/man', 'man', 'man/common', 'man/en_US', 'share/man'], + 'PATH': ['bin'], + 'TBBROOT': ['tbb'], + } + if self.cfg['m32']: # 32-bit toolchain - dirmap = { - 'PATH': ['bin', 'bin/ia32', 'tbb/bin/ia32'], - 'LD_LIBRARY_PATH': ['lib', 'lib/ia32'], - 'LIBRARY_PATH': ['lib', 'lib/ia32'], - 'MANPATH': ['man', 'share/man', 'man/en_US'], - 'IDB_HOME': ['bin/intel64'] - } + guesses['PATH'].extend(['bin/ia32', 'tbb/bin/ia32']) + # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' + guesses['LD_LIBRARY_PATH'].append('lib/ia32') + else: - # 64-bit toolit - dirmap = { - 'PATH': ['bin', 'bin/intel64', 'tbb/bin/emt64'], - 'LD_LIBRARY_PATH': ['lib', 'lib/intel64'], - 'LIBRARY_PATH': ['lib', 'lib/intel64'], - 'MANPATH': ['man', 'share/man', 'man/en_US'], - 'IDB_HOME': ['bin/intel64'] - } + # 64-bit toolkit + guesses['PATH'].extend([ + 'bin/intel64', + 'debugger/gdb/intel64/bin', + 'ipp/bin/intel64', + 'mpi/intel64/bin', + 'tbb/bin/emt64', + 'tbb/bin/intel64', + ]) + + # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' + guesses['LD_LIBRARY_PATH'].extend([ + 'compiler/lib/intel64', + 'debugger/ipt/intel64/lib', + 'ipp/lib/intel64', + 'mkl/lib/intel64', + 'mpi/intel64', + 'tbb/lib/intel64/%s' % get_tbb_gccprefix(), + ]) + + if LooseVersion(self.version) < LooseVersion('2016'): + prefix = 'composer-xe-%s' % self.version + + # debugger is dependent on $INTEL_PYTHONHOME since version 2015 and newer + if LooseVersion(self.version) >= LooseVersion('2015'): + self.debuggerpath = os.path.join('composer-xe-%s' % self.version.split('.')[0], 'debugger') + + else: + # new directory layout for Intel Parallel Studio XE 2016 + # https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016 + prefix = 'compilers_and_libraries_%s/linux' % self.version + # Debugger requires INTEL_PYTHONHOME, which only allows for a single value + self.debuggerpath = 'debugger_%s' % self.version.split('.')[0] + + guesses['LD_LIBRARY_PATH'].extend([ + os.path.join(self.debuggerpath, 'libipt/intel64/lib'), + 'daal/lib/intel64_lin', + ]) + + # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs + guesses['LD_LIBRARY_PATH'].append('lib/intel64') + + guesses['LIBRARY_PATH'] = guesses['LD_LIBRARY_PATH'] + + # set debugger path + if self.debuggerpath: + guesses['PATH'].append(os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin')) # in recent Intel compiler distributions, the actual binaries are # in deeper directories, and symlinked in top-level directories # however, not all binaries are symlinked (e.g. mcpcom is not) - if os.path.isdir("%s/composerxe-%s" % (self.installdir, self.version)): - prefix = "composerxe-%s" % self.version - oldmap = dirmap - dirmap = {} - for k, vs in oldmap.items(): - dirmap[k] = [] - if k == "LD_LIBRARY_PATH": - prefix = "composerxe-%s/compiler" % self.version - else: - prefix = "composerxe-%s" % self.version - for v in vs: - v2 = "%s/%s" % (prefix, v) - dirmap[k].append(v2) - - elif os.path.isdir("%s/compiler" % (self.installdir)): - prefix = "compiler" - oldmap = dirmap - dirmap = {} - for k, vs in oldmap.items(): - dirmap[k] = [] - prefix = '' - if k == "LD_LIBRARY_PATH": - prefix = "compiler/" - for v in vs: - v2 = "%s%s" % (prefix, v) - dirmap[k].append(v2) - - return dirmap + if prefix and os.path.isdir(os.path.join(self.installdir, prefix)): + for key, subdirs in guesses.items(): + guesses[key].extend([os.path.join(prefix, subdir) for subdir in subdirs]) + + return guesses + + def make_module_extra(self): + """Additional custom variables for icc: $INTEL_PYTHONHOME.""" + txt = super(EB_icc, self).make_module_extra() + + if self.debuggerpath: + intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64') + if os.path.isdir(intel_pythonhome): + txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome) + + return txt diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index 2cd51c75104..7e1ce53cde6 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -30,8 +30,10 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) +@author: Ward Poelmans (Ghent University) """ +import os from distutils.version import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase @@ -48,25 +50,24 @@ class EB_ifort(EB_icc, IntelBase): def sanity_check_step(self): """Custom sanity check paths for ifort.""" - binprefix = "bin/intel64" - libprefix = "lib/intel64/lib" - if LooseVersion(self.version) >= LooseVersion("2011"): - if LooseVersion(self.version) <= LooseVersion("2011.3.174"): - binprefix = "bin" - elif LooseVersion(self.version) >= LooseVersion("2013_sp1"): - binprefix = "bin" - libprefix = "lib/intel64/lib" + binprefix = 'bin/intel64' + libprefix = 'lib/intel64' + if LooseVersion(self.version) >= LooseVersion('2011'): + if LooseVersion(self.version) <= LooseVersion('2011.3.174'): + binprefix = 'bin' + elif LooseVersion(self.version) >= LooseVersion('2013_sp1'): + binprefix = 'bin' else: - libprefix = "compiler/lib/intel64/lib" + libprefix = 'compiler/lib/intel64' - bins = ["ifort"] + bins = ['ifort'] if LooseVersion(self.version) < LooseVersion('2013'): # idb is not shipped with ifort anymore in 2013.x versions (it is with icc though) - bins.append("idb") + bins.append('idb') custom_paths = { - 'files': ["%s/%s" % (binprefix, x) for x in bins] + - ["%s%s" % (libprefix, x) for x in ["ifcore.a", "ifcore.so", "iomp5.a", "iomp5.so"]], + 'files': [os.path.join(binprefix, x) for x in bins] + + [os.path.join(libprefix, 'lib%s' % x) for x in ['ifcore.a', 'ifcore.so', 'iomp5.a', 'iomp5.so']], 'dirs': [], } diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index e0b13888600..c5dbcbf4eeb 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Ward Poelmans (Ghent University) +@author: Lumir Jasiok (IT4Innovations) """ import itertools @@ -320,16 +321,16 @@ def sanity_check_step(self): mklfiles = None mkldirs = None ver = LooseVersion(self.version) - libs = ["libmkl_core.so", "libmkl_gnu_thread.so", "libmkl_intel_thread.so", "libmkl_sequential.so"] - extralibs = ["libmkl_blacs_intelmpi_%(suff)s.so", "libmkl_scalapack_%(suff)s.so"] + libs = ['libmkl_core.so', 'libmkl_gnu_thread.so', 'libmkl_intel_thread.so', 'libmkl_sequential.so'] + extralibs = ['libmkl_blacs_intelmpi_%(suff)s.so', 'libmkl_scalapack_%(suff)s.so'] if self.cfg['interfaces']: - compsuff = '_intel' - if get_software_root('icc') is None: - if get_software_root('GCC'): - compsuff = '_gnu' - else: - raise EasyBuildError("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.") + compsuff = '_intel' + if get_software_root('icc') is None: + if get_software_root('GCC'): + compsuff = '_gnu' + else: + raise EasyBuildError("Not using Intel/GCC, don't know compiler suffix for FFTW libraries.") precs = ['_double', '_single'] if ver < LooseVersion('11'): @@ -357,25 +358,27 @@ def sanity_check_step(self): if self.cfg['m32']: raise EasyBuildError("Sanity check for 32-bit not implemented yet for IMKL v%s (>= 10.3)", self.version) else: - mkldirs = ["bin", "mkl/bin", "mkl/bin/intel64", "mkl/lib/intel64", "mkl/include"] + mkldirs = ['bin', 'mkl/bin', 'mkl/lib/intel64', 'mkl/include'] + if ver < LooseVersion('11.3'): + mkldirs.append('mkl/bin/intel64') libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']] - mklfiles = ["mkl/lib/intel64/libmkl.so", "mkl/include/mkl.h"] + \ - ["mkl/lib/intel64/%s" % lib for lib in libs] + mklfiles = ['mkl/lib/intel64/libmkl.so', 'mkl/include/mkl.h'] + \ + ['mkl/lib/intel64/%s' % lib for lib in libs] if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'): - mkldirs += ["compiler/lib/intel64"] + mkldirs += ['compiler/lib/intel64'] else: - mkldirs += ["lib/intel64"] + mkldirs += ['lib/intel64'] else: if self.cfg['m32']: - mklfiles = ["lib/32/libmkl.so", "include/mkl.h"] + \ - ["lib/32/%s" % lib for lib in libs] - mkldirs = ["lib/32", "include/32", "interfaces"] + mklfiles = ['lib/32/libmkl.so', 'include/mkl.h'] + \ + ['lib/32/%s' % lib for lib in libs] + mkldirs = ['lib/32', 'include/32', 'interfaces'] else: libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']] - mklfiles = ["lib/em64t/libmkl.so", "include/mkl.h"] + \ - ["lib/em64t/%s" % lib for lib in libs] - mkldirs = ["lib/em64t", "include/em64t", "interfaces"] + mklfiles = ['lib/em64t/libmkl.so', 'include/mkl.h'] + \ + ['lib/em64t/%s' % lib for lib in libs] + mkldirs = ['lib/em64t', 'include/em64t', 'interfaces'] custom_paths = { 'files': mklfiles, diff --git a/easybuild/easyblocks/t/tbb.py b/easybuild/easyblocks/t/tbb.py index 23df1f3d617..2d399124794 100644 --- a/easybuild/easyblocks/t/tbb.py +++ b/easybuild/easyblocks/t/tbb.py @@ -33,15 +33,38 @@ @author: Lumir Jasiok (IT4Innovations) """ +import glob import os import shutil -import glob from distutils.version import LooseVersion from easybuild.easyblocks.generic.intelbase import INSTALL_MODE_NAME_2015, INSTALL_MODE_2015 from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.systemtools import get_platform_name +from easybuild.tools.modules import get_software_version +from easybuild.tools.systemtools import get_gcc_version, get_platform_name + + +def get_tbb_gccprefix(): + """ + Find the correct gcc version for the lib dir of TBB + """ + # using get_software_version('GCC') won't work, while the compiler toolchain is dummy:dummy, which does not + # load dependencies. + gccversion = get_software_version('GCC') + # manual approach to at least have the system version of gcc + if not gccversion: + gccversion = get_gcc_version() + + # TBB directory structure + # https://www.threadingbuildingblocks.org/docs/help/tbb_userguide/Linux_OS.htm + tbb_gccprefix = 'gcc4.4' # gcc version 4.4 or higher that may or may not support exception_ptr + if gccversion: + gccversion = LooseVersion(gccversion) + if gccversion >= LooseVersion("4.1") and gccversion < LooseVersion("4.4"): + tbb_gccprefix = 'gcc4.1' # gcc version number between 4.1 and 4.4 that do not support exception_ptr + + return tbb_gccprefix class EB_tbb(IntelBase): @@ -85,21 +108,19 @@ def install_step(self): super(EB_tbb, self).install_step(silent_cfg_names_map=silent_cfg_names_map, silent_cfg_extras=silent_cfg_extras) - # save libdir + # determine libdir os.chdir(self.installdir) if LooseVersion(self.version) < LooseVersion('4.1.0'): libglob = 'tbb/lib/intel64/cc*libc*_kernel*' + libs = sorted(glob.glob(libglob), key=LooseVersion) + if len(libs): + # take the last one, should be ordered by cc version + # we're only interested in the last bit + libdir = libs[-1].split('/')[-1] + else: + raise EasyBuildError("No libs found using %s in %s", libglob, self.installdir) else: - libglob = 'tbb/lib/intel64/gcc*' - libs = sorted(glob.glob(libglob), key=LooseVersion) - if len(libs): - libdir = libs[-1] # take the last one, should be ordered by cc get_version. - # we're only interested in the last bit - libdir = libdir.split('/')[-1] - else: - raise EasyBuildError("No libs found using %s in %s", libglob, self.installdir) - self.libdir = libdir - + libdir = 'tbb/lib/intel64/%s' % get_tbb_gccprefix() self.libpath = os.path.join('tbb', 'libs', 'intel64', libdir) self.log.debug("self.libpath: %s" % self.libpath)