add custom easyblock to support installation of new Intel compilers (v2021.x, oneAPI)

easybuild/easyblocks/generic/bundle.py

@@ -90,6 +90,9 @@ def __init__(self, *args, **kwargs):
 
             comp_cfg = self.cfg.copy()
 
+            comp_cfg['name'] = comp_name
+            comp_cfg['version'] = comp_version
+
             easyblock = comp_specs.get('easyblock') or self.cfg['default_easyblock']
             if easyblock is None:
                 raise EasyBuildError("No easyblock specified for component %s v%s", comp_cfg['name'],
@@ -103,8 +106,6 @@ def __init__(self, *args, **kwargs):
             extra_opts = comp_cfg.easyblock.extra_options()
             comp_cfg.extend_params(copy.deepcopy(extra_opts))
 
-            comp_cfg['name'] = comp_name
-            comp_cfg['version'] = comp_version
             comp_cfg.generate_template_values()
 
             # do not inherit easyblock to use from parent (since that would result in an infinite loop in install_step)

easybuild/easyblocks/generic/intelbase.py

@@ -38,6 +38,7 @@
 import os
 import re
 import shutil
+import stat
 import tempfile
 from distutils.version import LooseVersion
 
@@ -46,7 +47,8 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.types import ensure_iterable_license_specs
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import find_flexlm_license, mkdir, read_file, remove_file, write_file
+from easybuild.tools.filetools import adjust_permissions, find_flexlm_license
+from easybuild.tools.filetools import mkdir, read_file, remove_file, write_file
 from easybuild.tools.run import run_cmd
 
 
@@ -193,18 +195,18 @@ def parse_components_list(self):
 
     def clean_home_subdir(self):
         """Remove contents of (local) 'intel' directory home subdir, where stuff is cached."""
-
-        self.log.debug("Cleaning up %s..." % self.home_subdir_local)
-        try:
-            for tree in os.listdir(self.home_subdir_local):
-                self.log.debug("... removing %s subtree" % tree)
-                path = os.path.join(self.home_subdir_local, tree)
-                if os.path.isfile(path) or os.path.islink(path):
-                    remove_file(path)
-                else:
-                    shutil.rmtree(path)
-        except OSError as err:
-            raise EasyBuildError("Cleaning up intel dir %s failed: %s", self.home_subdir_local, err)
+        if os.path.exists(self.home_subdir_local):
+            self.log.debug("Cleaning up %s..." % self.home_subdir_local)
+            try:
+                for tree in os.listdir(self.home_subdir_local):
+                    self.log.debug("... removing %s subtree" % tree)
+                    path = os.path.join(self.home_subdir_local, tree)
+                    if os.path.isfile(path) or os.path.islink(path):
+                        remove_file(path)
+                    else:
+                        shutil.rmtree(path)
+            except OSError as err:
+                raise EasyBuildError("Cleaning up intel dir %s failed: %s", self.home_subdir_local, err)
 
     def setup_local_home_subdir(self):
         """
@@ -301,8 +303,8 @@ def build_step(self):
         """Binary installation files, so no building."""
         pass
 
-    def install_step(self, silent_cfg_names_map=None, silent_cfg_extras=None):
-        """Actual installation
+    def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None):
+        """Actual installation for versions prior to 2021.x
 
         - create silent cfg file
         - set environment parameters
@@ -407,6 +409,55 @@ def install_step(self, silent_cfg_names_map=None, silent_cfg_extras=None):
 
         return run_cmd(cmd, log_all=True, simple=True, log_output=True)
 
+    def install_step_oneapi(self, *args, **kwargs):
+        """
+        Actual installation for versions 2021.x onwards.
+        """
+        # require that EULA is accepted
+        intel_eula_url = 'https://software.intel.com/content/www/us/en/develop/articles/end-user-license-agreement.html'
+        self.check_accepted_eula(name='Intel-oneAPI', more_info=intel_eula_url)
+
+        # exactly one "source" file is expected: the (offline) installation script
+        if len(self.src) == 1:
+            install_script = self.src[0]['name']
+        else:
+            src_fns = ', '.join([x['name'] for x in self.src])
+            raise EasyBuildError("Expected to find exactly one 'source' file (installation script): %s", src_fns)
+
+        adjust_permissions(install_script, stat.S_IXUSR)
+
+        # see https://software.intel.com/content/www/us/en/develop/documentation/...
+        # .../installation-guide-for-intel-oneapi-toolkits-linux/top/...
+        # .../local-installer-full-package/install-with-command-line.html
+        cmd = [
+            self.cfg['preinstallopts'],
+            './' + install_script,
+            '-a',  # required to specify that following are options for installer
+            '--action install',
+            '--silent',
+            '--eula accept',
+            '--install-dir ' + self.installdir,
+        ]
+
+        if self.install_components:
+            cmd.extend([
+                '--components',
+                ':'.join(self.install_components),
+            ])
+
+        cmd.append(self.cfg['installopts'])
+
+        return run_cmd(' '.join(cmd), log_all=True, simple=True, log_output=True)
+
+    def install_step(self, *args, **kwargs):
+        """
+        Install Intel software
+        """
+        if LooseVersion(self.version) >= LooseVersion('2021'):
+            return self.install_step_oneapi(*args, **kwargs)
+        else:
+            return self.install_step_classic(*args, **kwargs)
+
     def move_after_install(self):
         """Move installed files to correct location after installation."""
         subdir = os.path.join(self.installdir, self.name, self.version)

easybuild/easyblocks/i/intel_compilers.py

@@ -0,0 +1,131 @@
+# #
+# Copyright 2021-2021 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+EasyBuild support for installing Intel compilers, implemented as an easyblock
+
+@author: Kenneth Hoste (Ghent University)
+"""
+import os
+from distutils.version import LooseVersion
+
+from easybuild.easyblocks.generic.intelbase import IntelBase
+from easybuild.tools.build_log import EasyBuildError, print_msg
+
+
+class EB_intel_minus_compilers(IntelBase):
+    """
+    Support for installing Intel compilers, starting with verion 2021.x (oneAPI)
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Easyblock constructor: check version
+        """
+        super(EB_intel_minus_compilers, self).__init__(*args, **kwargs)
+
+        # this easyblock is only valid for recent versions of the Intel compilers (2021.x, oneAPI)
+        if LooseVersion(self.version) < LooseVersion('2021'):
+            raise EasyBuildError("Invalid version %s, should be >= 2021.x" % self.version)
+
+        self.compilers_subdir = os.path.join('compiler', self.version, 'linux')
+
+    def prepare_step(self, *args, **kwargs):
+        """
+        Prepare environment for installing.
+
+        Specify that oneAPI versions of Intel compilers don't require a runtime license.
+        """
+        # avoid that IntelBase trips over not having license info specified
+        kwargs['requires_runtime_license'] = False
+
+        super(EB_intel_minus_compilers, self).prepare_step(*args, **kwargs)
+
+    def configure_step(self):
+        """Configure installation."""
+
+        # redefine $HOME for install step, to avoid that anything is stored in $HOME/intel
+        # (like the 'installercache' database)
+        self.cfg['preinstallopts'] += " HOME=%s " % self.builddir
+
+    def install_step(self):
+        """
+        Install step: install each 'source file' one by one.
+        To install a patch release of Intel oneAPI compilers, we need to install the HPC Toolkit first,
+        and then separate updates for the C++ and Fortran compilers...
+        """
+        srcs = self.src[:]
+        cnt = len(srcs)
+        for idx, src in enumerate(srcs):
+            print_msg("installing part %d/%s (%s)..." % (idx + 1, cnt, src['name']))
+            self.src = [src]
+            super(EB_intel_minus_compilers, self).install_step()
+
+    def sanity_check_step(self):
+        """
+        Custom sanity check for Intel compilers.
+        """
+
+        classic_compiler_cmds = ['icc', 'icpc', 'ifort']
+        oneapi_compiler_cmds = [
+            'dpcpp',  # Intel oneAPI Data Parallel C++ compiler
+            'icx',  # oneAPI Intel C compiler
+            'icpx',  # oneAPI Intel C++ compiler
+            'ifx',  # oneAPI Intel Fortran compiler
+        ]
+        bindir = os.path.join(self.compilers_subdir, 'bin')
+        classic_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds]
+        oneapi_compiler_paths = [os.path.join(bindir, 'intel64', x) for x in classic_compiler_cmds]
+
+        custom_paths = {
+            'files': classic_compiler_paths + oneapi_compiler_paths,
+            'dirs': [self.compilers_subdir],
+        }
+
+        all_compiler_cmds = classic_compiler_cmds + oneapi_compiler_cmds
+        custom_commands = ["which %s" % c for c in all_compiler_cmds]
+        custom_commands.extend("%s --version | grep %s" % (c, self.version) for c in all_compiler_cmds)
+
+        super(EB_intel_minus_compilers, self).sanity_check_step(custom_paths=custom_paths,
+                                                                custom_commands=custom_commands)
+
+    def make_module_req_guess(self):
+        """
+        Paths to consider for prepend-paths statements in module file
+        """
+        libdirs = [
+            'lib',
+            os.path.join('lib', 'x64'),
+            os.path.join('compiler', 'lib', 'intel64_lin'),
+        ]
+        libdirs = [os.path.join(self.compilers_subdir, x) for x in libdirs]
+        guesses = {
+            'PATH': [
+                os.path.join(self.compilers_subdir, 'bin'),
+                os.path.join(self.compilers_subdir, 'bin', 'intel64'),
+            ],
+            'LD_LIBRARY_PATH': libdirs,
+            'LIBRARY_PATH': libdirs,
+        }
+        return guesses

test/easyblocks/init_easyblocks.py

@@ -212,6 +212,9 @@ def innertest(self):
         elif os.path.basename(easyblock) == 'systemmpi.py':
             # use OpenMPI as name when testing SystemMPI easyblock
             innertest = make_inner_test(easyblock, name='OpenMPI', version='system')
+        elif os.path.basename(easyblock) == 'intel_compilers.py':
+            # custom easyblock for intel-compilers (oneAPI) requires v2021.x or newer
+            innertest = make_inner_test(easyblock, name='intel-compilers', version='2021.1')
         else:
             innertest = make_inner_test(easyblock)
 

test/easyblocks/module.py

@@ -396,6 +396,9 @@ def innertest(self):
             # exactly one dependency is included with ModuleRC generic easyblock (and name must match)
             extra_txt = 'dependencies = [("foo", "1.2.3.4.5")]'
             innertest = make_inner_test(easyblock, name='foo', version='1.2.3.4', extra_txt=extra_txt)
+        elif eb_fn == 'intel_compilers.py':
+            # custom easyblock for intel-compilers (oneAPI) requires v2021.x or newer
+            innertest = make_inner_test(easyblock, name='intel-compilers', version='2021.1')
         else:
             # Make up some unique name
             innertest = make_inner_test(easyblock, name=eb_fn.replace('.', '-') + '-sw')