From 91cee1036d35f229f132c94e361456b65598f581 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 13:30:33 +0100 Subject: [PATCH 01/10] Added GPU support to ELPA EasyBlock. If CUDA is specified in the EasyConfig, automatically enable nvidia GPU support --- easybuild/easyblocks/e/elpa.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 8ac4856207c..116bd2abea4 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -39,7 +39,7 @@ from easybuild.tools.systemtools import get_cpu_features, get_shared_lib_ext from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC from easybuild.tools.utilities import nub - +from easybuild.tools.modules import get_software_root ELPA_CPU_FEATURE_FLAGS = ['avx', 'avx2', 'avx512f', 'vsx', 'sse4_2'] @@ -176,6 +176,28 @@ def run_all_steps(self, *args, **kwargs): return super(EB_ELPA, self).run_all_steps(*args, **kwargs) + def configure_step(self): + """Configure step for ELPA""" + + # Add nvidia GPU support if requested + cuda_root = get_software_root('CUDA') + self.log.info("Got cuda root: %s", cuda_root) + if cuda_root: + self.cfg.update('configopts', '--enable-nvidia-gpu') + self.cfg.update('configopts', '--with-cuda-path=%s' % cuda_root) + self.cfg.update('configopts', '--with-cuda-sdk-path=%s' % cuda_root) + + cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + if not cuda_cc: + raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' + 'cuda_compute_capabilities easyconfig parameter or via ' + '--cuda-compute-capabilities') + cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) + + super(EB_ELPA, self).configure_step() + def patch_step(self, *args, **kwargs): """Patch manual_cpp script to avoid using hardcoded /usr/bin/python.""" super(EB_ELPA, self).patch_step(*args, **kwargs) From ca7775227ef921e480599428ef29ad8002c6480b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 13:46:58 +0100 Subject: [PATCH 02/10] Currently, ELPA only supports passing one architecture to --with-NVIDIA-GPU-compute-capability=VALUE --- easybuild/easyblocks/e/elpa.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 116bd2abea4..189c99068ce 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -192,6 +192,12 @@ def configure_step(self): raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' 'cuda_compute_capabilities easyconfig parameter or via ' '--cuda-compute-capabilities') + + # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture + if len(cuda_cc) > 1: + raise EasyBuildError('ELPA currently only supports specifying one architecture when ' + 'building. You specified cuda-compute-capabilities: %s', cuda_cc) + cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) From 4865037803d4b98837845f07c94a94f678995f1b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 17:35:27 +0100 Subject: [PATCH 03/10] Make sure to set CPP, as newer ELPA configures require it --- easybuild/easyblocks/e/elpa.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 189c99068ce..cdbce5f23c8 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -31,6 +31,7 @@ """ import os +import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError @@ -202,6 +203,17 @@ def configure_step(self): self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) + # From v2022.05.001 onwards, the config complains if CPP is not set + env_dict = env.read_environment({'cxx': 'CXX', 'cpp': 'CPP'}) + if 'cxx' in env_dict: + if 'cpp' in env_dict and env_dict['cxx'] != env_dict['cpp']: + self.log.warning("Overwriting value of CPP (%s) with the value for CXX (%s)", + env_dict['cpp'], env_dict['cxx']) + env.setvar('CPP', env_dict['cxx']) + else: + raise EasyBuildError('ELPA requires CPP to be set. EasyBuild tried setting it based on the value of CXX, ' + 'but could not retreive a value for CXX') + super(EB_ELPA, self).configure_step() def patch_step(self, *args, **kwargs): From 6642b40e99c847f3e30c3ba02e38e31fca2ca457 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 18:55:57 +0100 Subject: [PATCH 04/10] Set CPP env var to cpp. This is valid for GCC, but we might make to make this neater and read it from the compiler definition from easybuild-framework. Still discussing that on EB slack --- easybuild/easyblocks/e/elpa.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index cdbce5f23c8..51742b1d0d9 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -204,15 +204,9 @@ def configure_step(self): self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) # From v2022.05.001 onwards, the config complains if CPP is not set - env_dict = env.read_environment({'cxx': 'CXX', 'cpp': 'CPP'}) - if 'cxx' in env_dict: - if 'cpp' in env_dict and env_dict['cxx'] != env_dict['cpp']: - self.log.warning("Overwriting value of CPP (%s) with the value for CXX (%s)", - env_dict['cpp'], env_dict['cxx']) - env.setvar('CPP', env_dict['cxx']) - else: - raise EasyBuildError('ELPA requires CPP to be set. EasyBuild tried setting it based on the value of CXX, ' - 'but could not retreive a value for CXX') + # Need to make this neater so that it is either set by easybuild-framework, OR query the toolchain for something like COMPILER_CPP + # Discussing that right now on EB Slack... + env.setvar('CPP', 'cpp') super(EB_ELPA, self).configure_step() From c687f9095d5c0616cd33a963fd7244fb5db4b907 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 12:02:44 +0100 Subject: [PATCH 05/10] Fixed typo, added option to enable dedicated sm80 kernel, set CPP (C preprocessor environment variable) in a way that can easily be extended or modified per toolchain family --- easybuild/easyblocks/e/elpa.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 51742b1d0d9..255abe4ee24 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -30,10 +30,13 @@ @author: Kenneth Hoste (Ghent University) """ import os +from distutils.version import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions @@ -201,12 +204,24 @@ def configure_step(self): cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) - self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if '8.0' in cuda_cc and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') # From v2022.05.001 onwards, the config complains if CPP is not set - # Need to make this neater so that it is either set by easybuild-framework, OR query the toolchain for something like COMPILER_CPP - # Discussing that right now on EB Slack... - env.setvar('CPP', 'cpp') + # C preprocessor to use for given comp_fam + cpp_dict = { + TC_CONSTANT_GCC: 'cpp', + TC_CONSTANT_INTELCOMP: 'cpp', + } + comp_fam = self.toolchain.comp_family() + if comp_fam in cpp_dict: + env.setvar('CPP', cpp_dict[comp_fam]) + else: + raise EasyBuildError('ELPA EasyBlock does not know which C preprocessor to use for the ' + 'current compiler family (%s). Please add the correct preprocessor ' + 'for this compiler family to cpp_dict in the ELPA EasyBlock', comp_fam) super(EB_ELPA, self).configure_step() From 515e57df7b1184597e3545f58d13f9755bb488fb Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 15:39:43 +0100 Subject: [PATCH 06/10] Clarify comment --- easybuild/easyblocks/e/elpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 255abe4ee24..a69d6573452 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -209,7 +209,7 @@ def configure_step(self): if '8.0' in cuda_cc and LooseVersion(self.version) >= LooseVersion('2021.11.001'): self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') - # From v2022.05.001 onwards, the config complains if CPP is not set + # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure # C preprocessor to use for given comp_fam cpp_dict = { TC_CONSTANT_GCC: 'cpp', From 1f86084c67c75c6caacfaa772ba7a8ef06417c21 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 15:56:37 +0100 Subject: [PATCH 07/10] Pull logic inside the if statement that checks if only a single cuda compute capability has been specified. Also, now check if the cuda_cc is larger or equal to 8.0, since you probably also want to build the 8.0 optimized kernel if you have 8.5 capability in your system --- easybuild/easyblocks/e/elpa.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index a69d6573452..5bff3c695be 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -198,17 +198,18 @@ def configure_step(self): '--cuda-compute-capabilities') # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture - if len(cuda_cc) > 1: + if len(cuda_cc) == 1: + cuda_cc = cuda_cc[0] + cuda_cc_string = cuda_cc.replace('.', '') + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') + else: raise EasyBuildError('ELPA currently only supports specifying one architecture when ' 'building. You specified cuda-compute-capabilities: %s', cuda_cc) - cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) - self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) - self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) - # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards - if '8.0' in cuda_cc and LooseVersion(self.version) >= LooseVersion('2021.11.001'): - self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') - # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure # C preprocessor to use for given comp_fam cpp_dict = { From c3e7701f583c285d905b6879ef4f4c824d46849a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 15:57:59 +0100 Subject: [PATCH 08/10] Make sure sm_ gets prepended in the argument for --with-NVIDIA-GPU-compute-capability --- easybuild/easyblocks/e/elpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 5bff3c695be..39ea64b41eb 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -201,7 +201,7 @@ def configure_step(self): if len(cuda_cc) == 1: cuda_cc = cuda_cc[0] cuda_cc_string = cuda_cc.replace('.', '') - self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): From 8b154aba5c1e61db84f432000386907dcd18d811 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Feb 2023 11:55:00 +0100 Subject: [PATCH 09/10] Reordered if-else statement to reduce indentation. Clarified error message that is raised --- easybuild/easyblocks/e/elpa.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 39ea64b41eb..143952cbf78 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -198,17 +198,16 @@ def configure_step(self): '--cuda-compute-capabilities') # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture - if len(cuda_cc) == 1: - cuda_cc = cuda_cc[0] - cuda_cc_string = cuda_cc.replace('.', '') - self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) - self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) - # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards - if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): - self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') - else: - raise EasyBuildError('ELPA currently only supports specifying one architecture when ' + if len(cuda_cc) != 1: + raise EasyBuildError('ELPA currently only supports specifying one CUDA architecture when ' 'building. You specified cuda-compute-capabilities: %s', cuda_cc) + cuda_cc = cuda_cc[0] + cuda_cc_string = cuda_cc.replace('.', '') + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure # C preprocessor to use for given comp_fam From 40bc11790fef31a4f8d0499c5c2e35dc3308e8c1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Feb 2023 13:38:07 +0100 Subject: [PATCH 10/10] Moved up from .. import to maintain alphabetical order --- easybuild/easyblocks/e/elpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 143952cbf78..41b81e15e84 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -40,10 +40,10 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions +from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_features, get_shared_lib_ext from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC from easybuild.tools.utilities import nub -from easybuild.tools.modules import get_software_root ELPA_CPU_FEATURE_FLAGS = ['avx', 'avx2', 'avx512f', 'vsx', 'sse4_2']