From a41d61d9b8c5dc14eb8d0b2465f956093ec92c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Fri, 12 Nov 2021 08:47:49 +0100 Subject: [PATCH 1/5] Add Clang-AOMP easyblock --- easybuild/easyblocks/c/clang_aomp.py | 238 +++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 easybuild/easyblocks/c/clang_aomp.py diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py new file mode 100644 index 00000000000..39f20978e6c --- /dev/null +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -0,0 +1,238 @@ +## +# Copyright 2009-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing AOMP version of LLVM/Clang + +@author: Jørgen Nordmoen (University of Oslo, USIT) +""" +import glob +import os + +from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP +from easybuild.easyblocks.generic.bundle import Bundle +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning +from easybuild.tools.config import build_option +from easybuild.tools.modules import get_software_root +from easybuild.tools.systemtools import AARCH64, POWER, X86_64 +from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext + + +# Default AMD GPU architectures to build for +DEFAULT_GFX_ARCHS = ['gfx900', 'gfx902', 'gfx906', 'gfx908', 'gfx90a', 'gfx1030', 'gfx1031'] + + +class EB_Clang_AOMP(Bundle): + """ + Self-contained build of AOMP version of Clang + """ + + @staticmethod + def extra_options(): + extra_vars = { + 'gfx_list': [[], "AMD GPU architectures to build for", CUSTOM], + } + return Bundle.extra_options(extra_vars) + + def __init__(self, *args, **kwargs): + super(EB_Clang_AOMP, self).__init__(*args, **kwargs) + # List of LLVM target architectures to build for, extended in the 'prepare_step' + self.target_archs = ['AMDGPU'] + # Mapping from known ROCm components to their configure method + self.cfg_method = { + 'llvm-project': self._configure_llvm, + 'ROCm-Device-Libs': self._configure_device_libs, + 'Clang-OpenMP': self._configure_omp, + 'aomp-extras': self._configure_aomp_extras, + } + # Prepare configuration options that point to the expected Clang build + self.llvm_compiler_flags = [ + "-DCMAKE_C_COMPILER=%s" % os.path.join(self.installdir, 'bin', 'clang'), + "-DCMAKE_CXX_COMPILER=%s" % os.path.join(self.installdir, 'bin', 'clang++'), + "-DLLVM_INSTALL_PREFIX=%s" % self.installdir, + "-DLLVM_DIR=%s" % self.installdir, + ] + # Variables to be filled in the prepare step + self.amd_gfx_archs = [] + self.cuda_archs = [] + self.device_lib_path = None + + def prepare_step(self, *args, **kwargs): + """ + Setup target architectures for LLVM + """ + super(EB_Clang_AOMP, self).prepare_step(*args, **kwargs) + # Detect CPU architecture and setup build targets for LLVM + cpu_arch = get_cpu_architecture() + if cpu_arch not in LLVM_ARCH_MAP: + raise EasyBuildError('Unknown CPU architecture for LLVM: %s' % cpu_arch) + self.target_archs.append(LLVM_ARCH_MAP[cpu_arch][0]) + # If CUDA is loaded when building, try to build CUDA offload backend + if get_software_root('CUDA'): + self.target_archs.append('NVPTX') + ec_cuda_cc = self.cfg['cuda_compute_capabilities'] + cfg_cuda_cc = build_option('cuda_compute_capabilities') + cuda_cc = cfg_cuda_cc or ec_cuda_cc or [] + if not cuda_cc: + raise EasyBuildError("Can't build Clang-AOMP with CUDA support " + "without specifying 'cuda-compute-capabilities'") + self.cuda_archs = [cc.replace('.', '') for cc in cuda_cc] + self.log.info("Building offload support for the following CUDA architectures: '%s'", + ' '.join(self.amd_gfx_archs)) + self.log.info("Building LLVM for the following architectures: '%s'", ';'.join(self.target_archs)) + # Setup AMD GFX list to build for + if self.cfg['gfx_list']: + self.amd_gfx_archs = self.cfg['gfx_list'] + else: + self.amd_gfx_archs = DEFAULT_GFX_ARCHS + self.log.info("Building offload support for the following AMD architectures: '%s'", + ' '.join(self.amd_gfx_archs)) + # Ensure necessary libraries are downloaded and can be found + device_lib_dir_pattern = os.path.join(self.builddir, 'ROCm-Device-Libs-*') + hits = glob.glob(device_lib_dir_pattern) + if len(hits) == 1: + self.device_lib_path = hits[0] + else: + raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory") + + def configure_step(self): + """ + Go through each component and setup configuration for the later Bundle install step + """ + super(EB_Clang_AOMP, self).configure_step() + + num_comps = len(self.cfg['components']) + for idx, comp in enumerate(self.comp_cfgs): + name = comp['name'] + msg = "configuring bundle component %s %s (%d/%d)..." % (name, comp['version'], idx + 1, num_comps) + print_msg(msg) + if name in self.cfg_method: + self.cfg_method[name](comp) + self.log.info(msg) + else: + self.log.warn("Component %s has no configure method!" % name) + + def sanity_check_step(self): + """ + Custom sanity check for ROCm + """ + shlib_ext = get_shared_lib_ext() + custom_paths = { + 'files': ['bin/clang', 'bin/lld', 'lib/libomp.%s' % shlib_ext, + 'lib/libomptarget.rtl.amdgpu.%s' % shlib_ext, 'lib/libomptarget.%s' % shlib_ext], + 'dirs': ['amdgcn/bitcode', 'include/clang', 'include/lld', 'include/llvm', + 'lib/libdevice'], + } + custom_commands = ['clang --help', 'clang++ --help'] + # Check that all AMD GFX libraries were built + for gfx in self.amd_gfx_archs: + custom_paths['files'].append('lib/libdevice/libomptarget-amdgcn-%s.bc' % gfx) + custom_paths['files'].append('lib/libdevice/libaompextras-amdgcn-%s.bc' % gfx) + custom_paths['files'].append('lib/libdevice/libm-amdgcn-%s.bc' % gfx) + # Check that CPU target OpenMP offloading library was built + arch = get_cpu_architecture() + # Check architecture explicitly since Clang uses potentially + # different names + if arch == X86_64: + arch = 'x86_64' + elif arch == POWER: + arch = 'ppc64' + elif arch == AARCH64: + arch = 'aarch64' + else: + print_warning("Unknown CPU architecture (%s) for OpenMP offloading!" % arch) + custom_paths['files'].append('lib/libomptarget.rtl.%s.%s' % (arch, shlib_ext)) + # If CUDA offloading support was requested, check that correct omptarget was built + if get_software_root('CUDA'): + custom_paths['files'].append('lib/libomptarget.rtl.cuda.%s' % shlib_ext) + for arch in self.cuda_archs: + sm_arch = 'sm_%s' % arch + custom_paths['files'].append('lib/libdevice/libomptarget-nvptx-%s' % sm_arch) + custom_paths['files'].append('lib/libdevice/libaompextras-nvptx-%s' % sm_arch) + custom_paths['files'].append('lib/libdevice/libm-nvptx-%s' % sm_arch) + super(EB_Clang_AOMP, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def _configure_llvm(self, component): + """ + Setup configure options for building compiler_rt, Clang and lld + """ + comp_dir = '%s-%s' % (component['name'], component['version']) + component['srcdir'] = os.path.join(comp_dir, 'llvm') + # Need to unset $CPATH to avoid that libunwind is pulled in via Mesa + # dependency and interrupts building of LLVM + component['prebuildopts'] = "unset CPATH && " + # Setup configuration options for LLVM + component['configopts'] = "-DLLVM_ENABLE_PROJECTS='clang;lld;compiler-rt' " + component['configopts'] += "-DCLANG_DEFAULT_LINKER=lld " + component['configopts'] += "-DGCC_INSTALL_PREFIX=$EBROOTGCC " + component['configopts'] += "-DLLVM_ENABLE_ASSERTIONS=ON " + component['configopts'] += "-DLLVM_ENABLE_BINDINGS=OFF " + component['configopts'] += "-DLLVM_INCLUDE_BENCHMARKS=OFF " + component['configopts'] += "-DLLVM_TARGETS_TO_BUILD='%s'" % ';'.join(self.target_archs) + + def _configure_device_libs(self, component): + """ + Setup device libs such that it is built with the local LLVM build + """ + component['configopts'] = ' '.join(self.llvm_compiler_flags) + component['configopts'] += " -DBUILD_HC_LIB=OFF" + + def _configure_omp(self, component): + """ + Setup OpenMP configuration options, OMP uses the LLVM source + """ + llvm_include_dir = os.path.join(self.installdir, 'include', 'llvm') + comp_dir = 'llvm-project-%s' % component['version'] + component['srcdir'] = os.path.join(comp_dir, 'openmp') + component['preconfigopts'] = "export HSA_RUNTIME_PATH=%s && " % self.installdir + component['configopts'] = " ".join(self.llvm_compiler_flags) + component['configopts'] += " -DLIBOMPTARGET_AMDGCN_GFXLIST='%s'" % ';'.join(self.amd_gfx_archs) + component['configopts'] += " -DLIBOMPTARGET_ENABLE_DEBUG=ON" + component['configopts'] += " -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=%s" % llvm_include_dir + component['configopts'] += " -DLIBOMP_COPY_EXPORTS=OFF" + component['configopts'] += " -DLLVM_MAIN_INCLUDE_DIR=%s" % llvm_include_dir + component['configopts'] += " -DOPENMP_ENABLE_LIBOMPTARGET=ON" + component['configopts'] += " -DOPENMP_ENABLE_LIBOMPTARGET_HSA=ON" + component['configopts'] += " -DROCDL=%s" % self.device_lib_path + component['configopts'] += " -DROCM_DIR=%s" % self.installdir + component['configopts'] += " -DAOMP_STANDALONE_BUILD=1" + if get_software_root('CUDA'): + llvm_link = os.path.join(self.installdir, 'bin', 'llvm-link') + cuda_path = os.path.join(self.installdir, 'bin', 'clang++') + component['configopts'] += " -DLIBOMPTARGET_NVPTX_BC_LINKER=%s" % llvm_link + component['configopts'] += " -DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES='%s'" % ','.join(self.cuda_archs) + component['configopts'] += " -DLIBOMPTARGET_NVPTX_CUDA_COMPILER=%s" % cuda_path + component['configopts'] += " -DLIBOMPTARGET_NVPTX_ENABLE_BCLIB=ON" + + def _configure_aomp_extras(self, component): + """ + Setup AOMP extras compiler configurations + """ + component['preconfigopts'] = "export AOMP=%s && " % self.installdir + component['preconfigopts'] += "export GFXLIST='%s' && " % ';'.join(self.amd_gfx_archs) + component['configopts'] = " ".join(self.llvm_compiler_flags) + component['configopts'] += " -DAOMP_STANDALONE_BUILD=1" + component['configopts'] += " -DROCDL=%s" % self.device_lib_path + component['configopts'] += " -DROCM_DIR=%s" % self.installdir From b30a0c016c7a01c09fbcbebb2934852da9621834 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 13 Nov 2021 11:20:43 +0100 Subject: [PATCH 2/5] declare utf-8 encoding on top of Clang-AOMP easyblock --- easybuild/easyblocks/c/clang_aomp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 39f20978e6c..56f9ec62ca8 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- ## # Copyright 2009-2021 Ghent University # From a74233951cf1596e48825d67ef3c9b55f852b13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Mon, 15 Nov 2021 07:31:14 +0100 Subject: [PATCH 3/5] Added commen to GFX list and moved prepare_step --- easybuild/easyblocks/c/clang_aomp.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 56f9ec62ca8..7140f38f9fc 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -42,6 +42,11 @@ # Default AMD GPU architectures to build for +# +# AMD uses 'gfx' to identify the GPU, the first number identifies the generation, according to +# https://www.x.org/wiki/RadeonFeature/#index5h2 while the rest identifies the specific GPU. +# In the context of EasyBuild this identifier can be thought of as equivalent to the 'sm_' +# nomenclature of Nvidia. DEFAULT_GFX_ARCHS = ['gfx900', 'gfx902', 'gfx906', 'gfx908', 'gfx90a', 'gfx1030', 'gfx1031'] @@ -80,11 +85,10 @@ def __init__(self, *args, **kwargs): self.cuda_archs = [] self.device_lib_path = None - def prepare_step(self, *args, **kwargs): + def prepare_sources(self): """ Setup target architectures for LLVM """ - super(EB_Clang_AOMP, self).prepare_step(*args, **kwargs) # Detect CPU architecture and setup build targets for LLVM cpu_arch = get_cpu_architecture() if cpu_arch not in LLVM_ARCH_MAP: @@ -124,6 +128,7 @@ def configure_step(self): """ super(EB_Clang_AOMP, self).configure_step() + self.prepare_sources() num_comps = len(self.cfg['components']) for idx, comp in enumerate(self.comp_cfgs): name = comp['name'] From f8ff55270c4f9e2a29ead6671339545bc6e61745 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Dec 2021 21:33:28 +0100 Subject: [PATCH 4/5] fixes, enhancements and code cleanup for custom easyblock for Clang-AOMP --- easybuild/easyblocks/c/clang_aomp.py | 166 ++++++++++++++++----------- 1 file changed, 99 insertions(+), 67 deletions(-) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 7140f38f9fc..a85aacd1712 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -33,12 +33,11 @@ from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP from easybuild.easyblocks.generic.bundle import Bundle +from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning -from easybuild.tools.config import build_option from easybuild.tools.modules import get_software_root -from easybuild.tools.systemtools import AARCH64, POWER, X86_64 -from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext # Default AMD GPU architectures to build for @@ -50,29 +49,34 @@ DEFAULT_GFX_ARCHS = ['gfx900', 'gfx902', 'gfx906', 'gfx908', 'gfx90a', 'gfx1030', 'gfx1031'] -class EB_Clang_AOMP(Bundle): +class EB_Clang_minus_AOMP(Bundle): """ Self-contained build of AOMP version of Clang """ @staticmethod def extra_options(): + gfx_list_help_msg = "AMD GPU architectures to build for (if None, use defaults: %s)" extra_vars = { - 'gfx_list': [[], "AMD GPU architectures to build for", CUSTOM], + 'gfx_list': [None, gfx_list_help_msg % ', '.join(DEFAULT_GFX_ARCHS), CUSTOM], } return Bundle.extra_options(extra_vars) def __init__(self, *args, **kwargs): - super(EB_Clang_AOMP, self).__init__(*args, **kwargs) + """Easyblock constructor.""" + super(EB_Clang_minus_AOMP, self).__init__(*args, **kwargs) + # List of LLVM target architectures to build for, extended in the 'prepare_step' self.target_archs = ['AMDGPU'] + # Mapping from known ROCm components to their configure method self.cfg_method = { - 'llvm-project': self._configure_llvm, - 'ROCm-Device-Libs': self._configure_device_libs, - 'Clang-OpenMP': self._configure_omp, 'aomp-extras': self._configure_aomp_extras, + 'Clang-OpenMP': self._configure_omp, + 'llvm-project': self._configure_llvm, + 'ROCm-Device-Libs': self._configure_rocm_device_libs, } + # Prepare configuration options that point to the expected Clang build self.llvm_compiler_flags = [ "-DCMAKE_C_COMPILER=%s" % os.path.join(self.installdir, 'bin', 'clang'), @@ -80,55 +84,64 @@ def __init__(self, *args, **kwargs): "-DLLVM_INSTALL_PREFIX=%s" % self.installdir, "-DLLVM_DIR=%s" % self.installdir, ] + # Variables to be filled in the prepare step - self.amd_gfx_archs = [] self.cuda_archs = [] self.device_lib_path = None + # Setup AMD GFX list to build for + if self.cfg['gfx_list']: + self.amd_gfx_archs = self.cfg['gfx_list'] + else: + self.amd_gfx_archs = DEFAULT_GFX_ARCHS + def prepare_sources(self): """ Setup target architectures for LLVM """ # Detect CPU architecture and setup build targets for LLVM cpu_arch = get_cpu_architecture() - if cpu_arch not in LLVM_ARCH_MAP: - raise EasyBuildError('Unknown CPU architecture for LLVM: %s' % cpu_arch) - self.target_archs.append(LLVM_ARCH_MAP[cpu_arch][0]) - # If CUDA is loaded when building, try to build CUDA offload backend + if cpu_arch in LLVM_ARCH_MAP: + self.target_archs.append(LLVM_ARCH_MAP[cpu_arch][0]) + else: + raise EasyBuildError('Unknown CPU architecture for LLVM: %s', cpu_arch) + + # If CUDA is loaded when building, build CUDA offload backend if get_software_root('CUDA'): self.target_archs.append('NVPTX') - ec_cuda_cc = self.cfg['cuda_compute_capabilities'] - cfg_cuda_cc = build_option('cuda_compute_capabilities') - cuda_cc = cfg_cuda_cc or ec_cuda_cc or [] + + # a specific set of CUDA compute capabilities must be specified, + # via --cuda-compute-capabilities EasyBuild configuration option or + # via cuda_compute_capabilities easyconfig parameter + cuda_cc = self.cfg.get_cuda_cc_template_value('cuda_compute_capabilities').split(',') if not cuda_cc: raise EasyBuildError("Can't build Clang-AOMP with CUDA support " "without specifying 'cuda-compute-capabilities'") self.cuda_archs = [cc.replace('.', '') for cc in cuda_cc] self.log.info("Building offload support for the following CUDA architectures: '%s'", - ' '.join(self.amd_gfx_archs)) + ' '.join(self.cuda_archs)) + self.log.info("Building LLVM for the following architectures: '%s'", ';'.join(self.target_archs)) - # Setup AMD GFX list to build for - if self.cfg['gfx_list']: - self.amd_gfx_archs = self.cfg['gfx_list'] - else: - self.amd_gfx_archs = DEFAULT_GFX_ARCHS + self.log.info("Building offload support for the following AMD architectures: '%s'", ' '.join(self.amd_gfx_archs)) + # Ensure necessary libraries are downloaded and can be found device_lib_dir_pattern = os.path.join(self.builddir, 'ROCm-Device-Libs-*') hits = glob.glob(device_lib_dir_pattern) if len(hits) == 1: self.device_lib_path = hits[0] else: - raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory") + raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir) def configure_step(self): """ Go through each component and setup configuration for the later Bundle install step """ - super(EB_Clang_AOMP, self).configure_step() + super(EB_Clang_minus_AOMP, self).configure_step() self.prepare_sources() + num_comps = len(self.cfg['components']) for idx, comp in enumerate(self.comp_cfgs): name = comp['name'] @@ -152,15 +165,18 @@ def sanity_check_step(self): 'lib/libdevice'], } custom_commands = ['clang --help', 'clang++ --help'] + + libs = ['aompextras', 'omptarget'] + libdevice = os.path.join('lib', 'libdevice') + # Check that all AMD GFX libraries were built for gfx in self.amd_gfx_archs: - custom_paths['files'].append('lib/libdevice/libomptarget-amdgcn-%s.bc' % gfx) - custom_paths['files'].append('lib/libdevice/libaompextras-amdgcn-%s.bc' % gfx) - custom_paths['files'].append('lib/libdevice/libm-amdgcn-%s.bc' % gfx) + custom_paths['files'].extend([os.path.join(libdevice, 'lib%s-amdgcn-%s.bc' % (x, gfx)) for x in libs]) + # Check that CPU target OpenMP offloading library was built arch = get_cpu_architecture() - # Check architecture explicitly since Clang uses potentially - # different names + + # Check architecture explicitly since Clang uses potentially different names if arch == X86_64: arch = 'x86_64' elif arch == POWER: @@ -169,16 +185,20 @@ def sanity_check_step(self): arch = 'aarch64' else: print_warning("Unknown CPU architecture (%s) for OpenMP offloading!" % arch) - custom_paths['files'].append('lib/libomptarget.rtl.%s.%s' % (arch, shlib_ext)) + + custom_paths['files'].append(os.path.join('lib', 'libomptarget.rtl.%s.%s' % (arch, shlib_ext))) + # If CUDA offloading support was requested, check that correct omptarget was built if get_software_root('CUDA'): - custom_paths['files'].append('lib/libomptarget.rtl.cuda.%s' % shlib_ext) + custom_paths['files'].append(os.path.join('lib', 'libomptarget.rtl.cuda.%s' % shlib_ext)) + for arch in self.cuda_archs: sm_arch = 'sm_%s' % arch - custom_paths['files'].append('lib/libdevice/libomptarget-nvptx-%s' % sm_arch) - custom_paths['files'].append('lib/libdevice/libaompextras-nvptx-%s' % sm_arch) - custom_paths['files'].append('lib/libdevice/libm-nvptx-%s' % sm_arch) - super(EB_Clang_AOMP, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + custom_paths['files'].extend([os.path.join(libdevice, 'lib%s-nvptx-%s' % (x, sm_arch)) for x in libs]) + + # need to bypass sanity_check_step of Bundle, because it only loads the generated module + # unless custom paths or commands are specified in the easyconfig + EasyBlock.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) def _configure_llvm(self, component): """ @@ -186,24 +206,27 @@ def _configure_llvm(self, component): """ comp_dir = '%s-%s' % (component['name'], component['version']) component['srcdir'] = os.path.join(comp_dir, 'llvm') + # Need to unset $CPATH to avoid that libunwind is pulled in via Mesa # dependency and interrupts building of LLVM component['prebuildopts'] = "unset CPATH && " + # Setup configuration options for LLVM - component['configopts'] = "-DLLVM_ENABLE_PROJECTS='clang;lld;compiler-rt' " - component['configopts'] += "-DCLANG_DEFAULT_LINKER=lld " - component['configopts'] += "-DGCC_INSTALL_PREFIX=$EBROOTGCC " - component['configopts'] += "-DLLVM_ENABLE_ASSERTIONS=ON " - component['configopts'] += "-DLLVM_ENABLE_BINDINGS=OFF " - component['configopts'] += "-DLLVM_INCLUDE_BENCHMARKS=OFF " - component['configopts'] += "-DLLVM_TARGETS_TO_BUILD='%s'" % ';'.join(self.target_archs) - - def _configure_device_libs(self, component): + component['configopts'] = ' '.join([ + "-DLLVM_ENABLE_PROJECTS='clang;lld;compiler-rt'", + "-DCLANG_DEFAULT_LINKER=lld", + "-DGCC_INSTALL_PREFIX=$EBROOTGCC", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DLLVM_ENABLE_BINDINGS=OFF", + "-DLLVM_INCLUDE_BENCHMARKS=OFF", + "-DLLVM_TARGETS_TO_BUILD='%s'" % ';'.join(self.target_archs), + ]) + + def _configure_rocm_device_libs(self, component): """ - Setup device libs such that it is built with the local LLVM build + Setup ROCm device libs such that it is built with the local LLVM build """ - component['configopts'] = ' '.join(self.llvm_compiler_flags) - component['configopts'] += " -DBUILD_HC_LIB=OFF" + component['configopts'] = ' '.join(self.llvm_compiler_flags + ['-DBUILD_HC_LIB=OFF']) def _configure_omp(self, component): """ @@ -212,25 +235,32 @@ def _configure_omp(self, component): llvm_include_dir = os.path.join(self.installdir, 'include', 'llvm') comp_dir = 'llvm-project-%s' % component['version'] component['srcdir'] = os.path.join(comp_dir, 'openmp') + component['preconfigopts'] = "export HSA_RUNTIME_PATH=%s && " % self.installdir - component['configopts'] = " ".join(self.llvm_compiler_flags) - component['configopts'] += " -DLIBOMPTARGET_AMDGCN_GFXLIST='%s'" % ';'.join(self.amd_gfx_archs) - component['configopts'] += " -DLIBOMPTARGET_ENABLE_DEBUG=ON" - component['configopts'] += " -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=%s" % llvm_include_dir - component['configopts'] += " -DLIBOMP_COPY_EXPORTS=OFF" - component['configopts'] += " -DLLVM_MAIN_INCLUDE_DIR=%s" % llvm_include_dir - component['configopts'] += " -DOPENMP_ENABLE_LIBOMPTARGET=ON" - component['configopts'] += " -DOPENMP_ENABLE_LIBOMPTARGET_HSA=ON" - component['configopts'] += " -DROCDL=%s" % self.device_lib_path - component['configopts'] += " -DROCM_DIR=%s" % self.installdir - component['configopts'] += " -DAOMP_STANDALONE_BUILD=1" + + component['configopts'] = ' '.join(self.llvm_compiler_flags + [ + "-DLIBOMPTARGET_AMDGCN_GFXLIST='%s'" % ';'.join(self.amd_gfx_archs), + "-DLIBOMPTARGET_ENABLE_DEBUG=ON", + "-DLIBOMPTARGET_LLVM_INCLUDE_DIRS=%s" % llvm_include_dir, + "-DLIBOMP_COPY_EXPORTS=OFF", + "-DLLVM_MAIN_INCLUDE_DIR=%s" % llvm_include_dir, + "-DOPENMP_ENABLE_LIBOMPTARGET=ON", + "-DOPENMP_ENABLE_LIBOMPTARGET_HSA=ON", + "-DROCDL=%s" % self.device_lib_path, + "-DROCM_DIR=%s" % self.installdir, + "-DAOMP_STANDALONE_BUILD=1", + '', + ]) + if get_software_root('CUDA'): llvm_link = os.path.join(self.installdir, 'bin', 'llvm-link') cuda_path = os.path.join(self.installdir, 'bin', 'clang++') - component['configopts'] += " -DLIBOMPTARGET_NVPTX_BC_LINKER=%s" % llvm_link - component['configopts'] += " -DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES='%s'" % ','.join(self.cuda_archs) - component['configopts'] += " -DLIBOMPTARGET_NVPTX_CUDA_COMPILER=%s" % cuda_path - component['configopts'] += " -DLIBOMPTARGET_NVPTX_ENABLE_BCLIB=ON" + component['configopts'] += ' '.join([ + "-DLIBOMPTARGET_NVPTX_BC_LINKER=%s" % llvm_link, + "-DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES='%s'" % ','.join(self.cuda_archs), + "-DLIBOMPTARGET_NVPTX_CUDA_COMPILER=%s" % cuda_path, + "-DLIBOMPTARGET_NVPTX_ENABLE_BCLIB=ON", + ]) def _configure_aomp_extras(self, component): """ @@ -238,7 +268,9 @@ def _configure_aomp_extras(self, component): """ component['preconfigopts'] = "export AOMP=%s && " % self.installdir component['preconfigopts'] += "export GFXLIST='%s' && " % ';'.join(self.amd_gfx_archs) - component['configopts'] = " ".join(self.llvm_compiler_flags) - component['configopts'] += " -DAOMP_STANDALONE_BUILD=1" - component['configopts'] += " -DROCDL=%s" % self.device_lib_path - component['configopts'] += " -DROCM_DIR=%s" % self.installdir + + component['configopts'] = ' '.join(self.llvm_compiler_flags + [ + "-DAOMP_STANDALONE_BUILD=1", + "-DROCDL=%s" % self.device_lib_path, + "-DROCM_DIR=%s" % self.installdir, + ]) From 7557515028caf01bdcec7266d618fbb0fbd7d740 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Dec 2021 18:58:35 +0100 Subject: [PATCH 5/5] fix sanity check for Clang-AOMP with CUDA support + full sanity check with --module-only by defining self.cuda_archs in prepare_step --- easybuild/easyblocks/c/clang_aomp.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index a85aacd1712..50fafbfdb1a 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -95,10 +95,12 @@ def __init__(self, *args, **kwargs): else: self.amd_gfx_archs = DEFAULT_GFX_ARCHS - def prepare_sources(self): + def prepare_step(self, *args, **kwargs): """ - Setup target architectures for LLVM + Prepare build environment """ + super(EB_Clang_minus_AOMP, self).prepare_step(*args, **kwargs) + # Detect CPU architecture and setup build targets for LLVM cpu_arch = get_cpu_architecture() if cpu_arch in LLVM_ARCH_MAP: @@ -106,6 +108,7 @@ def prepare_sources(self): else: raise EasyBuildError('Unknown CPU architecture for LLVM: %s', cpu_arch) + # Set up target architectures for LLVM # If CUDA is loaded when building, build CUDA offload backend if get_software_root('CUDA'): self.target_archs.append('NVPTX') @@ -126,6 +129,12 @@ def prepare_sources(self): self.log.info("Building offload support for the following AMD architectures: '%s'", ' '.join(self.amd_gfx_archs)) + def configure_step(self): + """ + Go through each component and setup configuration for the later Bundle install step + """ + super(EB_Clang_minus_AOMP, self).configure_step() + # Ensure necessary libraries are downloaded and can be found device_lib_dir_pattern = os.path.join(self.builddir, 'ROCm-Device-Libs-*') hits = glob.glob(device_lib_dir_pattern) @@ -134,14 +143,6 @@ def prepare_sources(self): else: raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir) - def configure_step(self): - """ - Go through each component and setup configuration for the later Bundle install step - """ - super(EB_Clang_minus_AOMP, self).configure_step() - - self.prepare_sources() - num_comps = len(self.cfg['components']) for idx, comp in enumerate(self.comp_cfgs): name = comp['name'] @@ -194,7 +195,7 @@ def sanity_check_step(self): for arch in self.cuda_archs: sm_arch = 'sm_%s' % arch - custom_paths['files'].extend([os.path.join(libdevice, 'lib%s-nvptx-%s' % (x, sm_arch)) for x in libs]) + custom_paths['files'].append(os.path.join('lib', 'libomptarget-nvptx-%s.bc' % sm_arch)) # need to bypass sanity_check_step of Bundle, because it only loads the generated module # unless custom paths or commands are specified in the easyconfig