From 8d03645bbc2278262d8951543610a8208b45fbf5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Jun 2025 16:04:12 +0200 Subject: [PATCH 1/4] update numpy easyblock to support installing numpy >= 2.0 --- easybuild/easyblocks/n/numpy.py | 396 ++++++++++++++++++-------------- 1 file changed, 222 insertions(+), 174 deletions(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index eaf93cf4591..a9110fe2c79 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -71,203 +71,250 @@ def __init__(self, *args, **kwargs): def configure_step(self): """Configure numpy build by composing site.cfg contents.""" - # see e.g. https://github.com/numpy/numpy/pull/2809/files - self.sitecfg = '\n'.join([ - "[DEFAULT]", - "library_dirs = %(libs)s", - "include_dirs= %(includes)s", - "search_static_first=True", - ]) - - # If both FlexiBLAS and MKL are found, we assume that FlexiBLAS has a dependency on MKL. - # In this case we want to link to FlexiBLAS and not directly to MKL. - imkl_direct = get_software_root("imkl") and not get_software_root("FlexiBLAS") + if LooseVersion(self.version) >= LooseVersion('2.0'): + # for newer numpy verion (Meson-based), we need to use specific installation options + self.log.info(f"Using Meson-based procedure to configure build for numpy version {self.version}") - if imkl_direct: + # see https://numpy.org/devdocs/building/compilers_and_options.html#controlling-build-parallelism + self.cfg.update('installopts', f"-Ccompile-args='-j{self.cfg.parallel}'") - if self.toolchain.comp_family() == toolchain.GCC: - # see https://software.intel.com/en-us/articles/numpyscipy-with-intel-mkl, - # section Building with GNU Compiler chain - extrasiteconfig = '\n'.join([ - "[mkl]", - "lapack_libs = ", - "mkl_libs = mkl_rt", - ]) - else: - extrasiteconfig = '\n'.join([ - "[mkl]", - "lapack_libs = %(lapack)s", - "mkl_libs = %(blas)s", - ]) + # see https://numpy.org/devdocs/building/blas_lapack.html + self.cfg.update('installopts', "-Csetup-args=-Dallow-noblas=false") + blas_lapack_lib_used = None + for blas_lapack_lib in ('FlexiBLAS', 'imkl', 'BLIS', 'OpenBLAS'): + numpy_name = 'mkl' if blas_lapack_lib == 'imkl' else blas_lapack_lib.lower() + if get_software_root(blas_lapack_lib): + self.cfg.update('installopts', "-Csetup-args=-Dblas-order=" + numpy_name) + self.cfg.update('installopts', "-Csetup-args=-Dlapack-order=" + numpy_name) + blas_lapack_lib_used = blas_lapack_lib + break + # make sure that a BLAS/LAPACK library was found + if blas_lapack_lib_used is None: + raise EasyBuildError("No known BLAS/LAPACK library found when configuring numpy installation!") else: - # [atlas] the only real alternative, even for non-ATLAS BLAS libs (e.g., OpenBLAS, ACML, ...) - # using only the [blas] and [lapack] sections results in sub-optimal builds that don't provide _dotblas.so; - # it does require a CBLAS interface to be available for the BLAS library being used - # e.g. for ACML, the CBLAS module providing a C interface needs to be used - extrasiteconfig = '\n'.join([ - "[atlas]", - "atlas_libs = %(lapack)s", - "[lapack]", - "lapack_libs = %(lapack)s", # required by scipy, that uses numpy's site.cfg + self.log.info(f"Using classic procedure to configure build for numpy version {self.version}") + + # see e.g. https://github.com/numpy/numpy/pull/2809/files + self.sitecfg = '\n'.join([ + "[DEFAULT]", + "library_dirs = %(libs)s", + "include_dirs= %(includes)s", + "search_static_first=True", ]) - blas = None - lapack = None - fft = None - - if imkl_direct: - # with IMKL, no spaces and use '-Wl:' - # redefine 'Wl,' to 'Wl:' so that the patch file can do its job - def get_libs_for_mkl(varname): - """Get list of libraries as required for MKL patch file.""" - libs = self.toolchain.variables['LIB%s' % varname].copy() - libs.try_remove(['pthread', 'dl']) - tweaks = { - 'prefix': '', - 'prefix_begin_end': '-Wl:', - 'separator': ',', - 'separator_begin_end': ',', - } - libs.try_function_on_element('change', kwargs=tweaks) - libs.SEPARATOR = ',' - return str(libs) # str causes list concatenation and adding prefixes & separators - - blas = get_libs_for_mkl('BLAS_MT') - lapack = get_libs_for_mkl('LAPACK_MT') - fft = get_libs_for_mkl('FFT') - - # make sure the patch file is there - # we check for a typical characteristic of a patch file that cooperates with the above - # not fool-proof, but better than enforcing a particular patch filename - patch_found = False - patch_wl_regex = re.compile(r"replace\(':',\s*','\)") - for patch in self.patches: - # patches are either strings (extension) or dicts (easyblock) - if isinstance(patch, dict): - patch = patch['path'] - if patch_wl_regex.search(read_file(patch)): - patch_found = True - break - if not patch_found: - raise EasyBuildError("Building numpy on top of Intel MKL requires a patch to " - "handle -Wl linker flags correctly, which doesn't seem to be there.") + # If both FlexiBLAS and MKL are found, we assume that FlexiBLAS has a dependency on MKL. + # In this case we want to link to FlexiBLAS and not directly to MKL. + imkl_direct = get_software_root("imkl") and not get_software_root("FlexiBLAS") + + if imkl_direct: + + if self.toolchain.comp_family() == toolchain.GCC: + # see https://software.intel.com/en-us/articles/numpyscipy-with-intel-mkl, + # section Building with GNU Compiler chain + extrasiteconfig = '\n'.join([ + "[mkl]", + "lapack_libs = ", + "mkl_libs = mkl_rt", + ]) + else: + extrasiteconfig = '\n'.join([ + "[mkl]", + "lapack_libs = %(lapack)s", + "mkl_libs = %(blas)s", + ]) - else: - # unless Intel MKL is used, $ATLAS should be set to take full control, - # and to make sure a fully optimized version is built, including _dotblas.so - # which is critical for decent performance of the numpy.dot (matrix dot product) function! - env.setvar('ATLAS', '1') - - lapack = ', '.join([x for x in self.toolchain.get_variable('LIBLAPACK_MT', typ=list) if x != "pthread"]) - fft = ', '.join(self.toolchain.get_variable('LIBFFT', typ=list)) - - libs = ':'.join(self.toolchain.get_variable('LDFLAGS', typ=list)) - includes = ':'.join(self.toolchain.get_variable('CPPFLAGS', typ=list)) - - # CBLAS is required for ACML, because it doesn't offer a C interface to BLAS - if get_software_root('ACML'): - cblasroot = get_software_root('CBLAS') - if cblasroot: - lapack = ', '.join([lapack, "cblas"]) - cblaslib = os.path.join(cblasroot, 'lib') - # with numpy as extension, CBLAS might not be included in LDFLAGS because it's not part of a toolchain - if cblaslib not in libs: - libs = ':'.join([libs, cblaslib]) else: - raise EasyBuildError("CBLAS is required next to ACML to provide a C interface to BLAS, " - "but it's not loaded.") - - if fft: - extrasiteconfig += "\n[fftw]\nlibraries = %s" % fft - - suitesparseroot = get_software_root('SuiteSparse') - if suitesparseroot: - - extrasiteconfig += '\n'.join([ - "[amd]", - "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), - "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), - "amd_libs = amd", - "[umfpack]", - "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), - "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), - "umfpack_libs = umfpack", - ]) + # [atlas] the only real alternative, even for non-ATLAS BLAS libs (e.g., OpenBLAS, ACML, ...) + # using only the [blas] and [lapack] sections results in sub-optimal builds + # that don't provide _dotblas.so; + # it does require a CBLAS interface to be available for the BLAS library being used + # e.g. for ACML, the CBLAS module providing a C interface needs to be used + extrasiteconfig = '\n'.join([ + "[atlas]", + "atlas_libs = %(lapack)s", + "[lapack]", + "lapack_libs = %(lapack)s", # required by scipy, that uses numpy's site.cfg + ]) - self.sitecfg = '\n'.join([self.sitecfg, extrasiteconfig]) + blas = None + lapack = None + fft = None + + if imkl_direct: + # with IMKL, no spaces and use '-Wl:' + # redefine 'Wl,' to 'Wl:' so that the patch file can do its job + def get_libs_for_mkl(varname): + """Get list of libraries as required for MKL patch file.""" + libs = self.toolchain.variables['LIB%s' % varname].copy() + libs.try_remove(['pthread', 'dl']) + tweaks = { + 'prefix': '', + 'prefix_begin_end': '-Wl:', + 'separator': ',', + 'separator_begin_end': ',', + } + libs.try_function_on_element('change', kwargs=tweaks) + libs.SEPARATOR = ',' + return str(libs) # str causes list concatenation and adding prefixes & separators + + blas = get_libs_for_mkl('BLAS_MT') + lapack = get_libs_for_mkl('LAPACK_MT') + fft = get_libs_for_mkl('FFT') + + # make sure the patch file is there + # we check for a typical characteristic of a patch file that cooperates with the above + # not fool-proof, but better than enforcing a particular patch filename + patch_found = False + patch_wl_regex = re.compile(r"replace\(':',\s*','\)") + for patch in self.patches: + # patches are either strings (extension) or dicts (easyblock) + if isinstance(patch, dict): + patch = patch['path'] + if patch_wl_regex.search(read_file(patch)): + patch_found = True + break + if not patch_found: + raise EasyBuildError("Building numpy on top of Intel MKL requires a patch to " + "handle -Wl linker flags correctly, which doesn't seem to be there.") - self.sitecfg = self.sitecfg % { - 'blas': blas, - 'lapack': lapack, - 'libs': libs, - 'includes': includes, - } + else: + # unless Intel MKL is used, $ATLAS should be set to take full control, + # and to make sure a fully optimized version is built, including _dotblas.so + # which is critical for decent performance of the numpy.dot (matrix dot product) function! + env.setvar('ATLAS', '1') + + lapack = ', '.join([x for x in self.toolchain.get_variable('LIBLAPACK_MT', typ=list) if x != "pthread"]) + fft = ', '.join(self.toolchain.get_variable('LIBFFT', typ=list)) + + libs = ':'.join(self.toolchain.get_variable('LDFLAGS', typ=list)) + includes = ':'.join(self.toolchain.get_variable('CPPFLAGS', typ=list)) + + # CBLAS is required for ACML, because it doesn't offer a C interface to BLAS + if get_software_root('ACML'): + cblasroot = get_software_root('CBLAS') + if cblasroot: + lapack = ', '.join([lapack, "cblas"]) + cblaslib = os.path.join(cblasroot, 'lib') + # with numpy as extension, CBLAS might not be included in LDFLAGS + # because it's not part of a toolchain + if cblaslib not in libs: + libs = ':'.join([libs, cblaslib]) + else: + raise EasyBuildError("CBLAS is required next to ACML to provide a C interface to BLAS, " + "but it's not loaded.") + + if fft: + extrasiteconfig += "\n[fftw]\nlibraries = %s" % fft + + suitesparseroot = get_software_root('SuiteSparse') + if suitesparseroot: + + extrasiteconfig += '\n'.join([ + "[amd]", + "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), + "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), + "amd_libs = amd", + "[umfpack]", + "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), + "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), + "umfpack_libs = umfpack", + ]) - if LooseVersion(self.version) < LooseVersion('1.26'): - # NumPy detects the required math by trying to link a minimal code containing a call to `log(0.)`. - # The first try is without any libraries, which works with `gcc -fno-math-errno` (our optimization default) - # because the call gets removed due to not having any effect. So it concludes that `-lm` is not required. - # This then fails to detect availability of functions such as `acosh` which do not get removed in the same - # way and so less exact replacements are used instead which e.g. fail the tests on PPC. - # This variable makes it try `-lm` first and is supported until the Meson backend is used in 1.26+. - env.setvar('MATHLIB', 'm') + self.sitecfg = '\n'.join([self.sitecfg, extrasiteconfig]) - super().configure_step() + self.sitecfg = self.sitecfg % { + 'blas': blas, + 'lapack': lapack, + 'libs': libs, + 'includes': includes, + } - if LooseVersion(self.version) < LooseVersion('1.21'): - # check configuration (for debugging purposes) - cmd = "%s setup.py config" % self.python_cmd - run_shell_cmd(cmd) + if LooseVersion(self.version) < LooseVersion('1.26'): + # NumPy detects the required math by trying to link a minimal code containing a call to `log(0.)`. + # The first try is without any libraries, which works with `gcc -fno-math-errno` + # (our optimization default) + # because the call gets removed due to not having any effect. + # So it concludes that `-lm` is not required. + # This then fails to detect availability of functions such as `acosh` which do not get removed + # in the same way and so less exact replacements are used instead which e.g. fail the tests on PPC. + # This variable makes it try `-lm` first and is supported until the Meson backend is used in 1.26+. + env.setvar('MATHLIB', 'm') + + FortranPythonPackage.configure_step(self) + + if LooseVersion(self.version) < LooseVersion('1.21'): + # check configuration (for debugging purposes) + cmd = "%s setup.py config" % self.python_cmd + run_shell_cmd(cmd) + + if LooseVersion(self.version) >= LooseVersion('1.26'): + # control BLAS/LAPACK library being used + # https://github.com/numpy/numpy/blob/v1.26.2/doc/source/release/1.26.1-notes.rst#build-system-changes + # and 'blas-order' in https://github.com/numpy/numpy/blob/v1.26.2/meson_options.txt + blas_lapack_names = { + toolchain.BLIS: 'blis', + toolchain.FLEXIBLAS: 'flexiblas', + toolchain.LAPACK: 'lapack', + toolchain.INTELMKL: 'mkl', + toolchain.OPENBLAS: 'openblas', + } + blas_family = self.toolchain.blas_family() + if blas_family in blas_lapack_names: + self.cfg.update('installopts', "-Csetup-args=-Dblas=" + blas_lapack_names[blas_family]) + else: + raise EasyBuildError("Unknown BLAS library for numpy %s: %s", self.version, blas_family) - if LooseVersion(self.version) >= LooseVersion('1.26'): - # control BLAS/LAPACK library being used - # see https://github.com/numpy/numpy/blob/v1.26.2/doc/source/release/1.26.1-notes.rst#build-system-changes - # and 'blas-order' in https://github.com/numpy/numpy/blob/v1.26.2/meson_options.txt - blas_lapack_names = { - toolchain.BLIS: 'blis', - toolchain.FLEXIBLAS: 'flexiblas', - toolchain.LAPACK: 'lapack', - toolchain.INTELMKL: 'mkl', - toolchain.OPENBLAS: 'openblas', - } - blas_family = self.toolchain.blas_family() - if blas_family in blas_lapack_names: - self.cfg.update('installopts', "-Csetup-args=-Dblas=" + blas_lapack_names[blas_family]) - else: - raise EasyBuildError("Unknown BLAS library for numpy %s: %s", self.version, blas_family) + lapack_family = self.toolchain.lapack_family() + if lapack_family in blas_lapack_names: + self.cfg.update('installopts', "-Csetup-args=-Dlapack=" + blas_lapack_names[lapack_family]) + else: + raise EasyBuildError("Unknown LAPACK library for numpy %s: %s", self.version, lapack_family) - lapack_family = self.toolchain.lapack_family() - if lapack_family in blas_lapack_names: - self.cfg.update('installopts', "-Csetup-args=-Dlapack=" + blas_lapack_names[lapack_family]) - else: - raise EasyBuildError("Unknown LAPACK library for numpy %s: %s", self.version, lapack_family) + self.cfg.update('installopts', "-Csetup-args=-Dallow-noblas=false") - self.cfg.update('installopts', "-Csetup-args=-Dallow-noblas=false") + def build_step(self, *args, **kwargs): + """ + Custom build step for numpy + """ + # no need for separate build step for numpy >= 2.0 + if LooseVersion(self.version) < LooseVersion('2.0'): + super().build(self, *args, **kwargs) def test_step(self): """Run available numpy unit tests, and more.""" # determine command to use to run numpy test suite, # and whether test results should be ignored or not - if self.cfg['ignore_test_result']: - test_code = 'numpy.test(verbose=2)' + if LooseVersion(self.version) >= LooseVersion('2.0'): + + # spin requires that path to where numpy was installed is in 'build-install' when using --no-build; + # while the 'build' part can be customized, the '-install' part is hardcoded, + # so just specify path in which test installation of numpy is done to parent test step; + self.pypkg_test_installdir = os.path.join(os.getcwd(), 'build-install') + + # test suite should be run via 'spin' tool, + # see https://numpy.org/devdocs/dev/development_environment.html#testing-builds + self.testcmd = f"spin test --no-build --verbose" else: - if LooseVersion(self.version) >= LooseVersion('1.15'): - # Numpy 1.15+ returns a True on success. Hence invert to get a failure value - test_code = 'sys.exit(not numpy.test(verbose=2))' + if self.cfg['ignore_test_result']: + test_code = 'numpy.test(verbose=2)' else: - # Return value is a TextTestResult. Check the errors member for any error - test_code = 'sys.exit(len(numpy.test(verbose=2).errors) > 0)' + if LooseVersion(self.version) >= LooseVersion('1.15'): + # Numpy 1.15+ returns a True on success. Hence invert to get a failure value + test_code = 'sys.exit(not numpy.test(verbose=2))' + else: + # Return value is a TextTestResult. Check the errors member for any error + test_code = 'sys.exit(len(numpy.test(verbose=2).errors) > 0)' - # Prepend imports - test_code = "import sys; import numpy; " + test_code + # Prepend imports + test_code = "import sys; import numpy; " + test_code - # LDFLAGS should not be set when testing numpy/scipy, because it overwrites whatever numpy/scipy sets - # see http://projects.scipy.org/numpy/ticket/182 - self.testcmd = "unset LDFLAGS && cd .. && %%(python)s -c '%s'" % test_code + # LDFLAGS should not be set when testing numpy/scipy, because it overwrites whatever numpy/scipy sets + # see http://projects.scipy.org/numpy/ticket/182 + self.testcmd = "unset LDFLAGS && cd .. && %%(python)s -c '%s'" % test_code - super().test_step() + FortranPythonPackage.test_step(self) # temporarily install numpy, it doesn't alow to be used straight from the source dir tmpdir = tempfile.mkdtemp() @@ -280,7 +327,7 @@ def test_step(self): try: pwd = os.getcwd() - os.chdir(tmpdir) + change_dir(tmpdir) except OSError as err: raise EasyBuildError("Faild to change to %s: %s", tmpdir, err) @@ -322,19 +369,20 @@ def test_step(self): raise EasyBuildError("Time for %dx%d matrix dot product: %d msec >= %d msec => ERROR", size, size, time_msec, self.cfg['blas_test_time_limit']) try: - os.chdir(pwd) + change_dir(pwd) remove_dir(tmpdir) except OSError as err: raise EasyBuildError("Failed to change back to %s: %s", pwd, err) def install_step(self): """Install numpy and remove numpy build dir, so scipy doesn't find it by accident.""" - super().install_step() + + FortranPythonPackage.install_step(self) builddir = os.path.join(self.builddir, "numpy") try: if os.path.isdir(builddir): - os.chdir(self.builddir) + change_dir(self.builddir) remove_dir(builddir) else: self.log.debug("build dir %s already clean" % builddir) From 8866792790665d345005e45d4f224bfe3c6dbc96 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Jun 2025 11:28:18 +0200 Subject: [PATCH 2/4] fix trivial style issue in numpy easyblock --- easybuild/easyblocks/n/numpy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index a9110fe2c79..7c354f1d840 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -295,7 +295,7 @@ def test_step(self): # test suite should be run via 'spin' tool, # see https://numpy.org/devdocs/dev/development_environment.html#testing-builds - self.testcmd = f"spin test --no-build --verbose" + self.testcmd = "spin test --no-build --verbose" else: if self.cfg['ignore_test_result']: test_code = 'numpy.test(verbose=2)' From 8b58bf49b604e6059fffa66971565d0fee0161ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Jun 2025 13:36:46 +0200 Subject: [PATCH 3/4] stick to using super() when calling step methods of parent easyblock in custom easyblock for numpy --- easybuild/easyblocks/n/numpy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 7c354f1d840..410cd5f45ab 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -241,7 +241,7 @@ def get_libs_for_mkl(varname): # This variable makes it try `-lm` first and is supported until the Meson backend is used in 1.26+. env.setvar('MATHLIB', 'm') - FortranPythonPackage.configure_step(self) + super().configure_step() if LooseVersion(self.version) < LooseVersion('1.21'): # check configuration (for debugging purposes) @@ -279,7 +279,7 @@ def build_step(self, *args, **kwargs): """ # no need for separate build step for numpy >= 2.0 if LooseVersion(self.version) < LooseVersion('2.0'): - super().build(self, *args, **kwargs) + super().build(*args, **kwargs) def test_step(self): """Run available numpy unit tests, and more.""" @@ -314,7 +314,7 @@ def test_step(self): # see http://projects.scipy.org/numpy/ticket/182 self.testcmd = "unset LDFLAGS && cd .. && %%(python)s -c '%s'" % test_code - FortranPythonPackage.test_step(self) + super().test_step() # temporarily install numpy, it doesn't alow to be used straight from the source dir tmpdir = tempfile.mkdtemp() @@ -377,7 +377,7 @@ def test_step(self): def install_step(self): """Install numpy and remove numpy build dir, so scipy doesn't find it by accident.""" - FortranPythonPackage.install_step(self) + super().install_step() builddir = os.path.join(self.builddir, "numpy") try: From 1dac1404ef086ccafd5fc8e7df69eda9ef26c5ef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Jun 2025 22:07:31 +0200 Subject: [PATCH 4/4] correctly call parent build_step in numpy easyblock Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/easyblocks/n/numpy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 410cd5f45ab..6ef6a832046 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -279,7 +279,7 @@ def build_step(self, *args, **kwargs): """ # no need for separate build step for numpy >= 2.0 if LooseVersion(self.version) < LooseVersion('2.0'): - super().build(*args, **kwargs) + super().build_step(*args, **kwargs) def test_step(self): """Run available numpy unit tests, and more."""