diff --git a/experiments/laghos/experiment.py b/experiments/laghos/experiment.py index d99c21c62..db3997c2d 100644 --- a/experiments/laghos/experiment.py +++ b/experiments/laghos/experiment.py @@ -33,10 +33,21 @@ class Laghos( description="app version", ) + variant( + "gpu-aware-mpi", + default=False, + values=(True, False), + description="Use GPU-aware MPI", + ) + maintainers("wdhawkins") def compute_applications_section(self): # "zones" defined from mesh file, we are hardcoding it here + self.add_experiment_variable("nx", 2, True) + self.add_experiment_variable("ny", 2, True) + self.add_experiment_variable("nz", 2, True) + self.add_experiment_variable("tf", 0.0033, True) self.add_experiment_variable("zones", 1024, True) # resource_count is the number of resources used for this experiment: @@ -70,9 +81,19 @@ def compute_applications_section(self): if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): self.add_experiment_variable("n_gpus", "{n_resources}", True) + if self.spec.satisfies("+gpu-aware-mpi"): + self.add_experiment_variable("gam", "--gpu-aware-mpi") + else: + self.add_experiment_variable("gam", "--no-gpu-aware-mpi") else: self.add_experiment_variable("n_ranks", "{n_resources}", True) def compute_package_section(self): - self.add_package_spec(self.name, [f"laghos{self.determine_version()} +metis"]) + gam = "~gpu-aware-mpi" + if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): + if self.spec.satisfies("+gpu-aware-mpi"): + gam = "+gpu-aware-mpi" + self.add_package_spec( + self.name, [f"laghos{self.determine_version()} +metis {gam}"] + ) self.add_package_spec("hypre", ["hypre@2.32.0: +lapack"]) diff --git a/repo/caliper/for_aarch64.patch b/repo/caliper/for_aarch64.patch deleted file mode 100644 index d3fed9a09..000000000 --- a/repo/caliper/for_aarch64.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- spack-src/src/services/callpath/Callpath.cpp.bak 2020-10-28 14:38:19.668122844 +0900 -+++ spack-src/src/services/callpath/Callpath.cpp 2020-10-28 15:03:12.258061188 +0900 -@@ -63,7 +63,7 @@ - unw_context_t unw_ctx; - unw_cursor_t unw_cursor; - -- unw_getcontext(&unw_ctx); -+ unw_getcontext(unw_ctx); - - if (unw_init_local(&unw_cursor, &unw_ctx) < 0) { - Log(0).stream() << "callpath: unable to init libunwind cursor" << endl; diff --git a/repo/caliper/package.py b/repo/caliper/package.py deleted file mode 100644 index 94744913a..000000000 --- a/repo/caliper/package.py +++ /dev/null @@ -1,295 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack.package import * - - -class Caliper(CachedCMakePackage, CudaPackage, ROCmPackage): - """Caliper is a program instrumentation and performance measurement - framework. It is designed as a performance analysis toolbox in a - library, allowing one to bake performance analysis capabilities - directly into applications and activate them at runtime. - """ - - homepage = "https://github.com/LLNL/Caliper" - git = "https://github.com/LLNL/Caliper.git" - url = "https://github.com/LLNL/Caliper/archive/v2.12.1.tar.gz" - tags = ["e4s", "radiuss"] - - maintainers("daboehme", "adrienbernede") - - test_requires_compiler = True - - license("BSD-3-Clause") - - version("master", branch="master") - version("2.12.1", sha256="2b5a8f98382c94dc75cc3f4517c758eaf9a3f9cea0a8dbdc7b38506060d6955c") - version("2.11.0", sha256="b86b733cbb73495d5f3fe06e6a9885ec77365c8aa9195e7654581180adc2217c") - version("2.10.0", sha256="14c4fb5edd5e67808d581523b4f8f05ace8549698c0e90d84b53171a77f58565") - version("2.9.1", sha256="4771d630de505eff9227e0ec498d0da33ae6f9c34df23cb201b56181b8759e9e") - version("2.9.0", sha256="507ea74be64a2dfd111b292c24c4f55f459257528ba51a5242313fa50978371f") - version( - "2.8.0", - sha256="17807b364b5ac4b05997ead41bd173e773f9a26ff573ff2fe61e0e70eab496e4", - deprecated=True, - ) - version( - "2.7.0", - sha256="b3bf290ec2692284c6b4f54cc0c507b5700c536571d3e1a66e56626618024b2b", - deprecated=True, - ) - version( - "2.6.0", - sha256="6efcd3e4845cc9a6169e0d934840766b12182c6d09aa3ceca4ae776e23b6360f", - deprecated=True, - ) - version( - "2.5.0", - sha256="d553e60697d61c53de369b9ca464eb30710bda90fba9671201543b64eeac943c", - deprecated=True, - ) - version( - "2.4.0", tag="v2.4.0", commit="30577b4b8beae104b2b35ed487fec52590a99b3d", deprecated=True - ) - version( - "2.3.0", tag="v2.3.0", commit="9fd89bb0120750d1f9dfe37bd963e24e478a2a20", deprecated=True - ) - version( - "2.2.0", tag="v2.2.0", commit="c408e9b3642c7aa80eff37b0826d819c57e7bc04", deprecated=True - ) - version( - "2.1.1", tag="v2.1.1", commit="0593b0e01c1d8d3e50c990399cc0fee403485599", deprecated=True - ) - version( - "2.0.1", tag="v2.0.1", commit="4d7ff46381c53a461e62edd949e2d9dea9db7b08", deprecated=True - ) - version( - "1.9.1", tag="v1.9.1", commit="cfc1defbbee20b50dd3e3477badd09a92b1df970", deprecated=True - ) - version( - "1.9.0", tag="v1.9.0", commit="8356e747349b285aa621c5b74e71559f0babc4a1", deprecated=True - ) - version( - "1.8.0", tag="v1.8.0", commit="117c1ef596b617dc71407b8b67eebef094a654f8", deprecated=True - ) - version( - "1.7.0", tag="v1.7.0", commit="898277c93d884d4e7ca1ffcf3bbea81d22364f26", deprecated=True - ) - - #depends_on("c", type="build") # generated - #depends_on("cxx", type="build") # generated - #depends_on("fortran", type="build") # generated - - is_linux = sys.platform.startswith("linux") - variant("shared", default=True, description="Build shared libraries") - variant("adiak", default=True, description="Enable Adiak support") - variant("mpi", default=True, description="Enable MPI support") - # libunwind has some issues on Mac - variant( - "libunwind", default=sys.platform != "darwin", description="Enable stack unwind support" - ) - variant("libdw", default=is_linux, description="Enable DWARF symbol lookup") - # pthread_self() signature is incompatible with PAPI_thread_init() on Mac - variant("papi", default=sys.platform != "darwin", description="Enable PAPI service") - variant("libpfm", default=False, description="Enable libpfm (perf_events) service") - # Gotcha is Linux-only - variant("gotcha", default=is_linux, description="Enable GOTCHA support") - variant("sampler", default=is_linux, description="Enable sampling support on Linux") - variant("sosflow", default=False, description="Enable SOSflow support") - variant("fortran", default=False, description="Enable Fortran support") - variant("variorum", default=False, description="Enable Variorum support") - variant("vtune", default=False, description="Enable Intel Vtune support") - variant("kokkos", default=True, when="@2.3.0:", description="Enable Kokkos profiling support") - variant("tests", default=False, description="Enable tests") - variant("tools", default=True, description="Enable tools") - variant("python", default=False, when="@v2.12:", description="Build Python bindings") - - depends_on("c", type="build") - depends_on("cxx", type="build") - depends_on("fortran", type="build") - - depends_on("adiak@0.1:0", when="@2.2:2.10 +adiak") - depends_on("adiak@0.4:0", when="@2.11: +adiak") - - depends_on("papi@5.3:5", when="@:2.2 +papi") - depends_on("papi@5.3:", when="@2.3: +papi") - - depends_on("libpfm4@4.8:4", when="+libpfm") - - depends_on("mpi", when="+mpi") - depends_on("unwind@1.2:1", when="+libunwind") - depends_on("elfutils", when="+libdw") - depends_on("variorum", when="+variorum") - depends_on("intel-oneapi-vtune", when="+vtune") - - depends_on("sosflow@spack", when="@1.0:1+sosflow") - - depends_on("cmake", type="build") - depends_on("python", type="build") - - depends_on("python@3", when="+python", type=("build", "link", "run")) - depends_on("py-pybind11", when="+python", type=("build", "link", "run")) - - # sosflow support not yet in 2.0 - conflicts("+sosflow", "@2:") - conflicts("+adiak", "@:2.1") - conflicts("+libdw", "@:2.4") - conflicts("+rocm", "@:2.7") - conflicts("+rocm+cuda") - - patch("for_aarch64.patch", when="@:2.11 target=aarch64:") - patch( - "sampler-service-missing-libunwind-include-dir.patch", - when="@2.9.0:2.9.1 +libunwind +sampler", - ) - - def _get_sys_type(self, spec): - sys_type = spec.architecture - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - def initconfig_compiler_entries(self): - spec = self.spec - entries = super().initconfig_compiler_entries() - - if spec.satisfies("+rocm"): - entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) - - entries.append(cmake_cache_option("WITH_FORTRAN", spec.satisfies("+fortran"))) - - entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) - entries.append(cmake_cache_option("BUILD_TESTING", spec.satisfies("+tests"))) - entries.append(cmake_cache_option("WITH_TOOLS", spec.satisfies("+tools"))) - entries.append(cmake_cache_option("BUILD_DOCS", False)) - entries.append(cmake_cache_path("PYTHON_EXECUTABLE", spec["python"].command.path)) - - return entries - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super().initconfig_hardware_entries() - - if spec.satisfies("+cuda"): - entries.append(cmake_cache_option("WITH_CUPTI", True)) - entries.append(cmake_cache_option("WITH_NVTX", True)) - entries.append(cmake_cache_path("CUDA_TOOLKIT_ROOT_DIR", spec["cuda"].prefix)) - entries.append(cmake_cache_path("CUPTI_PREFIX", spec["cuda"].prefix)) - else: - entries.append(cmake_cache_option("WITH_CUPTI", False)) - entries.append(cmake_cache_option("WITH_NVTX", False)) - - if spec.satisfies("+rocm"): - entries.append(cmake_cache_option("WITH_ROCTRACER", True)) - entries.append(cmake_cache_option("WITH_ROCTX", True)) - else: - entries.append(cmake_cache_option("WITH_ROCTRACER", False)) - entries.append(cmake_cache_option("WITH_ROCTX", False)) - - return entries - - def initconfig_mpi_entries(self): - spec = self.spec - entries = super().initconfig_mpi_entries() - - entries.append(cmake_cache_option("WITH_MPI", spec.satisfies("+mpi"))) - - return entries - - def initconfig_package_entries(self): - spec = self.spec - entries = [] - - # TPL locations - entries.append("#------------------{0}".format("-" * 60)) - entries.append("# TPLs") - entries.append("#------------------{0}\n".format("-" * 60)) - - if spec.satisfies("+adiak"): - entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix)) - if spec.satisfies("+papi"): - entries.append(cmake_cache_path("PAPI_PREFIX", spec["papi"].prefix)) - if spec.satisfies("+libdw"): - entries.append(cmake_cache_path("LIBDW_PREFIX", spec["elfutils"].prefix)) - if spec.satisfies("+libpfm"): - entries.append(cmake_cache_path("LIBPFM_INSTALL", spec["libpfm4"].prefix)) - if spec.satisfies("+sosflow"): - entries.append(cmake_cache_path("SOS_PREFIX", spec["sosflow"].prefix)) - if spec.satisfies("+variorum"): - entries.append(cmake_cache_path("VARIORUM_PREFIX", spec["variorum"].prefix)) - if spec.satisfies("+vtune"): - itt_dir = join_path(spec["intel-oneapi-vtune"].prefix, "vtune", "latest") - entries.append(cmake_cache_path("ITT_PREFIX", itt_dir)) - if spec.satisfies("+libunwind"): - entries.append(cmake_cache_path("LIBUNWIND_PREFIX", spec["unwind"].prefix)) - - # Build options - entries.append("#------------------{0}".format("-" * 60)) - entries.append("# Build Options") - entries.append("#------------------{0}\n".format("-" * 60)) - - entries.append(cmake_cache_option("WITH_ADIAK", spec.satisfies("+adiak"))) - entries.append(cmake_cache_option("WITH_GOTCHA", spec.satisfies("+gotcha"))) - entries.append(cmake_cache_option("WITH_SAMPLER", spec.satisfies("+sampler"))) - entries.append(cmake_cache_option("WITH_PAPI", spec.satisfies("+papi"))) - entries.append(cmake_cache_option("WITH_LIBDW", spec.satisfies("+libdw"))) - entries.append(cmake_cache_option("WITH_LIBPFM", spec.satisfies("+libpfm"))) - entries.append(cmake_cache_option("WITH_SOSFLOW", spec.satisfies("+sosflow"))) - entries.append(cmake_cache_option("WITH_KOKKOS", spec.satisfies("+kokkos"))) - entries.append(cmake_cache_option("WITH_VARIORUM", spec.satisfies("+variorum"))) - entries.append(cmake_cache_option("WITH_VTUNE", spec.satisfies("+vtune"))) - entries.append(cmake_cache_option("WITH_PYTHON_BINDINGS", spec.satisfies("+python"))) - - # -DWITH_CALLPATH was renamed -DWITH_LIBUNWIND in 2.5 - callpath_flag = "LIBUNWIND" if spec.satisfies("@2.5:") else "CALLPATH" - entries.append(cmake_cache_option("WITH_%s" % callpath_flag, spec.satisfies("+libunwind"))) - - return entries - - def cmake_args(self): - return [] - - def setup_run_environment(self, env): - if self.spec.satisfies("+python"): - env.prepend_path("PYTHONPATH", self.spec.prefix.join(python_platlib)) - env.prepend_path("PYTHONPATH", self.spec.prefix.join(python_purelib)) - - @run_after("install") - def cache_test_sources(self): - """Copy the example source files after the package is installed to an - install test subdirectory for use during `spack test run`.""" - cache_extra_test_sources(self, [join_path("examples", "apps")]) - - def test_cxx_example(self): - """build and run cxx-example""" - - exe = "cxx-example" - source_file = "{0}.cpp".format(exe) - - source_path = find_required_file( - self.test_suite.current_test_cache_dir, source_file, expected=1, recursive=True - ) - - lib_dir = self.prefix.lib if os.path.exists(self.prefix.lib) else self.prefix.lib64 - - cxx = which(os.environ["CXX"]) - test_dir = os.path.dirname(source_path) - with working_dir(test_dir): - cxx( - "-L{0}".format(lib_dir), - "-I{0}".format(self.prefix.include), - source_path, - "-o", - exe, - "-std=c++11", - "-lcaliper", - "-lstdc++", - ) - - cxx_example = which(exe) - cxx_example() diff --git a/repo/caliper/sampler-service-missing-libunwind-include-dir.patch b/repo/caliper/sampler-service-missing-libunwind-include-dir.patch deleted file mode 100644 index 4acd660f6..000000000 --- a/repo/caliper/sampler-service-missing-libunwind-include-dir.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff -ruN spack-src/src/services/sampler/CMakeLists.txt spack-src-patched/src/services/sampler/CMakeLists.txt ---- spack-src/src/services/sampler/CMakeLists.txt 2022-11-30 13:52:42.000000000 -0500 -+++ spack-src-patched/src/services/sampler/CMakeLists.txt 2023-05-04 20:43:47.240310306 -0400 -@@ -17,6 +17,10 @@ - - include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - -+if (CALIPER_HAVE_LIBUNWIND) -+ include_directories(${LIBUNWIND_INCLUDE_DIRS}) -+endif() -+ - add_library(caliper-sampler OBJECT ${CALIPER_SAMPLER_SOURCES}) - - add_service_objlib("caliper-sampler") diff --git a/repo/laghos/application.py b/repo/laghos/application.py index de4b188ec..5fa2ed6ac 100644 --- a/repo/laghos/application.py +++ b/repo/laghos/application.py @@ -19,14 +19,26 @@ class Laghos(ExecutableApplication): 'lagrangian','spatial-discretization','unstructured-grid', 'network-latency-bound','network-collectives','unstructured-grid'] - executable('prob', 'laghos -p {problem} -m {mesh} -rs {rs} -rp {rp} -ms {ms} -d {device}', use_mpi=True) + executable('prob', 'laghos -p {problem} -m {mesh} -nx {nx} -ny {ny} -nz {nz} -rs {rs} -rp {rp} -ms {ms} --fom {gam} -d {device} {assembly} -tf {tf}', use_mpi=True) workload('triplept', executables=['prob']) - workload_variable('mesh', default='{laghos}/data/box01_hex.mesh', + workload_variable('mesh', default='default', description='mesh file', workloads=['triplept']) + workload_variable('nx', default='2', + description='Elements in x-dimension', + workloads=['triplept']) + + workload_variable('ny', default='2', + description='Elements in y-dimension', + workloads=['triplept']) + + workload_variable('nz', default='2', + description='Elements in z-dimension', + workloads=['triplept']) + workload_variable('problem', default='3', description='problem number', workloads=['triplept']) @@ -42,10 +54,23 @@ class Laghos(ExecutableApplication): workload_variable('ms', default='250', description='max number of steps', workloads=['triplept']) + workload_variable('device', default='cpu', - description='cpu or cuda', + description='cpu, cuda or hip', workloads=['triplept']) + workload_variable('gam', default='--no-gpu-aware-mpi', + description='--gpu-aware-mpi or --no-gpu-aware-mpi', + workloads=['triplept']) + + workload_variable('assembly', default='-pa', + description='Activate 1D tensor-based assembly (partial assembly).', + workloads=['triplept']) + + workload_variable('tf', default='0.6', + description='Final time; start time is 0.', + workloads=['triplept']) + figure_of_merit('Major kernels total time', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Major kernels total time \(seconds\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', diff --git a/repo/laghos/package.py b/repo/laghos/package.py index 2adf767a1..8414d99e6 100644 --- a/repo/laghos/package.py +++ b/repo/laghos/package.py @@ -16,17 +16,18 @@ class Laghos(MakefilePackage, CudaPackage, ROCmPackage): tags = ["proxy-app", "ecp-proxy-app"] homepage = "https://github.com/CEED/Laghos" - git = "https://github.com/wdhawkins/Laghos.git" + git = "https://github.com/CEED/Laghos.git" maintainers("wdhawkins") license("BSD-2-Clause") - version("develop", branch="caliper") + version("develop", branch="master") variant("metis", default=True, description="Enable/disable METIS support") variant("caliper", default=False, description="Enable/disable Caliper support") variant("ofast", default=False, description="Enable gcc optimization flags") + variant("gpu-aware-mpi", default=False, description="Enable GPU aware MPI") depends_on("c", type="build") depends_on("cxx", type="build") @@ -69,7 +70,7 @@ class Laghos(MakefilePackage, CudaPackage, ROCmPackage): depends_on(f"hypre amdgpu_target={target}", when=f"amdgpu_target={target}") depends_on(f"mfem amdgpu_target={target}", when=f"amdgpu_target={target}") - depends_on("hypre+gpu-aware-mpi", when="^cray-mpich+gtl") + depends_on("hypre+gpu-aware-mpi", when="+gpu-aware-mpi") # Replace MPI_Session patch( diff --git a/repo/mfem/package.py b/repo/mfem/package.py index db456df40..084f2e106 100644 --- a/repo/mfem/package.py +++ b/repo/mfem/package.py @@ -15,8 +15,6 @@ class Mfem(BuiltinMfem): variant("caliper", default=False, description="Build Caliper support") - depends_on("hipblas", when="@4.4.0:+rocm") - depends_on("caliper", when="+caliper") depends_on("adiak", when="+caliper") @@ -32,651 +30,15 @@ def configure(self, spec, prefix): def setup_build_environment(self, env): if "+cuda" in self.spec: env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") + if "+mpi" in self.spec: + if self.spec["mpi"].extra_attributes and "ldflags" in self.spec["mpi"].extra_attributes: + env.append_flags("LDFLAGS", self.spec["mpi"].extra_attributes["ldflags"]) - # - # Note: Although MFEM does support CMake configuration, MFEM - # development team indicates that vanilla GNU Make is the - # preferred mode of configuration of MFEM and the mode most - # likely to be up to date in supporting *all* of MFEM's - # configuration options. So, don't use CMake - # def get_make_config_options(self, spec, prefix): def yes_no(varstr): return "YES" if varstr in self.spec else "NO" - xcompiler = "" if "~cuda" in spec else "-Xcompiler=" - - # We need to add rpaths explicitly to allow proper export of link flags - # from within MFEM. We use the following two functions to do that. - ld_flags_from_library_list = self.ld_flags_from_library_list - ld_flags_from_dirs = self.ld_flags_from_dirs - - def find_optional_library(name, prefix): - for shared in [True, False]: - for path in ["lib64", "lib"]: - lib = find_libraries( - name, join_path(prefix, path), shared=shared, recursive=False - ) - if lib: - return lib - return LibraryList([]) - - # Determine how to run MPI tests, e.g. when using '--test=root', when - # Spack is run inside a batch system job. - mfem_mpiexec = "mpirun" - mfem_mpiexec_np = "-np" - if "SLURM_JOBID" in os.environ: - mfem_mpiexec = "srun" - mfem_mpiexec_np = "-n" - elif "LSB_JOBID" in os.environ: - if "LLNL_COMPUTE_NODES" in os.environ: - mfem_mpiexec = "lrun" - mfem_mpiexec_np = "-n" - else: - mfem_mpiexec = "jsrun" - mfem_mpiexec_np = "-p" - elif "FLUX_EXEC_PATH" in os.environ: - mfem_mpiexec = "flux run" - mfem_mpiexec_np = "-n" - elif "PBS_JOBID" in os.environ: - mfem_mpiexec = "mpiexec" - mfem_mpiexec_np = "-n" - - metis5_str = "NO" - if ("+metis" in spec) and spec["metis"].satisfies("@5:"): - metis5_str = "YES" - - zlib_var = "MFEM_USE_ZLIB" if (spec.satisfies("@4.1.0:")) else "MFEM_USE_GZSTREAM" - - options = [ - "PREFIX=%s" % prefix, - "MFEM_USE_MEMALLOC=YES", - "MFEM_DEBUG=%s" % yes_no("+debug"), - # NOTE: env["CXX"] is the spack c++ compiler wrapper. The real - # compiler is defined by env["SPACK_CXX"]. - "CXX=%s" % env["CXX"], - "MFEM_USE_LIBUNWIND=%s" % yes_no("+libunwind"), - "%s=%s" % (zlib_var, yes_no("+zlib")), - "MFEM_USE_METIS=%s" % yes_no("+metis"), - "MFEM_USE_METIS_5=%s" % metis5_str, - "MFEM_THREAD_SAFE=%s" % yes_no("+threadsafe"), - "MFEM_USE_MPI=%s" % yes_no("+mpi"), - "MFEM_USE_LAPACK=%s" % yes_no("+lapack"), - "MFEM_USE_SUPERLU=%s" % yes_no("+superlu-dist"), - "MFEM_USE_STRUMPACK=%s" % yes_no("+strumpack"), - "MFEM_USE_SUITESPARSE=%s" % yes_no("+suite-sparse"), - "MFEM_USE_SUNDIALS=%s" % yes_no("+sundials"), - "MFEM_USE_PETSC=%s" % yes_no("+petsc"), - "MFEM_USE_SLEPC=%s" % yes_no("+slepc"), - "MFEM_USE_PUMI=%s" % yes_no("+pumi"), - "MFEM_USE_GSLIB=%s" % yes_no("+gslib"), - "MFEM_USE_NETCDF=%s" % yes_no("+netcdf"), - "MFEM_USE_MPFR=%s" % yes_no("+mpfr"), - "MFEM_USE_GNUTLS=%s" % yes_no("+gnutls"), - "MFEM_USE_OPENMP=%s" % yes_no("+openmp"), - "MFEM_USE_CONDUIT=%s" % yes_no("+conduit"), - "MFEM_USE_CUDA=%s" % yes_no("+cuda"), - "MFEM_USE_HIP=%s" % yes_no("+rocm"), - "MFEM_USE_OCCA=%s" % yes_no("+occa"), - "MFEM_USE_RAJA=%s" % yes_no("+raja"), - "MFEM_USE_AMGX=%s" % yes_no("+amgx"), - "MFEM_USE_CEED=%s" % yes_no("+libceed"), - "MFEM_USE_UMPIRE=%s" % yes_no("+umpire"), - "MFEM_USE_FMS=%s" % yes_no("+fms"), - "MFEM_USE_GINKGO=%s" % yes_no("+ginkgo"), - "MFEM_USE_HIOP=%s" % yes_no("+hiop"), - "MFEM_MPIEXEC=%s" % mfem_mpiexec, - "MFEM_MPIEXEC_NP=%s" % mfem_mpiexec_np, - "MFEM_USE_EXCEPTIONS=%s" % yes_no("+exceptions"), - "MFEM_USE_MUMPS=%s" % yes_no("+mumps"), - ] - if spec.satisfies("@4.7.0:"): - options += ["MFEM_PRECISION=%s" % spec.variants["precision"].value] - - # Determine C++ standard to use: - cxxstd = None - if self.spec.satisfies("@4.0.0:"): - cxxstd = "11" - if self.spec.satisfies("^raja@2022.03.0:"): - cxxstd = "14" - if self.spec.satisfies("^umpire@2022.03.0:"): - cxxstd = "14" - if self.spec.satisfies("^sundials@6.4.0:"): - cxxstd = "14" - if self.spec.satisfies("^ginkgo"): - cxxstd = "14" - # When rocPRIM is used (e.g. by PETSc + ROCm) we need C++14: - if self.spec.satisfies("^rocprim@5.5.0:"): - cxxstd = "14" - cxxstd_req = spec.variants["cxxstd"].value - if cxxstd_req != "auto": - # Constraints for valid standard level should be imposed during - # concretization based on 'conflicts' or other directives. - cxxstd = cxxstd_req - cxxstd_flag = None - if cxxstd: - if "+cuda" in spec: - cxxstd_flag = "-std=c++" + cxxstd - else: - cxxstd_flag = getattr(self.compiler, "cxx" + cxxstd + "_flag") - - cuda_arch = None if "~cuda" in spec else spec.variants["cuda_arch"].value - - cxxflags = spec.compiler_flags["cxxflags"].copy() - - if cxxflags: - # Add opt/debug flags if they are not present in global cxx flags - opt_flag_found = any(f in self.compiler.opt_flags for f in cxxflags) - debug_flag_found = any(f in self.compiler.debug_flags for f in cxxflags) - - if "+debug" in spec: - if not debug_flag_found: - cxxflags.append("-g") - if not opt_flag_found: - cxxflags.append("-O0") - else: - if not opt_flag_found: - cxxflags.append("-O2") - - cxxflags = [(xcompiler + flag) for flag in cxxflags] - if "+cuda" in spec: - cxxflags += [ - "-x=cu --expt-extended-lambda -arch=sm_%s" % cuda_arch, - "-ccbin %s" % (spec["mpi"].mpicxx if "+mpi" in spec else env["CXX"]), - ] - if cxxstd_flag: - cxxflags.append(cxxstd_flag) - # The cxxflags are set by the spack c++ compiler wrapper. We also - # set CXXFLAGS explicitly, for clarity, and to properly export the - # cxxflags in the variable MFEM_CXXFLAGS in config.mk. - options += ["CXXFLAGS=%s" % " ".join(cxxflags)] - - elif cxxstd_flag: - options += ["BASE_FLAGS=%s" % cxxstd_flag] - - # Treat any 'CXXFLAGS' in the environment as extra c++ flags which are - # handled through the 'CPPFLAGS' makefile variable in MFEM. Also, unset - # 'CXXFLAGS' from the environment to prevent it from overriding the - # defaults. - if "CXXFLAGS" in env: - options += ["CPPFLAGS=%s" % env["CXXFLAGS"]] - del env["CXXFLAGS"] - - if "~static" in spec: - options += ["STATIC=NO"] - if "+shared" in spec: - options += ["SHARED=YES", "PICFLAG=%s" % (xcompiler + self.compiler.cxx_pic_flag)] - - if "+mpi" in spec: - options += ["MPICXX=%s" % spec["mpi"].mpicxx] - hypre = spec["hypre"] - all_hypre_libs = hypre.libs - if "+lapack" in hypre: - all_hypre_libs += hypre["lapack"].libs + hypre["blas"].libs - - hypre_gpu_libs = "" - if "+cuda" in hypre: - hypre_gpu_libs = " -lcusparse -lcurand -lcublas" - elif "+rocm" in hypre: - hypre_rocm_libs = LibraryList([]) - if "^rocsparse" in hypre: - hypre_rocm_libs += hypre["rocsparse"].libs - if "^rocrand" in hypre: - hypre_rocm_libs += hypre["rocrand"].libs - hypre_gpu_libs = " " + ld_flags_from_library_list(hypre_rocm_libs) - options += [ - "HYPRE_OPT=-I%s" % hypre.prefix.include, - "HYPRE_LIB=%s%s" % (ld_flags_from_library_list(all_hypre_libs), hypre_gpu_libs), - ] - - if "+metis" in spec: - options += [ - "METIS_OPT=-I%s" % spec["metis"].prefix.include, - "METIS_LIB=%s" % ld_flags_from_library_list(spec["metis"].libs), - ] - - if "+lapack" in spec: - lapack_blas = spec["lapack"].libs + spec["blas"].libs - options += [ - # LAPACK_OPT is not used - "LAPACK_LIB=%s" - % ld_flags_from_library_list(lapack_blas) - ] - - if "+superlu-dist" in spec: - lapack_blas = spec["lapack"].libs + spec["blas"].libs - options += [ - "SUPERLU_OPT=-I%s -I%s" - % (spec["superlu-dist"].prefix.include, spec["parmetis"].prefix.include), - "SUPERLU_LIB=%s %s" - % ( - ld_flags_from_dirs( - [spec["superlu-dist"].prefix.lib, spec["parmetis"].prefix.lib], - ["superlu_dist", "parmetis"], - ), - ld_flags_from_library_list(lapack_blas), - ), - ] - - if "+strumpack" in spec: - strumpack = spec["strumpack"] - sp_opt = ["-I%s" % strumpack.prefix.include] - sp_lib = [ld_flags_from_library_list(strumpack.libs)] - # Parts of STRUMPACK use fortran, so we need to link with the - # fortran library and also the MPI fortran library: - if "~shared" in strumpack: - if os.path.basename(env["FC"]) == "gfortran": - gfortran = Executable(env["FC"]) - libext = "dylib" if sys.platform == "darwin" else "so" - libfile = os.path.abspath( - gfortran("-print-file-name=libgfortran.%s" % libext, output=str).strip() - ) - gfortran_lib = LibraryList(libfile) - sp_lib += [ld_flags_from_library_list(gfortran_lib)] - if "+mpi" in strumpack: - mpi = strumpack["mpi"] - if ("^mpich" in strumpack) or ("^mvapich2" in strumpack): - sp_lib += [ld_flags_from_dirs([mpi.prefix.lib], ["mpifort"])] - elif "^openmpi" in strumpack: - sp_lib += [ld_flags_from_dirs([mpi.prefix.lib], ["mpi_mpifh"])] - elif "^spectrum-mpi" in strumpack: - sp_lib += [ld_flags_from_dirs([mpi.prefix.lib], ["mpi_ibm_mpifh"])] - if "+openmp" in strumpack: - # The "+openmp" in the spec means strumpack will TRY to find - # OpenMP; if not found, we should not add any flags -- how do - # we figure out if strumpack found OpenMP? - if not self.spec.satisfies("%apple-clang"): - sp_opt += [xcompiler + self.compiler.openmp_flag] - if "^parmetis" in strumpack: - parmetis = strumpack["parmetis"] - sp_opt += [parmetis.headers.cpp_flags] - sp_lib += [ld_flags_from_library_list(parmetis.libs)] - if "^netlib-scalapack" in strumpack: - scalapack = strumpack["scalapack"] - sp_opt += ["-I%s" % scalapack.prefix.include] - sp_lib += [ld_flags_from_dirs([scalapack.prefix.lib], ["scalapack"])] - elif "^scalapack" in strumpack: - scalapack = strumpack["scalapack"] - sp_opt += [scalapack.headers.cpp_flags] - sp_lib += [ld_flags_from_library_list(scalapack.libs)] - if "+butterflypack" in strumpack: - bp = strumpack["butterflypack"] - sp_opt += ["-I%s" % bp.prefix.include] - bp_libs = find_libraries( - ["libdbutterflypack", "libzbutterflypack"], - bp.prefix, - shared=("+shared" in bp), - recursive=True, - ) - sp_lib += [ld_flags_from_library_list(bp_libs)] - if "+zfp" in strumpack: - zfp = strumpack["zfp"] - sp_opt += ["-I%s" % zfp.prefix.include] - zfp_lib = find_libraries( - "libzfp", zfp.prefix, shared=("+shared" in zfp), recursive=True - ) - sp_lib += [ld_flags_from_library_list(zfp_lib)] - if "+cuda" in strumpack: - # assuming also ("+cuda" in spec) - sp_lib += ["-lcusolver", "-lcublas"] - options += [ - "STRUMPACK_OPT=%s" % " ".join(sp_opt), - "STRUMPACK_LIB=%s" % " ".join(sp_lib), - ] - - if "+suite-sparse" in spec: - ss_spec = "suite-sparse:" + self.suitesparse_components - options += [ - "SUITESPARSE_OPT=-I%s" % spec[ss_spec].prefix.include, - "SUITESPARSE_LIB=%s" % ld_flags_from_library_list(spec[ss_spec].libs), - ] - - if "+sundials" in spec: - sun_spec = "sundials:" + self.sundials_components - options += [ - "SUNDIALS_OPT=%s" % spec[sun_spec].headers.cpp_flags, - "SUNDIALS_LIB=%s" % ld_flags_from_library_list(spec[sun_spec].libs), - ] - - if "+petsc" in spec: - petsc = spec["petsc"] - if "+shared" in petsc: - options += [ - "PETSC_OPT=%s" % petsc.headers.cpp_flags, - "PETSC_LIB=%s" % ld_flags_from_library_list(petsc.libs), - ] - else: - options += ["PETSC_DIR=%s" % petsc.prefix] - - if "+slepc" in spec: - slepc = spec["slepc"] - options += [ - "SLEPC_OPT=%s" % slepc.headers.cpp_flags, - "SLEPC_LIB=%s" % ld_flags_from_library_list(slepc.libs), - ] - - if "+pumi" in spec: - pumi_libs = [ - "pumi", - "crv", - "ma", - "mds", - "apf", - "pcu", - "gmi", - "parma", - "lion", - "mth", - "apf_zoltan", - "spr", - ] - pumi_dep_zoltan = "" - pumi_dep_parmetis = "" - if "+zoltan" in spec["pumi"]: - pumi_dep_zoltan = ld_flags_from_dirs([spec["zoltan"].prefix.lib], ["zoltan"]) - if "+parmetis" in spec["zoltan"]: - pumi_dep_parmetis = ld_flags_from_dirs( - [spec["parmetis"].prefix.lib], ["parmetis"] - ) - options += [ - "PUMI_OPT=-I%s" % spec["pumi"].prefix.include, - "PUMI_LIB=%s %s %s" - % ( - ld_flags_from_dirs([spec["pumi"].prefix.lib], pumi_libs), - pumi_dep_zoltan, - pumi_dep_parmetis, - ), - ] - - if "+gslib" in spec: - options += [ - "GSLIB_OPT=-I%s" % spec["gslib"].prefix.include, - "GSLIB_LIB=%s" % ld_flags_from_dirs([spec["gslib"].prefix.lib], ["gs"]), - ] - - if "+netcdf" in spec: - lib_flags = ld_flags_from_dirs([spec["netcdf-c"].prefix.lib], ["netcdf"]) - hdf5 = spec["hdf5:hl"] - if hdf5.satisfies("~shared"): - hdf5_libs = hdf5.libs - hdf5_libs += LibraryList(find_system_libraries("libdl")) - lib_flags += " " + ld_flags_from_library_list(hdf5_libs) - options += [ - "NETCDF_OPT=-I%s" % spec["netcdf-c"].prefix.include, - "NETCDF_LIB=%s" % lib_flags, - ] - - if "+zlib" in spec: - if "@:3.3.2" in spec: - options += ["ZLIB_DIR=%s" % spec["zlib-api"].prefix] - else: - options += [ - "ZLIB_OPT=-I%s" % spec["zlib-api"].prefix.include, - "ZLIB_LIB=%s" % ld_flags_from_library_list(spec["zlib-api"].libs), - ] - - if "+mpfr" in spec: - options += [ - "MPFR_OPT=-I%s" % spec["mpfr"].prefix.include, - "MPFR_LIB=%s" % ld_flags_from_dirs([spec["mpfr"].prefix.lib], ["mpfr"]), - ] - - if "+gnutls" in spec: - options += [ - "GNUTLS_OPT=-I%s" % spec["gnutls"].prefix.include, - "GNUTLS_LIB=%s" % ld_flags_from_dirs([spec["gnutls"].prefix.lib], ["gnutls"]), - ] - - if "+libunwind" in spec: - libunwind = spec["unwind"] - headers = find_headers("libunwind", libunwind.prefix.include) - headers.add_macro("-g") - libs = find_optional_library("libunwind", libunwind.prefix) - # When mfem uses libunwind, it also needs "libdl". - libs += LibraryList(find_system_libraries("libdl")) - options += [ - "LIBUNWIND_OPT=%s" % headers.cpp_flags, - "LIBUNWIND_LIB=%s" % ld_flags_from_library_list(libs), - ] - - if "+openmp" in spec: - options += ["OPENMP_OPT=%s" % (xcompiler + self.compiler.openmp_flag)] - - if "+cuda" in spec: - options += [ - "CUDA_CXX=%s" % join_path(spec["cuda"].prefix, "bin", "nvcc"), - "CUDA_ARCH=sm_%s" % cuda_arch, - ] - # Check if we are using a CUDA installation where the math libs are - # in a separate directory: - culibs = ["libcusparse"] - cuda_libs = find_optional_library(culibs, spec["cuda"].prefix) - if not cuda_libs: - p0 = os.path.realpath(join_path(spec["cuda"].prefix, "bin", "nvcc")) - p0 = os.path.dirname(p0) - p1 = os.path.dirname(p0) - while p1 != p0: - cuda_libs = find_optional_library(culibs, join_path(p1, "math_libs")) - if cuda_libs: - break - p0, p1 = p1, os.path.dirname(p1) - if not cuda_libs: - raise InstallError("Required CUDA libraries not found: %s" % culibs) - options += ["CUDA_LIB=%s" % ld_flags_from_library_list(cuda_libs)] - - if "+rocm" in spec: - amdgpu_target = ",".join(spec.variants["amdgpu_target"].value) - options += ["HIP_CXX=%s" % spec["hip"].hipcc, "HIP_ARCH=%s" % amdgpu_target] - hip_headers = HeaderList([]) - hip_libs = LibraryList([]) - # To use a C++ compiler that supports -xhip flag one can use - # something like this: - # options += [ - # "HIP_CXX=%s" % (spec["mpi"].mpicxx if "+mpi" in spec else spack_cxx), - # "HIP_FLAGS=-xhip --offload-arch=%s" % amdgpu_target, - # ] - # hip_libs += find_libraries("libamdhip64", spec["hip"].prefix.lib) - if "^hipsparse" in spec: # hipsparse is needed @4.4.0:+rocm - hipsparse = spec["hipsparse"] - hip_headers += hipsparse.headers - hip_libs += hipsparse.libs - # Note: MFEM's defaults.mk wants to find librocsparse.* in - # $(HIP_DIR)/lib, so we set HIP_DIR to be $ROCM_PATH when using - # external HIP, or the prefix of rocsparse (which is a - # dependency of hipsparse) when using Spack-built HIP. - if spec["hip"].external: - options += ["HIP_DIR=%s" % env["ROCM_PATH"]] - else: - options += ["HIP_DIR=%s" % hipsparse["rocsparse"].prefix] - if "^rocthrust" in spec and not spec["hip"].external: - # petsc+rocm needs the rocthrust header path - hip_headers += spec["rocthrust"].headers - if "^rocprim" in spec and not spec["hip"].external: - # rocthrust [via petsc+rocm] has a dependency on rocprim - hip_headers += spec["rocprim"].headers - if "^hipblas" in spec: - hipblas = spec["hipblas"] - hip_headers += hipblas.headers - hip_libs += hipblas.libs - if "%cce" in spec: - # We assume the proper Cray CCE module (cce) is loaded: - proc = str(spec.target.family) - craylibs_var = "CRAYLIBS_" + proc.upper() - craylibs_path = env.get(craylibs_var, None) - if not craylibs_path: - raise InstallError( - f"The environment variable {craylibs_var} is not defined.\n" - "\tMake sure the 'cce' module is in the compiler spec." - ) - craylibs = [ - "libmodules", - "libfi", - "libcraymath", - "libf", - "libu", - "libcsup", - "libpgas-shmem", - ] - hip_libs += find_libraries(craylibs, craylibs_path) - craylibs_path2 = join_path(craylibs_path, "../../../cce-clang", proc, "lib") - hip_libs += find_libraries("libunwind", craylibs_path2) - - if hip_headers: - options += ["HIP_OPT=%s" % hip_headers.cpp_flags] - if hip_libs: - options += ["HIP_LIB=%s" % ld_flags_from_library_list(hip_libs)] - - if "+occa" in spec: - options += [ - "OCCA_OPT=-I%s" % spec["occa"].prefix.include, - "OCCA_LIB=%s" % ld_flags_from_dirs([spec["occa"].prefix.lib], ["occa"]), - ] - - if "+raja" in spec: - raja = spec["raja"] - raja_opt = "-I%s" % raja.prefix.include - raja_lib = find_libraries( - "libRAJA", raja.prefix, shared=("+shared" in raja), recursive=True - ) - if raja.satisfies("^camp"): - camp = raja["camp"] - raja_opt += " -I%s" % camp.prefix.include - raja_lib += find_optional_library("libcamp", camp.prefix) - options += [ - "RAJA_OPT=%s" % raja_opt, - "RAJA_LIB=%s" % ld_flags_from_library_list(raja_lib), - ] - - if "+amgx" in spec: - amgx = spec["amgx"] - if "+shared" in amgx: - options += [ - "AMGX_OPT=-I%s" % amgx.prefix.include, - "AMGX_LIB=%s" % ld_flags_from_library_list(amgx.libs), - ] - else: - options += ["AMGX_DIR=%s" % amgx.prefix] - - if "+libceed" in spec: - options += [ - "CEED_OPT=-I%s" % spec["libceed"].prefix.include, - "CEED_LIB=%s" % ld_flags_from_dirs([spec["libceed"].prefix.lib], ["ceed"]), - ] - - if "+umpire" in spec: - umpire = spec["umpire"] - umpire_opts = umpire.headers - umpire_libs = umpire.libs - if "^camp" in umpire: - umpire_opts += umpire["camp"].headers - if "^fmt" in umpire: - umpire_opts += umpire["fmt"].headers - umpire_libs += umpire["fmt"].libs - options += [ - "UMPIRE_OPT=%s" % umpire_opts.cpp_flags, - "UMPIRE_LIB=%s" % ld_flags_from_library_list(umpire_libs), - ] - - timer_ids = {"std": "0", "posix": "2", "mac": "4", "mpi": "6"} - timer = spec.variants["timer"].value - if timer != "auto": - options += ["MFEM_TIMER_TYPE=%s" % timer_ids[timer]] - - if "+conduit" in spec: - conduit = spec["conduit"] - headers = HeaderList(find(conduit.prefix.include, "conduit.hpp", recursive=True)) - conduit_libs = ["libconduit", "libconduit_relay", "libconduit_blueprint"] - libs = find_libraries(conduit_libs, conduit.prefix.lib, shared=("+shared" in conduit)) - libs += LibraryList(find_system_libraries("libdl")) - if "+hdf5" in conduit: - hdf5 = conduit["hdf5"] - headers += find_headers("hdf5", hdf5.prefix.include) - libs += hdf5.libs - - ################## - # cyrush note: - ################## - # spack's HeaderList is applying too much magic, undermining us: - # - # It applies a regex to strip back to the last "include" dir - # in the path. In our case we need to pass the following - # as part of the CONDUIT_OPT flags: - # - # -I/include/conduit - # - # I tried several ways to present this path to the HeaderList, - # but the regex always kills the trailing conduit dir - # breaking build. - # - # To resolve the issue, we simply join our own string with - # the headers results (which are important b/c they handle - # hdf5 paths when enabled). - ################## - - # construct proper include path - conduit_include_path = conduit.prefix.include.conduit - # add this path to the found flags - conduit_opt_flags = "-I{0} {1}".format(conduit_include_path, headers.cpp_flags) - - options += [ - "CONDUIT_OPT=%s" % conduit_opt_flags, - "CONDUIT_LIB=%s" % ld_flags_from_library_list(libs), - ] - - if "+fms" in spec: - libfms = spec["libfms"] - options += [ - "FMS_OPT=%s" % libfms.headers.cpp_flags, - "FMS_LIB=%s" % ld_flags_from_library_list(libfms.libs), - ] - - if "+ginkgo" in spec: - ginkgo = spec["ginkgo"] - options += [ - "GINKGO_DIR=%s" % ginkgo.prefix, - "GINKGO_BUILD_TYPE=%s" % ginkgo.variants["build_type"].value, - ] - - if "+hiop" in spec: - hiop = spec["hiop"] - hiop_hdrs = hiop.headers - hiop_libs = hiop.libs - hiop_hdrs += spec["lapack"].headers + spec["blas"].headers - hiop_libs += spec["lapack"].libs + spec["blas"].libs - hiop_opt_libs = ["magma", "umpire", "hipblas", "hiprand"] - for opt_lib in hiop_opt_libs: - if "^" + opt_lib in hiop: - hiop_hdrs += hiop[opt_lib].headers - hiop_libs += hiop[opt_lib].libs - # raja's libs property does not work - if "^raja" in hiop: - raja = hiop["raja"] - hiop_hdrs += raja.headers - hiop_libs += find_libraries( - "libRAJA", raja.prefix, shared=("+shared" in raja), recursive=True - ) - if raja.satisfies("^camp"): - camp = raja["camp"] - hiop_hdrs += camp.headers - hiop_libs += find_optional_library("libcamp", camp.prefix) - if hiop.satisfies("@0.6:+cuda"): - hiop_libs += LibraryList(["cublas", "curand"]) - options += [ - "HIOP_OPT=%s" % hiop_hdrs.cpp_flags, - "HIOP_LIB=%s" % ld_flags_from_library_list(hiop_libs), - ] - - if "+mumps" in spec: - mumps = spec["mumps"] - mumps_opt = ["-I%s" % mumps.prefix.include] - if "+openmp" in mumps: - if not self.spec.satisfies("%apple-clang"): - mumps_opt += [xcompiler + self.compiler.openmp_flag] - options += [ - "MUMPS_OPT=%s" % " ".join(mumps_opt), - "MUMPS_LIB=%s" % ld_flags_from_library_list(mumps.libs), - ] - + options = super().get_make_config_options(spec, prefix) options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper")) if "+caliper" in self.spec: options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix)