From cd946d88edc062e025d4de78aa5ec15e1e6a1264 Mon Sep 17 00:00:00 2001 From: Robert Chen Date: Tue, 4 Nov 2025 11:06:11 -0800 Subject: [PATCH 1/3] OpenMP Target build scripts for El Cap. --- scripts/lc-builds/toss4_amdclang_omptarget.sh | 98 +++++++++++++++++++ scripts/lc-builds/toss4_clang_omptarget.sh | 47 +++++++++ 2 files changed, 145 insertions(+) create mode 100755 scripts/lc-builds/toss4_amdclang_omptarget.sh create mode 100755 scripts/lc-builds/toss4_clang_omptarget.sh diff --git a/scripts/lc-builds/toss4_amdclang_omptarget.sh b/scripts/lc-builds/toss4_amdclang_omptarget.sh new file mode 100755 index 0000000000..48f7898951 --- /dev/null +++ b/scripts/lc-builds/toss4_amdclang_omptarget.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [[ $# -lt 2 ]]; then + echo + echo "You must pass 2 or more arguments to the script (in the following order): " + echo " 1) compiler version number" + echo " 2) HIP compute architecture" + echo " 3...) optional arguments to cmake" + echo + echo "For example: " + echo " toss4_amdclang.sh 4.1.0 gfx906" + exit +fi + +COMP_VER=$1 +COMP_ARCH=$2 +shift 2 + +HOSTCONFIG="hip_3_X" + +if [[ ${COMP_VER} == 4.* ]] +then +##HIP_CLANG_FLAGS="-mllvm -amdgpu-fixed-function-abi=1" + HOSTCONFIG="hip_4_link_X" +elif [[ ${COMP_VER} == 3.* ]] +then + HOSTCONFIG="hip_3_X" +else + echo "Unknown hip version, using ${HOSTCONFIG} host-config" +fi + +BUILD_SUFFIX=lc_toss4-amdclang-omptarget-${COMP_VER}-${COMP_ARCH} + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo +echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2" +echo " -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\"" +echo + +rm -rf build_${BUILD_SUFFIX} >/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + + +module load cmake/3.24.2 + +# unload rocm to avoid configuration problems where the loaded rocm and COMP_VER +# are inconsistent causing the rocprim from the module to be used unexpectedly +module unload rocm + + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DROCM_ROOT_DIR="/opt/rocm-${COMP_VER}" \ + -DHIP_ROOT_DIR="/opt/rocm-${COMP_VER}/hip" \ + -DHIP_PATH=/opt/rocm-${COMP_VER}/llvm/bin \ + -DENABLE_CLANGFORMAT=On \ + -DCLANGFORMAT_EXECUTABLE=/opt/rocm-5.2.3/llvm/bin/clang-format \ + -DCMAKE_C_COMPILER=/opt/rocm-${COMP_VER}/llvm/bin/amdclang \ + -DCMAKE_CXX_COMPILER=/opt/rocm-${COMP_VER}/llvm/bin/amdclang++ \ + -DCMAKE_HIP_ARCHITECTURES="${COMP_ARCH}" \ + -DGPU_TARGETS="${COMP_ARCH}" \ + -DAMDGPU_TARGETS="${COMP_ARCH}" \ + -DBLT_CXX_STD=c++17 \ + -C "../host-configs/lc-builds/toss4/${HOSTCONFIG}.cmake" \ + -DENABLE_HIP=OFF \ + -DENABLE_OPENMP=ON \ + -DENABLE_CUDA=OFF \ + -DRAJA_ENABLE_TARGET_OPENMP=ON \ + -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx942" \ + -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx942" \ + -DENABLE_BENCHMARKS=On \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. + +echo +echo "***********************************************************************" +echo +echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA" +echo +echo " Please note that you have to have a consistent build environment" +echo " when you make RAJA as cmake may reconfigure; unload the rocm module" +echo " or load the appropriate rocm module (${COMP_VER}) when building." +echo +echo " module unload rocm" +echo " srun -n1 make" +echo +echo "***********************************************************************" diff --git a/scripts/lc-builds/toss4_clang_omptarget.sh b/scripts/lc-builds/toss4_clang_omptarget.sh new file mode 100755 index 0000000000..9b6fce7f2c --- /dev/null +++ b/scripts/lc-builds/toss4_clang_omptarget.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [ "$1" == "" ]; then + echo + echo "You must pass a compiler version number to script. For example," + echo " toss4_clang.sh 10.3.1" + exit +fi + +COMP_VER=$1 +shift 1 + +BUILD_SUFFIX=lc_toss4-clang-${COMP_VER} + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo + +rm -rf build_${BUILD_SUFFIX} 2>/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + +module load cmake/3.24.2 + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DBLT_CXX_STD=c++17 \ + -DENABLE_CLANGFORMAT=On \ + -C ../host-configs/lc-builds/toss4/clang_X.cmake \ + -DENABLE_OPENMP=On \ + -DRAJA_ENABLE_TARGET_OPENMP=ON \ + -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa" \ + -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa" \ + -DENABLE_BENCHMARKS=On \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. +# -DCLANGFORMAT_EXECUTABLE=/usr/tce/packages/clang/clang-14.0.6/bin/clang-format \ From 9940386ad9c5ce5148a61631b1a0a9f99745c15a Mon Sep 17 00:00:00 2001 From: Robert Chen Date: Tue, 4 Nov 2025 11:06:47 -0800 Subject: [PATCH 2/3] Temporary setting of gcc-toolchain. --- host-configs/lc-builds/toss4/hip_3_X.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/host-configs/lc-builds/toss4/hip_3_X.cmake b/host-configs/lc-builds/toss4/hip_3_X.cmake index ffbca5dba7..f7efc302e2 100644 --- a/host-configs/lc-builds/toss4/hip_3_X.cmake +++ b/host-configs/lc-builds/toss4/hip_3_X.cmake @@ -7,9 +7,9 @@ set(RAJA_COMPILER "RAJA_COMPILER_CLANG" CACHE STRING "") -set(CMAKE_CXX_FLAGS_RELEASE "-O2" CACHE STRING "") -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g" CACHE STRING "") -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "") +set(CMAKE_CXX_FLAGS_RELEASE "--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr -O2" CACHE STRING "") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr -O2 -g" CACHE STRING "") +set(CMAKE_CXX_FLAGS_DEBUG "--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr -O0 -g" CACHE STRING "") set(HIP_COMMON_OPT_FLAGS ) set(HIP_COMMON_DEBUG_FLAGS) From cc9fae8bb034cc7f358f2c04585f2391b73fdaaa Mon Sep 17 00:00:00 2001 From: Robert Chen Date: Tue, 11 Nov 2025 16:08:49 -0800 Subject: [PATCH 3/3] Temporarily re-implement memcpy. --- include/RAJA/util/TypeConvert.hpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/include/RAJA/util/TypeConvert.hpp b/include/RAJA/util/TypeConvert.hpp index 51c94a9807..fefe370673 100644 --- a/include/RAJA/util/TypeConvert.hpp +++ b/include/RAJA/util/TypeConvert.hpp @@ -33,6 +33,18 @@ namespace RAJA namespace util { +void * custom_memcpy( void * dest, const void * src, size_t len ) +{ + char * customdest = (char *) dest; + const char * customsrc = (const char *) src; + + for ( size_t ii = 0; ii < len; ++ii ) + { + customdest[ii] = customsrc[ii]; + } + + return dest; +} /*! * Reinterpret any datatype as another datatype of the same size @@ -43,7 +55,8 @@ RAJA_INLINE RAJA_HOST_DEVICE constexpr B reinterp_A_as_B(A const& a) static_assert(sizeof(A) == sizeof(B), "A and B must be the same size"); B b; - memcpy(&b, &a, sizeof(A)); + //memcpy(&b, &a, sizeof(A)); + custom_memcpy(&b, &a, sizeof(A)); return b; }