DRAFT: OpenMP Target Build Scripts for El Cap

host-configs/lc-builds/toss4/hip_3_X.cmake

@@ -7,9 +7,9 @@
 
 set(RAJA_COMPILER "RAJA_COMPILER_CLANG" CACHE STRING "")
 
-set(CMAKE_CXX_FLAGS_RELEASE "-O2" CACHE STRING "")
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g" CACHE STRING "")
-set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE "--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr -O2" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr -O2 -g" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr -O0 -g" CACHE STRING "")
 
 set(HIP_COMMON_OPT_FLAGS )
 set(HIP_COMMON_DEBUG_FLAGS)

include/RAJA/util/TypeConvert.hpp

@@ -33,6 +33,18 @@ namespace RAJA
 namespace util
 {
 
+void * custom_memcpy( void * dest, const void * src, size_t len )
+{
+  char * customdest = (char *) dest;
+  const char * customsrc = (const char *) src;
+
+  for ( size_t ii = 0; ii < len; ++ii )
+  {
+    customdest[ii] = customsrc[ii];
+  }
+
+  return dest;
+}
 
 /*!
  * Reinterpret any datatype as another datatype of the same size
@@ -43,7 +55,8 @@ RAJA_INLINE RAJA_HOST_DEVICE constexpr B reinterp_A_as_B(A const& a)
   static_assert(sizeof(A) == sizeof(B), "A and B must be the same size");
 
   B b;
-  memcpy(&b, &a, sizeof(A));
+  //memcpy(&b, &a, sizeof(A));
+  custom_memcpy(&b, &a, sizeof(A));
   return b;
 }
 

scripts/lc-builds/toss4_amdclang_omptarget.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# Copyright (c) 2016-25, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+if [[ $# -lt 2 ]]; then
+  echo
+  echo "You must pass 2 or more arguments to the script (in the following order): "
+  echo "   1) compiler version number"
+  echo "   2) HIP compute architecture"
+  echo "   3...) optional arguments to cmake"
+  echo
+  echo "For example: "
+  echo "    toss4_amdclang.sh 4.1.0 gfx906"
+  exit
+fi
+
+COMP_VER=$1
+COMP_ARCH=$2
+shift 2
+
+HOSTCONFIG="hip_3_X"
+
+if [[ ${COMP_VER} == 4.* ]]
+then
+##HIP_CLANG_FLAGS="-mllvm -amdgpu-fixed-function-abi=1"
+  HOSTCONFIG="hip_4_link_X"
+elif [[ ${COMP_VER} == 3.* ]]
+then
+  HOSTCONFIG="hip_3_X"
+else
+  echo "Unknown hip version, using ${HOSTCONFIG} host-config"
+fi
+
+BUILD_SUFFIX=lc_toss4-amdclang-omptarget-${COMP_VER}-${COMP_ARCH}
+
+echo
+echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it"
+echo "Configuration extra arguments:"
+echo "   $@"
+echo
+echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2"
+echo "   -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\""
+echo
+
+rm -rf build_${BUILD_SUFFIX} >/dev/null
+mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX}
+
+
+module load cmake/3.24.2
+
+# unload rocm to avoid configuration problems where the loaded rocm and COMP_VER
+# are inconsistent causing the rocprim from the module to be used unexpectedly
+module unload rocm
+
+
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DROCM_ROOT_DIR="/opt/rocm-${COMP_VER}" \
+  -DHIP_ROOT_DIR="/opt/rocm-${COMP_VER}/hip" \
+  -DHIP_PATH=/opt/rocm-${COMP_VER}/llvm/bin \
+  -DENABLE_CLANGFORMAT=On \
+  -DCLANGFORMAT_EXECUTABLE=/opt/rocm-5.2.3/llvm/bin/clang-format \
+  -DCMAKE_C_COMPILER=/opt/rocm-${COMP_VER}/llvm/bin/amdclang \
+  -DCMAKE_CXX_COMPILER=/opt/rocm-${COMP_VER}/llvm/bin/amdclang++ \
+  -DCMAKE_HIP_ARCHITECTURES="${COMP_ARCH}" \
+  -DGPU_TARGETS="${COMP_ARCH}" \
+  -DAMDGPU_TARGETS="${COMP_ARCH}" \
+  -DBLT_CXX_STD=c++17 \
+  -C "../host-configs/lc-builds/toss4/${HOSTCONFIG}.cmake" \
+  -DENABLE_HIP=OFF \
+  -DENABLE_OPENMP=ON \
+  -DENABLE_CUDA=OFF \
+  -DRAJA_ENABLE_TARGET_OPENMP=ON \
+  -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx942" \
+  -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa;-Xopenmp-target=amdgcn-amd-amdhsa;-march=gfx942" \
+  -DENABLE_BENCHMARKS=On \
+  -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \
+  "$@" \
+  ..
+
+echo
+echo "***********************************************************************"
+echo
+echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA"
+echo
+echo "  Please note that you have to have a consistent build environment"
+echo "  when you make RAJA as cmake may reconfigure; unload the rocm module"
+echo "  or load the appropriate rocm module (${COMP_VER}) when building."
+echo
+echo "    module unload rocm"
+echo "    srun -n1 make"
+echo
+echo "***********************************************************************"

scripts/lc-builds/toss4_clang_omptarget.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# Copyright (c) 2016-25, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+if [ "$1" == "" ]; then
+  echo
+  echo "You must pass a compiler version number to script. For example,"
+  echo "    toss4_clang.sh 10.3.1"
+  exit
+fi
+
+COMP_VER=$1
+shift 1
+
+BUILD_SUFFIX=lc_toss4-clang-${COMP_VER}
+
+echo
+echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it"
+echo "Configuration extra arguments:"
+echo "   $@"
+echo
+
+rm -rf build_${BUILD_SUFFIX} 2>/dev/null
+mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX}
+
+module load cmake/3.24.2
+
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_COMPILER=clang++ \
+  -DBLT_CXX_STD=c++17 \
+  -DENABLE_CLANGFORMAT=On \
+  -C ../host-configs/lc-builds/toss4/clang_X.cmake \
+  -DENABLE_OPENMP=On \
+  -DRAJA_ENABLE_TARGET_OPENMP=ON \
+  -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa" \
+  -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=amdgcn-amd-amdhsa" \
+  -DENABLE_BENCHMARKS=On \
+  -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \
+  "$@" \
+  ..
+#  -DCLANGFORMAT_EXECUTABLE=/usr/tce/packages/clang/clang-14.0.6/bin/clang-format \