From 8db2d64f11ad02a29b1f7564fce4b5ba81ab0db7 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 31 Aug 2021 17:56:01 -0400 Subject: [PATCH] Updates to support ROCm 4.3 in GPU package --- cmake/Modules/Packages/GPU.cmake | 17 +++++++++--- cmake/presets/hip_amd.cmake | 30 ++++++++++++++++++++++ tools/singularity/ubuntu20.04_amd_rocm.def | 4 +-- 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 cmake/presets/hip_amd.cmake diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index 2b6977005d..01383aae72 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -219,8 +219,15 @@ elseif(GPU_API STREQUAL "HIP") set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed") endif() endif() - set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH}) - find_package(HIP REQUIRED) + if(NOT DEFINED ROCM_PATH) + if(NOT DEFINED ENV{ROCM_PATH}) + set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCM has been installed") + else() + set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed") + endif() + endif() + list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH} ${ROCM_PATH}) + find_package(hip REQUIRED) option(HIP_USE_DEVICE_SORT "Use GPU sorting" ON) if(NOT DEFINED HIP_PLATFORM) @@ -322,10 +329,11 @@ elseif(GPU_API STREQUAL "HIP") set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h ${LAMMPS_LIB_BINARY_DIR}/gpu/*.cu.cpp") - hip_add_library(gpu STATIC ${GPU_LIB_SOURCES}) + add_library(gpu STATIC ${GPU_LIB_SOURCES}) target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu) target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT) target_compile_definitions(gpu PRIVATE -DUSE_HIP) + target_link_libraries(gpu PRIVATE hip::host) if(HIP_USE_DEVICE_SORT) # add hipCUB @@ -374,8 +382,9 @@ elseif(GPU_API STREQUAL "HIP") endif() endif() - hip_add_executable(hip_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + add_executable(hip_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) target_compile_definitions(hip_get_devices PRIVATE -DUCL_HIP) + target_link_libraries(hip_get_devices hip::host) if(HIP_PLATFORM STREQUAL "nvcc") target_compile_definitions(gpu PRIVATE -D__HIP_PLATFORM_NVCC__) diff --git a/cmake/presets/hip_amd.cmake b/cmake/presets/hip_amd.cmake new file mode 100644 index 0000000000..4b8945e0c7 --- /dev/null +++ b/cmake/presets/hip_amd.cmake @@ -0,0 +1,30 @@ +# preset that will enable hip (clang/clang++) with support for MPI and OpenMP (on Linux boxes) + +# prefer flang over gfortran, if available +find_program(CLANG_FORTRAN NAMES flang gfortran f95) +set(ENV{OMPI_FC} ${CLANG_FORTRAN}) + +set(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "" FORCE) +set(CMAKE_C_COMPILER "hipcc" CACHE STRING "" FORCE) +set(CMAKE_Fortran_COMPILER ${CLANG_FORTRAN} CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_DEBUG "-Wall -Wextra -g -std=f2003" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG -std=f2003" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -std=f2003" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) + +set(MPI_CXX "hipcc" CACHE STRING "" FORCE) +set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) + +unset(HAVE_OMP_H_INCLUDE CACHE) +set(OpenMP_C "hipcc" CACHE STRING "" FORCE) +set(OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" FORCE) +set(OpenMP_C_LIB_NAMES "omp" CACHE STRING "" FORCE) +set(OpenMP_CXX "hipcc" CACHE STRING "" FORCE) +set(OpenMP_CXX_FLAGS "-fopenmp" CACHE STRING "" FORCE) +set(OpenMP_CXX_LIB_NAMES "omp" CACHE STRING "" FORCE) +set(OpenMP_omp_LIBRARY "libomp.so" CACHE PATH "" FORCE) diff --git a/tools/singularity/ubuntu20.04_amd_rocm.def b/tools/singularity/ubuntu20.04_amd_rocm.def index 9db8265629..d55bacec21 100644 --- a/tools/singularity/ubuntu20.04_amd_rocm.def +++ b/tools/singularity/ubuntu20.04_amd_rocm.def @@ -3,7 +3,7 @@ From: ubuntu:20.04 %environment export PATH=/usr/lib/ccache:${PATH}:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm-4.2.0/llvm/lib + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm-4.3.0/llvm/lib %post export DEBIAN_FRONTEND=noninteractive apt-get update @@ -91,7 +91,7 @@ From: ubuntu:20.04 ########################################################################### export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - git clone -b rocm-4.1.x https://github.com/ROCmSoftwarePlatform/hipCUB.git + git clone -b develop https://github.com/ROCmSoftwarePlatform/hipCUB.git mkdir hipCUB/build cd hipCUB/build CXX=hipcc cmake -D BUILD_TEST=off ..