diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index aaa784ca8b..a57715d294 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -217,13 +217,20 @@ elseif(GPU_API STREQUAL "OPENCL") elseif(GPU_API STREQUAL "HIP") if(NOT DEFINED HIP_PATH) if(NOT DEFINED ENV{HIP_PATH}) - set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") + set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to HIP installation") else() - set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed") + set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to HIP installation") endif() endif() - set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH}) - find_package(HIP REQUIRED) + if(NOT DEFINED ROCM_PATH) + if(NOT DEFINED ENV{ROCM_PATH}) + set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to ROCm installation") + else() + set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to ROCm installation") + endif() + endif() + list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH} ${ROCM_PATH}) + find_package(hip REQUIRED) option(HIP_USE_DEVICE_SORT "Use GPU sorting" ON) if(NOT DEFINED HIP_PLATFORM) @@ -325,10 +332,11 @@ elseif(GPU_API STREQUAL "HIP") set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h ${LAMMPS_LIB_BINARY_DIR}/gpu/*.cu.cpp") - hip_add_library(gpu STATIC ${GPU_LIB_SOURCES}) + add_library(gpu STATIC ${GPU_LIB_SOURCES}) target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu) target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT) target_compile_definitions(gpu PRIVATE -DUSE_HIP) + target_link_libraries(gpu PRIVATE hip::host) if(HIP_USE_DEVICE_SORT) # add hipCUB @@ -377,8 +385,9 @@ elseif(GPU_API STREQUAL "HIP") endif() endif() - hip_add_executable(hip_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + add_executable(hip_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) target_compile_definitions(hip_get_devices PRIVATE -DUCL_HIP) + target_link_libraries(hip_get_devices hip::host) if(HIP_PLATFORM STREQUAL "nvcc") target_compile_definitions(gpu PRIVATE -D__HIP_PLATFORM_NVCC__) diff --git a/cmake/presets/hip_amd.cmake b/cmake/presets/hip_amd.cmake new file mode 100644 index 0000000000..4b8945e0c7 --- /dev/null +++ b/cmake/presets/hip_amd.cmake @@ -0,0 +1,30 @@ +# preset that will enable hip (clang/clang++) with support for MPI and OpenMP (on Linux boxes) + +# prefer flang over gfortran, if available +find_program(CLANG_FORTRAN NAMES flang gfortran f95) +set(ENV{OMPI_FC} ${CLANG_FORTRAN}) + +set(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "" FORCE) +set(CMAKE_C_COMPILER "hipcc" CACHE STRING "" FORCE) +set(CMAKE_Fortran_COMPILER ${CLANG_FORTRAN} CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_DEBUG "-Wall -Wextra -g -std=f2003" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG -std=f2003" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -std=f2003" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) + +set(MPI_CXX "hipcc" CACHE STRING "" FORCE) +set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) + +unset(HAVE_OMP_H_INCLUDE CACHE) +set(OpenMP_C "hipcc" CACHE STRING "" FORCE) +set(OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" FORCE) +set(OpenMP_C_LIB_NAMES "omp" CACHE STRING "" FORCE) +set(OpenMP_CXX "hipcc" CACHE STRING "" FORCE) +set(OpenMP_CXX_FLAGS "-fopenmp" CACHE STRING "" FORCE) +set(OpenMP_CXX_LIB_NAMES "omp" CACHE STRING "" FORCE) +set(OpenMP_omp_LIBRARY "libomp.so" CACHE PATH "" FORCE) diff --git a/tools/singularity/ubuntu18.04_amd_rocm.def b/tools/singularity/ubuntu18.04_amd_rocm.def index 7b06970110..0692b1528f 100644 --- a/tools/singularity/ubuntu18.04_amd_rocm.def +++ b/tools/singularity/ubuntu18.04_amd_rocm.def @@ -3,7 +3,7 @@ From: ubuntu:18.04 %environment export PATH=/usr/lib/ccache:${PATH}:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm-4.3.0/llvm/lib %post export DEBIAN_FRONTEND=noninteractive apt-get update @@ -94,7 +94,7 @@ From: ubuntu:18.04 ########################################################################### export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - git clone -b rocm-4.1.x https://github.com/ROCmSoftwarePlatform/hipCUB.git + git clone -b release/rocm-rel-4.3 https://github.com/ROCmSoftwarePlatform/hipCUB.git mkdir hipCUB/build cd hipCUB/build CXX=hipcc cmake -D BUILD_TEST=off .. diff --git a/tools/singularity/ubuntu18.04_gpu.def b/tools/singularity/ubuntu18.04_gpu.def index cfd94c8531..d7f3ade1a1 100644 --- a/tools/singularity/ubuntu18.04_gpu.def +++ b/tools/singularity/ubuntu18.04_gpu.def @@ -5,7 +5,7 @@ From: ubuntu:18.04 export PATH=/usr/lib/ccache:/usr/local/cuda-11.0/bin:${PATH}:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 export CUDADIR=/usr/local/cuda-11.0 export CUDA_PATH=/usr/local/cuda-11.0 - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11.0/lib64 + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11.0/lib64:/opt/rocm/lib:/opt/rocm-4.3.0/llvm/lib export LIBRARY_PATH=/usr/local/cuda-11.0/lib64/stubs %post export DEBIAN_FRONTEND=noninteractive @@ -134,7 +134,7 @@ From: ubuntu:18.04 ########################################################################### export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - git clone -b rocm-3.7.x https://github.com/ROCmSoftwarePlatform/hipCUB.git + git clone -b release/rocm-rel-4.3 https://github.com/ROCmSoftwarePlatform/hipCUB.git mkdir hipCUB/build cd hipCUB/build CXX=hipcc cmake -D BUILD_TEST=off .. diff --git a/tools/singularity/ubuntu20.04_amd_rocm.def b/tools/singularity/ubuntu20.04_amd_rocm.def index 9db8265629..926a6dc321 100644 --- a/tools/singularity/ubuntu20.04_amd_rocm.def +++ b/tools/singularity/ubuntu20.04_amd_rocm.def @@ -3,7 +3,7 @@ From: ubuntu:20.04 %environment export PATH=/usr/lib/ccache:${PATH}:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm-4.2.0/llvm/lib + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib:/opt/rocm-4.3.0/llvm/lib %post export DEBIAN_FRONTEND=noninteractive apt-get update @@ -91,7 +91,7 @@ From: ubuntu:20.04 ########################################################################### export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - git clone -b rocm-4.1.x https://github.com/ROCmSoftwarePlatform/hipCUB.git + git clone -b release/rocm-rel-4.3 https://github.com/ROCmSoftwarePlatform/hipCUB.git mkdir hipCUB/build cd hipCUB/build CXX=hipcc cmake -D BUILD_TEST=off .. diff --git a/tools/singularity/ubuntu20.04_gpu.def b/tools/singularity/ubuntu20.04_gpu.def index 44f975d2c8..ceb0566b11 100644 --- a/tools/singularity/ubuntu20.04_gpu.def +++ b/tools/singularity/ubuntu20.04_gpu.def @@ -5,7 +5,7 @@ From: ubuntu:20.04 export PATH=/usr/lib/ccache:/usr/local/cuda-11.0/bin:${PATH}:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 export CUDADIR=/usr/local/cuda-11.0 export CUDA_PATH=/usr/local/cuda-11.0 - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11.0/lib64:/opt/rocm/lib:/opt/rocm-4.2.0/llvm/lib + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11.0/lib64:/opt/rocm/lib:/opt/rocm-4.3.0/llvm/lib export LIBRARY_PATH=/usr/local/cuda-11.0/lib64/stubs %post export DEBIAN_FRONTEND=noninteractive @@ -131,7 +131,7 @@ From: ubuntu:20.04 ########################################################################### export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - git clone -b rocm-4.2.x https://github.com/ROCmSoftwarePlatform/hipCUB.git + git clone -b release/rocm-rel-4.3 https://github.com/ROCmSoftwarePlatform/hipCUB.git mkdir hipCUB/build cd hipCUB/build CXX=hipcc cmake -D BUILD_TEST=off ..