Merge pull request #2167 from Vsevak/master

Update GPU CMake and Makefile scripts for ROCm v3.5

cmake/Modules/Packages/GPU.cmake

@@ -268,10 +268,17 @@ elseif(GPU_API STREQUAL "HIP")
     if(HIP_PLATFORM STREQUAL "hcc")
         configure_file(${CU_FILE} ${CU_CPP_FILE} COPYONLY)
 
-        add_custom_command(OUTPUT ${CUBIN_FILE}
+        if(HIP_COMPILER STREQUAL "clang")
-          VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE}
+          add_custom_command(OUTPUT ${CUBIN_FILE}
-          DEPENDS ${CU_CPP_FILE}
+            VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco --offload-arch=${HIP_ARCH} -O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_CPP_FILE}
-          COMMENT "Generating ${CU_NAME}.cubin")
+            DEPENDS ${CU_CPP_FILE}
+            COMMENT "Generating ${CU_NAME}.cubin")
+        else()
+          add_custom_command(OUTPUT ${CUBIN_FILE}
+            VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE}
+            DEPENDS ${CU_CPP_FILE}
+            COMMENT "Generating ${CU_NAME}.cubin")
+        endif()
     elseif(HIP_PLATFORM STREQUAL "nvcc")
         add_custom_command(OUTPUT ${CUBIN_FILE}
           VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --fatbin --use_fast_math -DUSE_HIP -D_${GPU_PREC_SETTING} ${HIP_CUDA_GENCODE} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_FILE}

doc/src/Speed_gpu.rst

@@ -88,7 +88,7 @@ GPUs/node to 1.
 
 Using the "-pk" switch explicitly allows for setting of the number of
 GPUs/node to use and additional options.  Its syntax is the same as
-same as the "package gpu" command.  See the :doc:`package <package>`
+the "package gpu" command.  See the :doc:`package <package>`
 command doc page for details, including the default values used for
 all its options if it is not specified.
 

lib/gpu/Makefile.hip

@@ -33,10 +33,10 @@ HIP_HOST_OPTS += -DMPI_GERYON -DUCL_NO_EXIT
 # this settings should match LAMMPS Makefile
 MPI_COMP_OPTS = $(shell mpicxx --showme:compile)
 MPI_LINK_OPTS = $(shell mpicxx --showme:link)
-#MPI_COMP_OPTS += -I/usr/include/mpi  -DMPICH_IGNORE_CXX_SEEK -DOMPI_SKIP_MPICXX=1
 
 HIP_PATH ?= $(wildcard /opt/rocm/hip)
-HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --compiler)
+HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --platform)
+HIP_COMPILER=$(shell $(HIP_PATH)/bin/hipconfig --compiler)
 
 ifeq (hcc,$(HIP_PLATFORM))
 	HIP_OPTS  += -ffast-math
@@ -48,8 +48,6 @@ else ifeq (nvcc,$(HIP_PLATFORM))
 		    -gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_53,code=[sm_53,compute_53]\
 			-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] -gencode arch=compute_62,code=[sm_62,compute_62]\
 			-gencode arch=compute_70,code=[sm_70,compute_70] -gencode arch=compute_72,code=[sm_72,compute_72] -gencode arch=compute_75,code=[sm_75,compute_75]
-else
-	$(error Specify HIP platform using 'export HIP_PLATFORM=(hcc,nvcc)')
 endif
 
 BIN_DIR = .
@@ -66,7 +64,15 @@ BSH = /bin/sh
 HIP_OPTS += -DUSE_HIP $(HIP_PRECISION)
 HIP_GPU_OPTS += $(HIP_OPTS) -I./
 
-ifeq (hcc,$(HIP_PLATFORM))
+ifeq (clang,$(HIP_COMPILER))
+	HIP_HOST_OPTS += -fPIC
+	HIP_GPU_CC  = $(HIP_PATH)/bin/hipcc --genco
+	HIP_GPU_OPTS_S = --offload-arch=$(HIP_ARCH)
+	HIP_GPU_OPTS_E =
+	HIP_KERNEL_SUFFIX = .cpp
+	HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH)
+	export HCC_AMDGPU_TARGET := $(HIP_ARCH)
+else ifeq (hcc,$(HIP_COMPILER))
 	HIP_HOST_OPTS += -fPIC
 	HIP_GPU_CC  = $(HIP_PATH)/bin/hipcc --genco
 	HIP_GPU_OPTS_S = -t="$(HIP_ARCH)" -f=\"

lib/gpu/lal_lj_tip4p_long.cpp

@@ -250,22 +250,22 @@ void LJTIP4PLongT::copy_relations_data(int n, tagint *tag, int *map_array,
   if (ago == 0) {
     hneight.zero();
 
+
     {
-      UCL_H_Vec<tagint> host_tag_write(nall,*(this->ucl_device),UCL_WRITE_ONLY);
+      UCL_H_Vec<tagint> host_tag_write;
+      host_tag_write.view(tag, nall, *(this->ucl_device));
       this->tag.resize_ib(nall);
-      for(int i=0; i<nall; ++i) host_tag_write[i] = tag[i];
+      ucl_copy(this->tag, host_tag_write, false);
-      ucl_copy(this->tag, host_tag_write, nall, false);
     }
 
-    UCL_H_Vec<int> host_write(max_same,*(this->ucl_device),UCL_WRITE_ONLY);
+    UCL_H_Vec<int> host_write;
+    host_write.view(sametag, max_same, *(this->ucl_device));
     this->atom_sametag.resize_ib(max_same);
-    for(int i=0; i<max_same; ++i) host_write[i] = sametag[i];
+    ucl_copy(this->atom_sametag, host_write, false);
-    ucl_copy(this->atom_sametag, host_write, max_same, false);
 
-    host_write.resize_ib(map_size);
     this->map_array.resize_ib(map_size);
-    for(int i=0; i<map_size; ++i) host_write[i] = map_array[i];
+    host_write.view(map_array, map_size, *(this->ucl_device));
-    ucl_copy(this->map_array, host_write, map_size, false);
+    ucl_copy(this->map_array, host_write, false);
   }
 }
 

tools/singularity/ubuntu18.04_amd_rocm.def

@@ -1,5 +1,5 @@
 BootStrap: docker
-From: rocm/dev-ubuntu-18.04
+From: ubuntu:18.04
 
 %environment
     export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64
@@ -8,7 +8,24 @@ From: rocm/dev-ubuntu-18.04
     export DEBIAN_FRONTEND=noninteractive
     apt-get update
     apt-get upgrade --no-install-recommends -y
+
+    apt-get install -y --no-install-recommends curl libnuma-dev gnupg
+
+    curl -sL http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | apt-key add -
+    printf "deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main" > /etc/apt/sources.list.d/rocm.list
+
+    apt-get update
+    apt-get install --no-install-recommends -y \
+        kmod \
+        file \
+        sudo \
+        libelf1 \
+        rocm-dev \
+        rocm-libs \
+        build-essential
+
     apt-get install --no-install-recommends -y software-properties-common
+
     apt-get install --no-install-recommends -y \
         bc \
         build-essential \
@@ -54,7 +71,6 @@ From: rocm/dev-ubuntu-18.04
         python3-pkg-resources \
         python3-setuptools \
         python3-virtualenv \
-        rocm-libs \
         rsync \
         ssh \
         vim-nox \
@@ -65,11 +81,12 @@ From: rocm/dev-ubuntu-18.04
         valgrind \
         gdb
 
+
     export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64
-    git clone -b master-rocm-3.3 https://github.com/ROCmSoftwarePlatform/hipCUB.git
+    git clone -b master-rocm-3.5 https://github.com/ROCmSoftwarePlatform/hipCUB.git
     mkdir hipCUB/build
     cd hipCUB/build
-    CXX=hcc cmake -D BUILD_TEST=off ..
+    CXX=hipcc cmake -D BUILD_TEST=off ..
     make -j4
     make package
     make install