From 3298a86bd9a731929c0351ebbdfeae86378bc769 Mon Sep 17 00:00:00 2001 From: Vsevak Date: Thu, 20 Feb 2020 18:32:52 +0300 Subject: [PATCH 01/10] Reduce redundant copying using UCL_H_Vec::view --- lib/gpu/lal_lj_tip4p_long.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/gpu/lal_lj_tip4p_long.cpp b/lib/gpu/lal_lj_tip4p_long.cpp index f60f162d7b..fa88725e85 100644 --- a/lib/gpu/lal_lj_tip4p_long.cpp +++ b/lib/gpu/lal_lj_tip4p_long.cpp @@ -250,22 +250,22 @@ void LJTIP4PLongT::copy_relations_data(int n, tagint *tag, int *map_array, if (ago == 0) { hneight.zero(); + { - UCL_H_Vec host_tag_write(nall,*(this->ucl_device),UCL_WRITE_ONLY); + UCL_H_Vec host_tag_write; + host_tag_write.view(tag, nall, *(this->ucl_device)); this->tag.resize_ib(nall); - for(int i=0; itag, host_tag_write, nall, false); + ucl_copy(this->tag, host_tag_write, false); } - UCL_H_Vec host_write(max_same,*(this->ucl_device),UCL_WRITE_ONLY); + UCL_H_Vec host_write; + host_write.view(sametag, max_same, *(this->ucl_device)); this->atom_sametag.resize_ib(max_same); - for(int i=0; iatom_sametag, host_write, max_same, false); + ucl_copy(this->atom_sametag, host_write, false); - host_write.resize_ib(map_size); this->map_array.resize_ib(map_size); - for(int i=0; imap_array, host_write, map_size, false); + host_write.view(map_array, map_size, *(this->ucl_device)); + ucl_copy(this->map_array, host_write, false); } } From d3baa414d2d686f0b9bc7b2f2be44f45937a5277 Mon Sep 17 00:00:00 2001 From: Vsevak Date: Wed, 17 Jun 2020 23:33:16 +0300 Subject: [PATCH 02/10] Fix cmake for GPU ROCm 3.5 backend --- cmake/Modules/Packages/GPU.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index d3a73100c0..bc890d6691 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -269,7 +269,7 @@ elseif(GPU_API STREQUAL "HIP") configure_file(${CU_FILE} ${CU_CPP_FILE} COPYONLY) add_custom_command(OUTPUT ${CUBIN_FILE} - VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco --offload-arch=${HIP_ARCH} -O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_CPP_FILE} DEPENDS ${CU_CPP_FILE} COMMENT "Generating ${CU_NAME}.cubin") elseif(HIP_PLATFORM STREQUAL "nvcc") @@ -298,6 +298,8 @@ elseif(GPU_API STREQUAL "HIP") # add hipCUB target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/../include) target_compile_definitions(gpu PRIVATE -DUSE_HIP_DEVICE_SORT) + find_package(rocprim REQUIRED CONFIG PATHS "/opt/rocm/rocprim") + find_package(hipcub REQUIRED CONFIG PATHS "/opt/rocm/hipcub") if(HIP_PLATFORM STREQUAL "nvcc") find_package(CUB) From f75b8e168845952958b006fee2d362aa59a3192b Mon Sep 17 00:00:00 2001 From: Vsevak Date: Thu, 18 Jun 2020 00:33:46 +0300 Subject: [PATCH 03/10] Fix Makefile for GPU ROCm 3.5 backend --- lib/gpu/Makefile.hip | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/gpu/Makefile.hip b/lib/gpu/Makefile.hip index 62d34592d1..833aa34933 100644 --- a/lib/gpu/Makefile.hip +++ b/lib/gpu/Makefile.hip @@ -36,7 +36,7 @@ MPI_LINK_OPTS = $(shell mpicxx --showme:link) #MPI_COMP_OPTS += -I/usr/include/mpi -DMPICH_IGNORE_CXX_SEEK -DOMPI_SKIP_MPICXX=1 HIP_PATH ?= $(wildcard /opt/rocm/hip) -HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --compiler) +#HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --compiler) ifeq (hcc,$(HIP_PLATFORM)) HIP_OPTS += -ffast-math @@ -48,8 +48,6 @@ else ifeq (nvcc,$(HIP_PLATFORM)) -gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_53,code=[sm_53,compute_53]\ -gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] -gencode arch=compute_62,code=[sm_62,compute_62]\ -gencode arch=compute_70,code=[sm_70,compute_70] -gencode arch=compute_72,code=[sm_72,compute_72] -gencode arch=compute_75,code=[sm_75,compute_75] -else - $(error Specify HIP platform using 'export HIP_PLATFORM=(hcc,nvcc)') endif BIN_DIR = . @@ -69,8 +67,8 @@ HIP_GPU_OPTS += $(HIP_OPTS) -I./ ifeq (hcc,$(HIP_PLATFORM)) HIP_HOST_OPTS += -fPIC HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --genco - HIP_GPU_OPTS_S = -t="$(HIP_ARCH)" -f=\" - HIP_GPU_OPTS_E = \" + HIP_GPU_OPTS_S = --offload-arch=$(HIP_ARCH) + HIP_GPU_OPTS_E = HIP_KERNEL_SUFFIX = .cpp HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH) export HCC_AMDGPU_TARGET := $(HIP_ARCH) From 2ea5d9956a9028f6995a4bb4219a015aa8d4f184 Mon Sep 17 00:00:00 2001 From: Vsevak Date: Fri, 19 Jun 2020 00:01:03 +0300 Subject: [PATCH 04/10] find_package is unnecessary for rocprim --- cmake/Modules/Packages/GPU.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index bc890d6691..65ca4e33ff 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -298,8 +298,6 @@ elseif(GPU_API STREQUAL "HIP") # add hipCUB target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/../include) target_compile_definitions(gpu PRIVATE -DUSE_HIP_DEVICE_SORT) - find_package(rocprim REQUIRED CONFIG PATHS "/opt/rocm/rocprim") - find_package(hipcub REQUIRED CONFIG PATHS "/opt/rocm/hipcub") if(HIP_PLATFORM STREQUAL "nvcc") find_package(CUB) From 8eed1b27ffe8f8332dba04ff10f369b2bf96fe1a Mon Sep 17 00:00:00 2001 From: Vsevak Date: Fri, 19 Jun 2020 00:07:10 +0300 Subject: [PATCH 05/10] GPU Makefile cleanup --- lib/gpu/Makefile.hip | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/gpu/Makefile.hip b/lib/gpu/Makefile.hip index 833aa34933..332670246e 100644 --- a/lib/gpu/Makefile.hip +++ b/lib/gpu/Makefile.hip @@ -33,10 +33,8 @@ HIP_HOST_OPTS += -DMPI_GERYON -DUCL_NO_EXIT # this settings should match LAMMPS Makefile MPI_COMP_OPTS = $(shell mpicxx --showme:compile) MPI_LINK_OPTS = $(shell mpicxx --showme:link) -#MPI_COMP_OPTS += -I/usr/include/mpi -DMPICH_IGNORE_CXX_SEEK -DOMPI_SKIP_MPICXX=1 HIP_PATH ?= $(wildcard /opt/rocm/hip) -#HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --compiler) ifeq (hcc,$(HIP_PLATFORM)) HIP_OPTS += -ffast-math From 723b9e676041f3d5e7c10f87285e8dcb44ccfbbc Mon Sep 17 00:00:00 2001 From: Vsevak Date: Sat, 20 Jun 2020 15:17:54 +0300 Subject: [PATCH 06/10] Try to keep compatibility with deprecated ROCm hcc compiler --- cmake/Modules/Packages/GPU.cmake | 15 +++++++++++---- lib/gpu/Makefile.hip | 12 +++++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index 65ca4e33ff..dce28c305a 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -268,10 +268,17 @@ elseif(GPU_API STREQUAL "HIP") if(HIP_PLATFORM STREQUAL "hcc") configure_file(${CU_FILE} ${CU_CPP_FILE} COPYONLY) - add_custom_command(OUTPUT ${CUBIN_FILE} - VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco --offload-arch=${HIP_ARCH} -O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_CPP_FILE} - DEPENDS ${CU_CPP_FILE} - COMMENT "Generating ${CU_NAME}.cubin") + if(HIP_COMPILER STREQUAL "clang") + add_custom_command(OUTPUT ${CUBIN_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco --offload-arch=${HIP_ARCH} -O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_CPP_FILE} + DEPENDS ${CU_CPP_FILE} + COMMENT "Generating ${CU_NAME}.cubin") + else() + add_custom_command(OUTPUT ${CUBIN_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE} + DEPENDS ${CU_CPP_FILE} + COMMENT "Generating ${CU_NAME}.cubin") + endif() elseif(HIP_PLATFORM STREQUAL "nvcc") add_custom_command(OUTPUT ${CUBIN_FILE} VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --fatbin --use_fast_math -DUSE_HIP -D_${GPU_PREC_SETTING} ${HIP_CUDA_GENCODE} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_FILE} diff --git a/lib/gpu/Makefile.hip b/lib/gpu/Makefile.hip index 332670246e..76e202da8c 100644 --- a/lib/gpu/Makefile.hip +++ b/lib/gpu/Makefile.hip @@ -35,6 +35,8 @@ MPI_COMP_OPTS = $(shell mpicxx --showme:compile) MPI_LINK_OPTS = $(shell mpicxx --showme:link) HIP_PATH ?= $(wildcard /opt/rocm/hip) +HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --platform) +HIP_COMPILER=$(shell $(HIP_PATH)/bin/hipconfig --compiler) ifeq (hcc,$(HIP_PLATFORM)) HIP_OPTS += -ffast-math @@ -62,7 +64,7 @@ BSH = /bin/sh HIP_OPTS += -DUSE_HIP $(HIP_PRECISION) HIP_GPU_OPTS += $(HIP_OPTS) -I./ -ifeq (hcc,$(HIP_PLATFORM)) +ifeq (clang,$(HIP_COMPILER)) HIP_HOST_OPTS += -fPIC HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --genco HIP_GPU_OPTS_S = --offload-arch=$(HIP_ARCH) @@ -70,6 +72,14 @@ ifeq (hcc,$(HIP_PLATFORM)) HIP_KERNEL_SUFFIX = .cpp HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH) export HCC_AMDGPU_TARGET := $(HIP_ARCH) +else ifeq (hcc,$(HIP_COMPILER)) + HIP_HOST_OPTS += -fPIC + HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --genco + HIP_GPU_OPTS_S = -t="$(HIP_ARCH)" -f=\" + HIP_GPU_OPTS_E = \" + HIP_KERNEL_SUFFIX = .cpp + HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH) + export HCC_AMDGPU_TARGET := $(HIP_ARCH) else ifeq (nvcc,$(HIP_PLATFORM)) HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --fatbin HIP_GPU_OPTS += $(HIP_ARCH) From 0ba701acd77d0100bcc03c6bfd7da5d9cb01f4d1 Mon Sep 17 00:00:00 2001 From: Vsevak Date: Sat, 20 Jun 2020 17:24:46 +0300 Subject: [PATCH 07/10] Remove trailing space --- lib/gpu/Makefile.hip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gpu/Makefile.hip b/lib/gpu/Makefile.hip index 76e202da8c..e2fd3c22d7 100644 --- a/lib/gpu/Makefile.hip +++ b/lib/gpu/Makefile.hip @@ -79,7 +79,7 @@ else ifeq (hcc,$(HIP_COMPILER)) HIP_GPU_OPTS_E = \" HIP_KERNEL_SUFFIX = .cpp HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH) - export HCC_AMDGPU_TARGET := $(HIP_ARCH) + export HCC_AMDGPU_TARGET := $(HIP_ARCH) else ifeq (nvcc,$(HIP_PLATFORM)) HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --fatbin HIP_GPU_OPTS += $(HIP_ARCH) From 7bb4a2a81a86a804d7359f0f5acc04198da14ace Mon Sep 17 00:00:00 2001 From: Vsevak Date: Sun, 21 Jun 2020 19:37:46 +0300 Subject: [PATCH 08/10] Fix a typo in the GPU Docs --- doc/src/Speed_gpu.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/Speed_gpu.rst b/doc/src/Speed_gpu.rst index 9981b22b03..647e40c28f 100644 --- a/doc/src/Speed_gpu.rst +++ b/doc/src/Speed_gpu.rst @@ -88,7 +88,7 @@ GPUs/node to 1. Using the "-pk" switch explicitly allows for setting of the number of GPUs/node to use and additional options. Its syntax is the same as -same as the "package gpu" command. See the :doc:`package ` +the "package gpu" command. See the :doc:`package ` command doc page for details, including the default values used for all its options if it is not specified. From e6b3611c2d33cda5fc3f5336a9fada3d287db30a Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Thu, 25 Jun 2020 19:05:28 -0400 Subject: [PATCH 09/10] Update ROCm container to version 3.5 --- tools/singularity/ubuntu18.04_amd_rocm.def | 24 ++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/singularity/ubuntu18.04_amd_rocm.def b/tools/singularity/ubuntu18.04_amd_rocm.def index df7d5be062..07b2bb400b 100644 --- a/tools/singularity/ubuntu18.04_amd_rocm.def +++ b/tools/singularity/ubuntu18.04_amd_rocm.def @@ -1,5 +1,5 @@ BootStrap: docker -From: rocm/dev-ubuntu-18.04 +From: ubuntu:18.04 %environment export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 @@ -8,7 +8,23 @@ From: rocm/dev-ubuntu-18.04 export DEBIAN_FRONTEND=noninteractive apt-get update apt-get upgrade --no-install-recommends -y + + apt-get install -y --no-install-recommends curl libnuma-dev gnupg + + curl -sL http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | apt-key add - + printf "deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main" > /etc/apt/sources.list.d/rocm.list + + apt-get update + apt-get install --no-install-recommends -y \ + file \ + sudo \ + libelf1 \ + rocm-dev \ + rocm-libs \ + build-essential + apt-get install --no-install-recommends -y software-properties-common + apt-get install --no-install-recommends -y \ bc \ build-essential \ @@ -54,7 +70,6 @@ From: rocm/dev-ubuntu-18.04 python3-pkg-resources \ python3-setuptools \ python3-virtualenv \ - rocm-libs \ rsync \ ssh \ vim-nox \ @@ -65,11 +80,12 @@ From: rocm/dev-ubuntu-18.04 valgrind \ gdb + export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64 - git clone -b master-rocm-3.3 https://github.com/ROCmSoftwarePlatform/hipCUB.git + git clone -b master-rocm-3.5 https://github.com/ROCmSoftwarePlatform/hipCUB.git mkdir hipCUB/build cd hipCUB/build - CXX=hcc cmake -D BUILD_TEST=off .. + CXX=hipcc cmake -D BUILD_TEST=off .. make -j4 make package make install From f9a202288ccf604d9e387a94c70705732835c5b7 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Fri, 26 Jun 2020 13:09:06 -0400 Subject: [PATCH 10/10] Add missing kmod package to AMD ROCm container --- tools/singularity/ubuntu18.04_amd_rocm.def | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/singularity/ubuntu18.04_amd_rocm.def b/tools/singularity/ubuntu18.04_amd_rocm.def index 07b2bb400b..4c58f690aa 100644 --- a/tools/singularity/ubuntu18.04_amd_rocm.def +++ b/tools/singularity/ubuntu18.04_amd_rocm.def @@ -16,6 +16,7 @@ From: ubuntu:18.04 apt-get update apt-get install --no-install-recommends -y \ + kmod \ file \ sudo \ libelf1 \