ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into tip4p_cornercase

Browse Source
This commit is contained in:
Vsevak
2022-08-25 17:58:17 +03:00
parent baf3e614fb 48732ff376
commit 8d77c1daee
1256 changed files with 199072 additions and 15226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
lib/gpu/Makefile.hip
View File

@ -1,6 +1,9 @@
# /* ----------------------------------------------------------------------
# Generic Linux Makefile for HIP
# - export HIP_PLATFORM=amd (or nvcc) before execution
# - export HIP_PATH=/path/to/HIP/install path to the HIP implementation
# such as hipamd or CHIP-SPV.
# - export HIP_PLATFORM=<amd/nvcc/spirv> specify the HIP platform to use.
# Optional. If not set, will be determined by ${HIP_PATH}/bin/hipconfig.
# - change HIP_ARCH for your GPU
# ------------------------------------------------------------------------- */
@ -20,41 +23,62 @@ HIP_OPTS = -O3
HIP_HOST_OPTS = -Wno-deprecated-declarations -fopenmp
HIP_HOST_INCLUDE =
ifndef HIP_PATH
$(error HIP_PATH is not set)
endif
ifndef HIP_PLATFORM
HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --platform)
endif
HIP_COMPILER=$(shell $(HIP_PATH)/bin/hipconfig --compiler)
# use device sort
# requires linking with hipcc and hipCUB + (rocPRIM or CUB for AMD or Nvidia respectively)
ifneq (spirv,$(HIP_PLATFORM))
# hipCUB not aviable for CHIP-SPV
HIP_HOST_OPTS += -DUSE_HIP_DEVICE_SORT
endif
# path to cub
HIP_HOST_INCLUDE += -I./
# path to hipcub
HIP_HOST_INCLUDE += -I$(HIP_PATH)/../include
ifeq (amd,$(HIP_PLATFORM))
# newer version of ROCm (5.1+) require c++14 for rocprim
HIP_OPTS += -std=c++14
# newer version of ROCm (5.1+) require c++14 for rocprim
HIP_OPTS += -std=c++14
endif
# use mpi
HIP_HOST_OPTS += -DMPI_GERYON -DUCL_NO_EXIT
# this settings should match LAMMPS Makefile
MPI_COMP_OPTS = $(shell mpicxx --showme:compile)
# automatic flag detection for OpenMPI
ifeq ($(shell mpicxx --showme:compile >/dev/null 2>&1; echo $$?), 0)
MPI_COMP_OPTS = $(shell mpicxx --showme:compile) -DOMPI_SKIP_MPICXX=1
MPI_LINK_OPTS = $(shell mpicxx --showme:link)
HIP_PATH ?= $(wildcard /opt/rocm/hip)
HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --platform)
HIP_COMPILER=$(shell $(HIP_PATH)/bin/hipconfig --compiler)
# automatic flag detection for MPICH
else ifeq ($(shell mpicxx -compile_info >/dev/null 2>&1; echo $$?),0)
MPI_COMP_OPTS = $(filter -I%,$(shell mpicxx -compile_info)) -DMPICH_IGNORE_CXX_SEEK
MPI_LINK_OPTS = $(filter -Wl%,$(shell mpicxx -link_info)) $(filter -L%,$(shell mpicxx -link_info)) $(filter -l%,$(shell mpicxx -link_info))
# for other MPI libs: must set flags manually, if needed
else
MPI_COMP_OPTS =
MPI_LINK_OPTS =
endif
ifeq (hcc,$(HIP_PLATFORM))
# possible values: gfx803,gfx900,gfx906
HIP_ARCH = gfx906
# possible values: gfx803,gfx900,gfx906
HIP_ARCH = gfx906
else ifeq (amd,$(HIP_PLATFORM))
# possible values: gfx803,gfx900,gfx906
HIP_ARCH = gfx906
# possible values: gfx803,gfx900,gfx906
HIP_ARCH = gfx906
else ifeq (nvcc,$(HIP_PLATFORM))
HIP_OPTS += --use_fast_math
HIP_ARCH = -gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_32,code=[sm_32,compute_32] -gencode arch=compute_35,code=[sm_35,compute_35] \
HIP_OPTS += --use_fast_math
HIP_ARCH = -gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_32,code=[sm_32,compute_32] -gencode arch=compute_35,code=[sm_35,compute_35] \
-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_53,code=[sm_53,compute_53]\
-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] -gencode arch=compute_62,code=[sm_62,compute_62]\
-gencode arch=compute_70,code=[sm_70,compute_70] -gencode arch=compute_72,code=[sm_72,compute_72] -gencode arch=compute_75,code=[sm_75,compute_75]
else ifeq (spirv,$(HIP_PLATFORM))
HIP_ARCH = spirv
endif
BIN_DIR = .
@ -71,7 +95,15 @@ BSH = /bin/sh
HIP_OPTS += -DUSE_HIP $(HIP_PRECISION)
HIP_GPU_OPTS += $(HIP_OPTS) -I./
ifeq (clang,$(HIP_COMPILER))
ifeq (spirv,$(HIP_PLATFORM))
HIP_HOST_OPTS += -fPIC
HIP_GPU_CC = $(HIP_PATH)/bin/hipcc -c
HIP_GPU_OPTS_S =
HIP_GPU_OPTS_E =
HIP_KERNEL_SUFFIX = .cpp
HIP_LIBS_TARGET =
export HCC_AMDGPU_TARGET := $(HIP_ARCH)
else ifeq (clang,$(HIP_COMPILER))
HIP_HOST_OPTS += -fPIC
HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --genco
HIP_GPU_OPTS_S = --offload-arch=$(HIP_ARCH)

36
lib/gpu/lal_lj_sdk.cpp → lib/gpu/lal_lj_spica.cpp
View File

@ -1,9 +1,9 @@
/***************************************************************************
lj_sdk.cpp
lj_spica.cpp
-------------------
W. Michael Brown (ORNL)
Class for acceleration of the lj/sdk/cut pair style
Class for acceleration of the lj/spica/cut pair style
__________________________________________________________________________
This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -14,14 +14,14 @@
***************************************************************************/
#if defined(USE_OPENCL)
#include "lj_sdk_cl.h"
#include "lj_spica_cl.h"
#elif defined(USE_CUDART)
const char *lj_sdk=0;
const char *lj_spica=0;
#else
#include "lj_sdk_cubin.h"
#include "lj_spica_cubin.h"
#endif
#include "lal_lj_sdk.h"
#include "lal_lj_spica.h"
#include <cassert>
namespace LAMMPS_AL {
#define CGCMMT CGCMM<numtyp, acctyp>
@ -53,33 +53,33 @@ int CGCMMT::init(const int ntypes, double **host_cutsq,
const double gpu_split, FILE *_screen) {
int success;
success=this->init_atomic(nlocal,nall,max_nbors,maxspecial,cell_size,gpu_split,
_screen,lj_sdk,"k_lj_sdk");
_screen,lj_spica,"k_lj_spica");
if (success!=0)
return success;
// If atom type constants fit in shared memory use fast kernel
int sdk_types=ntypes;
int spica_types=ntypes;
shared_types=false;
int max_shared_types=this->device->max_shared_types();
if (sdk_types<=max_shared_types && this->_block_size>=max_shared_types) {
sdk_types=max_shared_types;
if (spica_types<=max_shared_types && this->_block_size>=max_shared_types) {
spica_types=max_shared_types;
shared_types=true;
}
_sdk_types=sdk_types;
_spica_types=spica_types;
// Allocate a host write buffer for data initialization
UCL_H_Vec<numtyp> host_write(sdk_types*sdk_types*32,*(this->ucl_device),
UCL_H_Vec<numtyp> host_write(spica_types*spica_types*32,*(this->ucl_device),
UCL_WRITE_ONLY);
for (int i=0; i<sdk_types*sdk_types; i++)
for (int i=0; i<spica_types*spica_types; i++)
host_write[i]=0.0;
lj1.alloc(sdk_types*sdk_types,*(this->ucl_device),UCL_READ_ONLY);
this->atom->type_pack4(ntypes,sdk_types,lj1,host_write,host_cutsq,
lj1.alloc(spica_types*spica_types,*(this->ucl_device),UCL_READ_ONLY);
this->atom->type_pack4(ntypes,spica_types,lj1,host_write,host_cutsq,
host_cg_type,host_lj1,host_lj2);
lj3.alloc(sdk_types*sdk_types,*(this->ucl_device),UCL_READ_ONLY);
this->atom->type_pack4(ntypes,sdk_types,lj3,host_write,host_lj3,host_lj4,
lj3.alloc(spica_types*spica_types,*(this->ucl_device),UCL_READ_ONLY);
this->atom->type_pack4(ntypes,spica_types,lj3,host_write,host_lj3,host_lj4,
host_offset);
UCL_H_Vec<double> dview;
@ -132,7 +132,7 @@ int CGCMMT::loop(const int eflag, const int vflag) {
} else {
this->k_pair.set_size(GX,BX);
this->k_pair.run(&this->atom->x, &lj1, &lj3,
&_sdk_types, &sp_lj, &this->nbor->dev_nbor,
&_spica_types, &sp_lj, &this->nbor->dev_nbor,
&this->_nbor_data->begin(), &this->ans->force,
&this->ans->engv, &eflag, &vflag, &ainum,
&nbor_pitch, &this->_threads_per_atom);

14
lib/gpu/lal_lj_sdk.cu → lib/gpu/lal_lj_spica.cu
View File

@ -1,9 +1,9 @@
// **************************************************************************
// lj_sdk.cu
// lj_spica.cu
// -------------------
// W. Michael Brown (ORNL)
//
// Device code for acceleration of the lj/sdk pair style
// Device code for acceleration of the lj/spica pair style
//
// __________________________________________________________________________
// This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -24,7 +24,7 @@ _texture_2d( pos_tex,int4);
#define pos_tex x_
#endif
__kernel void k_lj_sdk(const __global numtyp4 *restrict x_,
__kernel void k_lj_spica(const __global numtyp4 *restrict x_,
const __global numtyp4 *restrict lj1,
const __global numtyp4 *restrict lj3,
const int lj_types,
@ -91,6 +91,9 @@ __kernel void k_lj_sdk(const __global numtyp4 *restrict x_,
} else if (lj1[mtype].y == 1) {
inv2=r2inv*ucl_sqrt(r2inv);
inv1=inv2*inv2;
} else if (lj1[mtype].y == 4) {
inv1=r2inv*r2inv*ucl_sqrt(r2inv);
inv2=inv1*r2inv;
} else {
inv1=r2inv*r2inv*r2inv;
inv2=inv1;
@ -119,7 +122,7 @@ __kernel void k_lj_sdk(const __global numtyp4 *restrict x_,
ans,engv);
}
__kernel void k_lj_sdk_fast(const __global numtyp4 *restrict x_,
__kernel void k_lj_spica_fast(const __global numtyp4 *restrict x_,
const __global numtyp4 *restrict lj1_in,
const __global numtyp4 *restrict lj3_in,
const __global numtyp *restrict sp_lj_in,
@ -192,6 +195,9 @@ __kernel void k_lj_sdk_fast(const __global numtyp4 *restrict x_,
} else if (lj1[mtype].y == (numtyp)1) {
inv2=r2inv*ucl_sqrt(r2inv);
inv1=inv2*inv2;
} else if (lj1[mtype].y == (numtyp)4) {
inv1=r2inv*r2inv*ucl_sqrt(r2inv);
inv2=inv1*r2inv;
} else {
inv1=r2inv*r2inv*r2inv;
inv2=inv1;

6
lib/gpu/lal_lj_sdk.h → lib/gpu/lal_lj_spica.h
View File

@ -1,9 +1,9 @@
/***************************************************************************
lj_sdk.h
lj_spica.h
-------------------
W. Michael Brown (ORNL)
Class for acceleration of the lj/sdk pair style
Class for acceleration of the lj/spica pair style
__________________________________________________________________________
This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -67,7 +67,7 @@ class CGCMM : public BaseAtomic<numtyp, acctyp> {
bool shared_types;
/// Number of atom types
int _sdk_types;
int _spica_types;
private:
bool _allocated;

18
lib/gpu/lal_lj_sdk_ext.cpp → lib/gpu/lal_lj_spica_ext.cpp
View File

@ -1,9 +1,9 @@
/***************************************************************************
lj_sdk.h
lj_spica.h
-------------------
W. Michael Brown (ORNL)
Functions for LAMMPS access to lj/sdk pair acceleration routines
Functions for LAMMPS access to lj/spica pair acceleration routines
__________________________________________________________________________
This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -17,7 +17,7 @@
#include <cassert>
#include <cmath>
#include "lal_lj_sdk.h"
#include "lal_lj_spica.h"
using namespace std;
using namespace LAMMPS_AL;
@ -27,7 +27,7 @@ static CGCMM<PRECISION,ACC_PRECISION> CMMMF;
// ---------------------------------------------------------------------------
// Allocate memory on host and device and copy constants to device
// ---------------------------------------------------------------------------
int sdk_gpu_init(const int ntypes, double **cutsq, int **cg_types,
int spica_gpu_init(const int ntypes, double **cutsq, int **cg_types,
double **host_lj1, double **host_lj2, double **host_lj3,
double **host_lj4, double **offset, double *special_lj,
const int inum, const int nall, const int max_nbors,
@ -42,7 +42,7 @@ int sdk_gpu_init(const int ntypes, double **cutsq, int **cg_types,
int gpu_rank=CMMMF.device->gpu_rank();
int procs_per_gpu=CMMMF.device->procs_per_gpu();
CMMMF.device->init_message(screen,"lj/sdk",first_gpu,last_gpu);
CMMMF.device->init_message(screen,"lj/spica",first_gpu,last_gpu);
bool message=false;
if (CMMMF.device->replica_me()==0 && screen)
@ -89,11 +89,11 @@ int sdk_gpu_init(const int ntypes, double **cutsq, int **cg_types,
return init_ok;
}
void sdk_gpu_clear() {
void spica_gpu_clear() {
CMMMF.clear();
}
int** sdk_gpu_compute_n(const int ago, const int inum_full,
int** spica_gpu_compute_n(const int ago, const int inum_full,
const int nall, double **host_x, int *host_type,
double *sublo, double *subhi, tagint *tag, int **nspecial,
tagint **special, const bool eflag, const bool vflag,
@ -105,7 +105,7 @@ int** sdk_gpu_compute_n(const int ago, const int inum_full,
vatom, host_start, ilist, jnum, cpu_time, success);
}
void sdk_gpu_compute(const int ago, const int inum_full, const int nall,
void spica_gpu_compute(const int ago, const int inum_full, const int nall,
double **host_x, int *host_type, int *ilist, int *numj,
int **firstneigh, const bool eflag, const bool vflag,
const bool eatom, const bool vatom, int &host_start,
@ -114,7 +114,7 @@ void sdk_gpu_compute(const int ago, const int inum_full, const int nall,
firstneigh,eflag,vflag,eatom,vatom,host_start,cpu_time,success);
}
double sdk_gpu_bytes() {
double spica_gpu_bytes() {
return CMMMF.host_memory_usage();
}

14
lib/gpu/lal_lj_sdk_long.cpp → lib/gpu/lal_lj_spica_long.cpp
View File

@ -1,9 +1,9 @@
/***************************************************************************
lj_sdk_long.cpp
lj_spica_long.cpp
-------------------
W. Michael Brown (ORNL)
Class for acceleration of the lj/sdk/coul/long pair style
Class for acceleration of the lj/spica/coul/long pair style
__________________________________________________________________________
This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -14,14 +14,14 @@
***************************************************************************/
#if defined(USE_OPENCL)
#include "lj_sdk_long_cl.h"
#include "lj_spica_long_cl.h"
#elif defined(USE_CUDART)
const char *lj_sdk_long=0;
const char *lj_spica_long=0;
#else
#include "lj_sdk_long_cubin.h"
#include "lj_spica_long_cubin.h"
#endif
#include "lal_lj_sdk_long.h"
#include "lal_lj_spica_long.h"
#include <cassert>
namespace LAMMPS_AL {
#define CGCMMLongT CGCMMLong<numtyp, acctyp>
@ -58,7 +58,7 @@ int CGCMMLongT::init(const int ntypes, double **host_cutsq,
const double g_ewald) {
int success;
success=this->init_atomic(nlocal,nall,max_nbors,maxspecial,cell_size,gpu_split,
_screen,lj_sdk_long,"k_lj_sdk_long");
_screen,lj_spica_long,"k_lj_spica_long");
if (success!=0)
return success;

14
lib/gpu/lal_lj_sdk_long.cu → lib/gpu/lal_lj_spica_long.cu
View File

@ -1,9 +1,9 @@
// **************************************************************************
// lj_sdk_long.cu
// lj_spica_long.cu
// -------------------
// W. Michael Brown (ORNL)
//
// Device code for acceleration of the lj/sdk/coul/long pair style
// Device code for acceleration of the lj/spica/coul/long pair style
//
// __________________________________________________________________________
// This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -29,7 +29,7 @@ _texture( q_tex,int2);
#define q_tex q_
#endif
__kernel void k_lj_sdk_long(const __global numtyp4 *restrict x_,
__kernel void k_lj_spica_long(const __global numtyp4 *restrict x_,
const __global numtyp4 *restrict lj1,
const __global numtyp4 *restrict lj3,
const int lj_types,
@ -107,6 +107,9 @@ __kernel void k_lj_sdk_long(const __global numtyp4 *restrict x_,
} else if (lj3[mtype].x == (numtyp)1) {
inv2=r2inv*ucl_rsqrt(rsq);
inv1=inv2*inv2;
} else if (lj3[mtype].x == (numtyp)4) {
inv1=r2inv*r2inv*ucl_rsqrt(rsq);
inv2=inv1*r2inv;
} else {
inv1=r2inv*r2inv*r2inv;
inv2=inv1;
@ -157,7 +160,7 @@ __kernel void k_lj_sdk_long(const __global numtyp4 *restrict x_,
vflag,ans,engv);
}
__kernel void k_lj_sdk_long_fast(const __global numtyp4 *restrict x_,
__kernel void k_lj_spica_long_fast(const __global numtyp4 *restrict x_,
const __global numtyp4 *restrict lj1_in,
const __global numtyp4 *restrict lj3_in,
const __global numtyp *restrict sp_lj_in,
@ -236,6 +239,9 @@ __kernel void k_lj_sdk_long_fast(const __global numtyp4 *restrict x_,
} else if (lj3[mtype].x == (numtyp)1) {
inv2=r2inv*ucl_rsqrt(rsq);
inv1=inv2*inv2;
} else if (lj3[mtype].x == (numtyp)4) {
inv1=r2inv*r2inv*ucl_rsqrt(rsq);
inv2=inv1*r2inv;
} else {
inv1=r2inv*r2inv*r2inv;
inv2=inv1;

4
lib/gpu/lal_lj_sdk_long.h → lib/gpu/lal_lj_spica_long.h
View File

@ -1,9 +1,9 @@
/***************************************************************************
lj_sdk_long.h
lj_spica_long.h
-------------------
W. Michael Brown (ORNL)
Class for acceleration of the lj/sdk/coul/long pair style
Class for acceleration of the lj/spica/coul/long pair style
__________________________________________________________________________
This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)

18
lib/gpu/lal_lj_sdk_long_ext.cpp → lib/gpu/lal_lj_spica_long_ext.cpp
View File

@ -1,9 +1,9 @@
/***************************************************************************
lj_sdk_long.h
lj_spica_long.h
-------------------
W. Michael Brown (ORNL)
Functions for LAMMPS access to lj/sdk/coul/long acceleration functions
Functions for LAMMPS access to lj/spica/coul/long acceleration functions
__________________________________________________________________________
This file is part of the LAMMPS Accelerator Library (LAMMPS_AL)
@ -17,7 +17,7 @@
#include <cassert>
#include <cmath>
#include "lal_lj_sdk_long.h"
#include "lal_lj_spica_long.h"
using namespace std;
using namespace LAMMPS_AL;
@ -27,7 +27,7 @@ static CGCMMLong<PRECISION,ACC_PRECISION> CMMLMF;
// ---------------------------------------------------------------------------
// Allocate memory on host and device and copy constants to device
// ---------------------------------------------------------------------------
int sdkl_gpu_init(const int ntypes, double **cutsq, int **cg_type,
int spical_gpu_init(const int ntypes, double **cutsq, int **cg_type,
double **host_lj1, double **host_lj2, double **host_lj3,
double **host_lj4, double **offset, double *special_lj,
const int inum, const int nall, const int max_nbors,
@ -44,7 +44,7 @@ int sdkl_gpu_init(const int ntypes, double **cutsq, int **cg_type,
int gpu_rank=CMMLMF.device->gpu_rank();
int procs_per_gpu=CMMLMF.device->procs_per_gpu();
CMMLMF.device->init_message(screen,"lj/sdk/coul/long",first_gpu,last_gpu);
CMMLMF.device->init_message(screen,"lj/spica/coul/long",first_gpu,last_gpu);
bool message=false;
if (CMMLMF.device->replica_me()==0 && screen)
@ -93,11 +93,11 @@ int sdkl_gpu_init(const int ntypes, double **cutsq, int **cg_type,
return init_ok;
}
void sdkl_gpu_clear() {
void spical_gpu_clear() {
CMMLMF.clear();
}
int** sdkl_gpu_compute_n(const int ago, const int inum_full,
int** spical_gpu_compute_n(const int ago, const int inum_full,
const int nall, double **host_x, int *host_type,
double *sublo, double *subhi, tagint *tag, int **nspecial,
tagint **special, const bool eflag, const bool vflag,
@ -111,7 +111,7 @@ int** sdkl_gpu_compute_n(const int ago, const int inum_full,
host_q,boxlo,prd);
}
void sdkl_gpu_compute(const int ago, const int inum_full, const int nall,
void spical_gpu_compute(const int ago, const int inum_full, const int nall,
double **host_x, int *host_type, int *ilist, int *numj,
int **firstneigh, const bool eflag, const bool vflag,
const bool eatom, const bool vatom, int &host_start,
@ -122,7 +122,7 @@ void sdkl_gpu_compute(const int ago, const int inum_full, const int nall,
host_q,nlocal,boxlo,prd);
}
double sdkl_gpu_bytes() {
double spical_gpu_bytes() {
return CMMLMF.host_memory_usage();
}

15
lib/gpu/lal_pre_cuda_hip.h
View File

@ -30,7 +30,7 @@
// -------------------------------------------------------------------------
#if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__)
#if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__) || defined(__HIP_PLATFORM_SPIRV__)
#define CONFIG_ID 303
#define SIMD_SIZE 64
#else
@ -112,7 +112,7 @@
// KERNEL MACROS - TEXTURES
// -------------------------------------------------------------------------
#if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__)
#if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__) || defined(__HIP_PLATFORM_SPIRV__)
#define _texture(name, type) __device__ type* name
#define _texture_2d(name, type) __device__ type* name
#else
@ -134,9 +134,12 @@
int2 qt = tex1Dfetch(q_tex,i); \
ans=__hiloint2double(qt.y, qt.x); \
}
#elseif defined(__HIP_PLATFORM_SPIRV__)
#define fetch4(ans,i,pos_tex) tex1Dfetch(&ans, pos_tex, i);
#define fetch(ans,i,q_tex) tex1Dfetch(&ans, q_tex,i);
#else
#define fetch4(ans,i,pos_tex) ans=tex1Dfetch(pos_tex, i);
#define fetch(ans,i,q_tex) ans=tex1Dfetch(q_tex,i);
#define fetch4(ans,i,pos_tex) ans=tex1Dfetch(pos_tex, i);
#define fetch(ans,i,q_tex) ans=tex1Dfetch(q_tex,i);
#endif
#else
#define fetch4(ans,i,x) ans=x[i]
@ -152,7 +155,7 @@
#define mu_tex mu_
#endif
#if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__)
#if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__) || defined(__HIP_PLATFORM_SPIRV__)
#undef fetch4
#undef fetch
@ -209,7 +212,7 @@
#endif
#endif
#if defined(CUDA_PRE_NINE) || defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__)
#if defined(CUDA_PRE_NINE) || defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__) || defined(__HIP_PLATFORM_SPIRV__)
#ifdef _SINGLE_SINGLE
#define shfl_down __shfl_down