diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index e97f5546cd..aea766b79c 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -79,6 +79,7 @@ target_compile_definitions(lammps PRIVATE -DLMP_KOKKOS) set(KOKKOS_PKG_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/KOKKOS) set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/atom_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/atom_map_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 0ac90a53f2..a28b17270a 100755 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -55,6 +55,7 @@ action angle_harmonic_kokkos.cpp angle_harmonic.cpp action angle_harmonic_kokkos.h angle_harmonic.h action atom_kokkos.cpp action atom_kokkos.h +action atom_map_kokkos.cpp action atom_vec_angle_kokkos.cpp atom_vec_angle.cpp action atom_vec_angle_kokkos.h atom_vec_angle.h action atom_vec_atomic_kokkos.cpp diff --git a/src/KOKKOS/atom_kokkos.cpp b/src/KOKKOS/atom_kokkos.cpp index ce97510d7a..02cd4ea708 100644 --- a/src/KOKKOS/atom_kokkos.cpp +++ b/src/KOKKOS/atom_kokkos.cpp @@ -1,4 +1,3 @@ -// clang-format off /* ---------------------------------------------------------------------- LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator https://www.lammps.org/, Sandia National Laboratories @@ -13,23 +12,25 @@ ------------------------------------------------------------------------- */ #include "atom_kokkos.h" -#include + +#include "atom_masks.h" #include "atom_vec.h" #include "atom_vec_kokkos.h" #include "comm_kokkos.h" -#include "update.h" #include "domain.h" -#include "atom_masks.h" -#include "memory_kokkos.h" #include "error.h" #include "kokkos.h" -#include "atom_masks.h" +#include "memory_kokkos.h" +#include "update.h" using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- */ -AtomKokkos::AtomKokkos(LAMMPS *lmp) : Atom(lmp) {} +AtomKokkos::AtomKokkos(LAMMPS *lmp) : Atom(lmp) +{ + k_error_flag = DAT::tdual_int_scalar("atom:error_flag"); +} /* ---------------------------------------------------------------------- */ @@ -77,6 +78,8 @@ AtomKokkos::~AtomKokkos() memoryKK->destroy_kokkos(k_improper_atom3, improper_atom3); memoryKK->destroy_kokkos(k_improper_atom4, improper_atom4); + AtomKokkos::map_delete(); + // SPIN package memoryKK->destroy_kokkos(k_sp, sp); @@ -84,16 +87,16 @@ AtomKokkos::~AtomKokkos() memoryKK->destroy_kokkos(k_fm_long, fm_long); // DPD-REACT package - memoryKK->destroy_kokkos(k_uCond,uCond); - memoryKK->destroy_kokkos(k_uMech,uMech); - memoryKK->destroy_kokkos(k_uChem,uChem); - memoryKK->destroy_kokkos(k_uCG,uCG); - memoryKK->destroy_kokkos(k_uCGnew,uCGnew); - memoryKK->destroy_kokkos(k_rho,rho); - memoryKK->destroy_kokkos(k_dpdTheta,dpdTheta); - memoryKK->destroy_kokkos(k_duChem,duChem); + memoryKK->destroy_kokkos(k_uCond, uCond); + memoryKK->destroy_kokkos(k_uMech, uMech); + memoryKK->destroy_kokkos(k_uChem, uChem); + memoryKK->destroy_kokkos(k_uCG, uCG); + memoryKK->destroy_kokkos(k_uCGnew, uCGnew); + memoryKK->destroy_kokkos(k_rho, rho); + memoryKK->destroy_kokkos(k_dpdTheta, dpdTheta); + memoryKK->destroy_kokkos(k_duChem, duChem); - memoryKK->destroy_kokkos(k_dvector,dvector); + memoryKK->destroy_kokkos(k_dvector, dvector); dvector = nullptr; } @@ -101,34 +104,32 @@ AtomKokkos::~AtomKokkos() void AtomKokkos::sync(const ExecutionSpace space, unsigned int mask) { - if (space == Device && lmp->kokkos->auto_sync) - ((AtomVecKokkos *) avec)->modified(Host,mask); + if (space == Device && lmp->kokkos->auto_sync) ((AtomVecKokkos *) avec)->modified(Host, mask); - ((AtomVecKokkos *) avec)->sync(space,mask); + ((AtomVecKokkos *) avec)->sync(space, mask); } /* ---------------------------------------------------------------------- */ void AtomKokkos::modified(const ExecutionSpace space, unsigned int mask) { - ((AtomVecKokkos *) avec)->modified(space,mask); + ((AtomVecKokkos *) avec)->modified(space, mask); - if (space == Device && lmp->kokkos->auto_sync) - ((AtomVecKokkos *) avec)->sync(Host,mask); + if (space == Device && lmp->kokkos->auto_sync) ((AtomVecKokkos *) avec)->sync(Host, mask); } void AtomKokkos::sync_overlapping_device(const ExecutionSpace space, unsigned int mask) { - ((AtomVecKokkos *) avec)->sync_overlapping_device(space,mask); + ((AtomVecKokkos *) avec)->sync_overlapping_device(space, mask); } /* ---------------------------------------------------------------------- */ void AtomKokkos::allocate_type_arrays() { if (avec->mass_type == AtomVec::PER_TYPE) { - k_mass = DAT::tdual_float_1d("Mass",ntypes+1); + k_mass = DAT::tdual_float_1d("Mass", ntypes + 1); mass = k_mass.h_view.data(); - mass_setflag = new int[ntypes+1]; + mass_setflag = new int[ntypes + 1]; for (int itype = 1; itype <= ntypes; itype++) mass_setflag[itype] = 0; k_mass.modify(); } @@ -138,11 +139,11 @@ void AtomKokkos::allocate_type_arrays() void AtomKokkos::sort() { - int i,m,n,ix,iy,iz,ibin,empty; + int i, m, n, ix, iy, iz, ibin, empty; // set next timestep for sorting to take place - nextsort = (update->ntimestep/sortfreq)*sortfreq + sortfreq; + nextsort = (update->ntimestep / sortfreq) * sortfreq + sortfreq; // re-setup sort bins if needed @@ -155,33 +156,33 @@ void AtomKokkos::sort() memory->destroy(next); memory->destroy(permute); maxnext = atom->nmax; - memory->create(next,maxnext,"atom:next"); - memory->create(permute,maxnext,"atom:permute"); + memory->create(next, maxnext, "atom:next"); + memory->create(permute, maxnext, "atom:permute"); } // insure there is one extra atom location at end of arrays for swaps if (nlocal == nmax) avec->grow(0); - sync(Host,ALL_MASK); - modified(Host,ALL_MASK); + sync(Host, ALL_MASK); + modified(Host, ALL_MASK); // bin atoms in reverse order so linked list will be in forward order for (i = 0; i < nbins; i++) binhead[i] = -1; HAT::t_x_array_const h_x = k_x.view(); - for (i = nlocal-1; i >= 0; i--) { - ix = static_cast ((h_x(i,0)-bboxlo[0])*bininvx); - iy = static_cast ((h_x(i,1)-bboxlo[1])*bininvy); - iz = static_cast ((h_x(i,2)-bboxlo[2])*bininvz); - ix = MAX(ix,0); - iy = MAX(iy,0); - iz = MAX(iz,0); - ix = MIN(ix,nbinx-1); - iy = MIN(iy,nbiny-1); - iz = MIN(iz,nbinz-1); - ibin = iz*nbiny*nbinx + iy*nbinx + ix; + for (i = nlocal - 1; i >= 0; i--) { + ix = static_cast((h_x(i, 0) - bboxlo[0]) * bininvx); + iy = static_cast((h_x(i, 1) - bboxlo[1]) * bininvy); + iz = static_cast((h_x(i, 2) - bboxlo[2]) * bininvz); + ix = MAX(ix, 0); + iy = MAX(iy, 0); + iz = MAX(iz, 0); + ix = MIN(ix, nbinx - 1); + iy = MIN(iy, nbiny - 1); + iz = MIN(iz, nbinz - 1); + ibin = iz * nbiny * nbinx + iy * nbinx + ix; next[i] = binhead[ibin]; binhead[ibin] = i; } @@ -213,13 +214,13 @@ void AtomKokkos::sort() for (i = 0; i < nlocal; i++) { if (current[i] == permute[i]) continue; - avec->copy(i,nlocal,0); + avec->copy(i, nlocal, 0); empty = i; while (permute[empty] != i) { - avec->copy(permute[empty],empty,0); + avec->copy(permute[empty], empty, 0); empty = current[empty] = permute[empty]; } - avec->copy(nlocal,empty,0); + avec->copy(nlocal, empty, 0); current[empty] = permute[empty]; } @@ -237,13 +238,14 @@ void AtomKokkos::sort() reallocate memory to the pointer selected by the mask ------------------------------------------------------------------------- */ -void AtomKokkos::grow(unsigned int mask) { +void AtomKokkos::grow(unsigned int mask) +{ if (mask & SPECIAL_MASK) { memoryKK->destroy_kokkos(k_special, special); sync(Device, mask); modified(Device, mask); - memoryKK->grow_kokkos(k_special,special,nmax,maxspecial,"atom:special"); + memoryKK->grow_kokkos(k_special, special, nmax, maxspecial, "atom:special"); avec->grow_pointers(); sync(Host, mask); } @@ -262,22 +264,18 @@ int AtomKokkos::add_custom(const char *name, int flag) if (flag == 0) { index = nivector; nivector++; - iname = (char **) memory->srealloc(iname,nivector*sizeof(char *), - "atom:iname"); + iname = (char **) memory->srealloc(iname, nivector * sizeof(char *), "atom:iname"); iname[index] = utils::strdup(name); - ivector = (int **) memory->srealloc(ivector,nivector*sizeof(int *), - "atom:ivector"); - memory->create(ivector[index],nmax,"atom:ivector"); + ivector = (int **) memory->srealloc(ivector, nivector * sizeof(int *), "atom:ivector"); + memory->create(ivector[index], nmax, "atom:ivector"); } else { index = ndvector; ndvector++; - dname = (char **) memory->srealloc(dname,ndvector*sizeof(char *), - "atom:dname"); + dname = (char **) memory->srealloc(dname, ndvector * sizeof(char *), "atom:dname"); dname[index] = utils::strdup(name); - this->sync(Device,DVECTOR_MASK); - memoryKK->grow_kokkos(k_dvector,dvector,ndvector,nmax, - "atom:dvector"); - this->modified(Device,DVECTOR_MASK); + this->sync(Device, DVECTOR_MASK); + memoryKK->grow_kokkos(k_dvector, dvector, ndvector, nmax, "atom:dvector"); + this->modified(Device, DVECTOR_MASK); } return index; @@ -294,12 +292,12 @@ void AtomKokkos::remove_custom(int flag, int index) if (flag == 0) { memory->destroy(ivector[index]); ivector[index] = nullptr; - delete [] iname[index]; + delete[] iname[index]; iname[index] = nullptr; } else { //memoryKK->destroy_kokkos(dvector); dvector[index] = nullptr; - delete [] dname[index]; + delete[] dname[index]; dname[index] = nullptr; } } @@ -335,19 +333,16 @@ void AtomKokkos::deallocate_topology() done at higher levels (Verlet,Modify,etc) ------------------------------------------------------------------------- */ -void AtomKokkos::sync_modify(ExecutionSpace execution_space, - unsigned int datamask_read, +void AtomKokkos::sync_modify(ExecutionSpace execution_space, unsigned int datamask_read, unsigned int datamask_modify) { - sync(execution_space,datamask_read); - modified(execution_space,datamask_modify); + sync(execution_space, datamask_read); + modified(execution_space, datamask_modify); } -AtomVec *AtomKokkos::new_avec(const std::string &style, - int trysuffix, int &sflag) +AtomVec *AtomKokkos::new_avec(const std::string &style, int trysuffix, int &sflag) { - AtomVec* avec = Atom::new_avec(style,trysuffix,sflag); - if (!avec->kokkosable) - error->all(FLERR,"KOKKOS package requires a kokkos enabled atom_style"); + AtomVec *avec = Atom::new_avec(style, trysuffix, sflag); + if (!avec->kokkosable) error->all(FLERR, "KOKKOS package requires a kokkos enabled atom_style"); return avec; } diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index 1c3953ddf3..6cf91a5ddc 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -67,7 +67,43 @@ class AtomKokkos : public Atom { AtomKokkos(class LAMMPS *); - ~AtomKokkos(); + virtual ~AtomKokkos(); + + void map_init(int check = 1); + void map_set(); + void map_delete(); + + DAT::tdual_int_1d k_sametag; + DAT::tdual_int_1d k_map_array; + DAT::tdual_int_scalar k_error_flag; + dual_hash_type k_map_hash; + + // map lookup function inlined for efficiency + // return -1 if no map defined + + template + KOKKOS_INLINE_FUNCTION + static int map_kokkos(tagint global, int map_style, DAT::tdual_int_1d k_map_array, dual_hash_type k_map_hash) + { + if (map_style == 1) + return k_map_array.view()(global); + else if (map_style == 2) + return AtomKokkos::map_find_hash_kokkos(global,k_map_hash); + else + return -1; + } + + template + KOKKOS_INLINE_FUNCTION + static int map_find_hash_kokkos(tagint global, dual_hash_type &k_map_hash) + { + int local = -1; + auto d_map_hash = k_map_hash.view(); + auto index = d_map_hash.find(global); + if (d_map_hash.valid_at(index)) + local = d_map_hash.value_at(index); + return local; + } virtual void allocate_type_arrays(); void sync(const ExecutionSpace space, unsigned int mask); diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp new file mode 100644 index 0000000000..8d0d1243ac --- /dev/null +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -0,0 +1,288 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "atom_kokkos.h" + +#include "atom_masks.h" +#include "comm.h" +#include "error.h" +#include "fix.h" +#include "memory_kokkos.h" +#include "modify.h" +#include "neighbor_kokkos.h" + +#include + +using namespace LAMMPS_NS; + +#define EXTRA 1000 + +/* ---------------------------------------------------------------------- + allocate and initialize array or hash table for global -> local map + for array option: + array length = 1 to map_tag_max + set entire array to -1 as initial values + for hash option: + map_nhash = length of hash table + map_nbucket = # of hash buckets, prime larger than map_nhash * 2 + so buckets will only be filled with 0 or 1 atoms on average +------------------------------------------------------------------------- */ + +void AtomKokkos::map_init(int check) +{ + // check for new map style if max atomID changed (check = 1 = default) + // recreate = 1 if must delete old map and create new map + // recreate = 0 if can re-use old map w/out realloc and just adjust settings + // map_maxarray/map_nhash initially -1, to force recreate even when no atoms + + int recreate = 0; + if (check) recreate = map_style_set(); + + if (map_style == MAP_ARRAY && map_tag_max > map_maxarray) + recreate = 1; + else if (map_style == MAP_HASH && nlocal + nghost > map_nhash) + recreate = 1; + + // if not recreating: + // for array, initialize current map_tag_max values + // for hash, set all buckets to empty, put all entries in free list + + if (!recreate) { + if (map_style == MAP_ARRAY) { + for (int i = 0; i <= map_tag_max; i++) map_array[i] = -1; + } else { + for (int i = 0; i < map_nbucket; i++) map_bucket[i] = -1; + map_nused = 0; + map_free = 0; + for (int i = 0; i < map_nhash; i++) map_hash[i].next = i + 1; + if (map_nhash > 0) map_hash[map_nhash - 1].next = -1; + } + + // recreating: delete old map and create new one for array or hash + + } else { + map_delete(); + + if (map_style == MAP_ARRAY) { + map_maxarray = map_tag_max; + memoryKK->create_kokkos(k_map_array, map_array, map_maxarray + 1, "atom:map_array"); + for (int i = 0; i <= map_tag_max; i++) map_array[i] = -1; + + } else { + + // map_nhash = max # of atoms that can be hashed on this proc + // set to max of ave atoms/proc or atoms I can store + // multiply by 2, require at least 1000 + // doubling means hash table will need to be re-init only rarely + + int nper = static_cast(natoms / comm->nprocs); + map_nhash = MAX(nper, nmax); + map_nhash *= 2; + map_nhash = MAX(map_nhash, 1000); + + // map_nbucket = prime just larger than map_nhash + // next_prime() should be fast enough, + // about 10% of odd integers are prime above 1M + + map_nbucket = next_prime(map_nhash); + + // set all buckets to empty + // set hash to map_nhash in length + // put all hash entries in free list and point them to each other + + map_bucket = new int[map_nbucket]; + for (int i = 0; i < map_nbucket; i++) map_bucket[i] = -1; + + map_hash = new HashElem[map_nhash]; + map_nused = 0; + map_free = 0; + for (int i = 0; i < map_nhash; i++) map_hash[i].next = i + 1; + map_hash[map_nhash - 1].next = -1; + + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); // get type + h_map_hash = decltype(h_map_hash)(map_nhash); + k_map_hash.view() = h_map_hash; + } + } + + k_sametag.modify_host(); + if (map_style == Atom::MAP_ARRAY) k_map_array.modify_host(); +} + +/* ---------------------------------------------------------------------- + set global -> local map for all of my own and ghost atoms + loop in reverse order so that nearby images take precedence over far ones + and owned atoms take precedence over images + this enables valid lookups of bond topology atoms + for hash table option: + if hash table too small, re-init + global ID may already be in table if image atom was set +------------------------------------------------------------------------- */ + +void AtomKokkos::map_set() +{ + int nall = nlocal + nghost; + + atomKK->sync(Host, TAG_MASK); + + k_sametag.sync_host(); + if (map_style == Atom::MAP_ARRAY) k_map_array.sync_host(); + + if (map_style == MAP_ARRAY) { + + // possible reallocation of sametag must come before loop over atoms + // since loop sets sametag + + if (nall > max_same) { + max_same = nall + EXTRA; + memoryKK->destroy_kokkos(k_sametag, sametag); + memoryKK->create_kokkos(k_sametag, sametag, max_same, "atom:sametag"); + } + + for (int i = nall - 1; i >= 0; i--) { + sametag[i] = map_array[tag[i]]; + map_array[tag[i]] = i; + } + + } else { + + // if this proc has more atoms than hash table size, call map_init() + // call with 0 since max atomID in system has not changed + // possible reallocation of sametag must come after map_init(), + // b/c map_init() may invoke map_delete(), whacking sametag + + if (nall > map_nhash) map_init(0); + if (nall > max_same) { + max_same = nall + EXTRA; + memoryKK->destroy_kokkos(k_sametag, sametag); + memoryKK->create_kokkos(k_sametag, sametag, max_same, "atom:sametag"); + } + + int previous, ibucket, index; + tagint global; + + for (int i = nall - 1; i >= 0; i--) { + sametag[i] = map_find_hash(tag[i]); + + // search for key + // if found it, just overwrite local value with index + + previous = -1; + global = tag[i]; + ibucket = global % map_nbucket; + index = map_bucket[ibucket]; + while (index > -1) { + if (map_hash[index].global == global) break; + previous = index; + index = map_hash[index].next; + } + if (index > -1) { + map_hash[index].local = i; + continue; + } + + // take one entry from free list + // add the new global/local pair as entry at end of bucket list + // special logic if this entry is 1st in bucket + + index = map_free; + map_free = map_hash[map_free].next; + if (previous == -1) + map_bucket[ibucket] = index; + else + map_hash[previous].next = index; + map_hash[index].global = global; + map_hash[index].local = i; + map_hash[index].next = -1; + map_nused++; + } + + // Copy to Kokkos hash + + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); + h_map_hash.clear(); + + for (int i = nall - 1; i >= 0; i--) { + + // search for key + // if don't find it, done + + previous = -1; + global = tag[i]; + ibucket = global % map_nbucket; + index = map_bucket[ibucket]; + while (index > -1) { + if (map_hash[index].global == global) break; + previous = index; + index = map_hash[index].next; + } + if (index == -1) continue; + + int local = map_hash[index].local; + + auto insert_result = h_map_hash.insert(global, local); + if (insert_result.failed()) error->one(FLERR, "Kokkos::UnorderedMap insertion failed"); + } + } + + k_sametag.modify_host(); + if (map_style == Atom::MAP_ARRAY) + k_map_array.modify_host(); + else if (map_style == Atom::MAP_HASH) { + + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); + auto d_map_hash = k_map_hash.view(); + + // check if fix shake or neigh bond needs a device hash + + int device_hash_flag = 0; + + auto neighborKK = (NeighborKokkos *) neighbor; + if (neighborKK->device_flag) device_hash_flag = 1; + + for (int n = 0; n < modify->nfix; n++) + if (utils::strmatch(modify->fix[n]->style, "^shake")) + if (modify->fix[n]->execution_space == Device) device_hash_flag = 1; + + if (device_hash_flag) { + Kokkos::deep_copy(d_map_hash,h_map_hash); + k_map_hash.view() = d_map_hash; + } + } +} + +/* ---------------------------------------------------------------------- + free the array or hash table for global to local mapping +------------------------------------------------------------------------- */ + +void AtomKokkos::map_delete() +{ + memoryKK->destroy_kokkos(k_sametag, sametag); + sametag = nullptr; + + if (map_style == MAP_ARRAY) { + memoryKK->destroy_kokkos(k_map_array, map_array); + map_array = nullptr; + } else { + k_map_hash.h_view = host_hash_type(); + k_map_hash.d_view = hash_type(); + } + + Atom::map_delete(); +} diff --git a/src/KOKKOS/atom_vec_dpd_kokkos.cpp b/src/KOKKOS/atom_vec_dpd_kokkos.cpp index 4358a49358..e02631f89e 100644 --- a/src/KOKKOS/atom_vec_dpd_kokkos.cpp +++ b/src/KOKKOS/atom_vec_dpd_kokkos.cpp @@ -13,15 +13,15 @@ ------------------------------------------------------------------------- */ #include "atom_vec_dpd_kokkos.h" + #include "atom_kokkos.h" +#include "atom_masks.h" #include "comm_kokkos.h" #include "domain.h" -#include "modify.h" -#include "fix.h" -#include "atom_masks.h" -#include "memory_kokkos.h" #include "error.h" - +#include "fix.h" +#include "memory_kokkos.h" +#include "modify.h" using namespace LAMMPS_NS; diff --git a/src/KOKKOS/atom_vec_full_kokkos.cpp b/src/KOKKOS/atom_vec_full_kokkos.cpp index 45ce316d3f..cd5316cc73 100644 --- a/src/KOKKOS/atom_vec_full_kokkos.cpp +++ b/src/KOKKOS/atom_vec_full_kokkos.cpp @@ -13,15 +13,15 @@ ------------------------------------------------------------------------- */ #include "atom_vec_full_kokkos.h" + #include "atom_kokkos.h" +#include "atom_masks.h" #include "comm_kokkos.h" #include "domain.h" -#include "modify.h" -#include "fix.h" -#include "atom_masks.h" -#include "memory_kokkos.h" #include "error.h" - +#include "fix.h" +#include "memory_kokkos.h" +#include "modify.h" using namespace LAMMPS_NS; diff --git a/src/KOKKOS/comm_kokkos.cpp b/src/KOKKOS/comm_kokkos.cpp index fd5318e1d1..588ff94d73 100644 --- a/src/KOKKOS/comm_kokkos.cpp +++ b/src/KOKKOS/comm_kokkos.cpp @@ -1128,9 +1128,10 @@ void CommKokkos::borders_device() { max = MAX(maxforward*rmax,maxreverse*smax); if (max > maxrecv) grow_recv_kokkos(max); + atomKK->modified(exec_space,ALL_MASK); + // reset global->local map - atomKK->modified(exec_space,ALL_MASK); if (map_style != Atom::MAP_NONE) { atomKK->sync(Host,TAG_MASK); atom->map_set(); diff --git a/src/KOKKOS/fix_shake_kokkos.cpp b/src/KOKKOS/fix_shake_kokkos.cpp index 804da9fd22..91d8458d21 100644 --- a/src/KOKKOS/fix_shake_kokkos.cpp +++ b/src/KOKKOS/fix_shake_kokkos.cpp @@ -201,6 +201,14 @@ void FixShakeKokkos::pre_neighbor() type = atom->type; nlocal = atom->nlocal; + map_style = atom->map_style; + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + } + k_shake_flag.sync(); k_shake_atom.sync(); @@ -213,21 +221,16 @@ void FixShakeKokkos::pre_neighbor() d_list = k_list.view(); } - // don't yet have atom_map_kokkos routines, so move data from host to device + // Atom Map - if (atom->map_style != Atom::MAP_ARRAY) - error->all(FLERR,"Must use atom map style array with Kokkos"); + map_style = atom->map_style; - int* map_array_host = atom->get_map_array(); - int map_size = atom->get_map_size(); - int map_maxarray = atom->get_map_maxarray(); - if (map_maxarray > (int)k_map_array.extent(0)) - k_map_array = DAT::tdual_int_1d("NeighBond:map_array",map_maxarray); - for (int i=0; i(); - k_map_array.template sync(); - map_array = k_map_array.view(); + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + } // build list of SHAKE clusters I compute @@ -241,14 +244,16 @@ void FixShakeKokkos::pre_neighbor() auto d_list = this->d_list; auto d_error_flag = this->d_error_flag; auto d_nlist = this->d_nlist; - auto map_array = this->map_array; + auto map_style = atom->map_style; + auto k_map_array = this->k_map_array; + auto k_map_hash = this->k_map_hash; Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal), LAMMPS_LAMBDA(const int& i) { if (d_shake_flag[i]) { if (d_shake_flag[i] == 2) { - const int atom1 = map_array(d_shake_atom(i,0)); - const int atom2 = map_array(d_shake_atom(i,1)); + const int atom1 = AtomKokkos::map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = AtomKokkos::map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1) { d_error_flag() = 1; } @@ -257,9 +262,9 @@ void FixShakeKokkos::pre_neighbor() d_list[nlist] = i; } } else if (d_shake_flag[i] % 2 == 1) { - const int atom1 = map_array(d_shake_atom(i,0)); - const int atom2 = map_array(d_shake_atom(i,1)); - const int atom3 = map_array(d_shake_atom(i,2)); + const int atom1 = AtomKokkos::map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = AtomKokkos::map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom3 = AtomKokkos::map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) d_error_flag() = 1; if (i <= atom1 && i <= atom2 && i <= atom3) { @@ -267,10 +272,10 @@ void FixShakeKokkos::pre_neighbor() d_list[nlist] = i; } } else { - const int atom1 = map_array(d_shake_atom(i,0)); - const int atom2 = map_array(d_shake_atom(i,1)); - const int atom3 = map_array(d_shake_atom(i,2)); - const int atom4 = map_array(d_shake_atom(i,3)); + const int atom1 = AtomKokkos::map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = AtomKokkos::map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom3 = AtomKokkos::map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); + const int atom4 = AtomKokkos::map_kokkos(d_shake_atom(i,3),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) d_error_flag() = 1; if (i <= atom1 && i <= atom2 && i <= atom3 && i <= atom4) { @@ -307,6 +312,14 @@ void FixShakeKokkos::post_force(int vflag) d_mass = atomKK->k_mass.view(); nlocal = atomKK->nlocal; + map_style = atom->map_style; + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + } + if (d_rmass.data()) atomKK->sync(execution_space,X_MASK|F_MASK|RMASS_MASK); else @@ -586,8 +599,8 @@ void FixShakeKokkos::shake(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; // r01 = distance vec between atoms, with PBC @@ -697,9 +710,9 @@ void FixShakeKokkos::shake3(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); - int i2 = map_array(d_shake_atom(m,2)); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = AtomKokkos::map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; @@ -880,10 +893,10 @@ void FixShakeKokkos::shake4(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); - int i2 = map_array(d_shake_atom(m,2)); - int i3 = map_array(d_shake_atom(m,3)); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = AtomKokkos::map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); + int i3 = AtomKokkos::map_kokkos(d_shake_atom(m,3),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; double bond3 = d_bond_distance[d_shake_type(m,2)]; @@ -1142,9 +1155,9 @@ void FixShakeKokkos::shake3angle(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); - int i2 = map_array(d_shake_atom(m,2)); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = AtomKokkos::map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; double bond12 = d_angle_distance[d_shake_type(m,2)]; diff --git a/src/KOKKOS/fix_shake_kokkos.h b/src/KOKKOS/fix_shake_kokkos.h index b95c042ac1..12e082a903 100644 --- a/src/KOKKOS/fix_shake_kokkos.h +++ b/src/KOKKOS/fix_shake_kokkos.h @@ -26,6 +26,7 @@ FixStyle(shake/kk/host,FixShakeKokkos); #include "fix_shake.h" #include "kokkos_type.h" #include "kokkos_base.h" +#include namespace LAMMPS_NS { @@ -172,9 +173,6 @@ class FixShakeKokkos : public FixShake, public KokkosBase { KOKKOS_INLINE_FUNCTION void v_tally(EV_FLOAT&, int, int *, double, double *) const; - DAT::tdual_int_1d k_map_array; - typename AT::t_int_1d_randomread map_array; - int iswap; int first; typename AT::t_int_2d d_sendlist; @@ -185,6 +183,10 @@ class FixShakeKokkos : public FixShake, public KokkosBase { tagint **shake_atom_tmp; int **shake_type_tmp; + int map_style; + DAT::tdual_int_1d k_map_array; + dual_hash_type k_map_hash; + // copied from Domain KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index c9e5736410..05c839db3f 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -23,6 +23,7 @@ #include #include #include +#include enum{FULL=1u,HALFTHREAD=2u,HALF=4u}; @@ -45,7 +46,9 @@ enum{FULL=1u,HALFTHREAD=2u,HALF=4u}; static constexpr LAMMPS_NS::bigint LMP_KOKKOS_AV_DELTA = 10; namespace Kokkos { - using NoInit = ViewAllocateWithoutInitializing; + static auto NoInit = [](std::string const& label) { + return Kokkos::view_alloc(Kokkos::WithoutInitializing, label); + }; } struct lmp_float3 { @@ -551,6 +554,23 @@ typedef int T_INT; // LAMMPS types +typedef Kokkos::UnorderedMap hash_type; +typedef hash_type::HostMirror host_hash_type; + +struct dual_hash_type { + hash_type d_view; + host_hash_type h_view; + + template + KOKKOS_INLINE_FUNCTION + std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type&> view() {return d_view;} + + template + KOKKOS_INLINE_FUNCTION + std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type&> view() {return h_view;} + +}; + template struct ArrayTypes; diff --git a/src/KOKKOS/neigh_bond_kokkos.cpp b/src/KOKKOS/neigh_bond_kokkos.cpp index a7db309b1d..a4cb6e7d40 100644 --- a/src/KOKKOS/neigh_bond_kokkos.cpp +++ b/src/KOKKOS/neigh_bond_kokkos.cpp @@ -197,8 +197,6 @@ template void NeighBondKokkos::build_topology_kk() { atomKK->sync(execution_space, X_MASK | TAG_MASK); - int nall = atom->nlocal + atom->nghost; - int nmax = atom->nmax; nlocal = atom->nlocal; x = atomKK->k_x.view(); @@ -207,30 +205,7 @@ void NeighBondKokkos::build_topology_kk() lostbond = output->thermo->lostbond; - // don't yet have atom_map_kokkos routines, so move data from host to device - - if (atom->map_style != Atom::MAP_ARRAY) - error->all(FLERR,"Must use atom map style array with Kokkos"); - - int* map_array_host = atom->get_map_array(); - int map_size = atom->get_map_size(); - int map_maxarray = atom->get_map_maxarray(); - if (map_maxarray > (int)k_map_array.extent(0)) - k_map_array = DAT::tdual_int_1d("NeighBond:map_array",map_maxarray); - for (int i=0; i(); - k_map_array.template sync(); - map_array = k_map_array.view(); - - int* sametag_host = atomKK->sametag; - if (nmax > (int)k_sametag.extent(0)) - k_sametag = DAT::tdual_int_1d("NeighBond:sametag",nmax); - for (int i=0; i(); - k_sametag.template sync(); - sametag = k_sametag.view(); + update_class_variables(); if (force->bond) (this->*bond_build_kk)(); if (force->angle) (this->*angle_build_kk)(); @@ -306,7 +281,7 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondBondAll, const int &i, int &nmissing) const { for (int m = 0; m < num_bond[i]; m++) { - int atom1 = map_array(bond_atom(i,m)); + int atom1 = AtomKokkos::map_kokkos(bond_atom(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -394,7 +369,7 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondBondPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_bond[i]; m++) { if (bond_type(i,m) <= 0) continue; - int atom1 = map_array(bond_atom(i,m)); + int atom1 = AtomKokkos::map_kokkos(bond_atom(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -420,7 +395,6 @@ void NeighBondKokkos::bond_check() { int flag = 0; - update_domain_variables(); atomKK->sync(execution_space, X_MASK); k_bondlist.sync(); @@ -507,9 +481,9 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondAngleAll, const int &i, int &nmissing) const { for (int m = 0; m < num_angle[i]; m++) { - int atom1 = map_array(angle_atom1(i,m)); - int atom2 = map_array(angle_atom2(i,m)); - int atom3 = map_array(angle_atom3(i,m)); + int atom1 = AtomKokkos::map_kokkos(angle_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(angle_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(angle_atom3(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -602,9 +576,9 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondAnglePartial, const int &i, int &nmissing) const { for (int m = 0; m < num_angle[i]; m++) { if (angle_type(i,m) <= 0) continue; - int atom1 = map_array(angle_atom1(i,m)); - int atom2 = map_array(angle_atom2(i,m)); - int atom3 = map_array(angle_atom3(i,m)); + int atom1 = AtomKokkos::map_kokkos(angle_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(angle_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(angle_atom3(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -636,7 +610,6 @@ void NeighBondKokkos::angle_check() // check all 3 distances // in case angle potential computes any of them - update_domain_variables(); atomKK->sync(execution_space, X_MASK); k_anglelist.sync(); @@ -735,10 +708,10 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondDihedralAll, const int &i, int &nmissing) const { for (int m = 0; m < num_dihedral[i]; m++) { - int atom1 = map_array(dihedral_atom1(i,m)); - int atom2 = map_array(dihedral_atom2(i,m)); - int atom3 = map_array(dihedral_atom3(i,m)); - int atom4 = map_array(dihedral_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(dihedral_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(dihedral_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(dihedral_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(dihedral_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -835,10 +808,10 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondDihedralPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_dihedral[i]; m++) { if (dihedral_type(i,m) <= 0) continue; - int atom1 = map_array(dihedral_atom1(i,m)); - int atom2 = map_array(dihedral_atom2(i,m)); - int atom3 = map_array(dihedral_atom3(i,m)); - int atom4 = map_array(dihedral_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(dihedral_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(dihedral_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(dihedral_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(dihedral_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -874,7 +847,6 @@ void NeighBondKokkos::dihedral_check(int nlist, typename AT::t_int_2 // check all 6 distances // in case dihedral/improper potential computes any of them - update_domain_variables(); atomKK->sync(execution_space, X_MASK); k_dihedrallist.sync(); @@ -990,10 +962,10 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondImproperAll, const int &i, int &nmissing) const { for (int m = 0; m < num_improper[i]; m++) { - int atom1 = map_array(improper_atom1(i,m)); - int atom2 = map_array(improper_atom2(i,m)); - int atom3 = map_array(improper_atom3(i,m)); - int atom4 = map_array(improper_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(improper_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(improper_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(improper_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(improper_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -1090,10 +1062,10 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondImproperPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_improper[i]; m++) { if (improper_type(i,m) <= 0) continue; - int atom1 = map_array(improper_atom1(i,m)); - int atom2 = map_array(improper_atom2(i,m)); - int atom3 = map_array(improper_atom3(i,m)); - int atom4 = map_array(improper_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(improper_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(improper_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(improper_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(improper_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -1137,8 +1109,8 @@ int NeighBondKokkos::closest_image(const int i, int j) const X_FLOAT rsqmin = delx*delx + dely*dely + delz*delz; X_FLOAT rsq; - while (sametag[j] >= 0) { - j = sametag[j]; + while (d_sametag[j] >= 0) { + j = d_sametag[j]; delx = xi0 - x(j,0); dely = xi1 - x(j,1); delz = xi2 - x(j,2); @@ -1218,8 +1190,10 @@ void NeighBondKokkos::minimum_image(X_FLOAT &dx, X_FLOAT &dy, X_FLOA /* ---------------------------------------------------------------------- */ template -void NeighBondKokkos::update_domain_variables() +void NeighBondKokkos::update_class_variables() { + // Domain + triclinic = domain->triclinic; xperiodic = domain->xperiodic; xprd_half = domain->xprd_half; @@ -1233,6 +1207,21 @@ void NeighBondKokkos::update_domain_variables() xy = domain->xy; xz = domain->xz; yz = domain->yz; + + // Atom Map + + map_style = atom->map_style; + + k_sametag = atomKK->k_sametag; + k_sametag.template sync(); + d_sametag = k_sametag.view(); + + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + } } /* ---------------------------------------------------------------------- */ diff --git a/src/KOKKOS/neigh_bond_kokkos.h b/src/KOKKOS/neigh_bond_kokkos.h index 887ead95f6..d2e5d0fc62 100644 --- a/src/KOKKOS/neigh_bond_kokkos.h +++ b/src/KOKKOS/neigh_bond_kokkos.h @@ -19,6 +19,7 @@ #include "kokkos_type.h" #include "domain_kokkos.h" #include "pointers.h" +#include namespace LAMMPS_NS { @@ -81,13 +82,11 @@ class NeighBondKokkos : protected Pointers { int me,nprocs; private: - - - DAT::tdual_int_1d k_map_array; - typename AT::t_int_1d_randomread map_array; - + int map_style; DAT::tdual_int_1d k_sametag; - typename AT::t_int_1d_randomread sametag; + typename AT::t_int_1d d_sametag; + DAT::tdual_int_1d k_map_array; + dual_hash_type k_map_hash; typename AT::t_int_2d v_bondlist; typename AT::t_int_2d v_anglelist; @@ -130,7 +129,7 @@ class NeighBondKokkos : protected Pointers { KOKKOS_INLINE_FUNCTION void minimum_image(X_FLOAT &dx, X_FLOAT &dy, X_FLOAT &dz) const; - void update_domain_variables(); + void update_class_variables(); // topology build functions @@ -174,10 +173,6 @@ class NeighBondKokkos : protected Pointers { /* ERROR/WARNING messages: -E: Must use atom map style array with Kokkos - -See the atom_modify map command. - E: Bond atoms missing on proc %d at step %ld The 2nd atom needed to compute a particular bond is missing on this diff --git a/src/KOKKOS/neighbor_kokkos.h b/src/KOKKOS/neighbor_kokkos.h index 1a560e2129..ef885535ed 100644 --- a/src/KOKKOS/neighbor_kokkos.h +++ b/src/KOKKOS/neighbor_kokkos.h @@ -64,13 +64,14 @@ class NeighborKokkos : public Neighbor { DAT::tdual_int_2d k_dihedrallist; DAT::tdual_int_2d k_improperlist; + int device_flag; + private: DAT::tdual_x_array x; DAT::tdual_x_array xhold; X_FLOAT deltasq; - int device_flag; void init_cutneighsq_kokkos(int); void create_kokkos_list(int); diff --git a/src/OPENMP/reaxc_init_md_omp.cpp b/src/OPENMP/reaxc_init_md_omp.cpp index 226bfaf53c..b1c6b5e3c6 100644 --- a/src/OPENMP/reaxc_init_md_omp.cpp +++ b/src/OPENMP/reaxc_init_md_omp.cpp @@ -111,7 +111,7 @@ int Init_ListsOMP(reax_system *system, control_params *control, free(hb_top); free(bond_top); - return SUCCESS; + return REAXC_SUCCESS; } /* ---------------------------------------------------------------------- */ diff --git a/src/REAXFF/reaxc_allocate.cpp b/src/REAXFF/reaxc_allocate.cpp index 1cb101b118..7a9fca4d08 100644 --- a/src/REAXFF/reaxc_allocate.cpp +++ b/src/REAXFF/reaxc_allocate.cpp @@ -66,7 +66,7 @@ int PreAllocate_Space( reax_system *system, control_params * /*control*/, LMP_UNUSED_PARAM(workspace); #endif - return SUCCESS; + return REAXC_SUCCESS; } @@ -78,7 +78,7 @@ int Allocate_System( reax_system *system, int /*local_cap*/, int total_cap, system->my_atoms = (reax_atom*) realloc( system->my_atoms, total_cap*sizeof(reax_atom) ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -315,7 +315,7 @@ int Allocate_Workspace( reax_system * /*system*/, control_params * control, LMP_UNUSED_PARAM(control); #endif - return SUCCESS; + return REAXC_SUCCESS; } @@ -392,7 +392,7 @@ static int Reallocate_Bonds_List( reax_system *system, reax_list *bonds, (double*) smalloc(system->error_ptr, sizeof(double)*nthreads, "CdboReduction"); #endif - return SUCCESS; + return REAXC_SUCCESS; } @@ -426,7 +426,7 @@ void ReAllocate( reax_system *system, control_params *control, if (Nflag) { /* system */ ret = Allocate_System( system, system->local_cap, system->total_cap, msg ); - if (ret != SUCCESS) { + if (ret != REAXC_SUCCESS) { char errmsg[256]; snprintf(errmsg, 256, "Not enough space for atom_list: total_cap=%d", system->total_cap); system->error_ptr->one(FLERR, errmsg); @@ -436,7 +436,7 @@ void ReAllocate( reax_system *system, control_params *control, DeAllocate_Workspace( control, workspace ); ret = Allocate_Workspace( system, control, workspace, system->local_cap, system->total_cap, msg ); - if (ret != SUCCESS) { + if (ret != REAXC_SUCCESS) { char errmsg[256]; snprintf(errmsg, 256, "Not enough space for workspace: local_cap=%d total_cap=%d", system->local_cap, system->total_cap); system->error_ptr->one(FLERR, errmsg); diff --git a/src/REAXFF/reaxc_control.cpp b/src/REAXFF/reaxc_control.cpp index 84c5feeec2..af24d393d8 100644 --- a/src/REAXFF/reaxc_control.cpp +++ b/src/REAXFF/reaxc_control.cpp @@ -387,5 +387,5 @@ char Read_Control_File( char *control_file, control_params* control, fclose(fp); - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/REAXFF/reaxc_defs.h b/src/REAXFF/reaxc_defs.h index b98d8a2828..a98e5b5300 100644 --- a/src/REAXFF/reaxc_defs.h +++ b/src/REAXFF/reaxc_defs.h @@ -34,8 +34,8 @@ #define inline __inline__ #endif /*IBMC*/ -#ifndef SUCCESS -#define SUCCESS 1 +#ifndef REAXC_SUCCESS +#define REAXC_SUCCESS 1 #endif #ifndef FAILURE #define FAILURE 0 diff --git a/src/REAXFF/reaxc_ffield.cpp b/src/REAXFF/reaxc_ffield.cpp index 0b771bf356..0a3db31fc3 100644 --- a/src/REAXFF/reaxc_ffield.cpp +++ b/src/REAXFF/reaxc_ffield.cpp @@ -736,5 +736,5 @@ char Read_Force_Field( FILE *fp, reax_interaction *reax, fclose(fp); - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/REAXFF/reaxc_init_md.cpp b/src/REAXFF/reaxc_init_md.cpp index 3d0a60c862..b08ca8a59d 100644 --- a/src/REAXFF/reaxc_init_md.cpp +++ b/src/REAXFF/reaxc_init_md.cpp @@ -67,7 +67,7 @@ int Init_System(reax_system *system, control_params *control, char * /*msg*/) } system->Hcap = (int)(MAX(system->numH * saferzone, mincap)); - return SUCCESS; + return REAXC_SUCCESS; } @@ -83,7 +83,7 @@ int Init_Simulation_Data(reax_system *system, control_params *control, data->step = data->prev_steps = 0; - return SUCCESS; + return REAXC_SUCCESS; } void Init_Taper(control_params *control, storage *workspace) @@ -134,7 +134,7 @@ int Init_Workspace(reax_system *system, control_params *control, ret = Allocate_Workspace(system, control, workspace, system->local_cap, system->total_cap, msg); - if (ret != SUCCESS) + if (ret != REAXC_SUCCESS) return ret; memset(&(workspace->realloc), 0, sizeof(reallocate_data)); @@ -143,7 +143,7 @@ int Init_Workspace(reax_system *system, control_params *control, /* Initialize the Taper function */ Init_Taper(control, workspace); - return SUCCESS; + return REAXC_SUCCESS; } @@ -156,7 +156,7 @@ int Init_MPI_Datatypes(reax_system *system, storage * /*workspace*/, mpi_data->world = comm; MPI_Comm_size(comm, &(system->wsize)); - return SUCCESS; + return REAXC_SUCCESS; } int Init_Lists(reax_system *system, control_params *control, @@ -216,7 +216,7 @@ int Init_Lists(reax_system *system, control_params *control, free(hb_top); free(bond_top); - return SUCCESS; + return REAXC_SUCCESS; } void Initialize(reax_system *system, control_params *control, diff --git a/src/REAXFF/reaxc_io_tools.cpp b/src/REAXFF/reaxc_io_tools.cpp index 0e254b2c88..dc9cf55789 100644 --- a/src/REAXFF/reaxc_io_tools.cpp +++ b/src/REAXFF/reaxc_io_tools.cpp @@ -78,7 +78,7 @@ int Init_Output_Files( reax_system *system, control_params *control, } } - return SUCCESS; + return REAXC_SUCCESS; } @@ -101,7 +101,7 @@ int Close_Output_Files( reax_system *system, control_params * /* control */, } } - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/REAXFF/reaxc_list.cpp b/src/REAXFF/reaxc_list.cpp index a02f7d2fd7..4620a7a3c8 100644 --- a/src/REAXFF/reaxc_list.cpp +++ b/src/REAXFF/reaxc_list.cpp @@ -95,7 +95,7 @@ int Make_List(int n, int num_intrs, int type, reax_list *l ) l->error_ptr->one(FLERR,errmsg); } - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/REAXFF/reaxc_traj.cpp b/src/REAXFF/reaxc_traj.cpp index 637a69ed0f..f6e2dafeee 100644 --- a/src/REAXFF/reaxc_traj.cpp +++ b/src/REAXFF/reaxc_traj.cpp @@ -48,7 +48,7 @@ int Reallocate_Output_Buffer( LAMMPS_NS::Error *error_ptr, output_controls *out_ error_ptr->one(FLERR,errmsg); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -259,7 +259,7 @@ int Write_Header( reax_system *system, control_params *control, if (system->my_rank == MASTER_NODE) fprintf( out_control->strj, "%s", out_control->buffer ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -311,7 +311,7 @@ int Write_Init_Desc( reax_system *system, control_params * /*control*/, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -355,7 +355,7 @@ int Init_Traj( reax_system *system, control_params *control, Write_Header( system, control, out_control, mpi_data ); Write_Init_Desc( system, control, out_control, mpi_data ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -480,7 +480,7 @@ int Write_Frame_Header( reax_system *system, control_params *control, if (system->my_rank == MASTER_NODE) fprintf( out_control->strj, "%s", out_control->buffer ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -557,7 +557,7 @@ int Write_Atoms( reax_system *system, control_params * /*control*/, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -646,7 +646,7 @@ int Write_Bonds(reax_system *system, control_params *control, reax_list *bonds, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -741,7 +741,7 @@ int Write_Angles( reax_system *system, control_params *control, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -761,7 +761,7 @@ int Append_Frame( reax_system *system, control_params *control, Write_Angles( system, control, (*lists + BONDS), (*lists + THREE_BODIES), out_control, mpi_data ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -773,5 +773,5 @@ int End_Traj( int my_rank, output_controls *out_control ) free( out_control->buffer ); free( out_control->line ); - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/atom.h b/src/atom.h index c881490b8e..c7273e1ff5 100644 --- a/src/atom.h +++ b/src/atom.h @@ -275,7 +275,7 @@ class Atom : protected Pointers { // functions Atom(class LAMMPS *); - ~Atom(); + virtual ~Atom(); void settings(class Atom *); void peratom_create(); @@ -364,12 +364,12 @@ class Atom : protected Pointers { return -1; }; - void map_init(int check = 1); - void map_clear(); - void map_set(); + virtual void map_init(int check = 1); + virtual void map_clear(); + virtual void map_set(); void map_one(tagint, int); int map_style_set(); - void map_delete(); + virtual void map_delete(); int map_find_hash(tagint); protected: diff --git a/src/atom_map.cpp b/src/atom_map.cpp index 95f09d2639..2eb4a08ea6 100644 --- a/src/atom_map.cpp +++ b/src/atom_map.cpp @@ -309,7 +309,7 @@ int Atom::map_style_set() if (map_user == MAP_ARRAY || map_user == MAP_HASH) { map_style = map_user; } else { // map_user == MAP_YES - if (map_tag_max > 1000000 && !lmp->kokkos) map_style = MAP_HASH; + if (map_tag_max > 1000000) map_style = MAP_HASH; else map_style = MAP_ARRAY; }