From a4c2bc13cf64d62ed4dfebd8887ff11b65d4d16a Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 28 Jun 2021 08:38:31 -0600 Subject: [PATCH 01/23] Add atom_map hash option for Kokkos package --- src/KOKKOS/Install.sh | 1 + src/KOKKOS/atom_kokkos.cpp | 6 +- src/KOKKOS/atom_kokkos.h | 27 +++ src/KOKKOS/atom_map_kokkos.cpp | 287 +++++++++++++++++++++++++++++++ src/KOKKOS/comm_kokkos.cpp | 3 +- src/KOKKOS/fix_shake_kokkos.cpp | 104 +++++++---- src/KOKKOS/fix_shake_kokkos.h | 15 +- src/KOKKOS/neigh_bond_kokkos.cpp | 118 +++++++------ src/KOKKOS/neigh_bond_kokkos.h | 19 +- src/atom.h | 12 +- src/atom_map.cpp | 2 +- 11 files changed, 487 insertions(+), 107 deletions(-) create mode 100644 src/KOKKOS/atom_map_kokkos.cpp diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 0ac90a53f2..a28b17270a 100755 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -55,6 +55,7 @@ action angle_harmonic_kokkos.cpp angle_harmonic.cpp action angle_harmonic_kokkos.h angle_harmonic.h action atom_kokkos.cpp action atom_kokkos.h +action atom_map_kokkos.cpp action atom_vec_angle_kokkos.cpp atom_vec_angle.cpp action atom_vec_angle_kokkos.h atom_vec_angle.h action atom_vec_atomic_kokkos.cpp diff --git a/src/KOKKOS/atom_kokkos.cpp b/src/KOKKOS/atom_kokkos.cpp index a8527989d7..c8d2268937 100644 --- a/src/KOKKOS/atom_kokkos.cpp +++ b/src/KOKKOS/atom_kokkos.cpp @@ -29,7 +29,9 @@ using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- */ -AtomKokkos::AtomKokkos(LAMMPS *lmp) : Atom(lmp) {} +AtomKokkos::AtomKokkos(LAMMPS *lmp) : Atom(lmp) { + k_error_flag = DAT::tdual_int_scalar("atom:error_flag"); +} /* ---------------------------------------------------------------------- */ @@ -77,6 +79,8 @@ AtomKokkos::~AtomKokkos() memoryKK->destroy_kokkos(k_improper_atom3, improper_atom3); memoryKK->destroy_kokkos(k_improper_atom4, improper_atom4); + map_delete(); + // SPIN package memoryKK->destroy_kokkos(k_sp, sp); diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index e807180f35..d2629b4441 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -14,6 +14,7 @@ #include "atom.h" // IWYU pragma: export #include "kokkos_type.h" +#include #ifndef LMP_ATOM_KOKKOS_H #define LMP_ATOM_KOKKOS_H @@ -69,6 +70,32 @@ class AtomKokkos : public Atom { AtomKokkos(class LAMMPS *); ~AtomKokkos(); + void map_init(int check = 1); + void map_clear(); + void map_set(); + void map_delete(); + + DAT::tdual_int_1d k_sametag; + DAT::tdual_int_1d k_map_array; + DAT::tdual_int_scalar k_error_flag; + + typedef Kokkos::UnorderedMap hash_type; + typedef Kokkos::DualView dual_hash_type; + typedef dual_hash_type::t_host::data_type host_hash_type; + dual_hash_type k_map_hash; + + template + KOKKOS_INLINE_FUNCTION + static int map_find_hash_kokkos(tagint global, dual_hash_type k_map_hash) + { + int local = -1; + auto d_map_hash = k_map_hash.view()(); + auto index = d_map_hash.find(global); + if (d_map_hash.valid_at(index)) + local = d_map_hash.value_at(index); + return local; + } + virtual void allocate_type_arrays(); void sync(const ExecutionSpace space, unsigned int mask); void modified(const ExecutionSpace space, unsigned int mask); diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp new file mode 100644 index 0000000000..4c3d5cd1e9 --- /dev/null +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -0,0 +1,287 @@ +// clang-format off +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "atom_kokkos.h" +#include "comm.h" +#include "error.h" +#include "memory_kokkos.h" +#include "atom_masks.h" + +#include + +using namespace LAMMPS_NS; + +#define EXTRA 1000 + +/* ---------------------------------------------------------------------- + allocate and initialize array or hash table for global -> local map + for array option: + array length = 1 to map_tag_max + set entire array to -1 as initial values + for hash option: + map_nhash = length of hash table + map_nbucket = # of hash buckets, prime larger than map_nhash * 2 + so buckets will only be filled with 0 or 1 atoms on average +------------------------------------------------------------------------- */ + +void AtomKokkos::map_init(int check) +{ + // check for new map style if max atomID changed (check = 1 = default) + // recreate = 1 if must delete old map and create new map + // recreate = 0 if can re-use old map w/out realloc and just adjust settings + // map_maxarray/map_nhash initially -1, to force recreate even when no atoms + + int recreate = 0; + if (check) recreate = map_style_set(); + + if (map_style == MAP_ARRAY && map_tag_max > map_maxarray) recreate = 1; + else if (map_style == MAP_HASH && nlocal+nghost > map_nhash) recreate = 1; + + // if not recreating: + // for array, initialize current map_tag_max values + // for hash, set all buckets to empty, put all entries in free list + + if (!recreate) { + if (map_style == MAP_ARRAY) { + for (int i = 0; i <= map_tag_max; i++) map_array[i] = -1; + } else { + for (int i = 0; i < map_nbucket; i++) map_bucket[i] = -1; + map_nused = 0; + map_free = 0; + for (int i = 0; i < map_nhash; i++) map_hash[i].next = i+1; + if (map_nhash > 0) map_hash[map_nhash-1].next = -1; + + k_map_hash.h_view().clear(); + } + + // recreating: delete old map and create new one for array or hash + + } else { + map_delete(); + + if (map_style == MAP_ARRAY) { + map_maxarray = map_tag_max; + memoryKK->create_kokkos(k_map_array,map_array,map_maxarray+1,"atom:map_array"); + for (int i = 0; i <= map_tag_max; i++) map_array[i] = -1; + + } else { + + // map_nhash = max # of atoms that can be hashed on this proc + // set to max of ave atoms/proc or atoms I can store + // multiply by 2, require at least 1000 + // doubling means hash table will need to be re-init only rarely + + int nper = static_cast (natoms/comm->nprocs); + map_nhash = MAX(nper,nmax); + map_nhash *= 2; + map_nhash = MAX(map_nhash,1000); + + // map_nbucket = prime just larger than map_nhash + // next_prime() should be fast enough, + // about 10% of odd integers are prime above 1M + + map_nbucket = next_prime(map_nhash); + + // set all buckets to empty + // set hash to map_nhash in length + // put all hash entries in free list and point them to each other + + map_bucket = new int[map_nbucket]; + for (int i = 0; i < map_nbucket; i++) map_bucket[i] = -1; + + map_hash = new HashElem[map_nhash]; + map_nused = 0; + map_free = 0; + for (int i = 0; i < map_nhash; i++) map_hash[i].next = i+1; + map_hash[map_nhash-1].next = -1; + + k_map_hash = dual_hash_type("atom:map_hash"); + k_map_hash.h_view() = host_hash_type(map_nhash); + } + } + + if (map_style == Atom::MAP_ARRAY) + k_map_array.modify_host(); + else if (map_style == Atom::MAP_HASH) + k_map_hash.modify_host(); +} + +/* ---------------------------------------------------------------------- + clear global -> local map for all of my own and ghost atoms + for hash table option: + global ID may not be in table if image atom was already cleared +------------------------------------------------------------------------- */ + +void AtomKokkos::map_clear() +{ + Atom::map_clear(); + + if (map_style == MAP_ARRAY) { + k_map_array.modify_host(); + } else { + k_map_hash.h_view().clear(); + k_map_hash.modify_host(); + } + k_sametag.modify_host(); +} + +/* ---------------------------------------------------------------------- + set global -> local map for all of my own and ghost atoms + loop in reverse order so that nearby images take precedence over far ones + and owned atoms take precedence over images + this enables valid lookups of bond topology atoms + for hash table option: + if hash table too small, re-init + global ID may already be in table if image atom was set +------------------------------------------------------------------------- */ + +void AtomKokkos::map_set() +{ + int nall = nlocal + nghost; + + atomKK->sync(Host,TAG_MASK); + + k_sametag.sync_host(); + if (map_style == Atom::MAP_ARRAY) + k_map_array.sync_host(); + + if (map_style == MAP_ARRAY) { + + // possible reallocation of sametag must come before loop over atoms + // since loop sets sametag + + if (nall > max_same) { + max_same = nall + EXTRA; + memoryKK->destroy_kokkos(k_sametag,sametag); + memoryKK->create_kokkos(k_sametag,sametag,max_same,"atom:sametag"); + } + + for (int i = nall-1; i >= 0 ; i--) { + sametag[i] = map_array[tag[i]]; + map_array[tag[i]] = i; + } + + } else { + + // if this proc has more atoms than hash table size, call map_init() + // call with 0 since max atomID in system has not changed + // possible reallocation of sametag must come after map_init(), + // b/c map_init() may invoke map_delete(), whacking sametag + + if (nall > map_nhash) map_init(0); + if (nall > max_same) { + max_same = nall + EXTRA; + memoryKK->destroy_kokkos(k_sametag,sametag); + memoryKK->create_kokkos(k_sametag,sametag,max_same,"atom:sametag"); + } + + int previous,ibucket,index; + tagint global; + + for (int i = nall-1; i >= 0 ; i--) { + sametag[i] = map_find_hash(tag[i]); + + // search for key + // if found it, just overwrite local value with index + + previous = -1; + global = tag[i]; + ibucket = global % map_nbucket; + index = map_bucket[ibucket]; + while (index > -1) { + if (map_hash[index].global == global) break; + previous = index; + index = map_hash[index].next; + } + if (index > -1) { + map_hash[index].local = i; + continue; + } + + // take one entry from free list + // add the new global/local pair as entry at end of bucket list + // special logic if this entry is 1st in bucket + + index = map_free; + map_free = map_hash[map_free].next; + if (previous == -1) map_bucket[ibucket] = index; + else map_hash[previous].next = index; + map_hash[index].global = global; + map_hash[index].local = i; + map_hash[index].next = -1; + map_nused++; + } + + // Copy to Kokkos hash + + k_map_hash.h_view().clear(); + auto h_map_hash = k_map_hash.h_view(); + + for (int i = nall-1; i >= 0 ; i--) { + + // search for key + // if don't find it, done + + previous = -1; + global = tag[i]; + ibucket = global % map_nbucket; + index = map_bucket[ibucket]; + while (index > -1) { + if (map_hash[index].global == global) break; + previous = index; + index = map_hash[index].next; + } + if (index == -1) continue; + + int local = map_hash[index].local; + + auto insert_result = h_map_hash.insert(global,local); + if (insert_result.failed()) + error->one(FLERR, "Kokkos::UnorderedMap insertion failed"); + } + } + + k_sametag.modify_host(); + if (map_style == Atom::MAP_ARRAY) + k_map_array.modify_host(); + else if (map_style == Atom::MAP_HASH) + k_map_hash.modify_host(); +} + + +/* ---------------------------------------------------------------------- + free the array or hash table for global to local mapping +------------------------------------------------------------------------- */ + +void AtomKokkos::map_delete() +{ + memoryKK->destroy_kokkos(k_sametag,sametag); + sametag = nullptr; + max_same = 0; + + if (map_style == MAP_ARRAY) { + memoryKK->destroy_kokkos(k_map_array,map_array); + map_array = nullptr; + } else { + if (map_nhash) { + delete [] map_bucket; + delete [] map_hash; + map_bucket = nullptr; + map_hash = nullptr; + k_map_hash = dual_hash_type(); + } + map_nhash = map_nbucket = 0; + } +} + diff --git a/src/KOKKOS/comm_kokkos.cpp b/src/KOKKOS/comm_kokkos.cpp index fd5318e1d1..588ff94d73 100644 --- a/src/KOKKOS/comm_kokkos.cpp +++ b/src/KOKKOS/comm_kokkos.cpp @@ -1128,9 +1128,10 @@ void CommKokkos::borders_device() { max = MAX(maxforward*rmax,maxreverse*smax); if (max > maxrecv) grow_recv_kokkos(max); + atomKK->modified(exec_space,ALL_MASK); + // reset global->local map - atomKK->modified(exec_space,ALL_MASK); if (map_style != Atom::MAP_NONE) { atomKK->sync(Host,TAG_MASK); atom->map_set(); diff --git a/src/KOKKOS/fix_shake_kokkos.cpp b/src/KOKKOS/fix_shake_kokkos.cpp index 804da9fd22..594c9c60db 100644 --- a/src/KOKKOS/fix_shake_kokkos.cpp +++ b/src/KOKKOS/fix_shake_kokkos.cpp @@ -201,6 +201,15 @@ void FixShakeKokkos::pre_neighbor() type = atom->type; nlocal = atom->nlocal; + map_style = atom->map_style; + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + k_map_hash.template sync(); + } + k_shake_flag.sync(); k_shake_atom.sync(); @@ -213,21 +222,17 @@ void FixShakeKokkos::pre_neighbor() d_list = k_list.view(); } - // don't yet have atom_map_kokkos routines, so move data from host to device + // Atom Map - if (atom->map_style != Atom::MAP_ARRAY) - error->all(FLERR,"Must use atom map style array with Kokkos"); + map_style = atom->map_style; - int* map_array_host = atom->get_map_array(); - int map_size = atom->get_map_size(); - int map_maxarray = atom->get_map_maxarray(); - if (map_maxarray > (int)k_map_array.extent(0)) - k_map_array = DAT::tdual_int_1d("NeighBond:map_array",map_maxarray); - for (int i=0; i(); - k_map_array.template sync(); - map_array = k_map_array.view(); + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + k_map_hash.template sync(); + } // build list of SHAKE clusters I compute @@ -241,14 +246,16 @@ void FixShakeKokkos::pre_neighbor() auto d_list = this->d_list; auto d_error_flag = this->d_error_flag; auto d_nlist = this->d_nlist; - auto map_array = this->map_array; + auto map_style = atom->map_style; + auto k_map_array = this->k_map_array; + auto k_map_hash = this->k_map_hash; Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal), LAMMPS_LAMBDA(const int& i) { if (d_shake_flag[i]) { if (d_shake_flag[i] == 2) { - const int atom1 = map_array(d_shake_atom(i,0)); - const int atom2 = map_array(d_shake_atom(i,1)); + const int atom1 = map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1) { d_error_flag() = 1; } @@ -257,9 +264,9 @@ void FixShakeKokkos::pre_neighbor() d_list[nlist] = i; } } else if (d_shake_flag[i] % 2 == 1) { - const int atom1 = map_array(d_shake_atom(i,0)); - const int atom2 = map_array(d_shake_atom(i,1)); - const int atom3 = map_array(d_shake_atom(i,2)); + const int atom1 = map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom3 = map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) d_error_flag() = 1; if (i <= atom1 && i <= atom2 && i <= atom3) { @@ -267,10 +274,10 @@ void FixShakeKokkos::pre_neighbor() d_list[nlist] = i; } } else { - const int atom1 = map_array(d_shake_atom(i,0)); - const int atom2 = map_array(d_shake_atom(i,1)); - const int atom3 = map_array(d_shake_atom(i,2)); - const int atom4 = map_array(d_shake_atom(i,3)); + const int atom1 = map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom3 = map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); + const int atom4 = map_kokkos(d_shake_atom(i,3),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) d_error_flag() = 1; if (i <= atom1 && i <= atom2 && i <= atom3 && i <= atom4) { @@ -307,6 +314,15 @@ void FixShakeKokkos::post_force(int vflag) d_mass = atomKK->k_mass.view(); nlocal = atomKK->nlocal; + map_style = atom->map_style; + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + k_map_hash.template sync(); + } + if (d_rmass.data()) atomKK->sync(execution_space,X_MASK|F_MASK|RMASS_MASK); else @@ -586,8 +602,8 @@ void FixShakeKokkos::shake(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); + int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; // r01 = distance vec between atoms, with PBC @@ -697,9 +713,9 @@ void FixShakeKokkos::shake3(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); - int i2 = map_array(d_shake_atom(m,2)); + int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; @@ -880,10 +896,10 @@ void FixShakeKokkos::shake4(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); - int i2 = map_array(d_shake_atom(m,2)); - int i3 = map_array(d_shake_atom(m,3)); + int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); + int i3 = map_kokkos(d_shake_atom(m,3),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; double bond3 = d_bond_distance[d_shake_type(m,2)]; @@ -1142,9 +1158,9 @@ void FixShakeKokkos::shake3angle(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_array(d_shake_atom(m,0)); - int i1 = map_array(d_shake_atom(m,1)); - int i2 = map_array(d_shake_atom(m,2)); + int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; double bond12 = d_angle_distance[d_shake_type(m,2)]; @@ -1905,6 +1921,24 @@ void FixShakeKokkos::minimum_image_once(double *delta) const /* ---------------------------------------------------------------------- */ +// functions for global to local ID mapping +// map lookup function inlined for efficiency +// return -1 if no map defined + +template +KOKKOS_INLINE_FUNCTION +int FixShakeKokkos::map_kokkos(tagint global, int map_style, DAT::tdual_int_1d k_map_array, dual_hash_type k_map_hash) +{ + if (map_style == 1) + return k_map_array.view()(global); + else if (map_style == 2) + return AtomKokkos::map_find_hash_kokkos(global,k_map_hash); + else + return -1; +} + +/* ---------------------------------------------------------------------- */ + namespace LAMMPS_NS { template class FixShakeKokkos; #ifdef LMP_KOKKOS_GPU diff --git a/src/KOKKOS/fix_shake_kokkos.h b/src/KOKKOS/fix_shake_kokkos.h index b95c042ac1..fbf6e1de88 100644 --- a/src/KOKKOS/fix_shake_kokkos.h +++ b/src/KOKKOS/fix_shake_kokkos.h @@ -26,6 +26,7 @@ FixStyle(shake/kk/host,FixShakeKokkos); #include "fix_shake.h" #include "kokkos_type.h" #include "kokkos_base.h" +#include namespace LAMMPS_NS { @@ -172,9 +173,6 @@ class FixShakeKokkos : public FixShake, public KokkosBase { KOKKOS_INLINE_FUNCTION void v_tally(EV_FLOAT&, int, int *, double, double *) const; - DAT::tdual_int_1d k_map_array; - typename AT::t_int_1d_randomread map_array; - int iswap; int first; typename AT::t_int_2d d_sendlist; @@ -185,6 +183,17 @@ class FixShakeKokkos : public FixShake, public KokkosBase { tagint **shake_atom_tmp; int **shake_type_tmp; + int map_style; + + DAT::tdual_int_1d k_map_array; + + typedef Kokkos::UnorderedMap hash_type; + typedef Kokkos::DualView dual_hash_type; + dual_hash_type k_map_hash; + + KOKKOS_INLINE_FUNCTION + static int map_kokkos(tagint, int, DAT::tdual_int_1d, dual_hash_type); + // copied from Domain KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/neigh_bond_kokkos.cpp b/src/KOKKOS/neigh_bond_kokkos.cpp index eed0026af3..c427d18216 100644 --- a/src/KOKKOS/neigh_bond_kokkos.cpp +++ b/src/KOKKOS/neigh_bond_kokkos.cpp @@ -207,30 +207,7 @@ void NeighBondKokkos::build_topology_kk() lostbond = output->thermo->lostbond; - // don't yet have atom_map_kokkos routines, so move data from host to device - - if (atom->map_style != Atom::MAP_ARRAY) - error->all(FLERR,"Must use atom map style array with Kokkos"); - - int* map_array_host = atom->get_map_array(); - int map_size = atom->get_map_size(); - int map_maxarray = atom->get_map_maxarray(); - if (map_maxarray > (int)k_map_array.extent(0)) - k_map_array = DAT::tdual_int_1d("NeighBond:map_array",map_maxarray); - for (int i=0; i(); - k_map_array.template sync(); - map_array = k_map_array.view(); - - int* sametag_host = atomKK->sametag; - if (nmax > (int)k_sametag.extent(0)) - k_sametag = DAT::tdual_int_1d("NeighBond:sametag",nmax); - for (int i=0; i(); - k_sametag.template sync(); - sametag = k_sametag.view(); + update_class_variables(); if (force->bond) (this->*bond_build_kk)(); if (force->angle) (this->*angle_build_kk)(); @@ -306,7 +283,7 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondBondAll, const int &i, int &nmissing) const { for (int m = 0; m < num_bond[i]; m++) { - int atom1 = map_array(bond_atom(i,m)); + int atom1 = map_kokkos(bond_atom(i,m)); if (atom1 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -394,7 +371,7 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondBondPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_bond[i]; m++) { if (bond_type(i,m) <= 0) continue; - int atom1 = map_array(bond_atom(i,m)); + int atom1 = map_kokkos(bond_atom(i,m)); if (atom1 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -420,7 +397,6 @@ void NeighBondKokkos::bond_check() { int flag = 0; - update_domain_variables(); atomKK->sync(execution_space, X_MASK); k_bondlist.sync(); @@ -507,9 +483,9 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondAngleAll, const int &i, int &nmissing) const { for (int m = 0; m < num_angle[i]; m++) { - int atom1 = map_array(angle_atom1(i,m)); - int atom2 = map_array(angle_atom2(i,m)); - int atom3 = map_array(angle_atom3(i,m)); + int atom1 = map_kokkos(angle_atom1(i,m)); + int atom2 = map_kokkos(angle_atom2(i,m)); + int atom3 = map_kokkos(angle_atom3(i,m)); if (atom1 == -1 || atom2 == -1 || atom3 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -602,9 +578,9 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondAnglePartial, const int &i, int &nmissing) const { for (int m = 0; m < num_angle[i]; m++) { if (angle_type(i,m) <= 0) continue; - int atom1 = map_array(angle_atom1(i,m)); - int atom2 = map_array(angle_atom2(i,m)); - int atom3 = map_array(angle_atom3(i,m)); + int atom1 = map_kokkos(angle_atom1(i,m)); + int atom2 = map_kokkos(angle_atom2(i,m)); + int atom3 = map_kokkos(angle_atom3(i,m)); if (atom1 == -1 || atom2 == -1 || atom3 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -636,7 +612,6 @@ void NeighBondKokkos::angle_check() // check all 3 distances // in case angle potential computes any of them - update_domain_variables(); atomKK->sync(execution_space, X_MASK); k_anglelist.sync(); @@ -735,10 +710,10 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondDihedralAll, const int &i, int &nmissing) const { for (int m = 0; m < num_dihedral[i]; m++) { - int atom1 = map_array(dihedral_atom1(i,m)); - int atom2 = map_array(dihedral_atom2(i,m)); - int atom3 = map_array(dihedral_atom3(i,m)); - int atom4 = map_array(dihedral_atom4(i,m)); + int atom1 = map_kokkos(dihedral_atom1(i,m)); + int atom2 = map_kokkos(dihedral_atom2(i,m)); + int atom3 = map_kokkos(dihedral_atom3(i,m)); + int atom4 = map_kokkos(dihedral_atom4(i,m)); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -835,10 +810,10 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondDihedralPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_dihedral[i]; m++) { if (dihedral_type(i,m) <= 0) continue; - int atom1 = map_array(dihedral_atom1(i,m)); - int atom2 = map_array(dihedral_atom2(i,m)); - int atom3 = map_array(dihedral_atom3(i,m)); - int atom4 = map_array(dihedral_atom4(i,m)); + int atom1 = map_kokkos(dihedral_atom1(i,m)); + int atom2 = map_kokkos(dihedral_atom2(i,m)); + int atom3 = map_kokkos(dihedral_atom3(i,m)); + int atom4 = map_kokkos(dihedral_atom4(i,m)); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -874,7 +849,6 @@ void NeighBondKokkos::dihedral_check(int nlist, typename AT::t_int_2 // check all 6 distances // in case dihedral/improper potential computes any of them - update_domain_variables(); atomKK->sync(execution_space, X_MASK); k_dihedrallist.sync(); @@ -990,10 +964,10 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondImproperAll, const int &i, int &nmissing) const { for (int m = 0; m < num_improper[i]; m++) { - int atom1 = map_array(improper_atom1(i,m)); - int atom2 = map_array(improper_atom2(i,m)); - int atom3 = map_array(improper_atom3(i,m)); - int atom4 = map_array(improper_atom4(i,m)); + int atom1 = map_kokkos(improper_atom1(i,m)); + int atom2 = map_kokkos(improper_atom2(i,m)); + int atom3 = map_kokkos(improper_atom3(i,m)); + int atom4 = map_kokkos(improper_atom4(i,m)); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -1090,10 +1064,10 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondImproperPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_improper[i]; m++) { if (improper_type(i,m) <= 0) continue; - int atom1 = map_array(improper_atom1(i,m)); - int atom2 = map_array(improper_atom2(i,m)); - int atom3 = map_array(improper_atom3(i,m)); - int atom4 = map_array(improper_atom4(i,m)); + int atom1 = map_kokkos(improper_atom1(i,m)); + int atom2 = map_kokkos(improper_atom2(i,m)); + int atom3 = map_kokkos(improper_atom3(i,m)); + int atom4 = map_kokkos(improper_atom4(i,m)); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -1137,8 +1111,8 @@ int NeighBondKokkos::closest_image(const int i, int j) const X_FLOAT rsqmin = delx*delx + dely*dely + delz*delz; X_FLOAT rsq; - while (sametag[j] >= 0) { - j = sametag[j]; + while (d_sametag[j] >= 0) { + j = d_sametag[j]; delx = xi0 - x(j,0); dely = xi1 - x(j,1); delz = xi2 - x(j,2); @@ -1217,9 +1191,29 @@ void NeighBondKokkos::minimum_image(X_FLOAT &dx, X_FLOAT &dy, X_FLOA /* ---------------------------------------------------------------------- */ +// functions for global to local ID mapping +// map lookup function inlined for efficiency +// return -1 if no map defined + template -void NeighBondKokkos::update_domain_variables() +KOKKOS_INLINE_FUNCTION +int NeighBondKokkos::map_kokkos(tagint global) const { + if (map_style == 1) + return k_map_array.view()(global); + else if (map_style == 2) + return AtomKokkos::map_find_hash_kokkos(global,k_map_hash); + else + return -1; +} + +/* ---------------------------------------------------------------------- */ + +template +void NeighBondKokkos::update_class_variables() +{ + // Domain + triclinic = domain->triclinic; xperiodic = domain->xperiodic; xprd_half = domain->xprd_half; @@ -1233,6 +1227,22 @@ void NeighBondKokkos::update_domain_variables() xy = domain->xy; xz = domain->xz; yz = domain->yz; + + // Atom Map + + map_style = atom->map_style; + + k_sametag = atomKK->k_sametag; + k_sametag.template sync(); + d_sametag = k_sametag.view(); + + if (map_style == Atom::MAP_ARRAY) { + k_map_array = atomKK->k_map_array; + k_map_array.template sync(); + } else if (map_style == Atom::MAP_HASH) { + k_map_hash = atomKK->k_map_hash; + k_map_hash.template sync(); + } } /* ---------------------------------------------------------------------- */ diff --git a/src/KOKKOS/neigh_bond_kokkos.h b/src/KOKKOS/neigh_bond_kokkos.h index 887ead95f6..defcf990e3 100644 --- a/src/KOKKOS/neigh_bond_kokkos.h +++ b/src/KOKKOS/neigh_bond_kokkos.h @@ -19,6 +19,7 @@ #include "kokkos_type.h" #include "domain_kokkos.h" #include "pointers.h" +#include namespace LAMMPS_NS { @@ -81,13 +82,19 @@ class NeighBondKokkos : protected Pointers { int me,nprocs; private: - - - DAT::tdual_int_1d k_map_array; - typename AT::t_int_1d_randomread map_array; + int map_style; DAT::tdual_int_1d k_sametag; - typename AT::t_int_1d_randomread sametag; + typename AT::t_int_1d d_sametag; + + DAT::tdual_int_1d k_map_array; + + typedef Kokkos::UnorderedMap hash_type; + typedef Kokkos::DualView dual_hash_type; + dual_hash_type k_map_hash; + + KOKKOS_INLINE_FUNCTION + int map_kokkos(tagint) const; typename AT::t_int_2d v_bondlist; typename AT::t_int_2d v_anglelist; @@ -130,7 +137,7 @@ class NeighBondKokkos : protected Pointers { KOKKOS_INLINE_FUNCTION void minimum_image(X_FLOAT &dx, X_FLOAT &dy, X_FLOAT &dz) const; - void update_domain_variables(); + void update_class_variables(); // topology build functions diff --git a/src/atom.h b/src/atom.h index 392e5c5d5c..e922630706 100644 --- a/src/atom.h +++ b/src/atom.h @@ -364,13 +364,13 @@ class Atom : protected Pointers { return -1; }; - void map_init(int check = 1); - void map_clear(); - void map_set(); - void map_one(tagint, int); + virtual void map_init(int check = 1); + virtual void map_clear(); + virtual void map_set(); + virtual void map_one(tagint, int); int map_style_set(); - void map_delete(); - int map_find_hash(tagint); + virtual void map_delete(); + virtual int map_find_hash(tagint); protected: // global to local ID mapping diff --git a/src/atom_map.cpp b/src/atom_map.cpp index 95f09d2639..2eb4a08ea6 100644 --- a/src/atom_map.cpp +++ b/src/atom_map.cpp @@ -309,7 +309,7 @@ int Atom::map_style_set() if (map_user == MAP_ARRAY || map_user == MAP_HASH) { map_style = map_user; } else { // map_user == MAP_YES - if (map_tag_max > 1000000 && !lmp->kokkos) map_style = MAP_HASH; + if (map_tag_max > 1000000) map_style = MAP_HASH; else map_style = MAP_ARRAY; } From 91e0614cefbea7442a3012cae55c1a18ecb61345 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 28 Jun 2021 08:51:28 -0600 Subject: [PATCH 02/23] Remove unnecessary virtual keyword --- src/atom.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atom.h b/src/atom.h index e922630706..96066a5ab1 100644 --- a/src/atom.h +++ b/src/atom.h @@ -367,10 +367,10 @@ class Atom : protected Pointers { virtual void map_init(int check = 1); virtual void map_clear(); virtual void map_set(); - virtual void map_one(tagint, int); + void map_one(tagint, int); int map_style_set(); virtual void map_delete(); - virtual int map_find_hash(tagint); + int map_find_hash(tagint); protected: // global to local ID mapping From 39b99afb56311e54469e02a2d88230c7decdcdae Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 28 Jun 2021 09:34:08 -0600 Subject: [PATCH 03/23] Remove error description in header file --- src/KOKKOS/neigh_bond_kokkos.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/KOKKOS/neigh_bond_kokkos.h b/src/KOKKOS/neigh_bond_kokkos.h index defcf990e3..9bf508ec7a 100644 --- a/src/KOKKOS/neigh_bond_kokkos.h +++ b/src/KOKKOS/neigh_bond_kokkos.h @@ -181,10 +181,6 @@ class NeighBondKokkos : protected Pointers { /* ERROR/WARNING messages: -E: Must use atom map style array with Kokkos - -See the atom_modify map command. - E: Bond atoms missing on proc %d at step %ld The 2nd atom needed to compute a particular bond is missing on this From 9a4735c6efd4f718c038adc23118267a7e8cbfc0 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 28 Jun 2021 11:11:15 -0600 Subject: [PATCH 04/23] Temporarily rename variable to avoid name collision until #2708 is merged --- src/USER-REAXC/reaxc_allocate.cpp | 12 ++++++------ src/USER-REAXC/reaxc_control.cpp | 2 +- src/USER-REAXC/reaxc_defs.h | 4 ++-- src/USER-REAXC/reaxc_ffield.cpp | 2 +- src/USER-REAXC/reaxc_init_md.cpp | 12 ++++++------ src/USER-REAXC/reaxc_io_tools.cpp | 4 ++-- src/USER-REAXC/reaxc_list.cpp | 2 +- src/USER-REAXC/reaxc_traj.cpp | 20 ++++++++++---------- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/USER-REAXC/reaxc_allocate.cpp b/src/USER-REAXC/reaxc_allocate.cpp index 1cb101b118..7a9fca4d08 100644 --- a/src/USER-REAXC/reaxc_allocate.cpp +++ b/src/USER-REAXC/reaxc_allocate.cpp @@ -66,7 +66,7 @@ int PreAllocate_Space( reax_system *system, control_params * /*control*/, LMP_UNUSED_PARAM(workspace); #endif - return SUCCESS; + return REAXC_SUCCESS; } @@ -78,7 +78,7 @@ int Allocate_System( reax_system *system, int /*local_cap*/, int total_cap, system->my_atoms = (reax_atom*) realloc( system->my_atoms, total_cap*sizeof(reax_atom) ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -315,7 +315,7 @@ int Allocate_Workspace( reax_system * /*system*/, control_params * control, LMP_UNUSED_PARAM(control); #endif - return SUCCESS; + return REAXC_SUCCESS; } @@ -392,7 +392,7 @@ static int Reallocate_Bonds_List( reax_system *system, reax_list *bonds, (double*) smalloc(system->error_ptr, sizeof(double)*nthreads, "CdboReduction"); #endif - return SUCCESS; + return REAXC_SUCCESS; } @@ -426,7 +426,7 @@ void ReAllocate( reax_system *system, control_params *control, if (Nflag) { /* system */ ret = Allocate_System( system, system->local_cap, system->total_cap, msg ); - if (ret != SUCCESS) { + if (ret != REAXC_SUCCESS) { char errmsg[256]; snprintf(errmsg, 256, "Not enough space for atom_list: total_cap=%d", system->total_cap); system->error_ptr->one(FLERR, errmsg); @@ -436,7 +436,7 @@ void ReAllocate( reax_system *system, control_params *control, DeAllocate_Workspace( control, workspace ); ret = Allocate_Workspace( system, control, workspace, system->local_cap, system->total_cap, msg ); - if (ret != SUCCESS) { + if (ret != REAXC_SUCCESS) { char errmsg[256]; snprintf(errmsg, 256, "Not enough space for workspace: local_cap=%d total_cap=%d", system->local_cap, system->total_cap); system->error_ptr->one(FLERR, errmsg); diff --git a/src/USER-REAXC/reaxc_control.cpp b/src/USER-REAXC/reaxc_control.cpp index 84c5feeec2..af24d393d8 100644 --- a/src/USER-REAXC/reaxc_control.cpp +++ b/src/USER-REAXC/reaxc_control.cpp @@ -387,5 +387,5 @@ char Read_Control_File( char *control_file, control_params* control, fclose(fp); - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/USER-REAXC/reaxc_defs.h b/src/USER-REAXC/reaxc_defs.h index b98d8a2828..a98e5b5300 100644 --- a/src/USER-REAXC/reaxc_defs.h +++ b/src/USER-REAXC/reaxc_defs.h @@ -34,8 +34,8 @@ #define inline __inline__ #endif /*IBMC*/ -#ifndef SUCCESS -#define SUCCESS 1 +#ifndef REAXC_SUCCESS +#define REAXC_SUCCESS 1 #endif #ifndef FAILURE #define FAILURE 0 diff --git a/src/USER-REAXC/reaxc_ffield.cpp b/src/USER-REAXC/reaxc_ffield.cpp index 0b771bf356..0a3db31fc3 100644 --- a/src/USER-REAXC/reaxc_ffield.cpp +++ b/src/USER-REAXC/reaxc_ffield.cpp @@ -736,5 +736,5 @@ char Read_Force_Field( FILE *fp, reax_interaction *reax, fclose(fp); - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/USER-REAXC/reaxc_init_md.cpp b/src/USER-REAXC/reaxc_init_md.cpp index 3d0a60c862..b08ca8a59d 100644 --- a/src/USER-REAXC/reaxc_init_md.cpp +++ b/src/USER-REAXC/reaxc_init_md.cpp @@ -67,7 +67,7 @@ int Init_System(reax_system *system, control_params *control, char * /*msg*/) } system->Hcap = (int)(MAX(system->numH * saferzone, mincap)); - return SUCCESS; + return REAXC_SUCCESS; } @@ -83,7 +83,7 @@ int Init_Simulation_Data(reax_system *system, control_params *control, data->step = data->prev_steps = 0; - return SUCCESS; + return REAXC_SUCCESS; } void Init_Taper(control_params *control, storage *workspace) @@ -134,7 +134,7 @@ int Init_Workspace(reax_system *system, control_params *control, ret = Allocate_Workspace(system, control, workspace, system->local_cap, system->total_cap, msg); - if (ret != SUCCESS) + if (ret != REAXC_SUCCESS) return ret; memset(&(workspace->realloc), 0, sizeof(reallocate_data)); @@ -143,7 +143,7 @@ int Init_Workspace(reax_system *system, control_params *control, /* Initialize the Taper function */ Init_Taper(control, workspace); - return SUCCESS; + return REAXC_SUCCESS; } @@ -156,7 +156,7 @@ int Init_MPI_Datatypes(reax_system *system, storage * /*workspace*/, mpi_data->world = comm; MPI_Comm_size(comm, &(system->wsize)); - return SUCCESS; + return REAXC_SUCCESS; } int Init_Lists(reax_system *system, control_params *control, @@ -216,7 +216,7 @@ int Init_Lists(reax_system *system, control_params *control, free(hb_top); free(bond_top); - return SUCCESS; + return REAXC_SUCCESS; } void Initialize(reax_system *system, control_params *control, diff --git a/src/USER-REAXC/reaxc_io_tools.cpp b/src/USER-REAXC/reaxc_io_tools.cpp index 0e254b2c88..dc9cf55789 100644 --- a/src/USER-REAXC/reaxc_io_tools.cpp +++ b/src/USER-REAXC/reaxc_io_tools.cpp @@ -78,7 +78,7 @@ int Init_Output_Files( reax_system *system, control_params *control, } } - return SUCCESS; + return REAXC_SUCCESS; } @@ -101,7 +101,7 @@ int Close_Output_Files( reax_system *system, control_params * /* control */, } } - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/USER-REAXC/reaxc_list.cpp b/src/USER-REAXC/reaxc_list.cpp index a02f7d2fd7..4620a7a3c8 100644 --- a/src/USER-REAXC/reaxc_list.cpp +++ b/src/USER-REAXC/reaxc_list.cpp @@ -95,7 +95,7 @@ int Make_List(int n, int num_intrs, int type, reax_list *l ) l->error_ptr->one(FLERR,errmsg); } - return SUCCESS; + return REAXC_SUCCESS; } diff --git a/src/USER-REAXC/reaxc_traj.cpp b/src/USER-REAXC/reaxc_traj.cpp index 637a69ed0f..f6e2dafeee 100644 --- a/src/USER-REAXC/reaxc_traj.cpp +++ b/src/USER-REAXC/reaxc_traj.cpp @@ -48,7 +48,7 @@ int Reallocate_Output_Buffer( LAMMPS_NS::Error *error_ptr, output_controls *out_ error_ptr->one(FLERR,errmsg); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -259,7 +259,7 @@ int Write_Header( reax_system *system, control_params *control, if (system->my_rank == MASTER_NODE) fprintf( out_control->strj, "%s", out_control->buffer ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -311,7 +311,7 @@ int Write_Init_Desc( reax_system *system, control_params * /*control*/, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -355,7 +355,7 @@ int Init_Traj( reax_system *system, control_params *control, Write_Header( system, control, out_control, mpi_data ); Write_Init_Desc( system, control, out_control, mpi_data ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -480,7 +480,7 @@ int Write_Frame_Header( reax_system *system, control_params *control, if (system->my_rank == MASTER_NODE) fprintf( out_control->strj, "%s", out_control->buffer ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -557,7 +557,7 @@ int Write_Atoms( reax_system *system, control_params * /*control*/, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -646,7 +646,7 @@ int Write_Bonds(reax_system *system, control_params *control, reax_list *bonds, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -741,7 +741,7 @@ int Write_Angles( reax_system *system, control_params *control, fprintf( out_control->strj, "%s", out_control->buffer ); } - return SUCCESS; + return REAXC_SUCCESS; } @@ -761,7 +761,7 @@ int Append_Frame( reax_system *system, control_params *control, Write_Angles( system, control, (*lists + BONDS), (*lists + THREE_BODIES), out_control, mpi_data ); - return SUCCESS; + return REAXC_SUCCESS; } @@ -773,5 +773,5 @@ int End_Traj( int my_rank, output_controls *out_control ) free( out_control->buffer ); free( out_control->line ); - return SUCCESS; + return REAXC_SUCCESS; } From 4f6e28a0d8fd757d28f98ef25f8a8d56f98ce6d7 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Tue, 6 Jul 2021 13:49:12 -0600 Subject: [PATCH 05/23] Refactor use of Kokkos::UnorderedMap to fix Cuda issue --- src/KOKKOS/atom_kokkos.h | 10 ++--- src/KOKKOS/atom_map_kokkos.cpp | 72 ++++++++++++-------------------- src/KOKKOS/fix_shake_kokkos.cpp | 3 -- src/KOKKOS/fix_shake_kokkos.h | 2 - src/KOKKOS/kokkos_type.h | 19 +++++++++ src/KOKKOS/neigh_bond_kokkos.cpp | 1 - src/KOKKOS/neigh_bond_kokkos.h | 2 - src/KOKKOS/neighbor_kokkos.h | 3 +- 8 files changed, 50 insertions(+), 62 deletions(-) diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index d2629b4441..bf2d74f511 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -14,7 +14,6 @@ #include "atom.h" // IWYU pragma: export #include "kokkos_type.h" -#include #ifndef LMP_ATOM_KOKKOS_H #define LMP_ATOM_KOKKOS_H @@ -71,25 +70,22 @@ class AtomKokkos : public Atom { ~AtomKokkos(); void map_init(int check = 1); - void map_clear(); void map_set(); void map_delete(); DAT::tdual_int_1d k_sametag; DAT::tdual_int_1d k_map_array; DAT::tdual_int_scalar k_error_flag; - - typedef Kokkos::UnorderedMap hash_type; - typedef Kokkos::DualView dual_hash_type; - typedef dual_hash_type::t_host::data_type host_hash_type; dual_hash_type k_map_hash; + hash_type d_map_hash; + host_hash_type h_map_hash; template KOKKOS_INLINE_FUNCTION static int map_find_hash_kokkos(tagint global, dual_hash_type k_map_hash) { int local = -1; - auto d_map_hash = k_map_hash.view()(); + auto d_map_hash = k_map_hash.view(); auto index = d_map_hash.find(global); if (d_map_hash.valid_at(index)) local = d_map_hash.value_at(index); diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 4c3d5cd1e9..486594e182 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -13,8 +13,11 @@ ------------------------------------------------------------------------- */ #include "atom_kokkos.h" +#include "neighbor_kokkos.h" #include "comm.h" #include "error.h" +#include "modify.h" +#include "fix.h" #include "memory_kokkos.h" #include "atom_masks.h" @@ -61,8 +64,6 @@ void AtomKokkos::map_init(int check) map_free = 0; for (int i = 0; i < map_nhash; i++) map_hash[i].next = i+1; if (map_nhash > 0) map_hash[map_nhash-1].next = -1; - - k_map_hash.h_view().clear(); } // recreating: delete old map and create new one for array or hash @@ -106,34 +107,9 @@ void AtomKokkos::map_init(int check) for (int i = 0; i < map_nhash; i++) map_hash[i].next = i+1; map_hash[map_nhash-1].next = -1; - k_map_hash = dual_hash_type("atom:map_hash"); - k_map_hash.h_view() = host_hash_type(map_nhash); + h_map_hash = host_hash_type(map_nhash); } } - - if (map_style == Atom::MAP_ARRAY) - k_map_array.modify_host(); - else if (map_style == Atom::MAP_HASH) - k_map_hash.modify_host(); -} - -/* ---------------------------------------------------------------------- - clear global -> local map for all of my own and ghost atoms - for hash table option: - global ID may not be in table if image atom was already cleared -------------------------------------------------------------------------- */ - -void AtomKokkos::map_clear() -{ - Atom::map_clear(); - - if (map_style == MAP_ARRAY) { - k_map_array.modify_host(); - } else { - k_map_hash.h_view().clear(); - k_map_hash.modify_host(); - } - k_sametag.modify_host(); } /* ---------------------------------------------------------------------- @@ -225,8 +201,7 @@ void AtomKokkos::map_set() // Copy to Kokkos hash - k_map_hash.h_view().clear(); - auto h_map_hash = k_map_hash.h_view(); + h_map_hash.clear(); for (int i = nall-1; i >= 0 ; i--) { @@ -252,13 +227,24 @@ void AtomKokkos::map_set() } } - k_sametag.modify_host(); - if (map_style == Atom::MAP_ARRAY) - k_map_array.modify_host(); - else if (map_style == Atom::MAP_HASH) - k_map_hash.modify_host(); -} + // check if fix shake or neigh bond needs a device hash + int device_hash_flag = 0; + + auto neighborKK = (NeighborKokkos*) neighbor; + if (neighborKK->device_flag) device_hash_flag = 1; + + for (int n = 0; n < modify->nfix; n++) + if (utils::strmatch(modify->fix[n]->style,"^shake")) + if (modify->fix[n]->execution_space == Device) + device_hash_flag = 1; + + if (device_hash_flag) + Kokkos::deep_copy(d_map_hash,h_map_hash); + + k_map_hash.h_view = h_map_hash; + k_map_hash.d_view = d_map_hash; +} /* ---------------------------------------------------------------------- free the array or hash table for global to local mapping @@ -268,20 +254,14 @@ void AtomKokkos::map_delete() { memoryKK->destroy_kokkos(k_sametag,sametag); sametag = nullptr; - max_same = 0; if (map_style == MAP_ARRAY) { memoryKK->destroy_kokkos(k_map_array,map_array); map_array = nullptr; } else { - if (map_nhash) { - delete [] map_bucket; - delete [] map_hash; - map_bucket = nullptr; - map_hash = nullptr; - k_map_hash = dual_hash_type(); - } - map_nhash = map_nbucket = 0; + h_map_hash = host_hash_type(); + d_map_hash = hash_type(); } -} + Atom::map_delete(); +} diff --git a/src/KOKKOS/fix_shake_kokkos.cpp b/src/KOKKOS/fix_shake_kokkos.cpp index 594c9c60db..a50ec573b4 100644 --- a/src/KOKKOS/fix_shake_kokkos.cpp +++ b/src/KOKKOS/fix_shake_kokkos.cpp @@ -207,7 +207,6 @@ void FixShakeKokkos::pre_neighbor() k_map_array.template sync(); } else if (map_style == Atom::MAP_HASH) { k_map_hash = atomKK->k_map_hash; - k_map_hash.template sync(); } k_shake_flag.sync(); @@ -231,7 +230,6 @@ void FixShakeKokkos::pre_neighbor() k_map_array.template sync(); } else if (map_style == Atom::MAP_HASH) { k_map_hash = atomKK->k_map_hash; - k_map_hash.template sync(); } // build list of SHAKE clusters I compute @@ -320,7 +318,6 @@ void FixShakeKokkos::post_force(int vflag) k_map_array.template sync(); } else if (map_style == Atom::MAP_HASH) { k_map_hash = atomKK->k_map_hash; - k_map_hash.template sync(); } if (d_rmass.data()) diff --git a/src/KOKKOS/fix_shake_kokkos.h b/src/KOKKOS/fix_shake_kokkos.h index fbf6e1de88..698298c65d 100644 --- a/src/KOKKOS/fix_shake_kokkos.h +++ b/src/KOKKOS/fix_shake_kokkos.h @@ -187,8 +187,6 @@ class FixShakeKokkos : public FixShake, public KokkosBase { DAT::tdual_int_1d k_map_array; - typedef Kokkos::UnorderedMap hash_type; - typedef Kokkos::DualView dual_hash_type; dual_hash_type k_map_hash; KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index c9e5736410..7a27e3f20f 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -23,6 +23,7 @@ #include #include #include +#include enum{FULL=1u,HALFTHREAD=2u,HALF=4u}; @@ -551,6 +552,20 @@ typedef int T_INT; // LAMMPS types +typedef Kokkos::UnorderedMap hash_type; +typedef hash_type::HostMirror host_hash_type; + +struct dual_hash_type { + hash_type d_view; + host_hash_type h_view; + + template + std::enable_if_t::value,hash_type> view() {return d_view;} + template + std::enable_if_t::value,host_hash_type> view() {return h_view;} + +}; + template struct ArrayTypes; @@ -837,6 +852,8 @@ typedef tdual_neighbors_2d::t_dev_um t_neighbors_2d_um; typedef tdual_neighbors_2d::t_dev_const_um t_neighbors_2d_const_um; typedef tdual_neighbors_2d::t_dev_const_randomread t_neighbors_2d_randomread; +typedef hash_type t_hash; + }; #ifdef LMP_KOKKOS_GPU @@ -1098,6 +1115,8 @@ typedef tdual_neighbors_2d::t_host_um t_neighbors_2d_um; typedef tdual_neighbors_2d::t_host_const_um t_neighbors_2d_const_um; typedef tdual_neighbors_2d::t_host_const_randomread t_neighbors_2d_randomread; +typedef host_hash_type t_hash; + }; #endif //default LAMMPS Types diff --git a/src/KOKKOS/neigh_bond_kokkos.cpp b/src/KOKKOS/neigh_bond_kokkos.cpp index c427d18216..228d521fdd 100644 --- a/src/KOKKOS/neigh_bond_kokkos.cpp +++ b/src/KOKKOS/neigh_bond_kokkos.cpp @@ -1241,7 +1241,6 @@ void NeighBondKokkos::update_class_variables() k_map_array.template sync(); } else if (map_style == Atom::MAP_HASH) { k_map_hash = atomKK->k_map_hash; - k_map_hash.template sync(); } } diff --git a/src/KOKKOS/neigh_bond_kokkos.h b/src/KOKKOS/neigh_bond_kokkos.h index 9bf508ec7a..d60ad2f1aa 100644 --- a/src/KOKKOS/neigh_bond_kokkos.h +++ b/src/KOKKOS/neigh_bond_kokkos.h @@ -89,8 +89,6 @@ class NeighBondKokkos : protected Pointers { DAT::tdual_int_1d k_map_array; - typedef Kokkos::UnorderedMap hash_type; - typedef Kokkos::DualView dual_hash_type; dual_hash_type k_map_hash; KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/neighbor_kokkos.h b/src/KOKKOS/neighbor_kokkos.h index 1a560e2129..ef885535ed 100644 --- a/src/KOKKOS/neighbor_kokkos.h +++ b/src/KOKKOS/neighbor_kokkos.h @@ -64,13 +64,14 @@ class NeighborKokkos : public Neighbor { DAT::tdual_int_2d k_dihedrallist; DAT::tdual_int_2d k_improperlist; + int device_flag; + private: DAT::tdual_x_array x; DAT::tdual_x_array xhold; X_FLOAT deltasq; - int device_flag; void init_cutneighsq_kokkos(int); void create_kokkos_list(int); From 33ac10df7ab87e10b8bdc64fcbd639748657701e Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Fri, 9 Jul 2021 14:00:19 -0600 Subject: [PATCH 06/23] Missed one instance of SUCCESS --- src/OPENMP/reaxc_init_md_omp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OPENMP/reaxc_init_md_omp.cpp b/src/OPENMP/reaxc_init_md_omp.cpp index 226bfaf53c..4a0cf6b926 100644 --- a/src/OPENMP/reaxc_init_md_omp.cpp +++ b/src/OPENMP/reaxc_init_md_omp.cpp @@ -111,7 +111,7 @@ int Init_ListsOMP(reax_system *system, control_params *control, free(hb_top); free(bond_top); - return SUCCESS; + return REAXC_SUCCES; } /* ---------------------------------------------------------------------- */ From 6c088b97ac1d1e7c5fa115b47a7eb4af218198ff Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Fri, 9 Jul 2021 14:08:32 -0600 Subject: [PATCH 07/23] Add missing data transfer for map_array --- src/KOKKOS/atom_map_kokkos.cpp | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 486594e182..2cdbe36d91 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -110,6 +110,10 @@ void AtomKokkos::map_init(int check) h_map_hash = host_hash_type(map_nhash); } } + + k_sametag.modify_host(); + if (map_style == Atom::MAP_ARRAY) + k_map_array.modify_host(); } /* ---------------------------------------------------------------------- @@ -227,23 +231,29 @@ void AtomKokkos::map_set() } } - // check if fix shake or neigh bond needs a device hash + k_sametag.modify_host(); + if (map_style == Atom::MAP_ARRAY) + k_map_array.modify_host(); + else if (map_style == Atom::MAP_ARRAY) { - int device_hash_flag = 0; + // check if fix shake or neigh bond needs a device hash - auto neighborKK = (NeighborKokkos*) neighbor; - if (neighborKK->device_flag) device_hash_flag = 1; + int device_hash_flag = 0; - for (int n = 0; n < modify->nfix; n++) - if (utils::strmatch(modify->fix[n]->style,"^shake")) - if (modify->fix[n]->execution_space == Device) - device_hash_flag = 1; + auto neighborKK = (NeighborKokkos*) neighbor; + if (neighborKK->device_flag) device_hash_flag = 1; - if (device_hash_flag) - Kokkos::deep_copy(d_map_hash,h_map_hash); + for (int n = 0; n < modify->nfix; n++) + if (utils::strmatch(modify->fix[n]->style,"^shake")) + if (modify->fix[n]->execution_space == Device) + device_hash_flag = 1; - k_map_hash.h_view = h_map_hash; - k_map_hash.d_view = d_map_hash; + if (device_hash_flag) + Kokkos::deep_copy(d_map_hash,h_map_hash); + + k_map_hash.h_view = h_map_hash; + k_map_hash.d_view = d_map_hash; + } } /* ---------------------------------------------------------------------- From be3c0ce5a0a2bec9399a1226c4bea62243a2d330 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Fri, 9 Jul 2021 14:13:24 -0600 Subject: [PATCH 08/23] Fix typo --- src/OPENMP/reaxc_init_md_omp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OPENMP/reaxc_init_md_omp.cpp b/src/OPENMP/reaxc_init_md_omp.cpp index 4a0cf6b926..b1c6b5e3c6 100644 --- a/src/OPENMP/reaxc_init_md_omp.cpp +++ b/src/OPENMP/reaxc_init_md_omp.cpp @@ -111,7 +111,7 @@ int Init_ListsOMP(reax_system *system, control_params *control, free(hb_top); free(bond_top); - return REAXC_SUCCES; + return REAXC_SUCCESS; } /* ---------------------------------------------------------------------- */ From 0eaacea67d25e918403aa618104a4ff9b951b721 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Fri, 9 Jul 2021 14:17:06 -0600 Subject: [PATCH 09/23] Whitespace --- src/KOKKOS/atom_map_kokkos.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 2cdbe36d91..90cebd17b4 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -112,7 +112,7 @@ void AtomKokkos::map_init(int check) } k_sametag.modify_host(); - if (map_style == Atom::MAP_ARRAY) + if (map_style == Atom::MAP_ARRAY) k_map_array.modify_host(); } From e8aa3823d310668df2e53c47c50e352fb9b8fb3c Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Fri, 9 Jul 2021 15:48:44 -0600 Subject: [PATCH 10/23] Add virtual keyword to AtomKokkos destructor --- src/KOKKOS/atom_kokkos.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index 7798c60321..a88c6b3ad8 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -67,7 +67,7 @@ class AtomKokkos : public Atom { AtomKokkos(class LAMMPS *); - ~AtomKokkos(); + virtual ~AtomKokkos(); void map_init(int check = 1); void map_set(); From 89d7e34540e16bd4ae7fddb56521e1e5fe3c8adc Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sat, 10 Jul 2021 10:53:36 -0400 Subject: [PATCH 11/23] destructor in polymorph base class should be virtual --- src/atom.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atom.h b/src/atom.h index 5d96be67bf..c7273e1ff5 100644 --- a/src/atom.h +++ b/src/atom.h @@ -275,7 +275,7 @@ class Atom : protected Pointers { // functions Atom(class LAMMPS *); - ~Atom(); + virtual ~Atom(); void settings(class Atom *); void peratom_create(); From 4456e8151f3d4c026f01ed5d0447a9cf043fde3c Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sat, 10 Jul 2021 10:54:02 -0400 Subject: [PATCH 12/23] use explicit scoping in destructor of polymorph class --- src/KOKKOS/atom_kokkos.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KOKKOS/atom_kokkos.cpp b/src/KOKKOS/atom_kokkos.cpp index 40db94311b..84419984b8 100644 --- a/src/KOKKOS/atom_kokkos.cpp +++ b/src/KOKKOS/atom_kokkos.cpp @@ -79,7 +79,7 @@ AtomKokkos::~AtomKokkos() memoryKK->destroy_kokkos(k_improper_atom3, improper_atom3); memoryKK->destroy_kokkos(k_improper_atom4, improper_atom4); - map_delete(); + AtomKokkos::map_delete(); // SPIN package From ac903ec291d14a5fc9ea3812e58e4fda2bfda26e Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sat, 10 Jul 2021 10:55:07 -0400 Subject: [PATCH 13/23] update list of non-style KOKKOS package sources with newly added file --- cmake/Modules/Packages/KOKKOS.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index e97f5546cd..aea766b79c 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -79,6 +79,7 @@ target_compile_definitions(lammps PRIVATE -DLMP_KOKKOS) set(KOKKOS_PKG_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/KOKKOS) set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/atom_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/atom_map_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp From 8f8dff758e07bb4f0101321a3684bfa3855da232 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sat, 10 Jul 2021 14:39:25 -0400 Subject: [PATCH 14/23] reformat --- src/KOKKOS/atom_kokkos.cpp | 129 +++++++++++++--------------- src/KOKKOS/atom_map_kokkos.cpp | 81 +++++++++-------- src/KOKKOS/atom_vec_dpd_kokkos.cpp | 10 +-- src/KOKKOS/atom_vec_full_kokkos.cpp | 10 +-- 4 files changed, 110 insertions(+), 120 deletions(-) diff --git a/src/KOKKOS/atom_kokkos.cpp b/src/KOKKOS/atom_kokkos.cpp index 84419984b8..02cd4ea708 100644 --- a/src/KOKKOS/atom_kokkos.cpp +++ b/src/KOKKOS/atom_kokkos.cpp @@ -1,4 +1,3 @@ -// clang-format off /* ---------------------------------------------------------------------- LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator https://www.lammps.org/, Sandia National Laboratories @@ -13,23 +12,23 @@ ------------------------------------------------------------------------- */ #include "atom_kokkos.h" -#include + +#include "atom_masks.h" #include "atom_vec.h" #include "atom_vec_kokkos.h" #include "comm_kokkos.h" -#include "update.h" #include "domain.h" -#include "atom_masks.h" -#include "memory_kokkos.h" #include "error.h" #include "kokkos.h" -#include "atom_masks.h" +#include "memory_kokkos.h" +#include "update.h" using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- */ -AtomKokkos::AtomKokkos(LAMMPS *lmp) : Atom(lmp) { +AtomKokkos::AtomKokkos(LAMMPS *lmp) : Atom(lmp) +{ k_error_flag = DAT::tdual_int_scalar("atom:error_flag"); } @@ -88,16 +87,16 @@ AtomKokkos::~AtomKokkos() memoryKK->destroy_kokkos(k_fm_long, fm_long); // DPD-REACT package - memoryKK->destroy_kokkos(k_uCond,uCond); - memoryKK->destroy_kokkos(k_uMech,uMech); - memoryKK->destroy_kokkos(k_uChem,uChem); - memoryKK->destroy_kokkos(k_uCG,uCG); - memoryKK->destroy_kokkos(k_uCGnew,uCGnew); - memoryKK->destroy_kokkos(k_rho,rho); - memoryKK->destroy_kokkos(k_dpdTheta,dpdTheta); - memoryKK->destroy_kokkos(k_duChem,duChem); + memoryKK->destroy_kokkos(k_uCond, uCond); + memoryKK->destroy_kokkos(k_uMech, uMech); + memoryKK->destroy_kokkos(k_uChem, uChem); + memoryKK->destroy_kokkos(k_uCG, uCG); + memoryKK->destroy_kokkos(k_uCGnew, uCGnew); + memoryKK->destroy_kokkos(k_rho, rho); + memoryKK->destroy_kokkos(k_dpdTheta, dpdTheta); + memoryKK->destroy_kokkos(k_duChem, duChem); - memoryKK->destroy_kokkos(k_dvector,dvector); + memoryKK->destroy_kokkos(k_dvector, dvector); dvector = nullptr; } @@ -105,34 +104,32 @@ AtomKokkos::~AtomKokkos() void AtomKokkos::sync(const ExecutionSpace space, unsigned int mask) { - if (space == Device && lmp->kokkos->auto_sync) - ((AtomVecKokkos *) avec)->modified(Host,mask); + if (space == Device && lmp->kokkos->auto_sync) ((AtomVecKokkos *) avec)->modified(Host, mask); - ((AtomVecKokkos *) avec)->sync(space,mask); + ((AtomVecKokkos *) avec)->sync(space, mask); } /* ---------------------------------------------------------------------- */ void AtomKokkos::modified(const ExecutionSpace space, unsigned int mask) { - ((AtomVecKokkos *) avec)->modified(space,mask); + ((AtomVecKokkos *) avec)->modified(space, mask); - if (space == Device && lmp->kokkos->auto_sync) - ((AtomVecKokkos *) avec)->sync(Host,mask); + if (space == Device && lmp->kokkos->auto_sync) ((AtomVecKokkos *) avec)->sync(Host, mask); } void AtomKokkos::sync_overlapping_device(const ExecutionSpace space, unsigned int mask) { - ((AtomVecKokkos *) avec)->sync_overlapping_device(space,mask); + ((AtomVecKokkos *) avec)->sync_overlapping_device(space, mask); } /* ---------------------------------------------------------------------- */ void AtomKokkos::allocate_type_arrays() { if (avec->mass_type == AtomVec::PER_TYPE) { - k_mass = DAT::tdual_float_1d("Mass",ntypes+1); + k_mass = DAT::tdual_float_1d("Mass", ntypes + 1); mass = k_mass.h_view.data(); - mass_setflag = new int[ntypes+1]; + mass_setflag = new int[ntypes + 1]; for (int itype = 1; itype <= ntypes; itype++) mass_setflag[itype] = 0; k_mass.modify(); } @@ -142,11 +139,11 @@ void AtomKokkos::allocate_type_arrays() void AtomKokkos::sort() { - int i,m,n,ix,iy,iz,ibin,empty; + int i, m, n, ix, iy, iz, ibin, empty; // set next timestep for sorting to take place - nextsort = (update->ntimestep/sortfreq)*sortfreq + sortfreq; + nextsort = (update->ntimestep / sortfreq) * sortfreq + sortfreq; // re-setup sort bins if needed @@ -159,33 +156,33 @@ void AtomKokkos::sort() memory->destroy(next); memory->destroy(permute); maxnext = atom->nmax; - memory->create(next,maxnext,"atom:next"); - memory->create(permute,maxnext,"atom:permute"); + memory->create(next, maxnext, "atom:next"); + memory->create(permute, maxnext, "atom:permute"); } // insure there is one extra atom location at end of arrays for swaps if (nlocal == nmax) avec->grow(0); - sync(Host,ALL_MASK); - modified(Host,ALL_MASK); + sync(Host, ALL_MASK); + modified(Host, ALL_MASK); // bin atoms in reverse order so linked list will be in forward order for (i = 0; i < nbins; i++) binhead[i] = -1; HAT::t_x_array_const h_x = k_x.view(); - for (i = nlocal-1; i >= 0; i--) { - ix = static_cast ((h_x(i,0)-bboxlo[0])*bininvx); - iy = static_cast ((h_x(i,1)-bboxlo[1])*bininvy); - iz = static_cast ((h_x(i,2)-bboxlo[2])*bininvz); - ix = MAX(ix,0); - iy = MAX(iy,0); - iz = MAX(iz,0); - ix = MIN(ix,nbinx-1); - iy = MIN(iy,nbiny-1); - iz = MIN(iz,nbinz-1); - ibin = iz*nbiny*nbinx + iy*nbinx + ix; + for (i = nlocal - 1; i >= 0; i--) { + ix = static_cast((h_x(i, 0) - bboxlo[0]) * bininvx); + iy = static_cast((h_x(i, 1) - bboxlo[1]) * bininvy); + iz = static_cast((h_x(i, 2) - bboxlo[2]) * bininvz); + ix = MAX(ix, 0); + iy = MAX(iy, 0); + iz = MAX(iz, 0); + ix = MIN(ix, nbinx - 1); + iy = MIN(iy, nbiny - 1); + iz = MIN(iz, nbinz - 1); + ibin = iz * nbiny * nbinx + iy * nbinx + ix; next[i] = binhead[ibin]; binhead[ibin] = i; } @@ -217,13 +214,13 @@ void AtomKokkos::sort() for (i = 0; i < nlocal; i++) { if (current[i] == permute[i]) continue; - avec->copy(i,nlocal,0); + avec->copy(i, nlocal, 0); empty = i; while (permute[empty] != i) { - avec->copy(permute[empty],empty,0); + avec->copy(permute[empty], empty, 0); empty = current[empty] = permute[empty]; } - avec->copy(nlocal,empty,0); + avec->copy(nlocal, empty, 0); current[empty] = permute[empty]; } @@ -241,13 +238,14 @@ void AtomKokkos::sort() reallocate memory to the pointer selected by the mask ------------------------------------------------------------------------- */ -void AtomKokkos::grow(unsigned int mask) { +void AtomKokkos::grow(unsigned int mask) +{ if (mask & SPECIAL_MASK) { memoryKK->destroy_kokkos(k_special, special); sync(Device, mask); modified(Device, mask); - memoryKK->grow_kokkos(k_special,special,nmax,maxspecial,"atom:special"); + memoryKK->grow_kokkos(k_special, special, nmax, maxspecial, "atom:special"); avec->grow_pointers(); sync(Host, mask); } @@ -266,22 +264,18 @@ int AtomKokkos::add_custom(const char *name, int flag) if (flag == 0) { index = nivector; nivector++; - iname = (char **) memory->srealloc(iname,nivector*sizeof(char *), - "atom:iname"); + iname = (char **) memory->srealloc(iname, nivector * sizeof(char *), "atom:iname"); iname[index] = utils::strdup(name); - ivector = (int **) memory->srealloc(ivector,nivector*sizeof(int *), - "atom:ivector"); - memory->create(ivector[index],nmax,"atom:ivector"); + ivector = (int **) memory->srealloc(ivector, nivector * sizeof(int *), "atom:ivector"); + memory->create(ivector[index], nmax, "atom:ivector"); } else { index = ndvector; ndvector++; - dname = (char **) memory->srealloc(dname,ndvector*sizeof(char *), - "atom:dname"); + dname = (char **) memory->srealloc(dname, ndvector * sizeof(char *), "atom:dname"); dname[index] = utils::strdup(name); - this->sync(Device,DVECTOR_MASK); - memoryKK->grow_kokkos(k_dvector,dvector,ndvector,nmax, - "atom:dvector"); - this->modified(Device,DVECTOR_MASK); + this->sync(Device, DVECTOR_MASK); + memoryKK->grow_kokkos(k_dvector, dvector, ndvector, nmax, "atom:dvector"); + this->modified(Device, DVECTOR_MASK); } return index; @@ -298,12 +292,12 @@ void AtomKokkos::remove_custom(int flag, int index) if (flag == 0) { memory->destroy(ivector[index]); ivector[index] = nullptr; - delete [] iname[index]; + delete[] iname[index]; iname[index] = nullptr; } else { //memoryKK->destroy_kokkos(dvector); dvector[index] = nullptr; - delete [] dname[index]; + delete[] dname[index]; dname[index] = nullptr; } } @@ -339,19 +333,16 @@ void AtomKokkos::deallocate_topology() done at higher levels (Verlet,Modify,etc) ------------------------------------------------------------------------- */ -void AtomKokkos::sync_modify(ExecutionSpace execution_space, - unsigned int datamask_read, +void AtomKokkos::sync_modify(ExecutionSpace execution_space, unsigned int datamask_read, unsigned int datamask_modify) { - sync(execution_space,datamask_read); - modified(execution_space,datamask_modify); + sync(execution_space, datamask_read); + modified(execution_space, datamask_modify); } -AtomVec *AtomKokkos::new_avec(const std::string &style, - int trysuffix, int &sflag) +AtomVec *AtomKokkos::new_avec(const std::string &style, int trysuffix, int &sflag) { - AtomVec* avec = Atom::new_avec(style,trysuffix,sflag); - if (!avec->kokkosable) - error->all(FLERR,"KOKKOS package requires a kokkos enabled atom_style"); + AtomVec *avec = Atom::new_avec(style, trysuffix, sflag); + if (!avec->kokkosable) error->all(FLERR, "KOKKOS package requires a kokkos enabled atom_style"); return avec; } diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 90cebd17b4..d7f91c04e2 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -1,4 +1,3 @@ -// clang-format off /* ---------------------------------------------------------------------- LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator https://www.lammps.org/, Sandia National Laboratories @@ -13,13 +12,14 @@ ------------------------------------------------------------------------- */ #include "atom_kokkos.h" -#include "neighbor_kokkos.h" + +#include "atom_masks.h" #include "comm.h" #include "error.h" -#include "modify.h" #include "fix.h" #include "memory_kokkos.h" -#include "atom_masks.h" +#include "modify.h" +#include "neighbor_kokkos.h" #include @@ -48,8 +48,10 @@ void AtomKokkos::map_init(int check) int recreate = 0; if (check) recreate = map_style_set(); - if (map_style == MAP_ARRAY && map_tag_max > map_maxarray) recreate = 1; - else if (map_style == MAP_HASH && nlocal+nghost > map_nhash) recreate = 1; + if (map_style == MAP_ARRAY && map_tag_max > map_maxarray) + recreate = 1; + else if (map_style == MAP_HASH && nlocal + nghost > map_nhash) + recreate = 1; // if not recreating: // for array, initialize current map_tag_max values @@ -62,18 +64,18 @@ void AtomKokkos::map_init(int check) for (int i = 0; i < map_nbucket; i++) map_bucket[i] = -1; map_nused = 0; map_free = 0; - for (int i = 0; i < map_nhash; i++) map_hash[i].next = i+1; - if (map_nhash > 0) map_hash[map_nhash-1].next = -1; + for (int i = 0; i < map_nhash; i++) map_hash[i].next = i + 1; + if (map_nhash > 0) map_hash[map_nhash - 1].next = -1; } - // recreating: delete old map and create new one for array or hash + // recreating: delete old map and create new one for array or hash } else { map_delete(); if (map_style == MAP_ARRAY) { map_maxarray = map_tag_max; - memoryKK->create_kokkos(k_map_array,map_array,map_maxarray+1,"atom:map_array"); + memoryKK->create_kokkos(k_map_array, map_array, map_maxarray + 1, "atom:map_array"); for (int i = 0; i <= map_tag_max; i++) map_array[i] = -1; } else { @@ -83,10 +85,10 @@ void AtomKokkos::map_init(int check) // multiply by 2, require at least 1000 // doubling means hash table will need to be re-init only rarely - int nper = static_cast (natoms/comm->nprocs); - map_nhash = MAX(nper,nmax); + int nper = static_cast(natoms / comm->nprocs); + map_nhash = MAX(nper, nmax); map_nhash *= 2; - map_nhash = MAX(map_nhash,1000); + map_nhash = MAX(map_nhash, 1000); // map_nbucket = prime just larger than map_nhash // next_prime() should be fast enough, @@ -104,16 +106,15 @@ void AtomKokkos::map_init(int check) map_hash = new HashElem[map_nhash]; map_nused = 0; map_free = 0; - for (int i = 0; i < map_nhash; i++) map_hash[i].next = i+1; - map_hash[map_nhash-1].next = -1; + for (int i = 0; i < map_nhash; i++) map_hash[i].next = i + 1; + map_hash[map_nhash - 1].next = -1; h_map_hash = host_hash_type(map_nhash); } } k_sametag.modify_host(); - if (map_style == Atom::MAP_ARRAY) - k_map_array.modify_host(); + if (map_style == Atom::MAP_ARRAY) k_map_array.modify_host(); } /* ---------------------------------------------------------------------- @@ -130,11 +131,10 @@ void AtomKokkos::map_set() { int nall = nlocal + nghost; - atomKK->sync(Host,TAG_MASK); + atomKK->sync(Host, TAG_MASK); k_sametag.sync_host(); - if (map_style == Atom::MAP_ARRAY) - k_map_array.sync_host(); + if (map_style == Atom::MAP_ARRAY) k_map_array.sync_host(); if (map_style == MAP_ARRAY) { @@ -143,11 +143,11 @@ void AtomKokkos::map_set() if (nall > max_same) { max_same = nall + EXTRA; - memoryKK->destroy_kokkos(k_sametag,sametag); - memoryKK->create_kokkos(k_sametag,sametag,max_same,"atom:sametag"); + memoryKK->destroy_kokkos(k_sametag, sametag); + memoryKK->create_kokkos(k_sametag, sametag, max_same, "atom:sametag"); } - for (int i = nall-1; i >= 0 ; i--) { + for (int i = nall - 1; i >= 0; i--) { sametag[i] = map_array[tag[i]]; map_array[tag[i]] = i; } @@ -162,14 +162,14 @@ void AtomKokkos::map_set() if (nall > map_nhash) map_init(0); if (nall > max_same) { max_same = nall + EXTRA; - memoryKK->destroy_kokkos(k_sametag,sametag); - memoryKK->create_kokkos(k_sametag,sametag,max_same,"atom:sametag"); + memoryKK->destroy_kokkos(k_sametag, sametag); + memoryKK->create_kokkos(k_sametag, sametag, max_same, "atom:sametag"); } - int previous,ibucket,index; + int previous, ibucket, index; tagint global; - for (int i = nall-1; i >= 0 ; i--) { + for (int i = nall - 1; i >= 0; i--) { sametag[i] = map_find_hash(tag[i]); // search for key @@ -195,8 +195,10 @@ void AtomKokkos::map_set() index = map_free; map_free = map_hash[map_free].next; - if (previous == -1) map_bucket[ibucket] = index; - else map_hash[previous].next = index; + if (previous == -1) + map_bucket[ibucket] = index; + else + map_hash[previous].next = index; map_hash[index].global = global; map_hash[index].local = i; map_hash[index].next = -1; @@ -207,7 +209,7 @@ void AtomKokkos::map_set() h_map_hash.clear(); - for (int i = nall-1; i >= 0 ; i--) { + for (int i = nall - 1; i >= 0; i--) { // search for key // if don't find it, done @@ -225,9 +227,8 @@ void AtomKokkos::map_set() int local = map_hash[index].local; - auto insert_result = h_map_hash.insert(global,local); - if (insert_result.failed()) - error->one(FLERR, "Kokkos::UnorderedMap insertion failed"); + auto insert_result = h_map_hash.insert(global, local); + if (insert_result.failed()) error->one(FLERR, "Kokkos::UnorderedMap insertion failed"); } } @@ -240,16 +241,14 @@ void AtomKokkos::map_set() int device_hash_flag = 0; - auto neighborKK = (NeighborKokkos*) neighbor; + auto neighborKK = (NeighborKokkos *) neighbor; if (neighborKK->device_flag) device_hash_flag = 1; for (int n = 0; n < modify->nfix; n++) - if (utils::strmatch(modify->fix[n]->style,"^shake")) - if (modify->fix[n]->execution_space == Device) - device_hash_flag = 1; + if (utils::strmatch(modify->fix[n]->style, "^shake")) + if (modify->fix[n]->execution_space == Device) device_hash_flag = 1; - if (device_hash_flag) - Kokkos::deep_copy(d_map_hash,h_map_hash); + if (device_hash_flag) Kokkos::deep_copy(d_map_hash, h_map_hash); k_map_hash.h_view = h_map_hash; k_map_hash.d_view = d_map_hash; @@ -262,11 +261,11 @@ void AtomKokkos::map_set() void AtomKokkos::map_delete() { - memoryKK->destroy_kokkos(k_sametag,sametag); + memoryKK->destroy_kokkos(k_sametag, sametag); sametag = nullptr; if (map_style == MAP_ARRAY) { - memoryKK->destroy_kokkos(k_map_array,map_array); + memoryKK->destroy_kokkos(k_map_array, map_array); map_array = nullptr; } else { h_map_hash = host_hash_type(); diff --git a/src/KOKKOS/atom_vec_dpd_kokkos.cpp b/src/KOKKOS/atom_vec_dpd_kokkos.cpp index 4358a49358..e02631f89e 100644 --- a/src/KOKKOS/atom_vec_dpd_kokkos.cpp +++ b/src/KOKKOS/atom_vec_dpd_kokkos.cpp @@ -13,15 +13,15 @@ ------------------------------------------------------------------------- */ #include "atom_vec_dpd_kokkos.h" + #include "atom_kokkos.h" +#include "atom_masks.h" #include "comm_kokkos.h" #include "domain.h" -#include "modify.h" -#include "fix.h" -#include "atom_masks.h" -#include "memory_kokkos.h" #include "error.h" - +#include "fix.h" +#include "memory_kokkos.h" +#include "modify.h" using namespace LAMMPS_NS; diff --git a/src/KOKKOS/atom_vec_full_kokkos.cpp b/src/KOKKOS/atom_vec_full_kokkos.cpp index 45ce316d3f..cd5316cc73 100644 --- a/src/KOKKOS/atom_vec_full_kokkos.cpp +++ b/src/KOKKOS/atom_vec_full_kokkos.cpp @@ -13,15 +13,15 @@ ------------------------------------------------------------------------- */ #include "atom_vec_full_kokkos.h" + #include "atom_kokkos.h" +#include "atom_masks.h" #include "comm_kokkos.h" #include "domain.h" -#include "modify.h" -#include "fix.h" -#include "atom_masks.h" -#include "memory_kokkos.h" #include "error.h" - +#include "fix.h" +#include "memory_kokkos.h" +#include "modify.h" using namespace LAMMPS_NS; From b3619922928a5465685c93ed36db29f7fc4e5275 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 12 Jul 2021 08:53:07 -0600 Subject: [PATCH 15/23] Remove deprecated Kokkos code ViewAllocateWithoutInitializing --- src/KOKKOS/kokkos_type.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index c9e5736410..775462f1ff 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -45,7 +45,9 @@ enum{FULL=1u,HALFTHREAD=2u,HALF=4u}; static constexpr LAMMPS_NS::bigint LMP_KOKKOS_AV_DELTA = 10; namespace Kokkos { - using NoInit = ViewAllocateWithoutInitializing; + auto NoInit = [](std::string const& label) { + return Kokkos::view_alloc(Kokkos::WithoutInitializing, label); + }; } struct lmp_float3 { From ad966e130b586dfac0e71afe31bee9072545ee74 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 12 Jul 2021 10:41:31 -0600 Subject: [PATCH 16/23] Fix typos --- src/KOKKOS/atom_map_kokkos.cpp | 2 +- src/KOKKOS/kokkos_type.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index d7f91c04e2..57761372c9 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -235,7 +235,7 @@ void AtomKokkos::map_set() k_sametag.modify_host(); if (map_style == Atom::MAP_ARRAY) k_map_array.modify_host(); - else if (map_style == Atom::MAP_ARRAY) { + else if (map_style == Atom::MAP_HASH) { // check if fix shake or neigh bond needs a device hash diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 7a27e3f20f..7fe2130e9b 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -562,7 +562,7 @@ struct dual_hash_type { template std::enable_if_t::value,hash_type> view() {return d_view;} template - std::enable_if_t::value,host_hash_type> view() {return h_view;} + std::enable_if_t::value,host_hash_type> view() {return h_view;} }; From b752bda1b9d1c80246832dc0f9b332e886396433 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Mon, 12 Jul 2021 13:06:54 -0600 Subject: [PATCH 17/23] Fix GPU issues --- src/KOKKOS/kokkos_type.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 7fe2130e9b..2ecde3adeb 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -560,9 +560,12 @@ struct dual_hash_type { host_hash_type h_view; template + KOKKOS_INLINE_FUNCTION std::enable_if_t::value,hash_type> view() {return d_view;} + template - std::enable_if_t::value,host_hash_type> view() {return h_view;} + KOKKOS_INLINE_FUNCTION + std::enable_if_t::value,host_hash_type> view() {return h_view;} }; From b4d3dbe0af9d793123b91d614e12bc5b18e6038d Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 12 Jul 2021 14:11:53 -0600 Subject: [PATCH 18/23] Avoid unnecessary deep_copy when only a single memory space --- bench/in.rhodo | 1 + src/KOKKOS/atom_kokkos.h | 2 -- src/KOKKOS/atom_map_kokkos.cpp | 22 +++++++++++++++------- src/KOKKOS/kokkos_type.h | 6 ++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/bench/in.rhodo b/bench/in.rhodo index bd7e3df7f5..1607e3ddd1 100644 --- a/bench/in.rhodo +++ b/bench/in.rhodo @@ -4,6 +4,7 @@ units real neigh_modify delay 5 every 1 atom_style full +atom_modify map array bond_style harmonic angle_style charmm dihedral_style charmm diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index a88c6b3ad8..00b8e53578 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -77,8 +77,6 @@ class AtomKokkos : public Atom { DAT::tdual_int_1d k_map_array; DAT::tdual_int_scalar k_error_flag; dual_hash_type k_map_hash; - hash_type d_map_hash; - host_hash_type h_map_hash; template KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 57761372c9..d498974db2 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -109,7 +109,10 @@ void AtomKokkos::map_init(int check) for (int i = 0; i < map_nhash; i++) map_hash[i].next = i + 1; map_hash[map_nhash - 1].next = -1; - h_map_hash = host_hash_type(map_nhash); + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); + h_map_hash = decltype(h_map_hash)(map_nhash); } } @@ -207,6 +210,9 @@ void AtomKokkos::map_set() // Copy to Kokkos hash + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); h_map_hash.clear(); for (int i = nall - 1; i >= 0; i--) { @@ -237,6 +243,11 @@ void AtomKokkos::map_set() k_map_array.modify_host(); else if (map_style == Atom::MAP_HASH) { + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); + auto d_map_hash = k_map_hash.view(); + // check if fix shake or neigh bond needs a device hash int device_hash_flag = 0; @@ -248,10 +259,7 @@ void AtomKokkos::map_set() if (utils::strmatch(modify->fix[n]->style, "^shake")) if (modify->fix[n]->execution_space == Device) device_hash_flag = 1; - if (device_hash_flag) Kokkos::deep_copy(d_map_hash, h_map_hash); - - k_map_hash.h_view = h_map_hash; - k_map_hash.d_view = d_map_hash; + if (device_hash_flag) Kokkos::deep_copy(d_map_hash,h_map_hash); } } @@ -268,8 +276,8 @@ void AtomKokkos::map_delete() memoryKK->destroy_kokkos(k_map_array, map_array); map_array = nullptr; } else { - h_map_hash = host_hash_type(); - d_map_hash = hash_type(); + k_map_hash.h_view = host_hash_type(); + k_map_hash.d_view = hash_type(); } Atom::map_delete(); diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 2ecde3adeb..1992024ba0 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -560,12 +560,10 @@ struct dual_hash_type { host_hash_type h_view; template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value,hash_type> view() {return d_view;} + std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type> view() {return d_view;} template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value,host_hash_type> view() {return h_view;} + std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type> view() {return h_view;} }; From 65e2a8f0700001c6ea8df11fa309dba19f5d4149 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 12 Jul 2021 14:21:05 -0600 Subject: [PATCH 19/23] Avoid unnecessary deep_copy when only a single memory space --- src/KOKKOS/atom_kokkos.h | 2 -- src/KOKKOS/atom_map_kokkos.cpp | 22 +++++++++++++++------- src/KOKKOS/kokkos_type.h | 6 ++---- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index a88c6b3ad8..00b8e53578 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -77,8 +77,6 @@ class AtomKokkos : public Atom { DAT::tdual_int_1d k_map_array; DAT::tdual_int_scalar k_error_flag; dual_hash_type k_map_hash; - hash_type d_map_hash; - host_hash_type h_map_hash; template KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 57761372c9..d498974db2 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -109,7 +109,10 @@ void AtomKokkos::map_init(int check) for (int i = 0; i < map_nhash; i++) map_hash[i].next = i + 1; map_hash[map_nhash - 1].next = -1; - h_map_hash = host_hash_type(map_nhash); + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); + h_map_hash = decltype(h_map_hash)(map_nhash); } } @@ -207,6 +210,9 @@ void AtomKokkos::map_set() // Copy to Kokkos hash + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); h_map_hash.clear(); for (int i = nall - 1; i >= 0; i--) { @@ -237,6 +243,11 @@ void AtomKokkos::map_set() k_map_array.modify_host(); else if (map_style == Atom::MAP_HASH) { + // use "view" template method to avoid unnecessary deep_copy + + auto h_map_hash = k_map_hash.view(); + auto d_map_hash = k_map_hash.view(); + // check if fix shake or neigh bond needs a device hash int device_hash_flag = 0; @@ -248,10 +259,7 @@ void AtomKokkos::map_set() if (utils::strmatch(modify->fix[n]->style, "^shake")) if (modify->fix[n]->execution_space == Device) device_hash_flag = 1; - if (device_hash_flag) Kokkos::deep_copy(d_map_hash, h_map_hash); - - k_map_hash.h_view = h_map_hash; - k_map_hash.d_view = d_map_hash; + if (device_hash_flag) Kokkos::deep_copy(d_map_hash,h_map_hash); } } @@ -268,8 +276,8 @@ void AtomKokkos::map_delete() memoryKK->destroy_kokkos(k_map_array, map_array); map_array = nullptr; } else { - h_map_hash = host_hash_type(); - d_map_hash = hash_type(); + k_map_hash.h_view = host_hash_type(); + k_map_hash.d_view = hash_type(); } Atom::map_delete(); diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 2ecde3adeb..1992024ba0 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -560,12 +560,10 @@ struct dual_hash_type { host_hash_type h_view; template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value,hash_type> view() {return d_view;} + std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type> view() {return d_view;} template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value,host_hash_type> view() {return h_view;} + std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type> view() {return h_view;} }; From 01d7afdaa4299d692f31eb312697136779d6e7bb Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Mon, 12 Jul 2021 15:54:02 -0600 Subject: [PATCH 20/23] Small refactor --- src/KOKKOS/atom_kokkos.h | 17 +++++++- src/KOKKOS/atom_map_kokkos.cpp | 3 +- src/KOKKOS/fix_shake_kokkos.cpp | 60 ++++++++++------------------ src/KOKKOS/fix_shake_kokkos.h | 5 --- src/KOKKOS/kokkos_type.h | 6 ++- src/KOKKOS/neigh_bond_kokkos.cpp | 68 +++++++++++--------------------- src/KOKKOS/neigh_bond_kokkos.h | 6 --- 7 files changed, 67 insertions(+), 98 deletions(-) diff --git a/src/KOKKOS/atom_kokkos.h b/src/KOKKOS/atom_kokkos.h index 00b8e53578..6cf91a5ddc 100644 --- a/src/KOKKOS/atom_kokkos.h +++ b/src/KOKKOS/atom_kokkos.h @@ -78,9 +78,24 @@ class AtomKokkos : public Atom { DAT::tdual_int_scalar k_error_flag; dual_hash_type k_map_hash; + // map lookup function inlined for efficiency + // return -1 if no map defined + template KOKKOS_INLINE_FUNCTION - static int map_find_hash_kokkos(tagint global, dual_hash_type k_map_hash) + static int map_kokkos(tagint global, int map_style, DAT::tdual_int_1d k_map_array, dual_hash_type k_map_hash) + { + if (map_style == 1) + return k_map_array.view()(global); + else if (map_style == 2) + return AtomKokkos::map_find_hash_kokkos(global,k_map_hash); + else + return -1; + } + + template + KOKKOS_INLINE_FUNCTION + static int map_find_hash_kokkos(tagint global, dual_hash_type &k_map_hash) { int local = -1; auto d_map_hash = k_map_hash.view(); diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index d498974db2..9ca361e963 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -111,8 +111,9 @@ void AtomKokkos::map_init(int check) // use "view" template method to avoid unnecessary deep_copy - auto h_map_hash = k_map_hash.view(); + auto h_map_hash = k_map_hash.view(); // get type h_map_hash = decltype(h_map_hash)(map_nhash); + k_map_hash.view() = h_map_hash; } } diff --git a/src/KOKKOS/fix_shake_kokkos.cpp b/src/KOKKOS/fix_shake_kokkos.cpp index a50ec573b4..91d8458d21 100644 --- a/src/KOKKOS/fix_shake_kokkos.cpp +++ b/src/KOKKOS/fix_shake_kokkos.cpp @@ -252,8 +252,8 @@ void FixShakeKokkos::pre_neighbor() LAMMPS_LAMBDA(const int& i) { if (d_shake_flag[i]) { if (d_shake_flag[i] == 2) { - const int atom1 = map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); - const int atom2 = map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom1 = AtomKokkos::map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = AtomKokkos::map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1) { d_error_flag() = 1; } @@ -262,9 +262,9 @@ void FixShakeKokkos::pre_neighbor() d_list[nlist] = i; } } else if (d_shake_flag[i] % 2 == 1) { - const int atom1 = map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); - const int atom2 = map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); - const int atom3 = map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); + const int atom1 = AtomKokkos::map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = AtomKokkos::map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom3 = AtomKokkos::map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) d_error_flag() = 1; if (i <= atom1 && i <= atom2 && i <= atom3) { @@ -272,10 +272,10 @@ void FixShakeKokkos::pre_neighbor() d_list[nlist] = i; } } else { - const int atom1 = map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); - const int atom2 = map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); - const int atom3 = map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); - const int atom4 = map_kokkos(d_shake_atom(i,3),map_style,k_map_array,k_map_hash); + const int atom1 = AtomKokkos::map_kokkos(d_shake_atom(i,0),map_style,k_map_array,k_map_hash); + const int atom2 = AtomKokkos::map_kokkos(d_shake_atom(i,1),map_style,k_map_array,k_map_hash); + const int atom3 = AtomKokkos::map_kokkos(d_shake_atom(i,2),map_style,k_map_array,k_map_hash); + const int atom4 = AtomKokkos::map_kokkos(d_shake_atom(i,3),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) d_error_flag() = 1; if (i <= atom1 && i <= atom2 && i <= atom3 && i <= atom4) { @@ -599,8 +599,8 @@ void FixShakeKokkos::shake(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); - int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; // r01 = distance vec between atoms, with PBC @@ -710,9 +710,9 @@ void FixShakeKokkos::shake3(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); - int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); - int i2 = map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = AtomKokkos::map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; @@ -893,10 +893,10 @@ void FixShakeKokkos::shake4(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); - int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); - int i2 = map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); - int i3 = map_kokkos(d_shake_atom(m,3),map_style,k_map_array,k_map_hash); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = AtomKokkos::map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); + int i3 = AtomKokkos::map_kokkos(d_shake_atom(m,3),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; double bond3 = d_bond_distance[d_shake_type(m,2)]; @@ -1155,9 +1155,9 @@ void FixShakeKokkos::shake3angle(int m, EV_FLOAT& ev) const // local atom IDs and constraint distances - int i0 = map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); - int i1 = map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); - int i2 = map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); + int i0 = AtomKokkos::map_kokkos(d_shake_atom(m,0),map_style,k_map_array,k_map_hash); + int i1 = AtomKokkos::map_kokkos(d_shake_atom(m,1),map_style,k_map_array,k_map_hash); + int i2 = AtomKokkos::map_kokkos(d_shake_atom(m,2),map_style,k_map_array,k_map_hash); double bond1 = d_bond_distance[d_shake_type(m,0)]; double bond2 = d_bond_distance[d_shake_type(m,1)]; double bond12 = d_angle_distance[d_shake_type(m,2)]; @@ -1918,24 +1918,6 @@ void FixShakeKokkos::minimum_image_once(double *delta) const /* ---------------------------------------------------------------------- */ -// functions for global to local ID mapping -// map lookup function inlined for efficiency -// return -1 if no map defined - -template -KOKKOS_INLINE_FUNCTION -int FixShakeKokkos::map_kokkos(tagint global, int map_style, DAT::tdual_int_1d k_map_array, dual_hash_type k_map_hash) -{ - if (map_style == 1) - return k_map_array.view()(global); - else if (map_style == 2) - return AtomKokkos::map_find_hash_kokkos(global,k_map_hash); - else - return -1; -} - -/* ---------------------------------------------------------------------- */ - namespace LAMMPS_NS { template class FixShakeKokkos; #ifdef LMP_KOKKOS_GPU diff --git a/src/KOKKOS/fix_shake_kokkos.h b/src/KOKKOS/fix_shake_kokkos.h index 698298c65d..12e082a903 100644 --- a/src/KOKKOS/fix_shake_kokkos.h +++ b/src/KOKKOS/fix_shake_kokkos.h @@ -184,14 +184,9 @@ class FixShakeKokkos : public FixShake, public KokkosBase { int **shake_type_tmp; int map_style; - DAT::tdual_int_1d k_map_array; - dual_hash_type k_map_hash; - KOKKOS_INLINE_FUNCTION - static int map_kokkos(tagint, int, DAT::tdual_int_1d, dual_hash_type); - // copied from Domain KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 1992024ba0..1a52571260 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -560,10 +560,12 @@ struct dual_hash_type { host_hash_type h_view; template - std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type> view() {return d_view;} + KOKKOS_INLINE_FUNCTION + std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type> view() {printf("Yes device\n"); return d_view;} template - std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type> view() {return h_view;} + KOKKOS_INLINE_FUNCTION + std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type> view() {printf("Yes host\n"); return h_view;} }; diff --git a/src/KOKKOS/neigh_bond_kokkos.cpp b/src/KOKKOS/neigh_bond_kokkos.cpp index e3e24801ae..a4cb6e7d40 100644 --- a/src/KOKKOS/neigh_bond_kokkos.cpp +++ b/src/KOKKOS/neigh_bond_kokkos.cpp @@ -197,8 +197,6 @@ template void NeighBondKokkos::build_topology_kk() { atomKK->sync(execution_space, X_MASK | TAG_MASK); - int nall = atom->nlocal + atom->nghost; - int nmax = atom->nmax; nlocal = atom->nlocal; x = atomKK->k_x.view(); @@ -283,7 +281,7 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondBondAll, const int &i, int &nmissing) const { for (int m = 0; m < num_bond[i]; m++) { - int atom1 = map_kokkos(bond_atom(i,m)); + int atom1 = AtomKokkos::map_kokkos(bond_atom(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -371,7 +369,7 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondBondPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_bond[i]; m++) { if (bond_type(i,m) <= 0) continue; - int atom1 = map_kokkos(bond_atom(i,m)); + int atom1 = AtomKokkos::map_kokkos(bond_atom(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -483,9 +481,9 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondAngleAll, const int &i, int &nmissing) const { for (int m = 0; m < num_angle[i]; m++) { - int atom1 = map_kokkos(angle_atom1(i,m)); - int atom2 = map_kokkos(angle_atom2(i,m)); - int atom3 = map_kokkos(angle_atom3(i,m)); + int atom1 = AtomKokkos::map_kokkos(angle_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(angle_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(angle_atom3(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -578,9 +576,9 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondAnglePartial, const int &i, int &nmissing) const { for (int m = 0; m < num_angle[i]; m++) { if (angle_type(i,m) <= 0) continue; - int atom1 = map_kokkos(angle_atom1(i,m)); - int atom2 = map_kokkos(angle_atom2(i,m)); - int atom3 = map_kokkos(angle_atom3(i,m)); + int atom1 = AtomKokkos::map_kokkos(angle_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(angle_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(angle_atom3(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -710,10 +708,10 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondDihedralAll, const int &i, int &nmissing) const { for (int m = 0; m < num_dihedral[i]; m++) { - int atom1 = map_kokkos(dihedral_atom1(i,m)); - int atom2 = map_kokkos(dihedral_atom2(i,m)); - int atom3 = map_kokkos(dihedral_atom3(i,m)); - int atom4 = map_kokkos(dihedral_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(dihedral_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(dihedral_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(dihedral_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(dihedral_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -810,10 +808,10 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondDihedralPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_dihedral[i]; m++) { if (dihedral_type(i,m) <= 0) continue; - int atom1 = map_kokkos(dihedral_atom1(i,m)); - int atom2 = map_kokkos(dihedral_atom2(i,m)); - int atom3 = map_kokkos(dihedral_atom3(i,m)); - int atom4 = map_kokkos(dihedral_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(dihedral_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(dihedral_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(dihedral_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(dihedral_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -964,10 +962,10 @@ template KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondImproperAll, const int &i, int &nmissing) const { for (int m = 0; m < num_improper[i]; m++) { - int atom1 = map_kokkos(improper_atom1(i,m)); - int atom2 = map_kokkos(improper_atom2(i,m)); - int atom3 = map_kokkos(improper_atom3(i,m)); - int atom4 = map_kokkos(improper_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(improper_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(improper_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(improper_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(improper_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -1064,10 +1062,10 @@ KOKKOS_INLINE_FUNCTION void NeighBondKokkos::operator()(TagNeighBondImproperPartial, const int &i, int &nmissing) const { for (int m = 0; m < num_improper[i]; m++) { if (improper_type(i,m) <= 0) continue; - int atom1 = map_kokkos(improper_atom1(i,m)); - int atom2 = map_kokkos(improper_atom2(i,m)); - int atom3 = map_kokkos(improper_atom3(i,m)); - int atom4 = map_kokkos(improper_atom4(i,m)); + int atom1 = AtomKokkos::map_kokkos(improper_atom1(i,m),map_style,k_map_array,k_map_hash); + int atom2 = AtomKokkos::map_kokkos(improper_atom2(i,m),map_style,k_map_array,k_map_hash); + int atom3 = AtomKokkos::map_kokkos(improper_atom3(i,m),map_style,k_map_array,k_map_hash); + int atom4 = AtomKokkos::map_kokkos(improper_atom4(i,m),map_style,k_map_array,k_map_hash); if (atom1 == -1 || atom2 == -1 || atom3 == -1 || atom4 == -1) { nmissing++; if (lostbond == Thermo::ERROR) return; @@ -1191,24 +1189,6 @@ void NeighBondKokkos::minimum_image(X_FLOAT &dx, X_FLOAT &dy, X_FLOA /* ---------------------------------------------------------------------- */ -// functions for global to local ID mapping -// map lookup function inlined for efficiency -// return -1 if no map defined - -template -KOKKOS_INLINE_FUNCTION -int NeighBondKokkos::map_kokkos(tagint global) const -{ - if (map_style == 1) - return k_map_array.view()(global); - else if (map_style == 2) - return AtomKokkos::map_find_hash_kokkos(global,k_map_hash); - else - return -1; -} - -/* ---------------------------------------------------------------------- */ - template void NeighBondKokkos::update_class_variables() { diff --git a/src/KOKKOS/neigh_bond_kokkos.h b/src/KOKKOS/neigh_bond_kokkos.h index d60ad2f1aa..d2e5d0fc62 100644 --- a/src/KOKKOS/neigh_bond_kokkos.h +++ b/src/KOKKOS/neigh_bond_kokkos.h @@ -83,17 +83,11 @@ class NeighBondKokkos : protected Pointers { private: int map_style; - DAT::tdual_int_1d k_sametag; typename AT::t_int_1d d_sametag; - DAT::tdual_int_1d k_map_array; - dual_hash_type k_map_hash; - KOKKOS_INLINE_FUNCTION - int map_kokkos(tagint) const; - typename AT::t_int_2d v_bondlist; typename AT::t_int_2d v_anglelist; typename AT::t_int_2d v_dihedrallist; From ac07253ed6e2546d89a3557fb7751c68a0ac6425 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Tue, 13 Jul 2021 10:29:37 -0600 Subject: [PATCH 21/23] Fix GPU issues --- src/KOKKOS/atom_map_kokkos.cpp | 5 ++++- src/KOKKOS/kokkos_type.h | 8 ++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/KOKKOS/atom_map_kokkos.cpp b/src/KOKKOS/atom_map_kokkos.cpp index 9ca361e963..8d0d1243ac 100644 --- a/src/KOKKOS/atom_map_kokkos.cpp +++ b/src/KOKKOS/atom_map_kokkos.cpp @@ -260,7 +260,10 @@ void AtomKokkos::map_set() if (utils::strmatch(modify->fix[n]->style, "^shake")) if (modify->fix[n]->execution_space == Device) device_hash_flag = 1; - if (device_hash_flag) Kokkos::deep_copy(d_map_hash,h_map_hash); + if (device_hash_flag) { + Kokkos::deep_copy(d_map_hash,h_map_hash); + k_map_hash.view() = d_map_hash; + } } } diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 1a52571260..9de2a80375 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -561,11 +561,11 @@ struct dual_hash_type { template KOKKOS_INLINE_FUNCTION - std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type> view() {printf("Yes device\n"); return d_view;} + std::enable_if_t<(std::is_same::value || Kokkos::SpaceAccessibility::accessible),hash_type&> view() {return d_view;} template KOKKOS_INLINE_FUNCTION - std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type> view() {printf("Yes host\n"); return h_view;} + std::enable_if_t::value || Kokkos::SpaceAccessibility::accessible),host_hash_type&> view() {return h_view;} }; @@ -855,8 +855,6 @@ typedef tdual_neighbors_2d::t_dev_um t_neighbors_2d_um; typedef tdual_neighbors_2d::t_dev_const_um t_neighbors_2d_const_um; typedef tdual_neighbors_2d::t_dev_const_randomread t_neighbors_2d_randomread; -typedef hash_type t_hash; - }; #ifdef LMP_KOKKOS_GPU @@ -1118,8 +1116,6 @@ typedef tdual_neighbors_2d::t_host_um t_neighbors_2d_um; typedef tdual_neighbors_2d::t_host_const_um t_neighbors_2d_const_um; typedef tdual_neighbors_2d::t_host_const_randomread t_neighbors_2d_randomread; -typedef host_hash_type t_hash; - }; #endif //default LAMMPS Types From fc5f91b44c70a1d2cdf1b79161afaf21663f480e Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Tue, 13 Jul 2021 10:34:50 -0600 Subject: [PATCH 22/23] Revert accidental change --- bench/in.rhodo | 1 - 1 file changed, 1 deletion(-) diff --git a/bench/in.rhodo b/bench/in.rhodo index 1607e3ddd1..bd7e3df7f5 100644 --- a/bench/in.rhodo +++ b/bench/in.rhodo @@ -4,7 +4,6 @@ units real neigh_modify delay 5 every 1 atom_style full -atom_modify map array bond_style harmonic angle_style charmm dihedral_style charmm From 3895ae194d1a3e3e60037d15251916bb74884e1f Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Tue, 13 Jul 2021 12:17:24 -0600 Subject: [PATCH 23/23] Make function static --- src/KOKKOS/kokkos_type.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 775462f1ff..4a764a38af 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -45,7 +45,7 @@ enum{FULL=1u,HALFTHREAD=2u,HALF=4u}; static constexpr LAMMPS_NS::bigint LMP_KOKKOS_AV_DELTA = 10; namespace Kokkos { - auto NoInit = [](std::string const& label) { + static auto NoInit = [](std::string const& label) { return Kokkos::view_alloc(Kokkos::WithoutInitializing, label); }; }