From 0c6707bf0c77b980507e3944de9844928e643303 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Tue, 5 Oct 2021 14:49:30 -0600 Subject: [PATCH 1/5] Fix compile issue with bond_class2_kokkos and UVM-enabled --- src/KOKKOS/bond_class2_kokkos.cpp | 20 ++++++++++---------- src/KOKKOS/bond_class2_kokkos.h | 9 +++++---- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/KOKKOS/bond_class2_kokkos.cpp b/src/KOKKOS/bond_class2_kokkos.cpp index e538c8e045..b4e87b4119 100644 --- a/src/KOKKOS/bond_class2_kokkos.cpp +++ b/src/KOKKOS/bond_class2_kokkos.cpp @@ -68,14 +68,14 @@ void BondClass2Kokkos::compute(int eflag_in, int vflag_in) //if(k_eatom.extent(0)destroy_kokkos(k_eatom,eatom); memoryKK->create_kokkos(k_eatom,eatom,maxeatom,"improper:eatom"); - d_eatom = k_eatom.template view(); + d_eatom = k_eatom.template view(); //} } if (vflag_atom) { //if(k_vatom.extent(0)destroy_kokkos(k_vatom,vatom); memoryKK->create_kokkos(k_vatom,vatom,maxvatom,"improper:vatom"); - d_vatom = k_vatom.template view(); + d_vatom = k_vatom.template view(); //} } @@ -210,10 +210,10 @@ void BondClass2Kokkos::coeff(int narg, char **arg) BondClass2::coeff(narg, arg); int n = atom->nbondtypes; - Kokkos::DualView k_k2("BondClass2::k2",n+1); - Kokkos::DualView k_k3("BondClass2::k3",n+1); - Kokkos::DualView k_k4("BondClass2::k4",n+1); - Kokkos::DualView k_r0("BondClass2::r0",n+1); + typename AT::tdual_ffloat_1d k_k2("BondClass2::k2",n+1); + typename AT::tdual_ffloat_1d k_k3("BondClass2::k3",n+1); + typename AT::tdual_ffloat_1d k_k4("BondClass2::k4",n+1); + typename AT::tdual_ffloat_1d k_r0("BondClass2::r0",n+1); d_k2 = k_k2.template view(); d_k3 = k_k3.template view(); @@ -247,10 +247,10 @@ void BondClass2Kokkos::read_restart(FILE *fp) BondClass2::read_restart(fp); int n = atom->nbondtypes; - Kokkos::DualView k_k2("BondClass2::k2",n+1); - Kokkos::DualView k_k3("BondClass2::k3",n+1); - Kokkos::DualView k_k4("BondClass2::k4",n+1); - Kokkos::DualView k_r0("BondClass2::r0",n+1); + typename AT::tdual_ffloat_1d k_k2("BondClass2::k2",n+1); + typename AT::tdual_ffloat_1d k_k3("BondClass2::k3",n+1); + typename AT::tdual_ffloat_1d k_k4("BondClass2::k4",n+1); + typename AT::tdual_ffloat_1d k_r0("BondClass2::r0",n+1); d_k2 = k_k2.template view(); d_k3 = k_k3.template view(); diff --git a/src/KOKKOS/bond_class2_kokkos.h b/src/KOKKOS/bond_class2_kokkos.h index 52136030aa..529046845f 100644 --- a/src/KOKKOS/bond_class2_kokkos.h +++ b/src/KOKKOS/bond_class2_kokkos.h @@ -67,10 +67,11 @@ class BondClass2Kokkos : public BondClass2 { typename Kokkos::View::value,Kokkos::MemoryTraits > f; typename AT::t_int_2d bondlist; - Kokkos::DualView k_eatom; - Kokkos::DualView k_vatom; - Kokkos::View::value,Kokkos::MemoryTraits > d_eatom; - Kokkos::View::value,Kokkos::MemoryTraits > d_vatom; + typedef typename KKDevice::value KKDeviceType; + Kokkos::DualView k_eatom; + Kokkos::DualView k_vatom; + Kokkos::View > d_eatom; + Kokkos::View > d_vatom; int nlocal,newton_bond; int eflag,vflag; From bbfb2d2712e66793438098174ce58f8e19084e9b Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Wed, 6 Oct 2021 08:27:25 -0700 Subject: [PATCH 2/5] Add missing code to modify_kokkos --- src/KOKKOS/modify_kokkos.cpp | 102 ++++++++++++++++++++++++++++++++--- src/KOKKOS/modify_kokkos.h | 3 ++ 2 files changed, 97 insertions(+), 8 deletions(-) diff --git a/src/KOKKOS/modify_kokkos.cpp b/src/KOKKOS/modify_kokkos.cpp index 868de3e11e..08bbfa762f 100644 --- a/src/KOKKOS/modify_kokkos.cpp +++ b/src/KOKKOS/modify_kokkos.cpp @@ -38,7 +38,20 @@ ModifyKokkos::ModifyKokkos(LAMMPS *lmp) : Modify(lmp) void ModifyKokkos::setup(int vflag) { // compute setup needs to come before fix setup - // b/c NH fixes need use DOF of temperature computes + // b/c NH fixes need DOF of temperature computes + // fix group setup() is special case since populates a dynamic group + // needs to be done before temperature compute setup + + for (int i = 0; i < nfix; i++) { + if (strcmp(fix[i]->style,"GROUP") == 0) { + atomKK->sync(fix[i]->execution_space,fix[i]->datamask_read); + int prev_auto_sync = lmp->kokkos->auto_sync; + if (!fix[i]->kokkosable) lmp->kokkos->auto_sync = 1; + fix[i]->setup(vflag); + lmp->kokkos->auto_sync = prev_auto_sync; + atomKK->modified(fix[i]->execution_space,fix[i]->datamask_modify); + } + } for (int i = 0; i < ncompute; i++) compute[i]->setup(); @@ -124,6 +137,37 @@ void ModifyKokkos::setup_pre_neighbor() } } +/* ---------------------------------------------------------------------- + setup post_neighbor call, only for fixes that define post_neighbor + called from Verlet, RESPA +------------------------------------------------------------------------- */ + +void ModifyKokkos::setup_post_neighbor() +{ + if (update->whichflag == 1) + for (int i = 0; i < n_post_neighbor; i++) { + atomKK->sync(fix[list_post_neighbor[i]]->execution_space, + fix[list_post_neighbor[i]]->datamask_read); + int prev_auto_sync = lmp->kokkos->auto_sync; + if (!fix[list_post_neighbor[i]]->kokkosable) lmp->kokkos->auto_sync = 1; + fix[list_post_neighbor[i]]->setup_post_neighbor(); + lmp->kokkos->auto_sync = prev_auto_sync; + atomKK->modified(fix[list_post_neighbor[i]]->execution_space, + fix[list_post_neighbor[i]]->datamask_modify); + } + else if (update->whichflag == 2) + for (int i = 0; i < n_min_post_neighbor; i++) { + atomKK->sync(fix[list_min_post_neighbor[i]]->execution_space, + fix[list_min_post_neighbor[i]]->datamask_read); + int prev_auto_sync = lmp->kokkos->auto_sync; + if (!fix[list_min_post_neighbor[i]]->kokkosable) lmp->kokkos->auto_sync = 1; + fix[list_min_post_neighbor[i]]->setup_post_neighbor(); + lmp->kokkos->auto_sync = prev_auto_sync; + atomKK->modified(fix[list_min_post_neighbor[i]]->execution_space, + fix[list_min_post_neighbor[i]]->datamask_modify); + } +} + /* ---------------------------------------------------------------------- setup pre_force call, only for fixes that define pre_force called from Verlet, RESPA, Min @@ -258,6 +302,24 @@ void ModifyKokkos::pre_neighbor() } } +/* ---------------------------------------------------------------------- + post_neighbor call, only for relevant fixes +------------------------------------------------------------------------- */ + +void ModifyKokkos::post_neighbor() +{ + for (int i = 0; i < n_post_neighbor; i++) { + atomKK->sync(fix[list_post_neighbor[i]]->execution_space, + fix[list_post_neighbor[i]]->datamask_read); + int prev_auto_sync = lmp->kokkos->auto_sync; + if (!fix[list_post_neighbor[i]]->kokkosable) lmp->kokkos->auto_sync = 1; + fix[list_post_neighbor[i]]->post_neighbor(); + lmp->kokkos->auto_sync = prev_auto_sync; + atomKK->modified(fix[list_post_neighbor[i]]->execution_space, + fix[list_post_neighbor[i]]->datamask_modify); + } +} + /* ---------------------------------------------------------------------- pre_force call, only for relevant fixes ------------------------------------------------------------------------- */ @@ -420,6 +482,12 @@ void ModifyKokkos::post_run() atomKK->modified(fix[i]->execution_space, fix[i]->datamask_modify); } + + // must reset this to its default value, since computes may be added + // or removed between runs and with this change we will redirect any + // calls to addstep_compute() to addstep_compute_all() instead. + n_timeflag = -1; + } /* ---------------------------------------------------------------------- @@ -567,6 +635,24 @@ void ModifyKokkos::min_pre_neighbor() } } +/* ---------------------------------------------------------------------- + minimizer post-neighbor call, only for relevant fixes +------------------------------------------------------------------------- */ + +void ModifyKokkos::min_post_neighbor() +{ + for (int i = 0; i < n_min_post_neighbor; i++) { + atomKK->sync(fix[list_min_post_neighbor[i]]->execution_space, + fix[list_min_post_neighbor[i]]->datamask_read); + int prev_auto_sync = lmp->kokkos->auto_sync; + if (!fix[list_min_post_neighbor[i]]->kokkosable) lmp->kokkos->auto_sync = 1; + fix[list_min_post_neighbor[i]]->min_post_neighbor(); + lmp->kokkos->auto_sync = prev_auto_sync; + atomKK->modified(fix[list_min_post_neighbor[i]]->execution_space, + fix[list_min_post_neighbor[i]]->datamask_modify); + } +} + /* ---------------------------------------------------------------------- minimizer pre-force call, only for relevant fixes ------------------------------------------------------------------------- */ @@ -646,7 +732,7 @@ double ModifyKokkos::min_energy(double *fextra) } /* ---------------------------------------------------------------------- - store current state of extra dof, only for relevant fixes + store current state of extra minimizer dof, only for relevant fixes ------------------------------------------------------------------------- */ void ModifyKokkos::min_store() @@ -664,7 +750,7 @@ void ModifyKokkos::min_store() } /* ---------------------------------------------------------------------- - mange state of extra dof on a stack, only for relevant fixes + manage state of extra minimizer dof on a stack, only for relevant fixes ------------------------------------------------------------------------- */ void ModifyKokkos::min_clearstore() @@ -710,7 +796,7 @@ void ModifyKokkos::min_popstore() } /* ---------------------------------------------------------------------- - displace extra dof along vector hextra, only for relevant fixes + displace extra minimizer dof along vector hextra, only for relevant fixes ------------------------------------------------------------------------- */ void ModifyKokkos::min_step(double alpha, double *hextra) @@ -755,7 +841,7 @@ double ModifyKokkos::max_alpha(double *hextra) } /* ---------------------------------------------------------------------- - extract extra dof for minimization, only for relevant fixes + extract extra minimizer dof, only for relevant fixes ------------------------------------------------------------------------- */ int ModifyKokkos::min_dof() @@ -775,7 +861,7 @@ int ModifyKokkos::min_dof() } /* ---------------------------------------------------------------------- - reset reference state of fix, only for relevant fixes + reset minimizer reference state of fix, only for relevant fixes ------------------------------------------------------------------------- */ int ModifyKokkos::min_reset_ref() @@ -788,10 +874,10 @@ int ModifyKokkos::min_reset_ref() int prev_auto_sync = lmp->kokkos->auto_sync; if (!fix[list_min_energy[i]]->kokkosable) lmp->kokkos->auto_sync = 1; itmp = fix[list_min_energy[i]]->min_reset_ref(); - lmp->kokkos->auto_sync = prev_auto_sync; if (itmp) itmpall = 1; + lmp->kokkos->auto_sync = prev_auto_sync; atomKK->modified(fix[list_min_energy[i]]->execution_space, fix[list_min_energy[i]]->datamask_modify); } - return itmpall; +return itmpall; } diff --git a/src/KOKKOS/modify_kokkos.h b/src/KOKKOS/modify_kokkos.h index 9b75b7c607..25911743b2 100644 --- a/src/KOKKOS/modify_kokkos.h +++ b/src/KOKKOS/modify_kokkos.h @@ -26,6 +26,7 @@ class ModifyKokkos : public Modify { void setup(int); void setup_pre_exchange(); void setup_pre_neighbor(); + void setup_post_neighbor(); void setup_pre_force(int); void setup_pre_reverse(int, int); void initial_integrate(int); @@ -33,6 +34,7 @@ class ModifyKokkos : public Modify { void pre_decide(); void pre_exchange(); void pre_neighbor(); + void post_neighbor(); void pre_force(int); void pre_reverse(int,int); void post_force(int); @@ -52,6 +54,7 @@ class ModifyKokkos : public Modify { void min_pre_exchange(); void min_pre_neighbor(); + void min_post_neighbor(); void min_pre_force(int); void min_pre_reverse(int,int); void min_post_force(int); From d9288ae7e9bf47da56d549ac228723cf8e2265b6 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Wed, 6 Oct 2021 08:33:02 -0700 Subject: [PATCH 3/5] whitespace --- src/KOKKOS/modify_kokkos.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KOKKOS/modify_kokkos.cpp b/src/KOKKOS/modify_kokkos.cpp index 08bbfa762f..5545351f53 100644 --- a/src/KOKKOS/modify_kokkos.cpp +++ b/src/KOKKOS/modify_kokkos.cpp @@ -879,5 +879,5 @@ int ModifyKokkos::min_reset_ref() atomKK->modified(fix[list_min_energy[i]]->execution_space, fix[list_min_energy[i]]->datamask_modify); } -return itmpall; + return itmpall; } From 3d86a0f5f661e1d1ac7c466c5232b088cebe64e4 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Wed, 6 Oct 2021 11:15:34 -0700 Subject: [PATCH 4/5] Fix two bugs in compute_orientorder_atom_kokkos --- src/KOKKOS/compute_orientorder_atom_kokkos.cpp | 14 ++++++++++---- src/MAKE/MACHINES/Makefile.summit_kokkos | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/KOKKOS/compute_orientorder_atom_kokkos.cpp b/src/KOKKOS/compute_orientorder_atom_kokkos.cpp index 131e9933e8..f7ec879e3c 100644 --- a/src/KOKKOS/compute_orientorder_atom_kokkos.cpp +++ b/src/KOKKOS/compute_orientorder_atom_kokkos.cpp @@ -173,8 +173,6 @@ void ComputeOrientOrderAtomKokkos::compute_peratom() x = atomKK->k_x.view(); mask = atomKK->k_mask.view(); - Kokkos::deep_copy(d_qnm,{0.0,0.0}); - int vector_length_default = 1; int team_size_default = 1; if (!host_flag) @@ -185,6 +183,8 @@ void ComputeOrientOrderAtomKokkos::compute_peratom() if (chunk_size > inum - chunk_offset) chunk_size = inum - chunk_offset; + Kokkos::deep_copy(d_qnm,{0.0,0.0}); + //Neigh { int vector_length = vector_length_default; @@ -286,7 +286,7 @@ void ComputeOrientOrderAtomKokkos::operator() (TagComputeOrientOrder const int i = d_ilist[ii + chunk_offset]; const int ncount = d_ncount(ii); - // if not nnn neighbors, order parameter = 0; + // if not nnn neighbors, order parameter = 0 if ((ncount == 0) || (ncount < nnn)) { for (int jj = 0; jj < ncol; jj++) @@ -316,7 +316,7 @@ void ComputeOrientOrderAtomKokkos::operator() (TagComputeOrientOrder const int ncount = d_ncount(ii); if (jj >= ncount) return; - // if not nnn neighbors, order parameter = 0; + // if not nnn neighbors, order parameter = 0 if ((ncount == 0) || (ncount < nnn)) return; @@ -328,6 +328,12 @@ template KOKKOS_INLINE_FUNCTION void ComputeOrientOrderAtomKokkos::operator() (TagComputeOrientOrderAtomBOOP2,const int& ii) const { const int ncount = d_ncount(ii); + + // if not nnn neighbors, order parameter = 0 + + if ((ncount == 0) || (ncount < nnn)) + return; + calc_boop2(ncount, ii); } diff --git a/src/MAKE/MACHINES/Makefile.summit_kokkos b/src/MAKE/MACHINES/Makefile.summit_kokkos index 95ee7e39a8..4bd74a89c0 100644 --- a/src/MAKE/MACHINES/Makefile.summit_kokkos +++ b/src/MAKE/MACHINES/Makefile.summit_kokkos @@ -9,7 +9,7 @@ SHELL = /bin/sh KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) CC = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper -CCFLAGS = -g -O3 +CCFLAGS = -g -O3 -DLAMMPS_BIGBIG SHFLAGS = -fPIC DEPFLAGS = -M @@ -68,9 +68,9 @@ FFT_LIB = -lcufft # PATH = path(s) for JPEG library and/or PNG library # LIB = name(s) of JPEG library and/or PNG library -JPG_INC = +JPG_INC = -DLAMMPS_PNG -I~/libpng-1.6.37/include JPG_PATH = -JPG_LIB = +JPG_LIB = -L~/libpng-1.6.37/lib -lpng # --------------------------------------------------------------------- # build rules and dependencies From a539c317b3297089c26c93917cf9c0af6a458252 Mon Sep 17 00:00:00 2001 From: Stan Moore Date: Wed, 6 Oct 2021 11:43:40 -0700 Subject: [PATCH 5/5] Revert changes to makefile --- src/MAKE/MACHINES/Makefile.summit_kokkos | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/MAKE/MACHINES/Makefile.summit_kokkos b/src/MAKE/MACHINES/Makefile.summit_kokkos index 4bd74a89c0..95ee7e39a8 100644 --- a/src/MAKE/MACHINES/Makefile.summit_kokkos +++ b/src/MAKE/MACHINES/Makefile.summit_kokkos @@ -9,7 +9,7 @@ SHELL = /bin/sh KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) CC = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper -CCFLAGS = -g -O3 -DLAMMPS_BIGBIG +CCFLAGS = -g -O3 SHFLAGS = -fPIC DEPFLAGS = -M @@ -68,9 +68,9 @@ FFT_LIB = -lcufft # PATH = path(s) for JPEG library and/or PNG library # LIB = name(s) of JPEG library and/or PNG library -JPG_INC = -DLAMMPS_PNG -I~/libpng-1.6.37/include +JPG_INC = JPG_PATH = -JPG_LIB = -L~/libpng-1.6.37/lib -lpng +JPG_LIB = # --------------------------------------------------------------------- # build rules and dependencies