From fa386c0e9652b38b5cbe2ec707229371024ece58 Mon Sep 17 00:00:00 2001 From: Joel Clemmer Date: Tue, 4 May 2021 17:12:17 -0600 Subject: [PATCH] Changing how multi/cutoff works, changing collection indexing in user args --- src/GRANULAR/pair_gran_hooke_history.cpp | 2 +- src/comm.cpp | 22 +++++--- src/comm.h | 7 +-- src/comm_brick.cpp | 71 +++++++++--------------- src/comm_tiled.cpp | 67 +++++++++------------- src/neighbor.cpp | 4 ++ src/nstencil_full_multi_2d.cpp | 5 +- src/nstencil_full_multi_3d.cpp | 5 +- src/nstencil_half_multi_2d.cpp | 5 +- src/nstencil_half_multi_2d_tri.cpp | 5 +- src/nstencil_half_multi_3d.cpp | 5 +- src/nstencil_half_multi_3d_tri.cpp | 5 +- 12 files changed, 100 insertions(+), 103 deletions(-) diff --git a/src/GRANULAR/pair_gran_hooke_history.cpp b/src/GRANULAR/pair_gran_hooke_history.cpp index 510d0713b1..95cfaad3e0 100644 --- a/src/GRANULAR/pair_gran_hooke_history.cpp +++ b/src/GRANULAR/pair_gran_hooke_history.cpp @@ -824,4 +824,4 @@ double PairGranHookeHistory::radii2cut(double r1, double r2) { double cut = r1+r2; return cut; -} \ No newline at end of file +} diff --git a/src/comm.cpp b/src/comm.cpp index dc6d75dd79..ca09af2238 100644 --- a/src/comm.cpp +++ b/src/comm.cpp @@ -35,7 +35,6 @@ #include "update.h" #include -#include #ifdef _OPENMP #include #endif @@ -58,9 +57,8 @@ Comm::Comm(LAMMPS *lmp) : Pointers(lmp) bordergroup = 0; cutghostuser = 0.0; cutusermulti = nullptr; - cutusermultiflag = 0; cutusermultiold = nullptr; - ncollections_prior = 0; + ncollections = 0; ghost_velocity = 0; user_procgrid[0] = user_procgrid[1] = user_procgrid[2] = 0; @@ -149,6 +147,7 @@ void Comm::copy_arrays(Comm *oldcomm) memcpy(zsplit,oldcomm->zsplit,(procgrid[2]+1)*sizeof(double)); } + ncollections = oldcomm->ncollections; if (oldcomm->cutusermulti) { memory->create(cutusermulti,ncollections,"comm:cutusermulti"); memcpy(cutusermulti,oldcomm->cutusermulti,ncollections); @@ -354,14 +353,23 @@ void Comm::modify_params(int narg, char **arg) error->all(FLERR, "Cannot set cutoff/multi before simulation box is defined"); - // save arguments so they can be parsed in comm->setup() - // ncollections can be changed by neigh_modify commands + // Check if # of collections has changed, if so erase any previously defined cutoffs + // Neighbor will reset ncollections if collections are redefined + if (ncollections_cutoff != neighbor->ncollections) { + ncollections_cutoff = neighbor->ncollections; + if (cutusermulti) memory->destroy(cutusermulti); + memory->create(cutusermulti,ncollections_cutoff,"comm:cutusermulti"); + for (i=0; i < ncollections_cutoff; ++i) + cutusermulti[i] = -1.0; + } + utils::bounds(FLERR,arg[iarg+1],1,ncollections_cutoff+1,nlo,nhi,error); cut = utils::numeric(FLERR,arg[iarg+2],false,lmp); cutghostuser = MAX(cutghostuser,cut); - cutusermultiflag = 1; if (cut < 0.0) error->all(FLERR,"Invalid cutoff in comm_modify command"); - usermultiargs.emplace_back(arg[iarg+1], cut); + // collections use 1-based indexing externally and 0-based indexing internally + for (i=nlo; i<=nhi; ++i) + cutusermulti[i-1] = cut; iarg += 3; } else if (strcmp(arg[iarg],"cutoff/multi/old") == 0) { int i,nlo,nhi; diff --git a/src/comm.h b/src/comm.h index 418983b6e6..5d3d0b3d53 100644 --- a/src/comm.h +++ b/src/comm.h @@ -33,10 +33,9 @@ class Comm : protected Pointers { double cutghost[3]; // cutoffs used for acquiring ghost atoms double cutghostuser; // user-specified ghost cutoff (mode == 0) double *cutusermulti; // per collection user ghost cutoff (mode == 1) - int cutusermultiflag; - std::vector> usermultiargs; - // collection args for custom ghost cutoffs double *cutusermultiold; // per type user ghost cutoff (mode == 2) + int ncollections; // # of collections known by comm, used to test if # has changed + int ncollections_cutoff; // # of collections stored b cutoff/multi int recv_from_partition; // recv proc layout from this partition int send_to_partition; // send my proc layout to this partition // -1 if no recv or send @@ -157,8 +156,6 @@ class Comm : protected Pointers { int coregrid[3]; // 3d grid of cores within a node int user_coregrid[3]; // user request for cores in each dim int multi_reduce; // 1 if multi cutoff is intra-collection cutoff - int ncollections; // number of collection cutoffs defined for multi - int ncollections_prior; // value of ncollections at last setup void init_exchange(); int rendezvous_irregular(int, char *, int, int, int *, diff --git a/src/comm_brick.cpp b/src/comm_brick.cpp index b6306eb042..184babccd2 100644 --- a/src/comm_brick.cpp +++ b/src/comm_brick.cpp @@ -141,19 +141,30 @@ void CommBrick::init() // memory for multi style communication // allocate in setup - if (mode == Comm::MULTI && multilo == nullptr) { - ncollections = neighbor->ncollections; - allocate_multi(maxswap); - memory->create(cutghostmulti,ncollections,3,"comm:cutghostmulti"); - if (cutusermultiflag) { - memory->grow(cutusermulti,ncollections,"comm:cutusermulti"); - for (int i = ncollections_prior; i < ncollections; i++) - cutusermulti[i] = -1.0; - } + if (mode == Comm::MULTI) { + // If inconsitent # of collections, destroy any preexisting arrays (may be missized) + if (ncollections != neighbor->ncollections) { + ncollections = neighbor->ncollections; + if (multilo != nullptr) { + free_multi(); + memory->destroy(cutghostmulti); + } + } - ncollections_prior = ncollections; + // delete any old user cutoffs if # of collections chanaged + if (cutusermulti && ncollections != ncollections_cutoff) { + if(me == 0) error->warning(FLERR, "cutoff/multi settings discarded, must be defined" + " after customizing collections in neigh_modify"); + memory->destroy(cutusermulti); + cutusermulti = nullptr; + } + + if (multilo == nullptr) { + allocate_multi(maxswap); + memory->create(cutghostmulti,ncollections,3,"comm:cutghostmulti"); + } } - if ((mode == Comm::SINGLE or mode == Comm::MULTIOLD) && multilo) { + if ((mode == Comm::SINGLE || mode == Comm::MULTIOLD) && multilo) { free_multi(); memory->destroy(cutghostmulti); } @@ -164,7 +175,7 @@ void CommBrick::init() allocate_multiold(maxswap); memory->create(cutghostmultiold,atom->ntypes+1,3,"comm:cutghostmultiold"); } - if ((mode == Comm::SINGLE or mode == Comm::MULTI) && multioldlo) { + if ((mode == Comm::SINGLE || mode == Comm::MULTI) && multioldlo) { free_multiold(); memory->destroy(cutghostmultiold); } @@ -201,39 +212,11 @@ void CommBrick::setup() "will be generated. Atoms may get lost."); if (mode == Comm::MULTI) { - // build initial collection array - neighbor->build_collection(0); - ncollections = neighbor->ncollections; - - // reallocate memory for multi-style communication at setup if ncollections change - if (ncollections_prior != ncollections) { - if (multilo) free_multi(); - if (cutghostmulti) memory->destroy(cutghostmulti); - - allocate_multi(maxswap); - memory->create(cutghostmulti,ncollections,3,"comm:cutghostmulti"); - if (cutusermultiflag) { - memory->grow(cutusermulti,ncollections,"comm:cutusermulti"); - for (i = ncollections_prior; i < ncollections; i++) - cutusermulti[i] = -1.0; - } - - ncollections_prior = ncollections; - } - - // parse any cutoff/multi commands - int nhi, nlo; - for (auto it = usermultiargs.begin(); it != usermultiargs.end(); it ++) { - utils::bounds(FLERR,it->first,0,ncollections,nlo,nhi,error); - if (nhi >= ncollections) - error->all(FLERR, "Unused collection id in comm_modify cutoff/multi command"); - for (j = nlo; j<=nhi; ++j) - cutusermulti[j] = it->second; - } - usermultiargs.clear(); - - double **cutcollectionsq = neighbor->cutcollectionsq; + + // build collection array for atom exchange + neighbor->build_collection(0); + // If using multi/reduce, communicate particles a distance equal // to the max cutoff with equally sized or smaller collections // If not, communicate the maximum cutoff of the entire collection diff --git a/src/comm_tiled.cpp b/src/comm_tiled.cpp index da3bb3a65f..277228522b 100644 --- a/src/comm_tiled.cpp +++ b/src/comm_tiled.cpp @@ -104,12 +104,8 @@ void CommTiled::init_buffers() cutghostmultiold = nullptr; sendbox_multi = nullptr; sendbox_multiold = nullptr; - - // initialize ncollections so grow_swap_send_multi() will not - // construct arrays in init() but will wait for setup() - ncollections = 0; - ncollections_prior = 0; + // Note this may skip growing multi arrays, will call again in init() maxswap = 6; allocate_swap(maxswap); } @@ -125,11 +121,32 @@ void CommTiled::init() nswap = 2*domain->dimension; + memory->destroy(cutghostmulti); + if (mode == Comm::MULTI) { + // If inconsitent # of collections, destroy any preexisting arrays (may be missized) + if (ncollections != neighbor->ncollections) { + ncollections = neighbor->ncollections; + } + + // delete any old user cutoffs if # of collections chanaged + if (cutusermulti && ncollections != ncollections_cutoff) { + if(me == 0) error->warning(FLERR, "cutoff/multi settings discarded, must be defined" + " after customizing collections in neigh_modify"); + memory->destroy(cutusermulti); + cutusermulti = nullptr; + } + + // grow sendbox_multi now that ncollections is known + for (int i = 0; i < maxswap; i ++) + grow_swap_send_multi(i,DELTA_PROCS); + + memory->create(cutghostmulti,ncollections,3,"comm:cutghostmulti"); + } + memory->destroy(cutghostmultiold); if (mode == Comm::MULTIOLD) memory->create(cutghostmultiold,atom->ntypes+1,3,"comm:cutghostmultiold"); - int bufextra_old = bufextra; init_exchange(); if (bufextra > bufextra_old) grow_send(maxsend+bufextra,2); @@ -186,44 +203,14 @@ void CommTiled::setup() // check that cutoff < any periodic box length if (mode == Comm::MULTI) { - // build collection from scratch as it is needed for atom exchange - neighbor->build_collection(0); - ncollections = neighbor->ncollections; - - // allocate memory for multi-style communication at setup as ncollections can change - if (ncollections_prior != ncollections) { - memory->destroy(cutghostmulti); - if (mode == Comm::MULTI) - memory->create(cutghostmulti,ncollections,3,"comm:cutghostmulti"); - - for (i = 0; i < maxswap; i ++) - grow_swap_send_multi(i,DELTA_PROCS); - if (cutusermultiflag) { - memory->grow(cutusermulti,ncollections,"comm:cutusermulti"); - for (i = ncollections_prior; i < ncollections; i++) - cutusermulti[i] = -1.0; - } - - ncollections_prior = ncollections; - } - - // parse any cutoff/multi commands - int nhi, nlo; - for (auto it = usermultiargs.begin(); it != usermultiargs.end(); it ++) { - utils::bounds(FLERR,it->first,0,ncollections,nlo,nhi,error); - if (nhi >= ncollections) - error->all(FLERR, "Unused collection id in comm_modify cutoff/multi command"); - - for (j = nlo; j <= nhi; ++j) - cutusermulti[j] = it->second; - } - usermultiargs.clear(); - double **cutcollectionsq = neighbor->cutcollectionsq; + + // build collection array for atom exchange + neighbor->build_collection(0); + // If using multi/reduce, communicate particles a distance equal // to the max cutoff with equally sized or smaller collections // If not, communicate the maximum cutoff of the entire collection - for (i = 0; i < ncollections; i++) { if (cutusermulti) { cutghostmulti[i][0] = cutusermulti[i]; diff --git a/src/neighbor.cpp b/src/neighbor.cpp index 5a4c574ce9..ab63a6874b 100644 --- a/src/neighbor.cpp +++ b/src/neighbor.cpp @@ -2514,6 +2514,8 @@ void Neighbor::modify_params(int narg, char **arg) int ntypes = atom->ntypes; int n, nlo, nhi, i, j; + // Invalidate old user cutoffs + comm->ncollections_cutoff = 0; interval_collection_flag = 1; custom_collection_flag = 1; memory->grow(collection2cut,ncollections,"neigh:collection2cut"); @@ -2546,6 +2548,8 @@ void Neighbor::modify_params(int narg, char **arg) int ntypes = atom->ntypes; int n, nlo, nhi, i, j, k; + // Invalidate old user cutoffs + comm->ncollections_cutoff = 0; interval_collection_flag = 0; custom_collection_flag = 1; if (not type2collection) diff --git a/src/nstencil_full_multi_2d.cpp b/src/nstencil_full_multi_2d.cpp index b27e3c2a9c..c3b0472c26 100644 --- a/src/nstencil_full_multi_2d.cpp +++ b/src/nstencil_full_multi_2d.cpp @@ -56,7 +56,10 @@ void NStencilFullMulti2d::create() for (icollection = 0; icollection < n; icollection++) { for (jcollection = 0; jcollection < n; jcollection++) { - if (flag_skip_multi[icollection][jcollection]) continue; + if (flag_skip_multi[icollection][jcollection]) { + nstencil_multi[icollection][jcollection] = 0; + continue; + } ns = 0; diff --git a/src/nstencil_full_multi_3d.cpp b/src/nstencil_full_multi_3d.cpp index 3d97bc10ad..194afa408c 100644 --- a/src/nstencil_full_multi_3d.cpp +++ b/src/nstencil_full_multi_3d.cpp @@ -57,7 +57,10 @@ void NStencilFullMulti3d::create() for (icollection = 0; icollection < n; icollection++) { for (jcollection = 0; jcollection < n; jcollection++) { - if (flag_skip_multi[icollection][jcollection]) continue; + if (flag_skip_multi[icollection][jcollection]) { + nstencil_multi[icollection][jcollection] = 0; + continue; + } ns = 0; diff --git a/src/nstencil_half_multi_2d.cpp b/src/nstencil_half_multi_2d.cpp index 32ee0297ff..0c8cd775e1 100644 --- a/src/nstencil_half_multi_2d.cpp +++ b/src/nstencil_half_multi_2d.cpp @@ -68,7 +68,10 @@ void NStencilHalfMulti2d::create() for (icollection = 0; icollection < n; icollection++) { for (jcollection = 0; jcollection < n; jcollection++) { - if (flag_skip_multi[icollection][jcollection]) continue; + if (flag_skip_multi[icollection][jcollection]) { + nstencil_multi[icollection][jcollection] = 0; + continue; + } ns = 0; diff --git a/src/nstencil_half_multi_2d_tri.cpp b/src/nstencil_half_multi_2d_tri.cpp index 6588248109..974f5611df 100755 --- a/src/nstencil_half_multi_2d_tri.cpp +++ b/src/nstencil_half_multi_2d_tri.cpp @@ -68,7 +68,10 @@ void NStencilHalfMulti2dTri::create() for (icollection = 0; icollection < n; icollection++) { for (jcollection = 0; jcollection < n; jcollection++) { - if (flag_skip_multi[icollection][jcollection]) continue; + if (flag_skip_multi[icollection][jcollection]) { + nstencil_multi[icollection][jcollection] = 0; + continue; + } ns = 0; diff --git a/src/nstencil_half_multi_3d.cpp b/src/nstencil_half_multi_3d.cpp index 19cd051392..070daa75da 100644 --- a/src/nstencil_half_multi_3d.cpp +++ b/src/nstencil_half_multi_3d.cpp @@ -68,7 +68,10 @@ void NStencilHalfMulti3d::create() for (icollection = 0; icollection < n; icollection++) { for (jcollection = 0; jcollection < n; jcollection++) { - if (flag_skip_multi[icollection][jcollection]) continue; + if (flag_skip_multi[icollection][jcollection]) { + nstencil_multi[icollection][jcollection] = 0; + continue; + } ns = 0; diff --git a/src/nstencil_half_multi_3d_tri.cpp b/src/nstencil_half_multi_3d_tri.cpp index 3c5efac463..f1dfe2a78f 100755 --- a/src/nstencil_half_multi_3d_tri.cpp +++ b/src/nstencil_half_multi_3d_tri.cpp @@ -68,7 +68,10 @@ void NStencilHalfMulti3dTri::create() for (icollection = 0; icollection < n; icollection++) { for (jcollection = 0; jcollection < n; jcollection++) { - if (flag_skip_multi[icollection][jcollection]) continue; + if (flag_skip_multi[icollection][jcollection]) { + nstencil_multi[icollection][jcollection] = 0; + continue; + } ns = 0;