diff --git a/src/RIGID/fix_rigid_small.cpp b/src/RIGID/fix_rigid_small.cpp index 7c6aab4861..8697f66056 100644 --- a/src/RIGID/fix_rigid_small.cpp +++ b/src/RIGID/fix_rigid_small.cpp @@ -28,6 +28,7 @@ #include "modify.h" #include "group.h" #include "comm.h" +#include "neighbor.h" #include "force.h" #include "input.h" #include "output.h" @@ -44,6 +45,8 @@ using namespace LAMMPS_NS; using namespace FixConst; using namespace MathConst; +#define RVOUS 1 // 0 for irregular, 1 for all2all + #define MAXLINE 1024 #define CHUNK 1024 #define ATTRIBUTE_PERBODY 20 @@ -585,12 +588,22 @@ void FixRigidSmall::init() if (rflag && (modify->fmask[i] & POST_FORCE) && !modify->fix[i]->rigid_flag) { char str[128]; - snprintf(str,128,"Fix %s alters forces after fix rigid",modify->fix[i]->id); + snprintf(str,128,"Fix %s alters forces after fix rigid", + modify->fix[i]->id); error->warning(FLERR,str); } } } + // error if maxextent > comm->cutghost + // NOTE: could just warn if an override flag set + // NOTE: this could fail for comm multi mode if user sets a wrong cutoff + // for atom types in rigid bodies - need a more careful test + + double cutghost = MAX(neighbor->cutneighmax,comm->cutghostuser); + if (maxextent > cutghost) + error->all(FLERR,"Rigid body extent > ghost cutoff - use comm_modify cutoff"); + // error if npt,nph fix comes before rigid fix for (i = 0; i < modify->nfix; i++) { @@ -1576,10 +1589,10 @@ void FixRigidSmall::create_bodies(tagint *bodyID) // func = compute bbox of each body, find atom closest to geometric center char *buf; - int nreturn = comm->rendezvous(1,ncount,(char *) inbuf,sizeof(InRvous), + int nreturn = comm->rendezvous(RVOUS,ncount,(char *) inbuf,sizeof(InRvous), 0,proclist, rendezvous_body,0,buf,sizeof(OutRvous), - (void *) this); + (void *) this,1); OutRvous *outbuf = (OutRvous *) buf; memory->destroy(proclist); diff --git a/src/RIGID/fix_shake.cpp b/src/RIGID/fix_shake.cpp index 46e478064c..2d3244885d 100644 --- a/src/RIGID/fix_shake.cpp +++ b/src/RIGID/fix_shake.cpp @@ -39,6 +39,8 @@ using namespace LAMMPS_NS; using namespace FixConst; using namespace MathConst; +#define RVOUS 1 // 0 for irregular, 1 for all2all + #define BIG 1.0e20 #define MASSDELTA 0.1 @@ -1068,7 +1070,7 @@ void FixShake::atom_owners() // each proc assigned every 1/Pth atom char *buf; - comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous), + comm->rendezvous(RVOUS,nlocal,(char *) idbuf,sizeof(IDRvous), 0,proclist, rendezvous_ids,0,buf,0,(void *) this,1); @@ -1174,7 +1176,7 @@ void FixShake::partner_info(int *npartner, tagint **partner_tag, // receives all data needed to populate un-owned partner 4 values char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PartnerInfo), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PartnerInfo), 0,proclist, rendezvous_partners_info, 0,buf,sizeof(PartnerInfo), @@ -1263,7 +1265,7 @@ void FixShake::nshake_info(int *npartner, tagint **partner_tag, // receives all data needed to populate un-owned partner nshake char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(NShakeInfo), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(NShakeInfo), 0,proclist, rendezvous_nshake,0,buf,sizeof(NShakeInfo), (void *) this,1); @@ -1356,7 +1358,7 @@ void FixShake::shake_info(int *npartner, tagint **partner_tag, // receives all data needed to populate un-owned shake info char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(ShakeInfo), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(ShakeInfo), 0,proclist, rendezvous_shake,0,buf,sizeof(ShakeInfo), (void *) this,1); diff --git a/src/comm.cpp b/src/comm.cpp index 54476078be..e8a796036c 100644 --- a/src/comm.cpp +++ b/src/comm.cpp @@ -796,7 +796,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs, "rendezvous:inbuf"); irregular->exchange_data(inbuf,insize,inbuf_rvous); - bigint irregular1_bytes = 0; //irregular->irregular_bytes; + bigint irregular1_bytes = irregular->memory_usage(); irregular->destroy_data(); delete irregular; @@ -826,7 +826,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs, "rendezvous:outbuf"); irregular->exchange_data(outbuf_rvous,outsize,outbuf); - bigint irregular2_bytes = 0; //irregular->irregular_bytes; + bigint irregular2_bytes = irregular->memory_usage(); irregular->destroy_data(); delete irregular; @@ -834,90 +834,11 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs, memory->sfree(outbuf_rvous); // return number of output datums + // last arg to stats() = memory for procs_rvous + irregular comm - if (!statflag) return nout; - - // memory info for caller and rendezvous decompositions - - bigint size_in_all,size_in_max,size_in_min; - bigint size_out_all,size_out_max,size_out_min; - bigint size_inrvous_all,size_inrvous_max,size_inrvous_min; - bigint size_outrvous_all,size_outrvous_max,size_outrvous_min; - - bigint size = (bigint) n*insize; - MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world); - MPI_Allreduce(&size,&size_in_max,1,MPI_LMP_BIGINT,MPI_MAX,world); - MPI_Allreduce(&size,&size_in_min,1,MPI_LMP_BIGINT,MPI_MIN,world); - - size = (bigint) nout*outsize; - MPI_Allreduce(&size,&size_out_all,1,MPI_LMP_BIGINT,MPI_SUM,world); - MPI_Allreduce(&size,&size_out_max,1,MPI_LMP_BIGINT,MPI_MAX,world); - MPI_Allreduce(&size,&size_out_min,1,MPI_LMP_BIGINT,MPI_MIN,world); - - size = (bigint) nrvous*insize; - MPI_Allreduce(&size,&size_inrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world); - MPI_Allreduce(&size,&size_inrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world); - MPI_Allreduce(&size,&size_inrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world); - - size = (bigint) nrvous_out*insize; - MPI_Allreduce(&size,&size_outrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world); - MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world); - MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world); - - int mbytes = 1024*1024; - - if (me == 0) { - if (screen) { - fprintf(screen,"Rendezvous balance and memory info:\n"); - fprintf(screen," input datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", - size_in_all/insize,1.0*size_in_all/nprocs/insize, - size_in_max/insize,size_in_min/insize); - fprintf(screen," input data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", - 1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes, - 1.0*size_in_max/mbytes,1.0*size_in_min/mbytes); - fprintf(screen," output datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", - size_out_all/outsize,1.0*size_out_all/nprocs/outsize, - size_out_max/outsize,size_out_min/outsize); - fprintf(screen," output data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", - 1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes, - 1.0*size_out_max/mbytes,1.0*size_out_min/mbytes); - fprintf(screen," input rvous datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", - size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize, - size_inrvous_max/insize,size_inrvous_min/insize); - fprintf(screen," input rvous data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", - 1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes, - 1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes); - fprintf(screen," output rvous datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", - size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize, - size_outrvous_max/outsize,size_outrvous_min/outsize); - fprintf(screen," output rvous data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", - 1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes, - 1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes); - } - } - - /* - bigint rvous_bytes = 0; - rvous_bytes += n*insize; // inbuf - rvous_bytes += nout*outsize; // outbuf - rvous_bytes += nrvous*insize; // inbuf_rvous - rvous_bytes += nrvous_out*outsize; // outbuf_rvous - rvous_bytes += nrvous_out*sizeof(int); // procs_rvous - rvous_bytes += MAX(irregular1_bytes,irregular2_bytes); // max of 2 comms - */ - + if (statflag) rendezvous_stats(n,nout,nrvous,nrvous_out,insize,outsize, + (bigint) nrvous_out*sizeof(int) + + MAX(irregular1_bytes,irregular2_bytes)); return nout; } @@ -1121,15 +1042,28 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs, memory->destroy(rdispls); // return number of output datums + // last arg to stats() = mem for procs_rvous + per-proc vecs + reordering ops - if (!statflag) return nout; + if (statflag) rendezvous_stats(n,nout,nrvous,nrvous_out,insize,outsize, + (bigint) nrvous_out*sizeof(int) + + 4*nprocs*sizeof(int) + + MAX(all2all1_bytes,all2all2_bytes)); + return nout; +} - // memory info for caller and rendezvous decompositions +/* ---------------------------------------------------------------------- + print balance and memory info for rendezvous operation + useful for debugging +------------------------------------------------------------------------- */ +void Comm::rendezvous_stats(int n, int nout, int nrvous, int nrvous_out, + int insize, int outsize, bigint commsize) +{ bigint size_in_all,size_in_max,size_in_min; bigint size_out_all,size_out_max,size_out_min; bigint size_inrvous_all,size_inrvous_max,size_inrvous_min; bigint size_outrvous_all,size_outrvous_max,size_outrvous_min; + bigint size_comm_all,size_comm_max,size_comm_min; bigint size = (bigint) n*insize; MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world); @@ -1151,62 +1085,49 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs, MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world); MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world); + size = commsize; + MPI_Allreduce(&size,&size_comm_all,1,MPI_LMP_BIGINT,MPI_SUM,world); + MPI_Allreduce(&size,&size_comm_max,1,MPI_LMP_BIGINT,MPI_MAX,world); + MPI_Allreduce(&size,&size_comm_min,1,MPI_LMP_BIGINT,MPI_MIN,world); + int mbytes = 1024*1024; if (me == 0) { if (screen) { - fprintf(screen,"Rendezvous balance and memory info:\n"); - fprintf(screen," input datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", + fprintf(screen,"Rendezvous balance and memory info: (tot,ave,max,min) \n"); + fprintf(screen," input datum count: " + BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n", size_in_all/insize,1.0*size_in_all/nprocs/insize, size_in_max/insize,size_in_min/insize); - fprintf(screen," input data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", + fprintf(screen," input data (MB): %g %g %g %g\n", 1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes, 1.0*size_in_max/mbytes,1.0*size_in_min/mbytes); - fprintf(screen," output datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", + fprintf(screen," output datum count: " + BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n", size_out_all/outsize,1.0*size_out_all/nprocs/outsize, size_out_max/outsize,size_out_min/outsize); - fprintf(screen," output data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", + fprintf(screen," output data (MB): %g %g %g %g\n", 1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes, 1.0*size_out_max/mbytes,1.0*size_out_min/mbytes); - fprintf(screen," input rvous datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", + fprintf(screen," input rvous datum count: " + BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n", size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize, size_inrvous_max/insize,size_inrvous_min/insize); - fprintf(screen," input rvous data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", + fprintf(screen," input rvous data (MB): %g %g %g %g\n", 1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes, 1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes); - fprintf(screen," output rvous datum count " - "(tot,ave,max,min): " BIGINT_FORMAT " %g " - BIGINT_FORMAT " " BIGINT_FORMAT "\n", + fprintf(screen," output rvous datum count: " + BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n", size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize, size_outrvous_max/outsize,size_outrvous_min/outsize); - fprintf(screen," output rvous data (MB) " - "(tot,ave,max,min): %g %g %g %g\n", + fprintf(screen," output rvous data (MB): %g %g %g %g\n", 1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes, 1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes); + fprintf(screen," rvous comm (MB): %g %g %g %g\n", + 1.0*size_comm_all/mbytes,1.0*size_comm_all/nprocs/mbytes, + 1.0*size_comm_max/mbytes,1.0*size_comm_min/mbytes); } } - - /* - bigint rvous_bytes = 0; - rvous_bytes += n*insize; // inbuf - rvous_bytes += nout*outsize; // outbuf - rvous_bytes += nrvous*insize; // inbuf_rvous - rvous_bytes += nrvous_out*outsize; // outbuf_rvous - rvous_bytes += nrvous_out*sizeof(int); // procs_rvous - rvous_bytes += 4*nprocs*sizeof(int); // all2all vectors - rvous_bytes += MAX(all2all1_bytes,all2all2_bytes); // reorder ops - */ - - return nout; } /* ---------------------------------------------------------------------- diff --git a/src/comm.h b/src/comm.h index 89889e3ebe..9c0112b4c4 100644 --- a/src/comm.h +++ b/src/comm.h @@ -153,6 +153,7 @@ class Comm : protected Pointers { int rendezvous_all2all(int, char *, int, int, int *, int (*)(int, char *, int &, int *&, char *&, void *), int, char *&, int, void *, int); + void rendezvous_stats(int, int, int, int, int, int, bigint); public: enum{MULTIPLE}; diff --git a/src/special.cpp b/src/special.cpp index 5e0f865488..a0739d6ccc 100644 --- a/src/special.cpp +++ b/src/special.cpp @@ -27,6 +27,8 @@ using namespace LAMMPS_NS; +#define RVOUS 1 // 0 for irregular, 1 for all2all + /* ---------------------------------------------------------------------- */ Special::Special(LAMMPS *lmp) : Pointers(lmp) @@ -185,7 +187,7 @@ void Special::atom_owners() // perform rendezvous operation char *buf; - comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist, + comm->rendezvous(RVOUS,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist, rendezvous_ids,0,buf,0,(void *) this,1); memory->destroy(proclist); @@ -241,7 +243,7 @@ void Special::onetwo_build_newton() // perform rendezvous operation char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous), 0,proclist, rendezvous_pairs,0,buf,sizeof(PairRvous), (void *) this,1); @@ -376,7 +378,7 @@ void Special::onethree_build() // perform rendezvous operation char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous), 0,proclist, rendezvous_pairs,0,buf,sizeof(PairRvous), (void *) this,1); @@ -481,7 +483,7 @@ void Special::onefour_build() // perform rendezvous operation char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous), 0,proclist, rendezvous_pairs,0,buf,sizeof(PairRvous), (void *) this,1); @@ -904,7 +906,7 @@ void Special::angle_trim() // perform rendezvous operation char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous), 0,proclist, rendezvous_pairs,0,buf,sizeof(PairRvous), (void *) this,1); @@ -1123,7 +1125,7 @@ void Special::dihedral_trim() // perform rendezvous operation char *buf; - int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous), + int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous), 0,proclist, rendezvous_pairs,0,buf,sizeof(PairRvous), (void *) this,1);