tx258/lammps-gran-kokkos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup up rendezvous diagnostic output

Browse Source
This commit is contained in:
Steve Plimpton
2019-01-24 09:56:21 -07:00
committed by Axel Kohlmeyer
parent bbe6c9bb63
commit 62045a9c52
5 changed files with 73 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/RIGID/fix_rigid_small.cpp
View File

@ -28,6 +28,7 @@
#include "modify.h"
#include "group.h"
#include "comm.h"
#include "neighbor.h"
#include "force.h"
#include "input.h"
#include "output.h"
@ -44,6 +45,8 @@ using namespace LAMMPS_NS;
using namespace FixConst;
using namespace MathConst;
#define RVOUS 1 // 0 for irregular, 1 for all2all
#define MAXLINE 1024
#define CHUNK 1024
#define ATTRIBUTE_PERBODY 20
@ -585,12 +588,22 @@ void FixRigidSmall::init()
if (rflag && (modify->fmask[i] & POST_FORCE) &&
!modify->fix[i]->rigid_flag) {
char str[128];
snprintf(str,128,"Fix %s alters forces after fix rigid",modify->fix[i]->id);
snprintf(str,128,"Fix %s alters forces after fix rigid",
modify->fix[i]->id);
error->warning(FLERR,str);
}
}
}
// error if maxextent > comm->cutghost
// NOTE: could just warn if an override flag set
// NOTE: this could fail for comm multi mode if user sets a wrong cutoff
// for atom types in rigid bodies - need a more careful test
double cutghost = MAX(neighbor->cutneighmax,comm->cutghostuser);
if (maxextent > cutghost)
error->all(FLERR,"Rigid body extent > ghost cutoff - use comm_modify cutoff");
// error if npt,nph fix comes before rigid fix
for (i = 0; i < modify->nfix; i++) {
@ -1576,10 +1589,10 @@ void FixRigidSmall::create_bodies(tagint *bodyID)
// func = compute bbox of each body, find atom closest to geometric center
char *buf;
int nreturn = comm->rendezvous(1,ncount,(char *) inbuf,sizeof(InRvous),
int nreturn = comm->rendezvous(RVOUS,ncount,(char *) inbuf,sizeof(InRvous),
0,proclist,
rendezvous_body,0,buf,sizeof(OutRvous),
(void *) this);
(void *) this,1);
OutRvous *outbuf = (OutRvous *) buf;
memory->destroy(proclist);

10
src/RIGID/fix_shake.cpp
View File

@ -39,6 +39,8 @@ using namespace LAMMPS_NS;
using namespace FixConst;
using namespace MathConst;
#define RVOUS 1 // 0 for irregular, 1 for all2all
#define BIG 1.0e20
#define MASSDELTA 0.1
@ -1068,7 +1070,7 @@ void FixShake::atom_owners()
// each proc assigned every 1/Pth atom
char *buf;
comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),
comm->rendezvous(RVOUS,nlocal,(char *) idbuf,sizeof(IDRvous),
0,proclist,
rendezvous_ids,0,buf,0,(void *) this,1);
@ -1174,7 +1176,7 @@ void FixShake::partner_info(int *npartner, tagint **partner_tag,
// receives all data needed to populate un-owned partner 4 values
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PartnerInfo),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PartnerInfo),
0,proclist,
rendezvous_partners_info,
0,buf,sizeof(PartnerInfo),
@ -1263,7 +1265,7 @@ void FixShake::nshake_info(int *npartner, tagint **partner_tag,
// receives all data needed to populate un-owned partner nshake
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(NShakeInfo),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(NShakeInfo),
0,proclist,
rendezvous_nshake,0,buf,sizeof(NShakeInfo),
(void *) this,1);
@ -1356,7 +1358,7 @@ void FixShake::shake_info(int *npartner, tagint **partner_tag,
// receives all data needed to populate un-owned shake info
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(ShakeInfo),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(ShakeInfo),
0,proclist,
rendezvous_shake,0,buf,sizeof(ShakeInfo),
(void *) this,1);

163
src/comm.cpp
View File

@ -796,7 +796,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,
"rendezvous:inbuf");
irregular->exchange_data(inbuf,insize,inbuf_rvous);
bigint irregular1_bytes = 0; //irregular->irregular_bytes;
bigint irregular1_bytes = irregular->memory_usage();
irregular->destroy_data();
delete irregular;
@ -826,7 +826,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,
"rendezvous:outbuf");
irregular->exchange_data(outbuf_rvous,outsize,outbuf);
bigint irregular2_bytes = 0; //irregular->irregular_bytes;
bigint irregular2_bytes = irregular->memory_usage();
irregular->destroy_data();
delete irregular;
@ -834,90 +834,11 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,
memory->sfree(outbuf_rvous);
// return number of output datums
// last arg to stats() = memory for procs_rvous + irregular comm
if (!statflag) return nout;
// memory info for caller and rendezvous decompositions
bigint size_in_all,size_in_max,size_in_min;
bigint size_out_all,size_out_max,size_out_min;
bigint size_inrvous_all,size_inrvous_max,size_inrvous_min;
bigint size_outrvous_all,size_outrvous_max,size_outrvous_min;
bigint size = (bigint) n*insize;
MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
MPI_Allreduce(&size,&size_in_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
MPI_Allreduce(&size,&size_in_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
size = (bigint) nout*outsize;
MPI_Allreduce(&size,&size_out_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
MPI_Allreduce(&size,&size_out_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
MPI_Allreduce(&size,&size_out_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
size = (bigint) nrvous*insize;
MPI_Allreduce(&size,&size_inrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
MPI_Allreduce(&size,&size_inrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
MPI_Allreduce(&size,&size_inrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
size = (bigint) nrvous_out*insize;
MPI_Allreduce(&size,&size_outrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
int mbytes = 1024*1024;
if (me == 0) {
if (screen) {
fprintf(screen,"Rendezvous balance and memory info:\n");
fprintf(screen," input datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_in_all/insize,1.0*size_in_all/nprocs/insize,
size_in_max/insize,size_in_min/insize);
fprintf(screen," input data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes,
1.0*size_in_max/mbytes,1.0*size_in_min/mbytes);
fprintf(screen," output datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_out_all/outsize,1.0*size_out_all/nprocs/outsize,
size_out_max/outsize,size_out_min/outsize);
fprintf(screen," output data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes,
1.0*size_out_max/mbytes,1.0*size_out_min/mbytes);
fprintf(screen," input rvous datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize,
size_inrvous_max/insize,size_inrvous_min/insize);
fprintf(screen," input rvous data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes,
1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes);
fprintf(screen," output rvous datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize,
size_outrvous_max/outsize,size_outrvous_min/outsize);
fprintf(screen," output rvous data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes,
1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes);
}
}
/*
bigint rvous_bytes = 0;
rvous_bytes += n*insize; // inbuf
rvous_bytes += nout*outsize; // outbuf
rvous_bytes += nrvous*insize; // inbuf_rvous
rvous_bytes += nrvous_out*outsize; // outbuf_rvous
rvous_bytes += nrvous_out*sizeof(int); // procs_rvous
rvous_bytes += MAX(irregular1_bytes,irregular2_bytes); // max of 2 comms
*/
if (statflag) rendezvous_stats(n,nout,nrvous,nrvous_out,insize,outsize,
(bigint) nrvous_out*sizeof(int) +
MAX(irregular1_bytes,irregular2_bytes));
return nout;
}
@ -1121,15 +1042,28 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs,
memory->destroy(rdispls);
// return number of output datums
// last arg to stats() = mem for procs_rvous + per-proc vecs + reordering ops
if (!statflag) return nout;
if (statflag) rendezvous_stats(n,nout,nrvous,nrvous_out,insize,outsize,
(bigint) nrvous_out*sizeof(int) +
4*nprocs*sizeof(int) +
MAX(all2all1_bytes,all2all2_bytes));
return nout;
}
// memory info for caller and rendezvous decompositions
/* ----------------------------------------------------------------------
print balance and memory info for rendezvous operation
useful for debugging
------------------------------------------------------------------------- */
void Comm::rendezvous_stats(int n, int nout, int nrvous, int nrvous_out,
int insize, int outsize, bigint commsize)
{
bigint size_in_all,size_in_max,size_in_min;
bigint size_out_all,size_out_max,size_out_min;
bigint size_inrvous_all,size_inrvous_max,size_inrvous_min;
bigint size_outrvous_all,size_outrvous_max,size_outrvous_min;
bigint size_comm_all,size_comm_max,size_comm_min;
bigint size = (bigint) n*insize;
MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
@ -1151,62 +1085,49 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs,
MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
size = commsize;
MPI_Allreduce(&size,&size_comm_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
MPI_Allreduce(&size,&size_comm_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
MPI_Allreduce(&size,&size_comm_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
int mbytes = 1024*1024;
if (me == 0) {
if (screen) {
fprintf(screen,"Rendezvous balance and memory info:\n");
fprintf(screen," input datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
fprintf(screen,"Rendezvous balance and memory info: (tot,ave,max,min) \n");
fprintf(screen," input datum count: "
BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_in_all/insize,1.0*size_in_all/nprocs/insize,
size_in_max/insize,size_in_min/insize);
fprintf(screen," input data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
fprintf(screen," input data (MB): %g %g %g %g\n",
1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes,
1.0*size_in_max/mbytes,1.0*size_in_min/mbytes);
fprintf(screen," output datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
fprintf(screen," output datum count: "
BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_out_all/outsize,1.0*size_out_all/nprocs/outsize,
size_out_max/outsize,size_out_min/outsize);
fprintf(screen," output data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
fprintf(screen," output data (MB): %g %g %g %g\n",
1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes,
1.0*size_out_max/mbytes,1.0*size_out_min/mbytes);
fprintf(screen," input rvous datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
fprintf(screen," input rvous datum count: "
BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize,
size_inrvous_max/insize,size_inrvous_min/insize);
fprintf(screen," input rvous data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
fprintf(screen," input rvous data (MB): %g %g %g %g\n",
1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes,
1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes);
fprintf(screen," output rvous datum count "
"(tot,ave,max,min): " BIGINT_FORMAT " %g "
BIGINT_FORMAT " " BIGINT_FORMAT "\n",
fprintf(screen," output rvous datum count: "
BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize,
size_outrvous_max/outsize,size_outrvous_min/outsize);
fprintf(screen," output rvous data (MB) "
"(tot,ave,max,min): %g %g %g %g\n",
fprintf(screen," output rvous data (MB): %g %g %g %g\n",
1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes,
1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes);
fprintf(screen," rvous comm (MB): %g %g %g %g\n",
1.0*size_comm_all/mbytes,1.0*size_comm_all/nprocs/mbytes,
1.0*size_comm_max/mbytes,1.0*size_comm_min/mbytes);
}
}
/*
bigint rvous_bytes = 0;
rvous_bytes += n*insize; // inbuf
rvous_bytes += nout*outsize; // outbuf
rvous_bytes += nrvous*insize; // inbuf_rvous
rvous_bytes += nrvous_out*outsize; // outbuf_rvous
rvous_bytes += nrvous_out*sizeof(int); // procs_rvous
rvous_bytes += 4*nprocs*sizeof(int); // all2all vectors
rvous_bytes += MAX(all2all1_bytes,all2all2_bytes); // reorder ops
*/
return nout;
}
/* ----------------------------------------------------------------------

1
src/comm.h
View File

@ -153,6 +153,7 @@ class Comm : protected Pointers {
int rendezvous_all2all(int, char *, int, int, int *,
int (*)(int, char *, int &, int *&, char *&, void *),
int, char *&, int, void *, int);
void rendezvous_stats(int, int, int, int, int, int, bigint);
public:
enum{MULTIPLE};

14
src/special.cpp
View File

@ -27,6 +27,8 @@
using namespace LAMMPS_NS;
#define RVOUS 1 // 0 for irregular, 1 for all2all
/* ---------------------------------------------------------------------- */
Special::Special(LAMMPS *lmp) : Pointers(lmp)
@ -185,7 +187,7 @@ void Special::atom_owners()
// perform rendezvous operation
char *buf;
comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist,
comm->rendezvous(RVOUS,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist,
rendezvous_ids,0,buf,0,(void *) this,1);
memory->destroy(proclist);
@ -241,7 +243,7 @@ void Special::onetwo_build_newton()
// perform rendezvous operation
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
0,proclist,
rendezvous_pairs,0,buf,sizeof(PairRvous),
(void *) this,1);
@ -376,7 +378,7 @@ void Special::onethree_build()
// perform rendezvous operation
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
0,proclist,
rendezvous_pairs,0,buf,sizeof(PairRvous),
(void *) this,1);
@ -481,7 +483,7 @@ void Special::onefour_build()
// perform rendezvous operation
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
0,proclist,
rendezvous_pairs,0,buf,sizeof(PairRvous),
(void *) this,1);
@ -904,7 +906,7 @@ void Special::angle_trim()
// perform rendezvous operation
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
0,proclist,
rendezvous_pairs,0,buf,sizeof(PairRvous),
(void *) this,1);
@ -1123,7 +1125,7 @@ void Special::dihedral_trim()
// perform rendezvous operation
char *buf;
int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
0,proclist,
rendezvous_pairs,0,buf,sizeof(PairRvous),
(void *) this,1);