cleanup up rendezvous diagnostic output

src/RIGID/fix_rigid_small.cpp

@@ -28,6 +28,7 @@
 #include "modify.h"
 #include "group.h"
 #include "comm.h"
+#include "neighbor.h"
 #include "force.h"
 #include "input.h"
 #include "output.h"
@@ -44,6 +45,8 @@ using namespace LAMMPS_NS;
 using namespace FixConst;
 using namespace MathConst;
 
+#define RVOUS 1   // 0 for irregular, 1 for all2all
+
 #define MAXLINE 1024
 #define CHUNK 1024
 #define ATTRIBUTE_PERBODY 20
@@ -585,12 +588,22 @@ void FixRigidSmall::init()
       if (rflag && (modify->fmask[i] & POST_FORCE) &&
           !modify->fix[i]->rigid_flag) {
         char str[128];
-        snprintf(str,128,"Fix %s alters forces after fix rigid",modify->fix[i]->id);
+        snprintf(str,128,"Fix %s alters forces after fix rigid",
+                 modify->fix[i]->id);
         error->warning(FLERR,str);
       }
     }
   }
 
+  // error if maxextent > comm->cutghost
+  // NOTE: could just warn if an override flag set
+  // NOTE: this could fail for comm multi mode if user sets a wrong cutoff
+  //       for atom types in rigid bodies - need a more careful test
+
+  double cutghost = MAX(neighbor->cutneighmax,comm->cutghostuser);
+  if (maxextent > cutghost) 
+    error->all(FLERR,"Rigid body extent > ghost cutoff - use comm_modify cutoff");
+
   // error if npt,nph fix comes before rigid fix
 
   for (i = 0; i < modify->nfix; i++) {
@@ -1576,10 +1589,10 @@ void FixRigidSmall::create_bodies(tagint *bodyID)
   // func = compute bbox of each body, find atom closest to geometric center
 
   char *buf;
-  int nreturn = comm->rendezvous(1,ncount,(char *) inbuf,sizeof(InRvous),
+  int nreturn = comm->rendezvous(RVOUS,ncount,(char *) inbuf,sizeof(InRvous),
                                  0,proclist,
                                  rendezvous_body,0,buf,sizeof(OutRvous),
-                                 (void *) this);
+                                 (void *) this,1);
   OutRvous *outbuf = (OutRvous *) buf;
   
   memory->destroy(proclist);

src/RIGID/fix_shake.cpp

@@ -39,6 +39,8 @@ using namespace LAMMPS_NS;
 using namespace FixConst;
 using namespace MathConst;
 
+#define RVOUS 1   // 0 for irregular, 1 for all2all
+
 #define BIG 1.0e20
 #define MASSDELTA 0.1
 
@@ -1068,7 +1070,7 @@ void FixShake::atom_owners()
   // each proc assigned every 1/Pth atom
   
   char *buf;
-  comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),
+  comm->rendezvous(RVOUS,nlocal,(char *) idbuf,sizeof(IDRvous),
                    0,proclist,
                    rendezvous_ids,0,buf,0,(void *) this,1);
 
@@ -1174,7 +1176,7 @@ void FixShake::partner_info(int *npartner, tagint **partner_tag,
   // receives all data needed to populate un-owned partner 4 values
 
   char *buf;
-  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PartnerInfo),
+  int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PartnerInfo),
                                  0,proclist,
                                  rendezvous_partners_info,
                                  0,buf,sizeof(PartnerInfo),
@@ -1263,7 +1265,7 @@ void FixShake::nshake_info(int *npartner, tagint **partner_tag,
   // receives all data needed to populate un-owned partner nshake
 
   char *buf;
-  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(NShakeInfo),
+  int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(NShakeInfo),
                                  0,proclist,
                                  rendezvous_nshake,0,buf,sizeof(NShakeInfo),
                                  (void *) this,1);
@@ -1356,7 +1358,7 @@ void FixShake::shake_info(int *npartner, tagint **partner_tag,
   // receives all data needed to populate un-owned shake info
 
   char *buf;
-  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(ShakeInfo),
+  int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(ShakeInfo),
                                  0,proclist,
                                  rendezvous_shake,0,buf,sizeof(ShakeInfo),
                                  (void *) this,1);

src/comm.cpp

@@ -796,7 +796,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,
                                                "rendezvous:inbuf");
   irregular->exchange_data(inbuf,insize,inbuf_rvous);
 
-  bigint irregular1_bytes = 0; //irregular->irregular_bytes;
+  bigint irregular1_bytes = irregular->memory_usage();
   irregular->destroy_data();
   delete irregular;
 
@@ -826,7 +826,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,
                                     "rendezvous:outbuf");
   irregular->exchange_data(outbuf_rvous,outsize,outbuf);
 
-  bigint irregular2_bytes = 0; //irregular->irregular_bytes;
+  bigint irregular2_bytes = irregular->memory_usage();
   irregular->destroy_data();
   delete irregular;
 
@@ -834,90 +834,11 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,
   memory->sfree(outbuf_rvous);
 
   // return number of output datums
+  // last arg to stats() = memory for procs_rvous + irregular comm
 
-  if (!statflag) return nout;
+  if (statflag) rendezvous_stats(n,nout,nrvous,nrvous_out,insize,outsize,
-
+                                 (bigint) nrvous_out*sizeof(int) + 
-  // memory info for caller and rendezvous decompositions
+                                 MAX(irregular1_bytes,irregular2_bytes));
-
-  bigint size_in_all,size_in_max,size_in_min;
-  bigint size_out_all,size_out_max,size_out_min;
-  bigint size_inrvous_all,size_inrvous_max,size_inrvous_min;
-  bigint size_outrvous_all,size_outrvous_max,size_outrvous_min;
-
-  bigint size = (bigint) n*insize;
-  MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
-  MPI_Allreduce(&size,&size_in_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
-  MPI_Allreduce(&size,&size_in_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
-
-  size = (bigint) nout*outsize;
-  MPI_Allreduce(&size,&size_out_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
-  MPI_Allreduce(&size,&size_out_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
-  MPI_Allreduce(&size,&size_out_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
-
-  size = (bigint) nrvous*insize;
-  MPI_Allreduce(&size,&size_inrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
-  MPI_Allreduce(&size,&size_inrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
-  MPI_Allreduce(&size,&size_inrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
-
-  size = (bigint) nrvous_out*insize;
-  MPI_Allreduce(&size,&size_outrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
-  MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
-  MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
-
-  int mbytes = 1024*1024;
-
-  if (me == 0) {
-    if (screen) {
-      fprintf(screen,"Rendezvous balance and memory info:\n");
-      fprintf(screen,"  input datum count "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              size_in_all/insize,1.0*size_in_all/nprocs/insize,
-              size_in_max/insize,size_in_min/insize);
-      fprintf(screen,"  input data (MB) "
-              "(tot,ave,max,min): %g %g %g %g\n",
-              1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes,
-              1.0*size_in_max/mbytes,1.0*size_in_min/mbytes);
-      fprintf(screen,"  output datum count "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              size_out_all/outsize,1.0*size_out_all/nprocs/outsize,
-              size_out_max/outsize,size_out_min/outsize);
-      fprintf(screen,"  output data (MB) "
-              "(tot,ave,max,min): %g %g %g %g\n",
-              1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes,
-              1.0*size_out_max/mbytes,1.0*size_out_min/mbytes);
-      fprintf(screen,"  input rvous datum count "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize,
-              size_inrvous_max/insize,size_inrvous_min/insize);
-      fprintf(screen,"  input rvous data (MB) "
-              "(tot,ave,max,min): %g %g %g %g\n",
-              1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes,
-              1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes);
-      fprintf(screen,"  output rvous datum count "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize,
-              size_outrvous_max/outsize,size_outrvous_min/outsize);
-      fprintf(screen,"  output rvous data (MB) "
-              "(tot,ave,max,min): %g %g %g %g\n",
-              1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes,
-              1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes);
-    }
-  }
-
-  /*
-  bigint rvous_bytes = 0;
-  rvous_bytes += n*insize;                                // inbuf
-  rvous_bytes += nout*outsize;                            // outbuf
-  rvous_bytes += nrvous*insize;                           // inbuf_rvous
-  rvous_bytes += nrvous_out*outsize;                      // outbuf_rvous
-  rvous_bytes += nrvous_out*sizeof(int);                  // procs_rvous
-  rvous_bytes += MAX(irregular1_bytes,irregular2_bytes);  // max of 2 comms
-  */
-
   return nout;
 }
 
@@ -1121,15 +1042,28 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs,
   memory->destroy(rdispls);
 
   // return number of output datums
+  // last arg to stats() = mem for procs_rvous + per-proc vecs + reordering ops
 
-  if (!statflag) return nout;
+  if (statflag) rendezvous_stats(n,nout,nrvous,nrvous_out,insize,outsize,
+                                 (bigint) nrvous_out*sizeof(int) + 
+                                 4*nprocs*sizeof(int) + 
+                                 MAX(all2all1_bytes,all2all2_bytes));
+  return nout;
+}
 
-  // memory info for caller and rendezvous decompositions
+/* ----------------------------------------------------------------------
+   print balance and memory info for rendezvous operation
+   useful for debugging
+------------------------------------------------------------------------- */
 
+void Comm::rendezvous_stats(int n, int nout, int nrvous, int nrvous_out,
+                            int insize, int outsize, bigint commsize)
+{
   bigint size_in_all,size_in_max,size_in_min;
   bigint size_out_all,size_out_max,size_out_min;
   bigint size_inrvous_all,size_inrvous_max,size_inrvous_min;
   bigint size_outrvous_all,size_outrvous_max,size_outrvous_min;
+  bigint size_comm_all,size_comm_max,size_comm_min;
 
   bigint size = (bigint) n*insize;
   MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
@@ -1151,62 +1085,49 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs,
   MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
   MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
 
+  size = commsize;
+  MPI_Allreduce(&size,&size_comm_all,1,MPI_LMP_BIGINT,MPI_SUM,world);
+  MPI_Allreduce(&size,&size_comm_max,1,MPI_LMP_BIGINT,MPI_MAX,world);
+  MPI_Allreduce(&size,&size_comm_min,1,MPI_LMP_BIGINT,MPI_MIN,world);
+
   int mbytes = 1024*1024;
 
   if (me == 0) {
     if (screen) {
-      fprintf(screen,"Rendezvous balance and memory info:\n");
+      fprintf(screen,"Rendezvous balance and memory info: (tot,ave,max,min) \n");
-      fprintf(screen,"  input datum count "
+      fprintf(screen,"  input datum count: "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
+              BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
               size_in_all/insize,1.0*size_in_all/nprocs/insize,
               size_in_max/insize,size_in_min/insize);
-      fprintf(screen,"  input data (MB) "
+      fprintf(screen,"  input data (MB): %g %g %g %g\n",
-              "(tot,ave,max,min): %g %g %g %g\n",
               1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes,
               1.0*size_in_max/mbytes,1.0*size_in_min/mbytes);
-      fprintf(screen,"  output datum count "
+      fprintf(screen,"  output datum count: "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
+              BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
               size_out_all/outsize,1.0*size_out_all/nprocs/outsize,
               size_out_max/outsize,size_out_min/outsize);
-      fprintf(screen,"  output data (MB) "
+      fprintf(screen,"  output data (MB): %g %g %g %g\n",
-              "(tot,ave,max,min): %g %g %g %g\n",
               1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes,
               1.0*size_out_max/mbytes,1.0*size_out_min/mbytes);
-      fprintf(screen,"  input rvous datum count "
+      fprintf(screen,"  input rvous datum count: "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
+              BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
               size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize,
               size_inrvous_max/insize,size_inrvous_min/insize);
-      fprintf(screen,"  input rvous data (MB) "
+      fprintf(screen,"  input rvous data (MB): %g %g %g %g\n",
-              "(tot,ave,max,min): %g %g %g %g\n",
               1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes,
               1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes);
-      fprintf(screen,"  output rvous datum count "
+      fprintf(screen,"  output rvous datum count: "
-              "(tot,ave,max,min): " BIGINT_FORMAT " %g "
+              BIGINT_FORMAT " %g " BIGINT_FORMAT " " BIGINT_FORMAT "\n",
-              BIGINT_FORMAT " " BIGINT_FORMAT "\n",
               size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize,
               size_outrvous_max/outsize,size_outrvous_min/outsize);
-      fprintf(screen,"  output rvous data (MB) "
+      fprintf(screen,"  output rvous data (MB): %g %g %g %g\n",
-              "(tot,ave,max,min): %g %g %g %g\n",
               1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes,
               1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes);
+      fprintf(screen,"  rvous comm (MB): %g %g %g %g\n",
+              1.0*size_comm_all/mbytes,1.0*size_comm_all/nprocs/mbytes,
+              1.0*size_comm_max/mbytes,1.0*size_comm_min/mbytes);
     }
   }
-
-  /*
-  bigint rvous_bytes = 0;
-  rvous_bytes += n*insize;                                // inbuf
-  rvous_bytes += nout*outsize;                            // outbuf
-  rvous_bytes += nrvous*insize;                           // inbuf_rvous
-  rvous_bytes += nrvous_out*outsize;                      // outbuf_rvous
-  rvous_bytes += nrvous_out*sizeof(int);                  // procs_rvous
-  rvous_bytes += 4*nprocs*sizeof(int);                    // all2all vectors
-  rvous_bytes += MAX(all2all1_bytes,all2all2_bytes);      // reorder ops
-  */
-
-  return nout;
 }
 
 /* ----------------------------------------------------------------------

src/comm.h

@@ -153,6 +153,7 @@ class Comm : protected Pointers {
   int rendezvous_all2all(int, char *, int, int, int *, 
                          int (*)(int, char *, int &, int *&, char *&, void *), 
                          int, char *&, int, void *, int);
+  void rendezvous_stats(int, int, int, int, int, int, bigint);
 
  public:
   enum{MULTIPLE};

src/special.cpp

@@ -27,6 +27,8 @@
 
 using namespace LAMMPS_NS;
 
+#define RVOUS 1   // 0 for irregular, 1 for all2all
+
 /* ---------------------------------------------------------------------- */
 
 Special::Special(LAMMPS *lmp) : Pointers(lmp)
@@ -185,7 +187,7 @@ void Special::atom_owners()
   // perform rendezvous operation
   
   char *buf;
-  comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist,
+  comm->rendezvous(RVOUS,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist,
                    rendezvous_ids,0,buf,0,(void *) this,1);
 
   memory->destroy(proclist);
@@ -241,7 +243,7 @@ void Special::onetwo_build_newton()
   // perform rendezvous operation
 
   char *buf;
-  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
+  int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
                                  0,proclist,
                                  rendezvous_pairs,0,buf,sizeof(PairRvous),
                                  (void *) this,1);
@@ -376,7 +378,7 @@ void Special::onethree_build()
   // perform rendezvous operation
 
   char *buf;
-  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
+  int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
                                  0,proclist,
                                  rendezvous_pairs,0,buf,sizeof(PairRvous),
                                  (void *) this,1);
@@ -481,7 +483,7 @@ void Special::onefour_build()
   // perform rendezvous operation
 
   char *buf;
-  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
+  int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
                                  0,proclist,
                                  rendezvous_pairs,0,buf,sizeof(PairRvous),
                                  (void *) this,1);
@@ -904,7 +906,7 @@ void Special::angle_trim()
     // perform rendezvous operation
     
     char *buf;
-    int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
+    int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
                                    0,proclist,
                                    rendezvous_pairs,0,buf,sizeof(PairRvous),
                                    (void *) this,1);
@@ -1123,7 +1125,7 @@ void Special::dihedral_trim()
     // perform rendezvous operation
     
     char *buf;
-    int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),
+    int nreturn = comm->rendezvous(RVOUS,nsend,(char *) inbuf,sizeof(PairRvous),
                                    0,proclist,
                                    rendezvous_pairs,0,buf,sizeof(PairRvous),
                                    (void *) this,1);