git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@12281 f3b2605a-c512-4ea7-a41b-209d697bcdaa

src/comm.cpp

@@ -17,8 +17,15 @@
 #include "comm.h"
 #include "universe.h"
 #include "atom.h"
+#include "atom_vec.h"
 #include "force.h"
+#include "pair.h"
+#include "modify.h"
+#include "fix.h"
+#include "compute.h"
 #include "domain.h"
+#include "output.h"
+#include "dump.h"
 #include "group.h"
 #include "procmap.h"
 #include "accelerator_kokkos.h"
@@ -31,6 +38,8 @@
 
 using namespace LAMMPS_NS;
 
+#define BUFMIN 1000             // also in comm styles
+
 enum{SINGLE,MULTI};             // same as in Comm sub-styles
 enum{MULTIPLE};                   // same as in ProcMap
 enum{ONELEVEL,TWOLEVEL,NUMA,CUSTOM};
@@ -143,6 +152,67 @@ void Comm::copy_arrays(Comm *oldcomm)
   }
 }
 
+/* ----------------------------------------------------------------------
+   common to all Comm styles
+------------------------------------------------------------------------- */
+
+void Comm::init()
+{
+  triclinic = domain->triclinic;
+  map_style = atom->map_style;
+
+  // comm_only = 1 if only x,f are exchanged in forward/reverse comm
+  // comm_x_only = 0 if ghost_velocity since velocities are added
+
+  comm_x_only = atom->avec->comm_x_only;
+  comm_f_only = atom->avec->comm_f_only;
+  if (ghost_velocity) comm_x_only = 0;
+
+  // set per-atom sizes for forward/reverse/border comm
+  // augment by velocity and fix quantities if needed
+
+  size_forward = atom->avec->size_forward;
+  size_reverse = atom->avec->size_reverse;
+  size_border = atom->avec->size_border;
+
+  if (ghost_velocity) size_forward += atom->avec->size_velocity;
+  if (ghost_velocity) size_border += atom->avec->size_velocity;
+
+  for (int i = 0; i < modify->nfix; i++)
+    size_border += modify->fix[i]->comm_border;
+  
+  // maxexchange = max # of datums/atom in exchange communication
+  // maxforward = # of datums in largest forward communication
+  // maxreverse = # of datums in largest reverse communication
+  // query pair,fix,compute,dump for their requirements
+  // pair style can force reverse comm even if newton off
+
+  maxexchange = BUFMIN + maxexchange_fix;
+  maxforward = MAX(size_forward,size_border);
+  maxreverse = size_reverse;
+
+  if (force->pair) maxforward = MAX(maxforward,force->pair->comm_forward);
+  if (force->pair) maxreverse = MAX(maxreverse,force->pair->comm_reverse);
+
+  for (int i = 0; i < modify->nfix; i++) {
+    maxforward = MAX(maxforward,modify->fix[i]->comm_forward);
+    maxreverse = MAX(maxreverse,modify->fix[i]->comm_reverse);
+  }
+
+  for (int i = 0; i < modify->ncompute; i++) {
+    maxforward = MAX(maxforward,modify->compute[i]->comm_forward);
+    maxreverse = MAX(maxreverse,modify->compute[i]->comm_reverse);
+  }
+
+  for (int i = 0; i < output->ndump; i++) {
+    maxforward = MAX(maxforward,output->dump[i]->comm_forward);
+    maxreverse = MAX(maxreverse,output->dump[i]->comm_reverse);
+  }
+
+  if (force->newton == 0) maxreverse = 0;
+  if (force->pair) maxreverse = MAX(maxreverse,force->pair->comm_reverse_off);
+}
+
 /* ----------------------------------------------------------------------
    modify communication params
    invoked from input script by comm_modify command

src/comm.h

@@ -60,12 +60,12 @@ class Comm : protected Pointers {
   Comm(class LAMMPS *);
   virtual ~Comm();
   void copy_arrays(class Comm *);
+  virtual void init();
   void modify_params(int, char **);
 
   void set_processors(int, char **);      // set 3d processor grid attributes
   virtual void set_proc_grid(int outflag = 1); // setup 3d grid of procs
 
-  virtual void init() = 0;
   virtual void setup() = 0;                      // setup 3d comm pattern
   virtual void forward_comm(int dummy = 0) = 0;  // forward comm of atom coords
   virtual void reverse_comm() = 0;               // reverse comm of forces
@@ -104,6 +104,17 @@ class Comm : protected Pointers {
   int mode;                  // 0 = single cutoff, 1 = multi-type cutoff
   int bordergroup;           // only communicate this group in borders
 
+  int triclinic;                    // 0 if domain is orthog, 1 if triclinic
+  int map_style;                    // non-0 if global->local mapping is done
+  int comm_x_only,comm_f_only;      // 1 if only exchange x,f in for/rev comm
+
+  int size_forward;                 // # of per-atom datums in forward comm
+  int size_reverse;                 // # of datums in reverse comm
+  int size_border;                  // # of datums in forward border comm
+
+  int maxforward,maxreverse;        // max # of datums in forward/reverse comm
+  int maxexchange;                  // max # of datums/atom in exchange comm 
+
   int gridflag;                     // option for creating 3d grid
   int mapflag;                      // option for mapping procs to 3d grid
   char xyz[4];                      // xyz mapping of procs to 3d grid

src/comm_brick.cpp

@@ -122,59 +122,7 @@ void CommBrick::init_buffers()
 
 void CommBrick::init()
 {
-  triclinic = domain->triclinic;
+  Comm::init();
-  map_style = atom->map_style;
-
-  // comm_only = 1 if only x,f are exchanged in forward/reverse comm
-  // comm_x_only = 0 if ghost_velocity since velocities are added
-
-  comm_x_only = atom->avec->comm_x_only;
-  comm_f_only = atom->avec->comm_f_only;
-  if (ghost_velocity) comm_x_only = 0;
-
-  // set per-atom sizes for forward/reverse/border comm
-  // augment by velocity and fix quantities if needed
-
-  size_forward = atom->avec->size_forward;
-  size_reverse = atom->avec->size_reverse;
-  size_border = atom->avec->size_border;
-
-  if (ghost_velocity) size_forward += atom->avec->size_velocity;
-  if (ghost_velocity) size_border += atom->avec->size_velocity;
-
-  for (int i = 0; i < modify->nfix; i++)
-    size_border += modify->fix[i]->comm_border;
-  
-  // maxexchange = max # of datums/atom in exchange communication
-  // maxforward = # of datums in largest forward communication
-  // maxreverse = # of datums in largest reverse communication
-  // query pair,fix,compute,dump for their requirements
-  // pair style can force reverse comm even if newton off
-
-  maxexchange = BUFMIN + maxexchange_fix;
-  maxforward = MAX(size_forward,size_border);
-  maxreverse = size_reverse;
-
-  if (force->pair) maxforward = MAX(maxforward,force->pair->comm_forward);
-  if (force->pair) maxreverse = MAX(maxreverse,force->pair->comm_reverse);
-
-  for (int i = 0; i < modify->nfix; i++) {
-    maxforward = MAX(maxforward,modify->fix[i]->comm_forward);
-    maxreverse = MAX(maxreverse,modify->fix[i]->comm_reverse);
-  }
-
-  for (int i = 0; i < modify->ncompute; i++) {
-    maxforward = MAX(maxforward,modify->compute[i]->comm_forward);
-    maxreverse = MAX(maxreverse,modify->compute[i]->comm_reverse);
-  }
-
-  for (int i = 0; i < output->ndump; i++) {
-    maxforward = MAX(maxforward,output->dump[i]->comm_forward);
-    maxreverse = MAX(maxreverse,output->dump[i]->comm_reverse);
-  }
-
-  if (force->newton == 0) maxreverse = 0;
-  if (force->pair) maxreverse = MAX(maxreverse,force->pair->comm_reverse_off);
 
   // memory for multi-style communication
 
@@ -741,7 +689,7 @@ void CommBrick::exchange()
 
 void CommBrick::borders()
 {
-  int i,n,itype,iswap,dim,ineed,twoneed,smax,rmax;
+  int i,n,itype,iswap,dim,ineed,twoneed;
   int nsend,nrecv,sendflag,nfirst,nlast,ngroup;
   double lo,hi;
   int *type;
@@ -1310,7 +1258,14 @@ void CommBrick::forward_comm_array(int nsize, double **array)
   MPI_Request request;
   MPI_Status status;
 
-  // NOTE: should check that buf_send and buf_recv are big enough
+  // insure send/recv bufs are big enough for nsize
+  // based on smax/rmax from most recent borders() invocation
+
+  if (nsize > maxforward) {
+    maxforward = nsize;
+    if (maxforward*smax > maxsend) grow_send(maxforward*smax,0);
+    if (maxforward*rmax > maxrecv) grow_recv(maxforward*rmax);
+  }
 
   for (iswap = 0; iswap < nswap; iswap++) {
 

src/comm_brick.h

@@ -51,11 +51,7 @@ class CommBrick : public Comm {
   int recvneed[3][2];               // # of procs away I recv atoms from
   int sendneed[3][2];               // # of procs away I send atoms to
   int maxneed[3];                   // max procs away any proc needs, per dim
-  int triclinic;                    // 0 if domain is orthog, 1 if triclinic
   int maxswap;                      // max # of swaps memory is allocated for
-  int size_forward;                 // # of per-atom datums in forward comm
-  int size_reverse;                 // # of datums in reverse comm
-  int size_border;                  // # of datums in forward border comm
   int *sendnum,*recvnum;            // # of atoms to send/recv in each swap
   int *sendproc,*recvproc;          // proc to send/recv to/from at each swap
   int *size_forward_recv;           // # of values to recv in each forward comm
@@ -66,8 +62,6 @@ class CommBrick : public Comm {
   double **cutghostmulti;           // cutghost on a per-type basis
   int *pbc_flag;                    // general flag for sending atoms thru PBC
   int **pbc;                        // dimension flags for PBC adjustments
-  int comm_x_only,comm_f_only;      // 1 if only exchange x,f in for/rev comm
-  int map_style;                    // non-0 if global->local mapping is done
 
   int *firstrecv;                   // where to put 1st recv atom in each swap
   int **sendlist;                   // list of atoms to send in each swap
@@ -76,10 +70,8 @@ class CommBrick : public Comm {
   double *buf_send;                 // send buffer for all comm
   double *buf_recv;                 // recv buffer for all comm
   int maxsend,maxrecv;              // current size of send/recv buffer
-  int maxforward,maxreverse;        // max # of datums in forward/reverse comm
-
-  int maxexchange;                  // max # of datums/atom in exchange comm 
   int bufextra;                     // extra space beyond maxsend in send buffer
+  int smax,rmax;             // max size in atoms of single borders send/recv
 
   void init_buffers();
   int updown(int, int, int, double, int, double *);

src/comm_tiled.cpp

@@ -37,9 +37,7 @@ using namespace LAMMPS_NS;
 #define BUFEXTRA 1000
 #define EPSILON 1.0e-6
 
-// NOTE: change this to 16 after debugged
+#define DELTA_PROCS 16
-
-#define DELTA_PROCS 1
 
 enum{SINGLE,MULTI};               // same as in Comm
 enum{LAYOUT_UNIFORM,LAYOUT_NONUNIFORM,LAYOUT_TILED};    // several files
@@ -50,6 +48,11 @@ CommTiled::CommTiled(LAMMPS *lmp) : Comm(lmp)
 {
   error->all(FLERR,"Comm_style tiled is not yet supported");
 
+  if (lmp->cuda)
+    error->all(FLERR,"USER-CUDA package does not yet support comm_style tiled");
+  if (lmp->kokkos)
+    error->all(FLERR,"KOKKOS package does not yet support comm_style tiled");
+
   style = 1;
   layout = LAYOUT_UNIFORM;
   init_buffers();
@@ -61,6 +64,11 @@ CommTiled::CommTiled(LAMMPS *lmp, Comm *oldcomm) : Comm(*oldcomm)
 {
   error->all(FLERR,"Comm_style tiled is not yet supported");
 
+  if (lmp->cuda)
+    error->all(FLERR,"USER-CUDA package does not yet support comm_style tiled");
+  if (lmp->kokkos)
+    error->all(FLERR,"KOKKOS package does not yet support comm_style tiled");
+
   style = 1;
   layout = oldcomm->layout;
   copy_arrays(oldcomm);
@@ -80,7 +88,6 @@ CommTiled::~CommTiled()
 
 /* ----------------------------------------------------------------------
    initialize comm buffers and other data structs local to CommTiled
-   NOTE: if this is identical to CommBrick, put it into Comm ??
 ------------------------------------------------------------------------- */
 
 void CommTiled::init_buffers()
@@ -102,14 +109,11 @@ void CommTiled::init_buffers()
   rcbinfo = NULL;
 }
 
-/* ----------------------------------------------------------------------
+/* ---------------------------------------------------------------------- */
-   NOTE: if this is nearly identical to CommBrick, put it into Comm ??
-------------------------------------------------------------------------- */
 
 void CommTiled::init()
 {
-  triclinic = domain->triclinic;
+  Comm::init();
-  map_style = atom->map_style;
 
   // temporary restrictions
 
@@ -117,57 +121,6 @@ void CommTiled::init()
     error->all(FLERR,"Cannot yet use comm_style tiled with triclinic box");
   if (mode == MULTI)
     error->all(FLERR,"Cannot yet use comm_style tiled with multi-mode comm");
-
-  // comm_only = 1 if only x,f are exchanged in forward/reverse comm
-  // comm_x_only = 0 if ghost_velocity since velocities are added
-
-  comm_x_only = atom->avec->comm_x_only;
-  comm_f_only = atom->avec->comm_f_only;
-  if (ghost_velocity) comm_x_only = 0;
-
-  // set per-atom sizes for forward/reverse/border comm
-  // augment by velocity and fix quantities if needed
-
-  size_forward = atom->avec->size_forward;
-  size_reverse = atom->avec->size_reverse;
-  size_border = atom->avec->size_border;
-
-  if (ghost_velocity) size_forward += atom->avec->size_velocity;
-  if (ghost_velocity) size_border += atom->avec->size_velocity;
-
-  for (int i = 0; i < modify->nfix; i++)
-    size_border += modify->fix[i]->comm_border;
-
-  // maxexchange = max # of datums/atom in exchange communication
-  // maxforward = # of datums in largest forward communication
-  // maxreverse = # of datums in largest reverse communication
-  // query pair,fix,compute,dump for their requirements
-  // pair style can force reverse comm even if newton off
-
-  maxexchange = BUFMIN + maxexchange_fix;
-  maxforward = MAX(size_forward,size_border);
-  maxreverse = size_reverse;
-
-  if (force->pair) maxforward = MAX(maxforward,force->pair->comm_forward);
-  if (force->pair) maxreverse = MAX(maxreverse,force->pair->comm_reverse);
-
-  for (int i = 0; i < modify->nfix; i++) {
-    maxforward = MAX(maxforward,modify->fix[i]->comm_forward);
-    maxreverse = MAX(maxreverse,modify->fix[i]->comm_reverse);
-  }
-
-  for (int i = 0; i < modify->ncompute; i++) {
-    maxforward = MAX(maxforward,modify->compute[i]->comm_forward);
-    maxreverse = MAX(maxreverse,modify->compute[i]->comm_reverse);
-  }
-
-  for (int i = 0; i < output->ndump; i++) {
-    maxforward = MAX(maxforward,output->dump[i]->comm_forward);
-    maxreverse = MAX(maxreverse,output->dump[i]->comm_reverse);
-  }
-
-  if (force->newton == 0) maxreverse = 0;
-  if (force->pair) maxreverse = MAX(maxreverse,force->pair->comm_reverse_off);
 }
 
 /* ----------------------------------------------------------------------
@@ -475,122 +428,6 @@ void CommTiled::setup()
     requests = new MPI_Request[maxreqstat];
     statuses = new MPI_Status[maxreqstat];
   }
-
-  // DEBUG
-
-  /*
-  MPI_Barrier(world);
-
-  printf("SUBBOX %d: %g %g %g: %g %g %g\n",
-         me,sublo[0],sublo[1],sublo[2],subhi[0],subhi[1],subhi[2]);
-  MPI_Barrier(world);
-
-  for (i = 0; i < nswap; i++) {
-    if (nsendproc[i] == 1)
-      printf("SETUP SEND %d %d: nsend %d self %d sproc0 %d: "
-             "%g %g %g: %g %g %g\n",
-             i,me,nsendproc[i],sendself[i],sendproc[i][0],
-             sendbox[i][0][0],
-             sendbox[i][0][1],
-             sendbox[i][0][2],
-             sendbox[i][0][3],
-             sendbox[i][0][4],
-             sendbox[i][0][5]);
-    else if (nsendproc[i] == 2) 
-      printf("SETUP SEND %d %d: nsend %d self %d sprocs %d %d: "
-             "%g %g %g: %g %g %g: %g %g %g: %g %g %g\n",
-             i,me,nsendproc[i],sendself[i],sendproc[i][0],sendproc[i][1],
-             sendbox[i][0][0],
-             sendbox[i][0][1],
-             sendbox[i][0][2],
-             sendbox[i][0][3],
-             sendbox[i][0][4],
-             sendbox[i][0][5],
-             sendbox[i][1][0],
-             sendbox[i][1][1],
-             sendbox[i][1][2],
-             sendbox[i][1][3],
-             sendbox[i][1][4],
-             sendbox[i][1][5]);
-    else if (nsendproc[i] == 3) 
-      printf("SETUP SEND %d %d: nsend %d self %d sprocs %d %d %d: "
-             "%g %g %g: %g %g %g: %g %g %g: %g %g %g: %g %g %g: %g %g %g\n",
-             i,me,nsendproc[i],sendself[i],sendproc[i][0],sendproc[i][1],
-             sendproc[i][2],
-             sendbox[i][0][0],
-             sendbox[i][0][1],
-             sendbox[i][0][2],
-             sendbox[i][0][3],
-             sendbox[i][0][4],
-             sendbox[i][0][5],
-             sendbox[i][1][0],
-             sendbox[i][1][1],
-             sendbox[i][1][2],
-             sendbox[i][1][3],
-             sendbox[i][1][4],
-             sendbox[i][1][5],
-             sendbox[i][2][0],
-             sendbox[i][2][1],
-             sendbox[i][2][2],
-             sendbox[i][2][3],
-             sendbox[i][2][4],
-             sendbox[i][2][5]);
-    else if (nsendproc[i] >= 4)
-      printf("SETUP SEND %d %d: nsend %d self %d sprocs %d %d %d %d: "
-             "%g %g %g: %g %g %g: %g %g %g: %g %g %g: %g %g %g: %g %g %g"
-             " %g %g %g: %g %g %g\n",
-             i,me,nsendproc[i],sendself[i],sendproc[i][0],sendproc[i][1],
-             sendproc[i][2],sendproc[i][3],
-             sendbox[i][0][0],
-             sendbox[i][0][1],
-             sendbox[i][0][2],
-             sendbox[i][0][3],
-             sendbox[i][0][4],
-             sendbox[i][0][5],
-             sendbox[i][1][0],
-             sendbox[i][1][1],
-             sendbox[i][1][2],
-             sendbox[i][1][3],
-             sendbox[i][1][4],
-             sendbox[i][1][5],
-             sendbox[i][2][0],
-             sendbox[i][2][1],
-             sendbox[i][2][2],
-             sendbox[i][2][3],
-             sendbox[i][2][4],
-             sendbox[i][2][5],
-             sendbox[i][3][0],
-             sendbox[i][3][1],
-             sendbox[i][3][2],
-             sendbox[i][3][3],
-             sendbox[i][3][4],
-             sendbox[i][3][5]);
-    if (nrecvproc[i] == 1)
-      printf("SETUP RECV %d %d: nrecv %d other %d rproc0 %d\n",
-             i,me,nrecvproc[i],sendother[i],recvproc[i][0]);
-    else if (nrecvproc[i] >= 2)
-      printf("SETUP RECV %d %d: nrecv %d other %d rprocs %d %d\n",
-             i,me,nrecvproc[i],sendother[i],recvproc[i][0],recvproc[i][1]);
-  }
-
-  for (i = 0; i < dimension; i++) {
-    if (nexchproc[i] == 1)
-      printf("SETUP EXCH %d %d: nexch %d exch %d\n",
-             i,me,nexchproc[i],exchproc[i][0]);
-    else if (nexchproc[i] == 2)
-      printf("SETUP EXCH %d %d: nexch %d exch %d %d\n",
-             i,me,nexchproc[i],exchproc[i][0],exchproc[i][1]);
-    else if (nexchproc[i] == 3)
-      printf("SETUP EXCH %d %d: nexch %d exch %d %d %d\n",
-             i,me,nexchproc[i],exchproc[i][0],exchproc[i][1],exchproc[i][2]);
-    else if (nexchproc[i] >= 4)
-      printf("SETUP EXCH %d %d: nexch %d exch %d %d %d %d\n",
-             i,me,nexchproc[i],exchproc[i][0],exchproc[i][1],
-             exchproc[i][2],exchproc[i][3]);
-  }
-
-  MPI_Barrier(world);
-  */
 }
 
 /* ----------------------------------------------------------------------
@@ -923,73 +760,6 @@ void CommTiled::exchange()
   }
 
   if (atom->firstgroupname) atom->first_reorder();
-
-  // DEBUG
-
-  // check that total count of atoms is correct
-
-  /*
-  nlocal = atom->nlocal;
-  int ntotal;
-  MPI_Allreduce(&nlocal,&ntotal,1,MPI_INT,MPI_SUM,world);
-  if (ntotal != atom->natoms) {
-    if (me == 0) printf("Atom counts: old %d new %d\n",atom->natoms,ntotal);
-    //error->all(FLERR,"Post-exchange atom count wrong");
-  }
-
-  // check that each proc owns exact atoms it should in its sub-box
-  // lo/hi = first/last owned atom in lattice
-
-  int ilo,jlo,klo,ihi,jhi,khi;
-  bounds(sublo[0],sublo[1],sublo[2],subhi[0],subhi[1],subhi[2],
-         ilo,jlo,klo,ihi,jhi,khi);
-  int ncount = (ihi-ilo+1) * (jhi-jlo+1) * (khi-klo+1);
-
-  if (ncount != nlocal) 
-    printf("EXCH proc %d: correct %d actual %d\n",me,ncount,nlocal);
-  int flag = 0;
-  if (ncount != nlocal) {
-    flag = 1;
-    //for (int i = 0; i < nlocal; i++)
-    //  printf("MISSING %d %d: %g %g %g\n",i,atom->tag[i],
-    //         atom->x[i][0],atom->x[i][1],atom->x[i][2]);
-  }
-  int flagall;
-  MPI_Allreduce(&flag,&flagall,1,MPI_INT,MPI_SUM,world);
-  if (flagall) error->all(FLERR,"Bad exchange results");
-  else if (me == 0) printf("EXCH is correct\n");
-  */
-}
-
-/* ----------------------------------------------------------------------
-   convert box bounds into lattice indices
-   assume static_int can be below/above neg/pos floating point value
-   integer lo = integer >= floating point value
-   integer hi = integer < floating point value
-------------------------------------------------------------------------- */
-
-void CommTiled::bounds(double xlo, double ylo, double zlo, 
-                       double xhi, double yhi, double zhi,
-                       int &ilo, int &jlo, int &klo, 
-                       int &ihi, int &jhi, int &khi)
-{
-  ilo = static_cast<int> (xlo);
-  if (ilo < xlo) ilo++;
-
-  jlo = static_cast<int> (ylo);
-  if (jlo < ylo) jlo++;
-
-  klo = static_cast<int> (zlo);
-  if (klo < zlo) klo++;
-
-  ihi = static_cast<int> (xhi);
-  if (ihi >= xhi) ihi--;
-
-  jhi = static_cast<int> (yhi);
-  if (jhi >= yhi) jhi--;
-
-  khi = static_cast<int> (zhi);
-  if (khi >= zhi) khi--;
 }
 
 /* ----------------------------------------------------------------------
@@ -1014,10 +784,8 @@ void CommTiled::borders()
   // send/recv max one = max # of atoms in single send/recv for any swap
   // send/recv max all = max # of atoms in all sends/recvs within any swap
 
-  int smaxone = 0;
+  smaxone = smaxall = 0;
-  int smaxall = 0;
+  rmaxone = rmaxall = 0;
-  int rmaxone = 0;
-  int rmaxall = 0;
 
   // loop over swaps in all dimensions
 
@@ -1189,99 +957,6 @@ void CommTiled::borders()
   // reset global->local map
 
   if (map_style) atom->map_set();
-
-  // DEBUG
-
-  /*
-  MPI_Barrier(world);
-
-  for (i = 0; i < nswap; i++) {
-    if (nsendproc[i] == 1)
-      printf("BORDERS SEND %d %d: proc %d snum0 %d\n",
-             i,me,sendproc[i][0],sendnum[i][0]);
-    else if (nsendproc[i] == 2)
-      printf("BORDERS SEND %d %d: proc %d %d snums %d %d\n",
-             i,me,sendproc[i][0],sendproc[i][1],sendnum[i][0],sendnum[i][1]);
-    else if (nsendproc[i] == 3)
-      printf("BORDERS SEND %d %d: proc %d %d %d snums %d %d %d\n",
-             i,me,sendproc[i][0],sendproc[i][1],sendproc[i][2],
-             sendnum[i][0],sendnum[i][1],sendnum[i][2]);
-    else if (nsendproc[i] == 8)
-      printf("BORDERS SEND %d %d: proc %d %d %d %d: %d %d %d %d "
-             "snums %d %d %d %d: %d %d %d %d\n",
-             i,me,
-             sendproc[i][0],sendproc[i][1],sendproc[i][2],sendproc[i][3],
-             sendproc[i][4],sendproc[i][5],sendproc[i][6],sendproc[i][7],
-             sendnum[i][0],sendnum[i][1],sendnum[i][2],sendnum[i][3],
-             sendnum[i][4],sendnum[i][5],sendnum[i][6],sendnum[i][7]);
-    else if (nsendproc[i] >= 4)
-      printf("BORDERS SEND %d %d: proc %d %d %d %d snums %d %d %d %d\n",
-             i,me,sendproc[i][0],sendproc[i][1],sendproc[i][2],
-             sendproc[i][3],
-             sendnum[i][0],sendnum[i][1],sendnum[i][2],sendnum[i][3]);
-    if (nrecvproc[i] == 1)
-      printf("BORDERS RECV %d %d: proc %d rnum0 %d\n",
-             i,me,recvproc[i][0],recvnum[i][0]);
-    else if (nrecvproc[i] == 2)
-      printf("BORDERS RECV %d %d: proc %d %d rnums %d %d\n",
-             i,me,recvproc[i][0],recvproc[i][1],recvnum[i][0],recvnum[i][1]);
-    else if (nrecvproc[i] == 3)
-      printf("BORDERS RECV %d %d: proc %d %d %d rnums %d %d %d\n",
-             i,me,recvproc[i][0],recvproc[i][1],recvproc[i][2],
-             recvnum[i][0],recvnum[i][1],recvnum[i][2]);
-    else if (nrecvproc[i] == 8)
-      printf("BORDERS RECV %d %d: proc %d %d %d %d: %d %d %d %d "
-             "rnums %d %d %d %d: %d %d %d %d\n",
-             i,me,
-             recvproc[i][0],recvproc[i][1],recvproc[i][2],recvproc[i][3],
-             recvproc[i][4],recvproc[i][5],recvproc[i][6],recvproc[i][7],
-             recvnum[i][0],recvnum[i][1],recvnum[i][2],recvnum[i][3],
-             recvnum[i][4],recvnum[i][5],recvnum[i][6],recvnum[i][7]);
-    else if (nrecvproc[i] >= 4)
-      printf("BORDERS RECV %d %d: proc %d %d %d %d rnums %d %d %d %d\n",
-             i,me,recvproc[i][0],recvproc[i][1],recvproc[i][2],recvproc[i][3],
-             recvnum[i][0],recvnum[i][1],recvnum[i][2],recvnum[i][3]);
-  }
-
-  MPI_Barrier(world);
-  /*
-
-  // DEBUG
-
-  // check that each proc owns exact atoms it should in its sub-box + ghosts
-  // lo/hi = first/last owned atom in lattice
-
-  /*
-  int ilo,jlo,klo,ihi,jhi,khi;
-  double ghostlo[3],ghosthi[3];
-
-  ghostlo[0] = sublo[0] - cutghost[0];
-  ghostlo[1] = sublo[1] - cutghost[1];
-  ghosthi[0] = subhi[0] + cutghost[0];
-  ghosthi[1] = subhi[1] + cutghost[1];
-  if (domain->dimension == 3) {
-    ghostlo[2] = sublo[2] - cutghost[2];
-    ghosthi[2] = subhi[2] + cutghost[2];
-  } else {
-    ghostlo[2] = sublo[2];
-    ghosthi[2] = subhi[2];
-  }
-
-  bounds(ghostlo[0],ghostlo[1],ghostlo[2],ghosthi[0],ghosthi[1],ghosthi[2],
-         ilo,jlo,klo,ihi,jhi,khi);
-  ncount = (ihi-ilo+1) * (jhi-jlo+1) * (khi-klo+1);
-  ncount -= atom->nlocal;
-
-  int nghost = atom->nghost;
-  if (ncount != nghost) 
-    printf("BORDERS proc %d: correct %d actual %d\n",me,ncount,nghost);
-  int flag = 0;
-  if (ncount != nghost) flag = 1;
-  int flagall;
-  MPI_Allreduce(&flag,&flagall,1,MPI_INT,MPI_SUM,world);
-  if (flagall) error->all(FLERR,"Bad borders results");
-  else if (me == 0) printf("BORDERS is correct\n");
-  */
 }
 
 /* ----------------------------------------------------------------------
@@ -1469,8 +1144,6 @@ void CommTiled::reverse_comm_fix(Fix *fix)
   }
 }
 
-// NOTE: these two forward/reverse methods still need to be updated
-
 /* ----------------------------------------------------------------------
    forward communication invoked by a Fix
    n = total datums for all atoms, allows for variable number/atom
@@ -1479,6 +1152,8 @@ void CommTiled::reverse_comm_fix(Fix *fix)
 
 void CommTiled::forward_comm_variable_fix(Fix *fix)
 {
+// NOTE: these two forward/reverse methods still need to be updated
+
 }
 
 /* ----------------------------------------------------------------------
@@ -1689,7 +1364,14 @@ void CommTiled::forward_comm_array(int nsize, double **array)
   int i,j,k,m,n,iatom,last,irecv,nsend,nrecv;
   MPI_Status status;
 
-  // NOTE: should check that buf_send and buf_recv are big enough
+  // insure send/recv bufs are big enough for nsize
+  // based on smaxone/rmaxall from most recent borders() invocation
+
+  if (nsize > maxforward) {
+    maxforward = nsize;
+    if (maxforward*smaxone > maxsend) grow_send(maxforward*smaxone,0);
+    if (maxforward*rmaxall > maxrecv) grow_recv(maxforward*rmaxall);
+  }
 
   for (int iswap = 0; iswap < nswap; iswap++) {
     nsend = nsendproc[iswap] - sendself[iswap];
@@ -1742,10 +1424,9 @@ void CommTiled::forward_comm_array(int nsize, double **array)
   }
 }
 
-// NOTE: this one is not used ???
-
 /* ----------------------------------------------------------------------
    exchange info provided with all 6 stencil neighbors
+   NOTE: this method is currently not used
 ------------------------------------------------------------------------- */
 
 int CommTiled::exchange_variable(int n, double *inbuf, double *&outbuf)
@@ -1764,6 +1445,7 @@ void CommTiled::box_drop_brick(int idim, double *lo, double *hi, int &indexme)
 {
   // NOTE: this is not triclinic compatible
   // NOTE: there error messages are internal - should not occur
+  //       can remove at some point
 
   int index,dir;
   if (hi[idim] == sublo[idim]) {
@@ -1778,7 +1460,7 @@ void CommTiled::box_drop_brick(int idim, double *lo, double *hi, int &indexme)
   } else if (lo[idim] == boxlo[idim]) {
     index = 0;
     dir = 1;
-  } else error->one(FLERR,"Comm tiled: no match in box drop brick");
+  } else error->one(FLERR,"Comm tiled mis-match in box drop brick");
 
   int other1,other2,proc;
   double lower,upper;
@@ -1796,7 +1478,7 @@ void CommTiled::box_drop_brick(int idim, double *lo, double *hi, int &indexme)
   }
 
   if (index < 0 || index > procgrid[idim]) 
-    error->one(FLERR,"Comm tiled: index error in box drop brick");
+    error->one(FLERR,"Comm tiled invalid index in box drop brick");
 
   while (1) {
     lower = boxlo[idim] + prd[idim]*split[index];

src/comm_tiled.h

@@ -47,12 +47,6 @@ class CommTiled : public Comm {
   bigint memory_usage();
 
  private:
-  int triclinic;                    // 0 if domain is orthog, 1 if triclinic
-  int map_style;                    // non-0 if global->local mapping is done
-  int size_forward;                 // # of per-atom datums in forward comm
-  int size_reverse;                 // # of datums in reverse comm
-  int size_border;                  // # of datums in forward border comm
-
   int nswap;                    // # of swaps to perform = 2*dim
 
   // forward/reverse comm info, proc lists include self
@@ -87,17 +81,15 @@ class CommTiled : public Comm {
   double *buf_send;             // send buffer for all comm
   double *buf_recv;             // recv buffer for all comm
   int maxsend,maxrecv;          // current size of send/recv buffer
-  int maxforward,maxreverse;    // max # of datums in forward/reverse comm
-
-  int maxexchange;              // max # of datums/atom in exchange comm 
   int bufextra;                 // extra space beyond maxsend in send buffer
+  int smaxone,rmaxone;          // max size in atoms of single borders send/recv
+  int smaxall,rmaxall;          // max size in atoms of any borders send/recv
+                                //   for comm to all procs in one swap
 
   int maxreqstat;               // max size of Request and Status vectors
   MPI_Request *requests;
   MPI_Status *statuses;
 
-  int comm_x_only,comm_f_only;      // 1 if only exchange x,f in for/rev comm
-
   struct RCBinfo {
     double mysplit[3][2];      // fractional RCB bounding box for one proc
     double cutfrac;    	       // fractional position of cut this proc owns
@@ -149,12 +141,6 @@ class CommTiled : public Comm {
   void grow_swap_send(int, int, int);  // grow swap arrays for send and recv
   void grow_swap_recv(int, int);
   void deallocate_swap(int);           // deallocate swap arrays
-
-
-  // DEBUG
-
-  void bounds(double, double, double, double, double, double,
-              int &,int &,int &,int &,int &,int &);
 };
 
 }

src/rcb.cpp

@@ -22,10 +22,7 @@ using namespace LAMMPS_NS;
 
 #define MYHUGE 1.0e30
 #define TINY 1.0e-6
-
+#define DELTA 16384
-// set this to bigger number after debugging
-
-#define DELTA 10
 
 // prototypes for non-class functions