add user option for performing message sorting during load balancing

doc/src/balance.rst

@@ -53,6 +53,7 @@ Syntax
              name = name of the atom-style variable
            *store* name = store weight in custom atom property defined by :doc:`fix property/atom <fix_property_atom>` command
              name = atom property name (without d\_ prefix)
+       *sort* arg = *no* or *yes*
        *out* arg = filename
          filename = write each processor's subdomain to a file
 
@@ -492,6 +493,14 @@ different kinds of custom atom vectors or arrays as arguments.
 
 ----------
 
+The *sort* keyword determines whether the communication of per-atom
+data to other processors during load-balancing will be random or
+deterministic.  Random is generally faster; deterministic will ensure
+the new ordering of atoms on each processor is the same each time the
+same simulation is run.  This can be useful for debugging purposes.
+Since the balance commmand is a one-time operation, the default is
+*yes* to perform sorting.
+
 The *out* keyword writes a text file to the specified *filename* with
 the results of the balancing operation.  The file contains the bounds
 of the subdomain for each processor after the balancing operation
@@ -569,4 +578,5 @@ Related commands
 Default
 """""""
 
-none
+The default setting is sort = yes.
+

doc/src/fix_balance.rst

@@ -43,6 +43,7 @@ Syntax
              name = name of the atom-style variable
            *store* name = store weight in custom atom property defined by :doc:`fix property/atom <fix_property_atom>` command
              name = atom property name (without d\_ prefix)
+       *sort* arg = *no* or *yes*
        *out* arg = filename
          filename = write each processor's subdomain to a file, at each re-balancing
 
@@ -308,6 +309,14 @@ in that sub-box.
 
 ----------
 
+The *sort* keyword determines whether the communication of per-atom
+data to other processors during load-balancing will be random or
+deterministic.  Random is generally faster; deterministic will ensure
+the new ordering of atoms on each processor is the same each time the
+same simulation is run.  This can be useful for debugging purposes.
+Since the fix balance commmand is performed during timestepping, the
+default is *no* so that sorting is not performed.
+
 The *out* keyword writes text to the specified *filename* with the
 results of each re-balancing operation.  The file contains the bounds
 of the subdomain for each processor after the balancing operation
@@ -415,4 +424,4 @@ Related commands
 Default
 """""""
 
-none
+The default setting is sort = no.

src/balance.cpp

@@ -252,7 +252,7 @@ void Balance::command(int narg, char **arg)
 
   // process remaining optional args
 
-  options(iarg,narg,arg);
+  options(iarg,narg,arg,1);
   if (wtflag) weight_storage(nullptr);
 
   // ensure particles are in current box & update box via shrink-wrap
@@ -344,7 +344,7 @@ void Balance::command(int narg, char **arg)
 
   if (style == BISECTION) {
     comm->layout = Comm::LAYOUT_TILED;
-    bisection(1);
+    bisection();
   }
 
   // reset proc sub-domains
@@ -359,8 +359,8 @@ void Balance::command(int narg, char **arg)
   if (domain->triclinic) domain->x2lamda(atom->nlocal);
   auto irregular = new Irregular(lmp);
   if (wtflag) fixstore->disable = 0;
-  if (style == BISECTION) irregular->migrate_atoms(1,1,rcb->sendproc);
+  if (style == BISECTION) irregular->migrate_atoms(sortflag,1,rcb->sendproc);
-  else irregular->migrate_atoms(1);
+  else irregular->migrate_atoms(sortflag);
   delete irregular;
   if (domain->triclinic) domain->lamda2x(atom->nlocal);
 
@@ -421,9 +421,10 @@ void Balance::command(int narg, char **arg)
 
 /* ----------------------------------------------------------------------
    process optional command args for Balance and FixBalance
+   sortflag_default is different for the 2 classes
 ------------------------------------------------------------------------- */
 
-void Balance::options(int iarg, int narg, char **arg)
+void Balance::options(int iarg, int narg, char **arg, int sortflag_default)
 {
   // count max number of weight settings
 
@@ -435,10 +436,11 @@ void Balance::options(int iarg, int narg, char **arg)
 
   wtflag = 0;
   varflag = 0;
-  oldrcb = 0;
+  sortflag = sortflag_default;
   outflag = 0;
   int outarg = 0;
   fp = nullptr;
+  oldrcb = 0;
 
   while (iarg < narg) {
     if (strcmp(arg[iarg],"weight") == 0) {
@@ -471,14 +473,20 @@ void Balance::options(int iarg, int narg, char **arg)
       }
       iarg += 2+nopt;
 
-    } else if (strcmp(arg[iarg],"old") == 0) {
+    } else if (strcmp(arg[iarg+1],"sort") == 0) {
-      oldrcb = 1;
+      if (iarg+2 > narg) utils::missing_cmd_args(FLERR, "balance sort", error);
-      iarg++;
+      sortflag = utils::logical(FLERR,arg[iarg+1],false,lmp);
+      iarg += 2;
     } else if (strcmp(arg[iarg],"out") == 0) {
       if (iarg+2 > narg) error->all(FLERR,"Illegal (fix) balance command");
       outflag = 1;
       outarg = iarg+1;
       iarg += 2;
+      
+    } else if (strcmp(arg[iarg],"old") == 0) {
+      oldrcb = 1;
+      iarg++;
+      
     } else error->all(FLERR,"Illegal (fix) balance command");
   }
 
@@ -569,11 +577,10 @@ double Balance::imbalance_factor(double &maxcost)
 
 /* ----------------------------------------------------------------------
    perform balancing via RCB class
-   sortflag = flag for sorting order of received messages by proc ID
    return list of procs to send my atoms to
 ------------------------------------------------------------------------- */
 
-int *Balance::bisection(int sortflag)
+int *Balance::bisection()
 {
   if (!rcb) rcb = new RCB(lmp);
 
@@ -641,6 +648,7 @@ int *Balance::bisection(int sortflag)
 
   // invoke RCB
   // then invert() to create list of proc assignments for my atoms
+  // sortflag = flag for sorting order of received messages by proc ID
   // if triclinic, RCB operates on lamda coords
   // NOTE: (3/2017) can remove undocumented "old" option at some point
   //       ditto in rcb.cpp, or make it an option

src/balance.h

@@ -30,19 +30,20 @@ class Balance : public Command {
   class FixStorePeratom *fixstore;    // per-atom weights stored in FixStorePeratom
   int wtflag;                         // 1 if particle weighting is used
   int varflag;                        // 1 if weight style var(iable) is used
+  int sortflag;                       // 1 if sorting of comm messages is done
   int outflag;                        // 1 for output of balance results to file
 
   Balance(class LAMMPS *);
   ~Balance() override;
   void command(int, char **) override;
-  void options(int, int, char **);
+  void options(int, int, char **, int);
   void weight_storage(char *);
   void init_imbalance(int);
   void set_weights();
   double imbalance_factor(double &);
   void shift_setup(char *, int, double);
   int shift();
-  int *bisection(int sortflag = 0);
+  int *bisection();
   void dumpout(bigint);
 
   static constexpr int BSTR_SIZE = 3;

src/fix_balance.cpp

@@ -104,8 +104,9 @@ FixBalance::FixBalance(LAMMPS *lmp, int narg, char **arg) :
 
   balance = new Balance(lmp);
   if (lbstyle == SHIFT) balance->shift_setup(bstr,nitermax,thresh);
-  balance->options(iarg,narg,arg);
+  balance->options(iarg,narg,arg,0);
   wtflag = balance->wtflag;
+  sortflag = balance->sortflag;
   
   if (balance->varflag && nevery == 0)
     error->all(FLERR,"Fix balance nevery = 0 cannot be used with weight var");
@@ -295,12 +296,14 @@ void FixBalance::rebalance()
   // set disable = 0, so weights migrate with atoms
   //   important to delay disable = 1 until after pre_neighbor imbfinal calc
   //   b/c atoms may migrate again in comm->exchange()
-  // NOTE: for reproducible debug runs, set 1st arg of migrate_atoms() to 1
+  // sortflag determines whether irregular sorts its
+  //   comm messages for reproducibility
+  //   if not, message order is random, atom order is non-deterministic
   
   if (domain->triclinic) domain->x2lamda(atom->nlocal);
   if (wtflag) balance->fixstore->disable = 0;
-  if (lbstyle == BISECTION) irregular->migrate_atoms(0,1,sendproc);
+  if (lbstyle == BISECTION) irregular->migrate_atoms(sortflag,1,sendproc);
-  else if (irregular->migrate_check()) irregular->migrate_atoms();
+  else if (irregular->migrate_check()) irregular->migrate_atoms(sortflag);
   if (domain->triclinic) domain->lamda2x(atom->nlocal);
 
   // notify all classes that store distributed grids

src/fix_balance.h

@@ -44,6 +44,7 @@ class FixBalance : public Fix {
   double thresh, stopthresh;
   char bstr[4];
   int wtflag;               // 1 for weighted balancing
+  int sortflag;             // 1 for sorting comm messages
 
   double imbnow;            // current imbalance factor
   double imbprev;           // imbalance factor before last rebalancing