diff --git a/doc/src/balance.rst b/doc/src/balance.rst index a18d25c7ce..c30c133a8b 100644 --- a/doc/src/balance.rst +++ b/doc/src/balance.rst @@ -53,6 +53,7 @@ Syntax name = name of the atom-style variable *store* name = store weight in custom atom property defined by :doc:`fix property/atom ` command name = atom property name (without d\_ prefix) + *sort* arg = *no* or *yes* *out* arg = filename filename = write each processor's subdomain to a file @@ -492,6 +493,14 @@ different kinds of custom atom vectors or arrays as arguments. ---------- +The *sort* keyword determines whether the communication of per-atom +data to other processors during load-balancing will be random or +deterministic. Random is generally faster; deterministic will ensure +the new ordering of atoms on each processor is the same each time the +same simulation is run. This can be useful for debugging purposes. +Since the balance commmand is a one-time operation, the default is +*yes* to perform sorting. + The *out* keyword writes a text file to the specified *filename* with the results of the balancing operation. The file contains the bounds of the subdomain for each processor after the balancing operation @@ -569,4 +578,5 @@ Related commands Default """"""" -none +The default setting is sort = yes. + diff --git a/doc/src/fix_balance.rst b/doc/src/fix_balance.rst index d3f4c48248..8c65e8341d 100644 --- a/doc/src/fix_balance.rst +++ b/doc/src/fix_balance.rst @@ -43,6 +43,7 @@ Syntax name = name of the atom-style variable *store* name = store weight in custom atom property defined by :doc:`fix property/atom ` command name = atom property name (without d\_ prefix) + *sort* arg = *no* or *yes* *out* arg = filename filename = write each processor's subdomain to a file, at each re-balancing @@ -308,6 +309,14 @@ in that sub-box. ---------- +The *sort* keyword determines whether the communication of per-atom +data to other processors during load-balancing will be random or +deterministic. Random is generally faster; deterministic will ensure +the new ordering of atoms on each processor is the same each time the +same simulation is run. This can be useful for debugging purposes. +Since the fix balance commmand is performed during timestepping, the +default is *no* so that sorting is not performed. + The *out* keyword writes text to the specified *filename* with the results of each re-balancing operation. The file contains the bounds of the subdomain for each processor after the balancing operation @@ -415,4 +424,4 @@ Related commands Default """"""" -none +The default setting is sort = no. diff --git a/src/balance.cpp b/src/balance.cpp index 10a3c320c5..c64f3f4c25 100644 --- a/src/balance.cpp +++ b/src/balance.cpp @@ -252,7 +252,7 @@ void Balance::command(int narg, char **arg) // process remaining optional args - options(iarg,narg,arg); + options(iarg,narg,arg,1); if (wtflag) weight_storage(nullptr); // ensure particles are in current box & update box via shrink-wrap @@ -344,7 +344,7 @@ void Balance::command(int narg, char **arg) if (style == BISECTION) { comm->layout = Comm::LAYOUT_TILED; - bisection(1); + bisection(); } // reset proc sub-domains @@ -359,8 +359,8 @@ void Balance::command(int narg, char **arg) if (domain->triclinic) domain->x2lamda(atom->nlocal); auto irregular = new Irregular(lmp); if (wtflag) fixstore->disable = 0; - if (style == BISECTION) irregular->migrate_atoms(1,1,rcb->sendproc); - else irregular->migrate_atoms(1); + if (style == BISECTION) irregular->migrate_atoms(sortflag,1,rcb->sendproc); + else irregular->migrate_atoms(sortflag); delete irregular; if (domain->triclinic) domain->lamda2x(atom->nlocal); @@ -421,9 +421,10 @@ void Balance::command(int narg, char **arg) /* ---------------------------------------------------------------------- process optional command args for Balance and FixBalance + sortflag_default is different for the 2 classes ------------------------------------------------------------------------- */ -void Balance::options(int iarg, int narg, char **arg) +void Balance::options(int iarg, int narg, char **arg, int sortflag_default) { // count max number of weight settings @@ -435,10 +436,11 @@ void Balance::options(int iarg, int narg, char **arg) wtflag = 0; varflag = 0; - oldrcb = 0; + sortflag = sortflag_default; outflag = 0; int outarg = 0; fp = nullptr; + oldrcb = 0; while (iarg < narg) { if (strcmp(arg[iarg],"weight") == 0) { @@ -471,14 +473,20 @@ void Balance::options(int iarg, int narg, char **arg) } iarg += 2+nopt; - } else if (strcmp(arg[iarg],"old") == 0) { - oldrcb = 1; - iarg++; + } else if (strcmp(arg[iarg+1],"sort") == 0) { + if (iarg+2 > narg) utils::missing_cmd_args(FLERR, "balance sort", error); + sortflag = utils::logical(FLERR,arg[iarg+1],false,lmp); + iarg += 2; } else if (strcmp(arg[iarg],"out") == 0) { if (iarg+2 > narg) error->all(FLERR,"Illegal (fix) balance command"); outflag = 1; outarg = iarg+1; iarg += 2; + + } else if (strcmp(arg[iarg],"old") == 0) { + oldrcb = 1; + iarg++; + } else error->all(FLERR,"Illegal (fix) balance command"); } @@ -569,11 +577,10 @@ double Balance::imbalance_factor(double &maxcost) /* ---------------------------------------------------------------------- perform balancing via RCB class - sortflag = flag for sorting order of received messages by proc ID return list of procs to send my atoms to ------------------------------------------------------------------------- */ -int *Balance::bisection(int sortflag) +int *Balance::bisection() { if (!rcb) rcb = new RCB(lmp); @@ -641,6 +648,7 @@ int *Balance::bisection(int sortflag) // invoke RCB // then invert() to create list of proc assignments for my atoms + // sortflag = flag for sorting order of received messages by proc ID // if triclinic, RCB operates on lamda coords // NOTE: (3/2017) can remove undocumented "old" option at some point // ditto in rcb.cpp, or make it an option @@ -658,7 +666,7 @@ int *Balance::bisection(int sortflag) } if (triclinic) domain->lamda2x(nlocal); - + rcb->invert(sortflag); // reset RCB lo/hi bounding box to full simulation box as needed diff --git a/src/balance.h b/src/balance.h index a21f8992f2..3a6e8bf457 100644 --- a/src/balance.h +++ b/src/balance.h @@ -30,19 +30,20 @@ class Balance : public Command { class FixStorePeratom *fixstore; // per-atom weights stored in FixStorePeratom int wtflag; // 1 if particle weighting is used int varflag; // 1 if weight style var(iable) is used + int sortflag; // 1 if sorting of comm messages is done int outflag; // 1 for output of balance results to file Balance(class LAMMPS *); ~Balance() override; void command(int, char **) override; - void options(int, int, char **); + void options(int, int, char **, int); void weight_storage(char *); void init_imbalance(int); void set_weights(); double imbalance_factor(double &); void shift_setup(char *, int, double); int shift(); - int *bisection(int sortflag = 0); + int *bisection(); void dumpout(bigint); static constexpr int BSTR_SIZE = 3; diff --git a/src/fix_balance.cpp b/src/fix_balance.cpp index 82c38d8650..0ada3d3b1f 100644 --- a/src/fix_balance.cpp +++ b/src/fix_balance.cpp @@ -104,9 +104,10 @@ FixBalance::FixBalance(LAMMPS *lmp, int narg, char **arg) : balance = new Balance(lmp); if (lbstyle == SHIFT) balance->shift_setup(bstr,nitermax,thresh); - balance->options(iarg,narg,arg); + balance->options(iarg,narg,arg,0); wtflag = balance->wtflag; - + sortflag = balance->sortflag; + if (balance->varflag && nevery == 0) error->all(FLERR,"Fix balance nevery = 0 cannot be used with weight var"); @@ -295,12 +296,14 @@ void FixBalance::rebalance() // set disable = 0, so weights migrate with atoms // important to delay disable = 1 until after pre_neighbor imbfinal calc // b/c atoms may migrate again in comm->exchange() - // NOTE: for reproducible debug runs, set 1st arg of migrate_atoms() to 1 - + // sortflag determines whether irregular sorts its + // comm messages for reproducibility + // if not, message order is random, atom order is non-deterministic + if (domain->triclinic) domain->x2lamda(atom->nlocal); if (wtflag) balance->fixstore->disable = 0; - if (lbstyle == BISECTION) irregular->migrate_atoms(0,1,sendproc); - else if (irregular->migrate_check()) irregular->migrate_atoms(); + if (lbstyle == BISECTION) irregular->migrate_atoms(sortflag,1,sendproc); + else if (irregular->migrate_check()) irregular->migrate_atoms(sortflag); if (domain->triclinic) domain->lamda2x(atom->nlocal); // notify all classes that store distributed grids diff --git a/src/fix_balance.h b/src/fix_balance.h index f1a8db57ae..074548ac0c 100644 --- a/src/fix_balance.h +++ b/src/fix_balance.h @@ -44,6 +44,7 @@ class FixBalance : public Fix { double thresh, stopthresh; char bstr[4]; int wtflag; // 1 for weighted balancing + int sortflag; // 1 for sorting comm messages double imbnow; // current imbalance factor double imbprev; // imbalance factor before last rebalancing