Merge branch 'develop' into fix-pair-dump-skip

doc/src/restart.rst

@@ -12,7 +12,7 @@ Syntax
    restart N root keyword value ...
    restart N file1 file2 keyword value ...
 
-* N = write a restart file every this many timesteps
+* N = write a restart file on timesteps which are multipls of N
 * N can be a variable (see below)
 * root = filename to which timestep # is appended
 * file1,file2 = two full filenames, toggle between them when writing file
@@ -42,13 +42,14 @@ Description
 """""""""""
 
 Write out a binary restart file with the current state of the
-simulation every so many timesteps, in either or both of two modes, as
+simulation on timesteps which are a multiple of N.  A value of N = 0
-a run proceeds.  A value of 0 means do not write out any restart
+means do not write out any restart files, which is the default.
-files.  The two modes are as follows.  If one filename is specified, a
+Restart files are written in one (or both) of two modes as a run
-series of filenames will be created which include the timestep in the
+proceeds.  If one filename is specified, a series of filenames will be
-filename.  If two filenames are specified, only 2 restart files will
+created which include the timestep in the filename.  If two filenames
-be created, with those names.  LAMMPS will toggle between the 2 names
+are specified, only 2 restart files will be created, with those names.
-as it writes successive restart files.
+LAMMPS will toggle between the 2 names as it writes successive restart
+files.
 
 Note that you can specify the restart command twice, once with a
 single filename and once with two filenames.  This would allow you,

lib/gpu/geryon/nvd_device.h

@@ -316,6 +316,9 @@ class UCL_Device {
   std::vector<CUstream> _cq;
   CUdevice _cu_device;
   CUcontext _context;
+#if GERYON_NVD_PRIMARY_CONTEXT
+  CUcontext _old_context;
+#endif
 };
 
 // Grabs the properties for all devices
@@ -391,8 +394,14 @@ int UCL_Device::set_platform(const int pid) {
 int UCL_Device::set(int num) {
   clear();
   _device=_properties[num].device_id;
+#if GERYON_NVD_PRIMARY_CONTEXT
+  CU_SAFE_CALL_NS(cuCtxGetCurrent(&_old_context));
+  CU_SAFE_CALL_NS(cuDeviceGet(&_cu_device,_device));
+  CUresult err=cuDevicePrimaryCtxRetain(&_context,_cu_device);
+#else
   CU_SAFE_CALL_NS(cuDeviceGet(&_cu_device,_device));
   CUresult err=cuCtxCreate(&_context,0,_cu_device);
+#endif
   if (err!=CUDA_SUCCESS) {
     #ifndef UCL_NO_EXIT
     std::cerr << "UCL Error: Could not access accelerator number " << num
@@ -401,13 +410,23 @@ int UCL_Device::set(int num) {
     #endif
     return UCL_ERROR;
   }
+#if GERYON_NVD_PRIMARY_CONTEXT
+  if (_context != _old_context) {
+    CU_SAFE_CALL_NS(cuCtxSetCurrent(_context));
+  }
+#endif
   return UCL_SUCCESS;
 }
 
 void UCL_Device::clear() {
   if (_device>-1) {
     for (int i=1; i<num_queues(); i++) pop_command_queue();
-    cuCtxDestroy(_context);
+#if GERYON_NVD_PRIMARY_CONTEXT
+    CU_SAFE_CALL_NS(cuCtxSetCurrent(_old_context));
+    CU_SAFE_CALL_NS(cuDevicePrimaryCtxRelease(_cu_device));
+#else
+    cuCtxDestroy(_context));
+#endif
   }
   _device=-1;
 }

lib/gpu/geryon/nvd_macros.h

@@ -7,6 +7,13 @@
 
 #define CUDA_INT_TYPE size_t
 
+// Use the primary context for GPU access to enable compatibility with tools
+// like OpenMPI accessing the GPU through the runtime interface.
+// Set to 0 to revert to old behavior
+#ifndef GERYON_NVD_PRIMARY_CONTEXT
+#define GERYON_NVD_PRIMARY_CONTEXT 1
+#endif
+
 #ifdef MPI_GERYON
 #include "mpi.h"
 #define NVD_GERYON_EXIT do {                                               \

src/output.cpp

@@ -237,11 +237,9 @@ void Output::setup(int memflag)
         last_dump[idump] = ntimestep;
       }
 
-      // calculate timestep and/or time for next dump
+      // calculate timestep or time for next dump
-      // set next_dump and next_time_dump, 0 arg for setup()
+      // set next_dump and next_time_dump
-      // only do this if dump written or dump has not been written yet
 
-      if (writeflag || last_dump[idump] < 0)
       calculate_next_dump(SETUP,idump,ntimestep);
 
       // if dump not written now, use addstep_compute_all()
@@ -268,11 +266,11 @@ void Output::setup(int memflag)
 
   if (restart_flag && update->restrict_output == 0) {
     if (restart_flag_single) {
-       if (restart_every_single)
+      if (restart_every_single) {
         next_restart_single =
           (ntimestep/restart_every_single)*restart_every_single +
           restart_every_single;
-       else {
+      } else {
         auto  nextrestart = static_cast<bigint>
           (input->variable->compute_equal(ivar_restart_single));
         if (nextrestart <= ntimestep)
@@ -326,16 +324,16 @@ void Output::setup(int memflag)
 
   next = MIN(next_dump_any,next_restart);
   next = MIN(next,next_thermo);
- }
+}
 
- /* ----------------------------------------------------------------------
+/* ----------------------------------------------------------------------
    perform all output for this timestep
    only perform output if next matches current step and last output doesn't
    do dump/restart before thermo so thermo CPU time will include them
- ------------------------------------------------------------------------- */
+------------------------------------------------------------------------- */
 
- void Output::write(bigint ntimestep)
+void Output::write(bigint ntimestep)
- {
+{
   // perform dump if its next_dump = current ntimestep
   //   but not if it was already written on this step
   // set next_dump and also next_time_dump for mode_dump = 1
@@ -365,7 +363,7 @@ void Output::setup(int memflag)
           modify->clearstep_compute();
 
         // perform dump
-         // reset next_dump and next_time_dump, 1 arg for write()
+        // set next_dump and next_time_dump
 
         dump[idump]->write();
         last_dump[idump] = ntimestep;
@@ -388,7 +386,6 @@ void Output::setup(int memflag)
 
   if (next_restart == ntimestep) {
     if (next_restart_single == ntimestep) {
-
       std::string file = restart1;
       std::size_t found = file.find('*');
       if (found != std::string::npos)
@@ -407,6 +404,7 @@ void Output::setup(int memflag)
         modify->addstep_compute(next_restart_single);
       }
     }
+
     if (next_restart_double == ntimestep) {
       if (last_restart != ntimestep) {
         if (restart_toggle == 0) {
@@ -417,6 +415,7 @@ void Output::setup(int memflag)
           restart_toggle = 0;
         }
       }
+
       if (restart_every_double) next_restart_double += restart_every_double;
       else {
         modify->clearstep_compute();
@@ -454,22 +453,22 @@ void Output::setup(int memflag)
 
   next = MIN(next_dump_any,next_restart);
   next = MIN(next,next_thermo);
- }
+}
 
- /* ----------------------------------------------------------------------
+/* ----------------------------------------------------------------------
    force a snapshot to be written for all dumps
    called from PRD and TAD
- ------------------------------------------------------------------------- */
+------------------------------------------------------------------------- */
 
- void Output::write_dump(bigint ntimestep)
+void Output::write_dump(bigint ntimestep)
- {
+{
   for (int idump = 0; idump < ndump; idump++) {
     dump[idump]->write();
     last_dump[idump] = ntimestep;
   }
- }
+}
 
- /* ----------------------------------------------------------------------
+/* ----------------------------------------------------------------------
    calculate when next dump occurs for Dump instance idump
    operates in one of two modes, based on mode_dump flag
    for timestep mode, set next_dump
@@ -478,10 +477,10 @@ void Output::setup(int memflag)
    SETUP = from setup() at start of run
    WRITE = from write() during run each time a dump file is written
    RESET_DT = from reset_dt() called from fix dt/reset when it changes timestep size
- ------------------------------------------------------------------------- */
+------------------------------------------------------------------------- */
 
 void Output::calculate_next_dump(int which, int idump, bigint ntimestep)
- {
+{
   // dump mode is by timestep
   // just set next_dump
 
@@ -489,8 +488,9 @@ void Output::calculate_next_dump(int which, int idump, bigint ntimestep)
 
     if (every_dump[idump]) {
 
-       // which = SETUP: nextdump = next multiple of every_dump
+      // which = SETUP: next_dump = next multiple of every_dump
-       // which = WRITE: increment nextdump by every_dump
+      // which = WRITE: increment next_dump by every_dump
+      //                current step is already multiple of every_dump
 
       if (which == SETUP)
         next_dump[idump] = (ntimestep/every_dump[idump])*every_dump[idump] + every_dump[idump];
@@ -576,7 +576,7 @@ void Output::calculate_next_dump(int which, int idump, bigint ntimestep)
     next_time_dump[idump] = nexttime;
     next_dump[idump] = nextdump;
   }
- }
+}
 
 /* ---------------------------------------------------------------------- */
 
@@ -589,13 +589,13 @@ int Output::check_time_dumps(bigint ntimestep)
   return nowflag;
 }
 
- /* ----------------------------------------------------------------------
+/* ----------------------------------------------------------------------
    force restart file(s) to be written
    called from PRD and TAD
- ------------------------------------------------------------------------- */
+------------------------------------------------------------------------- */
 
- void Output::write_restart(bigint ntimestep)
+void Output::write_restart(bigint ntimestep)
- {
+{
   if (restart_flag_single) {
     std::string file = restart1;
     std::size_t found = file.find('*');
@@ -615,9 +615,9 @@ int Output::check_time_dumps(bigint ntimestep)
   }
 
   last_restart = ntimestep;
- }
+}
 
- /* ----------------------------------------------------------------------
+/* ----------------------------------------------------------------------
    timestep is being changed, called by update->reset_timestep()
    for dumps, require that no dump is "active"
    meaning that a snapshot has already been output
@@ -627,10 +627,10 @@ int Output::check_time_dumps(bigint ntimestep)
    eval for ntimestep-1, so current ntimestep can be returned if needed
    no guarantee that variable can be evaluated for ntimestep-1
    e.g. if it depends on computes, but live with that rare case for now
- ------------------------------------------------------------------------- */
+------------------------------------------------------------------------- */
 
- void Output::reset_timestep(bigint ntimestep)
+void Output::reset_timestep(bigint ntimestep)
- {
+{
   next_dump_any = MAXBIGINT;
   for (int idump = 0; idump < ndump; idump++)
     if ((last_dump[idump] >= 0) && !update->whichflag && !dump[idump]->multifile)
@@ -694,7 +694,7 @@ int Output::check_time_dumps(bigint ntimestep)
 
   next = MIN(next_dump_any,next_restart);
   next = MIN(next,next_thermo);
- }
+}
 
 /* ----------------------------------------------------------------------
    timestep size is being changed
@@ -728,7 +728,6 @@ void Output::reset_dt()
   next = MIN(next,next_thermo);
 }
 
-
 /* ----------------------------------------------------------------------
    add a Dump to list of Dumps
 ------------------------------------------------------------------------- */

unittest/formats/test_dump_atom.cpp

@@ -699,6 +699,66 @@ TEST_F(DumpAtomTest, binary_write_dump)
     delete_file(dump_file);
 }
 
+TEST_F(DumpAtomTest, frequency)
+{
+    auto dump_file = dump_filename("frequency");
+    BEGIN_HIDE_OUTPUT();
+    command("dump id all atom 5 " + dump_file);
+    command("run 15 post no");
+    command("run 12 post no");
+    END_HIDE_OUTPUT();
+
+    // NOTE: must reset to current timestep (27) to avoid unexpected issues with following
+    TEST_FAILURE(".*ERROR: Cannot reset timestep with active dump - must undump first.*",
+                 command("reset_timestep 27"););
+
+    BEGIN_HIDE_OUTPUT();
+    command("run 3 post no");
+    command("undump id");
+    command("reset_timestep 5");
+    command("dump id all atom 10 " + dump_file);
+    command("dump_modify id append yes");
+    command("run 20 post no");
+    command("undump id");
+    END_HIDE_OUTPUT();
+
+    std::vector<std::string> expected, values;
+    values   = extract_items(dump_file, "TIMESTEP");
+    expected = {"0", "5", "10", "15", "20", "25", "30", "10", "20"};
+    ASSERT_EQ(values.size(), expected.size());
+    for (int i = 0; i < expected.size(); ++i)
+        ASSERT_THAT(values[i], Eq(expected[i]));
+
+    BEGIN_HIDE_OUTPUT();
+    command("reset_timestep 10");
+    command("dump id all atom 10 " + dump_file);
+    command("run 20 post no");
+    command("undump id");
+    END_HIDE_OUTPUT();
+
+    values   = extract_items(dump_file, "TIMESTEP");
+    expected = {"10", "20", "30"};
+    ASSERT_EQ(values.size(), expected.size());
+    for (int i = 0; i < expected.size(); ++i)
+        ASSERT_THAT(values[i], Eq(expected[i]));
+
+    BEGIN_HIDE_OUTPUT();
+    command("reset_timestep 0");
+    command("dump id all atom 10 " + dump_file);
+    command("minimize 0.0 0.0 15 30");
+    command("run 20 post no");
+    command("undump id");
+    END_HIDE_OUTPUT();
+
+    values   = extract_items(dump_file, "TIMESTEP");
+    expected = {"0", "10", "15", "20", "30"};
+    ASSERT_EQ(values.size(), expected.size());
+    for (int i = 0; i < expected.size(); ++i)
+        ASSERT_THAT(values[i], Eq(expected[i]));
+
+    delete_file(dump_file);
+}
+
 //-------------------------------------------------------------------------------------------------
 // dump_modify
 //-------------------------------------------------------------------------------------------------