From 294a1c216893cd6439ad6236be4822d95c101dea Mon Sep 17 00:00:00 2001 From: Ben Menadue Date: Tue, 6 Sep 2022 08:54:59 +1000 Subject: [PATCH 1/6] Use primary context in CUDA GPU code. Since LAMMPS uses the low-level driver API of CUDA, it needs to ensure that it is in the correct context when invoking such functions. At the moment it creates and switches to its own context inside `UCL_Device::set` but then assumes that the driver is still in that context for subsequent calls into CUDA; if another part of the program uses a different context (such as the CUDA runtime using the "primary" context) this will cause failures inside LAMMPS. This patch changes the context creation to instead use the primary context for the requested device. While it's not perfect, in that it still doesn't ensure that it's in the correct context before making driver API calls, it at least allows it to work with libraries that use the runtime API. --- lib/gpu/geryon/nvd_device.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/gpu/geryon/nvd_device.h b/lib/gpu/geryon/nvd_device.h index 80414c9873..36e34cb0f1 100644 --- a/lib/gpu/geryon/nvd_device.h +++ b/lib/gpu/geryon/nvd_device.h @@ -316,6 +316,7 @@ class UCL_Device { std::vector _cq; CUdevice _cu_device; CUcontext _context; + CUcontext _old_context; }; // Grabs the properties for all devices @@ -391,8 +392,9 @@ int UCL_Device::set_platform(const int pid) { int UCL_Device::set(int num) { clear(); _device=_properties[num].device_id; + CU_SAFE_CALL_NS(cuCtxGetCurrent(&_old_context)); CU_SAFE_CALL_NS(cuDeviceGet(&_cu_device,_device)); - CUresult err=cuCtxCreate(&_context,0,_cu_device); + CUresult err=cuDevicePrimaryCtxRetain(&_context,_cu_device); if (err!=CUDA_SUCCESS) { #ifndef UCL_NO_EXIT std::cerr << "UCL Error: Could not access accelerator number " << num @@ -401,13 +403,17 @@ int UCL_Device::set(int num) { #endif return UCL_ERROR; } + if (_context != _old_context) { + CU_SAFE_CALL_NS(cuCtxSetCurrent(_context)); + } return UCL_SUCCESS; } void UCL_Device::clear() { if (_device>-1) { for (int i=1; i Date: Fri, 9 Sep 2022 10:28:33 -0600 Subject: [PATCH 2/6] fix issue with dump snapshots in a run after a minimize possibly being non-multiples of N --- doc/src/restart.rst | 17 +- src/output.cpp | 585 ++++++++++++++++++++++---------------------- 2 files changed, 301 insertions(+), 301 deletions(-) diff --git a/doc/src/restart.rst b/doc/src/restart.rst index 1a746a280d..91b920f377 100644 --- a/doc/src/restart.rst +++ b/doc/src/restart.rst @@ -12,7 +12,7 @@ Syntax restart N root keyword value ... restart N file1 file2 keyword value ... -* N = write a restart file every this many timesteps +* N = write a restart file on timesteps which are multipls of N * N can be a variable (see below) * root = filename to which timestep # is appended * file1,file2 = two full filenames, toggle between them when writing file @@ -42,13 +42,14 @@ Description """"""""""" Write out a binary restart file with the current state of the -simulation every so many timesteps, in either or both of two modes, as -a run proceeds. A value of 0 means do not write out any restart -files. The two modes are as follows. If one filename is specified, a -series of filenames will be created which include the timestep in the -filename. If two filenames are specified, only 2 restart files will -be created, with those names. LAMMPS will toggle between the 2 names -as it writes successive restart files. +simulation on timesteps which are a multiple of N. A value of N = 0 +means do not write out any restart files, which is the default. +Restart files are written in one (or both) of two modes as a run +proceeds. If one filename is specified, a series of filenames will be +created which include the timestep in the filename. If two filenames +are specified, only 2 restart files will be created, with those names. +LAMMPS will toggle between the 2 names as it writes successive restart +files. Note that you can specify the restart command twice, once with a single filename and once with two filenames. This would allow you, diff --git a/src/output.cpp b/src/output.cpp index 4035de3529..4df19b0faa 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -237,12 +237,10 @@ void Output::setup(int memflag) last_dump[idump] = ntimestep; } - // calculate timestep and/or time for next dump - // set next_dump and next_time_dump, 0 arg for setup() - // only do this if dump written or dump has not been written yet + // calculate timestep or time for next dump + // set next_dump and next_time_dump - if (writeflag || last_dump[idump] < 0) - calculate_next_dump(SETUP,idump,ntimestep); + calculate_next_dump(SETUP,idump,ntimestep); // if dump not written now, use addstep_compute_all() // since don't know what computes the dump will invoke @@ -261,224 +259,225 @@ void Output::setup(int memflag) } else next_dump_any = update->laststep + 1; - // do not write restart files at start of run - // set next_restart values to multiple of every or variable value - // wrap variable eval with clear/add - // if no restarts, set next_restart to last+1 so will not influence next + // do not write restart files at start of run + // set next_restart values to multiple of every or variable value + // wrap variable eval with clear/add + // if no restarts, set next_restart to last+1 so will not influence next - if (restart_flag && update->restrict_output == 0) { - if (restart_flag_single) { - if (restart_every_single) - next_restart_single = - (ntimestep/restart_every_single)*restart_every_single + - restart_every_single; - else { - auto nextrestart = static_cast - (input->variable->compute_equal(ivar_restart_single)); - if (nextrestart <= ntimestep) - error->all(FLERR,"Restart variable returned a bad timestep"); - next_restart_single = nextrestart; - } - } else next_restart_single = update->laststep + 1; - if (restart_flag_double) { - if (restart_every_double) - next_restart_double = - (ntimestep/restart_every_double)*restart_every_double + - restart_every_double; - else { - auto nextrestart = static_cast - (input->variable->compute_equal(ivar_restart_double)); - if (nextrestart <= ntimestep) - error->all(FLERR,"Restart variable returned a bad timestep"); - next_restart_double = nextrestart; - } - } else next_restart_double = update->laststep + 1; - next_restart = MIN(next_restart_single,next_restart_double); - } else next_restart = update->laststep + 1; + if (restart_flag && update->restrict_output == 0) { + if (restart_flag_single) { + if (restart_every_single) { + next_restart_single = + (ntimestep/restart_every_single)*restart_every_single + + restart_every_single; + } else { + auto nextrestart = static_cast + (input->variable->compute_equal(ivar_restart_single)); + if (nextrestart <= ntimestep) + error->all(FLERR,"Restart variable returned a bad timestep"); + next_restart_single = nextrestart; + } + } else next_restart_single = update->laststep + 1; + if (restart_flag_double) { + if (restart_every_double) + next_restart_double = + (ntimestep/restart_every_double)*restart_every_double + + restart_every_double; + else { + auto nextrestart = static_cast + (input->variable->compute_equal(ivar_restart_double)); + if (nextrestart <= ntimestep) + error->all(FLERR,"Restart variable returned a bad timestep"); + next_restart_double = nextrestart; + } + } else next_restart_double = update->laststep + 1; + next_restart = MIN(next_restart_single,next_restart_double); + } else next_restart = update->laststep + 1; - // print memory usage unless being called between multiple runs - - if (memflag) memory_usage(); + // print memory usage unless being called between multiple runs + + if (memflag) memory_usage(); // set next_thermo to multiple of every or variable eval if var defined // insure thermo output on last step of run // thermo may invoke computes so wrap with clear/add - modify->clearstep_compute(); + modify->clearstep_compute(); + + thermo->header(); + thermo->compute(0); + last_thermo = ntimestep; + + if (var_thermo) { + next_thermo = static_cast + (input->variable->compute_equal(ivar_thermo)); + if (next_thermo <= ntimestep) + error->all(FLERR,"Thermo every variable returned a bad timestep"); + } else if (thermo_every) { + next_thermo = (ntimestep/thermo_every)*thermo_every + thermo_every; + next_thermo = MIN(next_thermo,update->laststep); + } else next_thermo = update->laststep; + + modify->addstep_compute(next_thermo); + + // next = next timestep any output will be done + + next = MIN(next_dump_any,next_restart); + next = MIN(next,next_thermo); +} - thermo->header(); - thermo->compute(0); - last_thermo = ntimestep; +/* ---------------------------------------------------------------------- + perform all output for this timestep + only perform output if next matches current step and last output doesn't + do dump/restart before thermo so thermo CPU time will include them +------------------------------------------------------------------------- */ - if (var_thermo) { - next_thermo = static_cast - (input->variable->compute_equal(ivar_thermo)); - if (next_thermo <= ntimestep) - error->all(FLERR,"Thermo every variable returned a bad timestep"); - } else if (thermo_every) { - next_thermo = (ntimestep/thermo_every)*thermo_every + thermo_every; - next_thermo = MIN(next_thermo,update->laststep); - } else next_thermo = update->laststep; +void Output::write(bigint ntimestep) +{ + // perform dump if its next_dump = current ntimestep + // but not if it was already written on this step + // set next_dump and also next_time_dump for mode_dump = 1 + // set next_dump_any to smallest next_dump + // wrap step dumps that invoke computes or do variable eval with clear/add + // NOTE: + // not wrapping time dumps means that Integrate::ev_set() + // needs to trigger all per-atom eng/virial computes + // on a timestep where any time dump will be output + // could wrap time dumps as well, if timestep size did not vary + // if wrap when timestep size varies frequently, + // then can do many unneeded addstep() --> inefficient + // hard to know if timestep varies, since run every could change it + // can't remove an uneeded addstep from a compute, b/c don't know + // what other command may have added it - modify->addstep_compute(next_thermo); + if (next_dump_any == ntimestep) { + next_dump_any = next_time_dump_any = MAXBIGINT; - // next = next timestep any output will be done + for (int idump = 0; idump < ndump; idump++) { - next = MIN(next_dump_any,next_restart); - next = MIN(next,next_thermo); - } + if (next_dump[idump] == ntimestep) { + if (last_dump[idump] == ntimestep) continue; - /* ---------------------------------------------------------------------- - perform all output for this timestep - only perform output if next matches current step and last output doesn't - do dump/restart before thermo so thermo CPU time will include them - ------------------------------------------------------------------------- */ + if (mode_dump[idump] == 0 && + (dump[idump]->clearstep || var_dump[idump])) + modify->clearstep_compute(); - void Output::write(bigint ntimestep) - { - // perform dump if its next_dump = current ntimestep - // but not if it was already written on this step - // set next_dump and also next_time_dump for mode_dump = 1 - // set next_dump_any to smallest next_dump - // wrap step dumps that invoke computes or do variable eval with clear/add - // NOTE: - // not wrapping time dumps means that Integrate::ev_set() - // needs to trigger all per-atom eng/virial computes - // on a timestep where any time dump will be output - // could wrap time dumps as well, if timestep size did not vary - // if wrap when timestep size varies frequently, - // then can do many unneeded addstep() --> inefficient - // hard to know if timestep varies, since run every could change it - // can't remove an uneeded addstep from a compute, b/c don't know - // what other command may have added it + // perform dump + // set next_dump and next_time_dump - if (next_dump_any == ntimestep) { - next_dump_any = next_time_dump_any = MAXBIGINT; + dump[idump]->write(); + last_dump[idump] = ntimestep; + calculate_next_dump(WRITE,idump,ntimestep); - for (int idump = 0; idump < ndump; idump++) { + if (mode_dump[idump] == 0 && + (dump[idump]->clearstep || var_dump[idump])) + modify->addstep_compute(next_dump[idump]); + } - if (next_dump[idump] == ntimestep) { - if (last_dump[idump] == ntimestep) continue; + if (mode_dump[idump] && (dump[idump]->clearstep || var_dump[idump])) + next_time_dump_any = MIN(next_time_dump_any,next_dump[idump]); + next_dump_any = MIN(next_dump_any,next_dump[idump]); + } + } - if (mode_dump[idump] == 0 && - (dump[idump]->clearstep || var_dump[idump])) - modify->clearstep_compute(); + // next_restart does not force output on last step of run + // for toggle = 0, replace "*" with current timestep in restart filename + // next restart variable may invoke computes so wrap with clear/add - // perform dump - // reset next_dump and next_time_dump, 1 arg for write() + if (next_restart == ntimestep) { + if (next_restart_single == ntimestep) { + std::string file = restart1; + std::size_t found = file.find('*'); + if (found != std::string::npos) + file.replace(found,1,fmt::format("{}",update->ntimestep)); - dump[idump]->write(); - last_dump[idump] = ntimestep; - calculate_next_dump(WRITE,idump,ntimestep); + if (last_restart != ntimestep) restart->write(file); - if (mode_dump[idump] == 0 && - (dump[idump]->clearstep || var_dump[idump])) - modify->addstep_compute(next_dump[idump]); - } + if (restart_every_single) next_restart_single += restart_every_single; + else { + modify->clearstep_compute(); + auto nextrestart = static_cast + (input->variable->compute_equal(ivar_restart_single)); + if (nextrestart <= ntimestep) + error->all(FLERR,"Restart variable returned a bad timestep"); + next_restart_single = nextrestart; + modify->addstep_compute(next_restart_single); + } + } - if (mode_dump[idump] && (dump[idump]->clearstep || var_dump[idump])) - next_time_dump_any = MIN(next_time_dump_any,next_dump[idump]); - next_dump_any = MIN(next_dump_any,next_dump[idump]); - } - } + if (next_restart_double == ntimestep) { + if (last_restart != ntimestep) { + if (restart_toggle == 0) { + restart->write(restart2a); + restart_toggle = 1; + } else { + restart->write(restart2b); + restart_toggle = 0; + } + } - // next_restart does not force output on last step of run - // for toggle = 0, replace "*" with current timestep in restart filename - // next restart variable may invoke computes so wrap with clear/add + if (restart_every_double) next_restart_double += restart_every_double; + else { + modify->clearstep_compute(); + auto nextrestart = static_cast + (input->variable->compute_equal(ivar_restart_double)); + if (nextrestart <= ntimestep) + error->all(FLERR,"Restart variable returned a bad timestep"); + next_restart_double = nextrestart; + modify->addstep_compute(next_restart_double); + } + } + last_restart = ntimestep; + next_restart = MIN(next_restart_single,next_restart_double); + } - if (next_restart == ntimestep) { - if (next_restart_single == ntimestep) { + // insure next_thermo forces output on last step of run + // thermo may invoke computes so wrap with clear/add - std::string file = restart1; - std::size_t found = file.find('*'); - if (found != std::string::npos) - file.replace(found,1,fmt::format("{}",update->ntimestep)); + if (next_thermo == ntimestep) { + modify->clearstep_compute(); + if (last_thermo != ntimestep) thermo->compute(1); + last_thermo = ntimestep; + if (var_thermo) { + next_thermo = static_cast + (input->variable->compute_equal(ivar_thermo)); + if (next_thermo <= ntimestep) + error->all(FLERR,"Thermo every variable returned a bad timestep"); + } else if (thermo_every) next_thermo += thermo_every; + else next_thermo = update->laststep; + next_thermo = MIN(next_thermo,update->laststep); + modify->addstep_compute(next_thermo); + } - if (last_restart != ntimestep) restart->write(file); + // next = next timestep any output will be done - if (restart_every_single) next_restart_single += restart_every_single; - else { - modify->clearstep_compute(); - auto nextrestart = static_cast - (input->variable->compute_equal(ivar_restart_single)); - if (nextrestart <= ntimestep) - error->all(FLERR,"Restart variable returned a bad timestep"); - next_restart_single = nextrestart; - modify->addstep_compute(next_restart_single); - } - } - if (next_restart_double == ntimestep) { - if (last_restart != ntimestep) { - if (restart_toggle == 0) { - restart->write(restart2a); - restart_toggle = 1; - } else { - restart->write(restart2b); - restart_toggle = 0; - } - } - if (restart_every_double) next_restart_double += restart_every_double; - else { - modify->clearstep_compute(); - auto nextrestart = static_cast - (input->variable->compute_equal(ivar_restart_double)); - if (nextrestart <= ntimestep) - error->all(FLERR,"Restart variable returned a bad timestep"); - next_restart_double = nextrestart; - modify->addstep_compute(next_restart_double); - } - } - last_restart = ntimestep; - next_restart = MIN(next_restart_single,next_restart_double); - } + next = MIN(next_dump_any,next_restart); + next = MIN(next,next_thermo); +} - // insure next_thermo forces output on last step of run - // thermo may invoke computes so wrap with clear/add +/* ---------------------------------------------------------------------- + force a snapshot to be written for all dumps + called from PRD and TAD +------------------------------------------------------------------------- */ - if (next_thermo == ntimestep) { - modify->clearstep_compute(); - if (last_thermo != ntimestep) thermo->compute(1); - last_thermo = ntimestep; - if (var_thermo) { - next_thermo = static_cast - (input->variable->compute_equal(ivar_thermo)); - if (next_thermo <= ntimestep) - error->all(FLERR,"Thermo every variable returned a bad timestep"); - } else if (thermo_every) next_thermo += thermo_every; - else next_thermo = update->laststep; - next_thermo = MIN(next_thermo,update->laststep); - modify->addstep_compute(next_thermo); - } +void Output::write_dump(bigint ntimestep) +{ + for (int idump = 0; idump < ndump; idump++) { + dump[idump]->write(); + last_dump[idump] = ntimestep; + } +} - // next = next timestep any output will be done - - next = MIN(next_dump_any,next_restart); - next = MIN(next,next_thermo); - } - - /* ---------------------------------------------------------------------- - force a snapshot to be written for all dumps - called from PRD and TAD - ------------------------------------------------------------------------- */ - - void Output::write_dump(bigint ntimestep) - { - for (int idump = 0; idump < ndump; idump++) { - dump[idump]->write(); - last_dump[idump] = ntimestep; - } - } - - /* ---------------------------------------------------------------------- - calculate when next dump occurs for Dump instance idump - operates in one of two modes, based on mode_dump flag - for timestep mode, set next_dump - for simulation time mode, set next_time_dump and next_dump - which flag depends on caller - SETUP = from setup() at start of run - WRITE = from write() during run each time a dump file is written - RESET_DT = from reset_dt() called from fix dt/reset when it changes timestep size - ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + calculate when next dump occurs for Dump instance idump + operates in one of two modes, based on mode_dump flag + for timestep mode, set next_dump + for simulation time mode, set next_time_dump and next_dump + which flag depends on caller + SETUP = from setup() at start of run + WRITE = from write() during run each time a dump file is written + RESET_DT = from reset_dt() called from fix dt/reset when it changes timestep size +------------------------------------------------------------------------- */ void Output::calculate_next_dump(int which, int idump, bigint ntimestep) { @@ -489,10 +488,11 @@ void Output::calculate_next_dump(int which, int idump, bigint ntimestep) if (every_dump[idump]) { - // which = SETUP: nextdump = next multiple of every_dump - // which = WRITE: increment nextdump by every_dump + // which = SETUP: next_dump = next multiple of every_dump + // which = WRITE: increment next_dump by every_dump + // current step is already multiple of every_dump - if (which == SETUP) + if (which == SETUP) next_dump[idump] = (ntimestep/every_dump[idump])*every_dump[idump] + every_dump[idump]; else if (which == WRITE) next_dump[idump] += every_dump[idump]; @@ -589,112 +589,112 @@ int Output::check_time_dumps(bigint ntimestep) return nowflag; } - /* ---------------------------------------------------------------------- - force restart file(s) to be written - called from PRD and TAD - ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + force restart file(s) to be written + called from PRD and TAD +------------------------------------------------------------------------- */ - void Output::write_restart(bigint ntimestep) - { - if (restart_flag_single) { - std::string file = restart1; - std::size_t found = file.find('*'); - if (found != std::string::npos) - file.replace(found,1,fmt::format("{}",update->ntimestep)); - restart->write(file); - } +void Output::write_restart(bigint ntimestep) +{ + if (restart_flag_single) { + std::string file = restart1; + std::size_t found = file.find('*'); + if (found != std::string::npos) + file.replace(found,1,fmt::format("{}",update->ntimestep)); + restart->write(file); + } - if (restart_flag_double) { - if (restart_toggle == 0) { - restart->write(restart2a); - restart_toggle = 1; - } else { - restart->write(restart2b); - restart_toggle = 0; - } - } + if (restart_flag_double) { + if (restart_toggle == 0) { + restart->write(restart2a); + restart_toggle = 1; + } else { + restart->write(restart2b); + restart_toggle = 0; + } + } - last_restart = ntimestep; - } + last_restart = ntimestep; +} - /* ---------------------------------------------------------------------- - timestep is being changed, called by update->reset_timestep() - for dumps, require that no dump is "active" - meaning that a snapshot has already been output - reset next output values for restart and thermo - reset to smallest value >= new timestep - if next timestep set by variable evaluation, - eval for ntimestep-1, so current ntimestep can be returned if needed - no guarantee that variable can be evaluated for ntimestep-1 - e.g. if it depends on computes, but live with that rare case for now - ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + timestep is being changed, called by update->reset_timestep() + for dumps, require that no dump is "active" + meaning that a snapshot has already been output + reset next output values for restart and thermo + reset to smallest value >= new timestep + if next timestep set by variable evaluation, + eval for ntimestep-1, so current ntimestep can be returned if needed + no guarantee that variable can be evaluated for ntimestep-1 + e.g. if it depends on computes, but live with that rare case for now +------------------------------------------------------------------------- */ - void Output::reset_timestep(bigint ntimestep) - { - next_dump_any = MAXBIGINT; - for (int idump = 0; idump < ndump; idump++) - if ((last_dump[idump] >= 0) && !update->whichflag && !dump[idump]->multifile) - error->all(FLERR, "Cannot reset timestep with active dump - must undump first"); +void Output::reset_timestep(bigint ntimestep) +{ + next_dump_any = MAXBIGINT; + for (int idump = 0; idump < ndump; idump++) + if ((last_dump[idump] >= 0) && !update->whichflag && !dump[idump]->multifile) + error->all(FLERR, "Cannot reset timestep with active dump - must undump first"); - if (restart_flag_single) { - if (restart_every_single) { - next_restart_single = - (ntimestep/restart_every_single)*restart_every_single; - if (next_restart_single < ntimestep) - next_restart_single += restart_every_single; - } else { - modify->clearstep_compute(); - update->ntimestep--; - auto nextrestart = static_cast - (input->variable->compute_equal(ivar_restart_single)); - if (nextrestart < ntimestep) - error->all(FLERR,"Restart variable returned a bad timestep"); - update->ntimestep++; - next_restart_single = nextrestart; - modify->addstep_compute(next_restart_single); - } - } else next_restart_single = update->laststep + 1; + if (restart_flag_single) { + if (restart_every_single) { + next_restart_single = + (ntimestep/restart_every_single)*restart_every_single; + if (next_restart_single < ntimestep) + next_restart_single += restart_every_single; + } else { + modify->clearstep_compute(); + update->ntimestep--; + auto nextrestart = static_cast + (input->variable->compute_equal(ivar_restart_single)); + if (nextrestart < ntimestep) + error->all(FLERR,"Restart variable returned a bad timestep"); + update->ntimestep++; + next_restart_single = nextrestart; + modify->addstep_compute(next_restart_single); + } + } else next_restart_single = update->laststep + 1; - if (restart_flag_double) { - if (restart_every_double) { - next_restart_double = - (ntimestep/restart_every_double)*restart_every_double; - if (next_restart_double < ntimestep) - next_restart_double += restart_every_double; - } else { - modify->clearstep_compute(); - update->ntimestep--; - auto nextrestart = static_cast - (input->variable->compute_equal(ivar_restart_double)); - if (nextrestart < ntimestep) - error->all(FLERR,"Restart variable returned a bad timestep"); - update->ntimestep++; - next_restart_double = nextrestart; - modify->addstep_compute(next_restart_double); - } - } else next_restart_double = update->laststep + 1; + if (restart_flag_double) { + if (restart_every_double) { + next_restart_double = + (ntimestep/restart_every_double)*restart_every_double; + if (next_restart_double < ntimestep) + next_restart_double += restart_every_double; + } else { + modify->clearstep_compute(); + update->ntimestep--; + auto nextrestart = static_cast + (input->variable->compute_equal(ivar_restart_double)); + if (nextrestart < ntimestep) + error->all(FLERR,"Restart variable returned a bad timestep"); + update->ntimestep++; + next_restart_double = nextrestart; + modify->addstep_compute(next_restart_double); + } + } else next_restart_double = update->laststep + 1; - next_restart = MIN(next_restart_single,next_restart_double); + next_restart = MIN(next_restart_single,next_restart_double); - if (var_thermo) { - modify->clearstep_compute(); - update->ntimestep--; - next_thermo = static_cast - (input->variable->compute_equal(ivar_thermo)); - if (next_thermo < ntimestep) - error->all(FLERR,"Thermo_modify every variable returned a bad timestep"); - update->ntimestep++; - next_thermo = MIN(next_thermo,update->laststep); - modify->addstep_compute(next_thermo); - } else if (thermo_every) { - next_thermo = (ntimestep/thermo_every)*thermo_every; - if (next_thermo < ntimestep) next_thermo += thermo_every; - next_thermo = MIN(next_thermo,update->laststep); - } else next_thermo = update->laststep; + if (var_thermo) { + modify->clearstep_compute(); + update->ntimestep--; + next_thermo = static_cast + (input->variable->compute_equal(ivar_thermo)); + if (next_thermo < ntimestep) + error->all(FLERR,"Thermo_modify every variable returned a bad timestep"); + update->ntimestep++; + next_thermo = MIN(next_thermo,update->laststep); + modify->addstep_compute(next_thermo); + } else if (thermo_every) { + next_thermo = (ntimestep/thermo_every)*thermo_every; + if (next_thermo < ntimestep) next_thermo += thermo_every; + next_thermo = MIN(next_thermo,update->laststep); + } else next_thermo = update->laststep; - next = MIN(next_dump_any,next_restart); - next = MIN(next,next_thermo); - } + next = MIN(next_dump_any,next_restart); + next = MIN(next,next_thermo); +} /* ---------------------------------------------------------------------- timestep size is being changed @@ -728,7 +728,6 @@ void Output::reset_dt() next = MIN(next,next_thermo); } - /* ---------------------------------------------------------------------- add a Dump to list of Dumps ------------------------------------------------------------------------- */ From 34d0c417f81b795502c5e843bf8b6f63ddf6e398 Mon Sep 17 00:00:00 2001 From: Steve Plimpton Date: Fri, 9 Sep 2022 10:39:07 -0600 Subject: [PATCH 3/6] another indentation reformat --- src/output.cpp | 148 ++++++++++++++++++++++++------------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/src/output.cpp b/src/output.cpp index 4df19b0faa..fd5bde5624 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -480,103 +480,103 @@ void Output::write_dump(bigint ntimestep) ------------------------------------------------------------------------- */ void Output::calculate_next_dump(int which, int idump, bigint ntimestep) - { - // dump mode is by timestep - // just set next_dump +{ + // dump mode is by timestep + // just set next_dump - if (mode_dump[idump] == 0) { + if (mode_dump[idump] == 0) { - if (every_dump[idump]) { + if (every_dump[idump]) { - // which = SETUP: next_dump = next multiple of every_dump - // which = WRITE: increment next_dump by every_dump - // current step is already multiple of every_dump + // which = SETUP: next_dump = next multiple of every_dump + // which = WRITE: increment next_dump by every_dump + // current step is already multiple of every_dump - if (which == SETUP) - next_dump[idump] = (ntimestep/every_dump[idump])*every_dump[idump] + every_dump[idump]; - else if (which == WRITE) - next_dump[idump] += every_dump[idump]; + if (which == SETUP) + next_dump[idump] = (ntimestep/every_dump[idump])*every_dump[idump] + every_dump[idump]; + else if (which == WRITE) + next_dump[idump] += every_dump[idump]; - } else { - next_dump[idump] = static_cast - (input->variable->compute_equal(ivar_dump[idump])); - if (next_dump[idump] <= ntimestep) - error->all(FLERR,"Dump every variable returned a bad timestep"); - } + } else { + next_dump[idump] = static_cast + (input->variable->compute_equal(ivar_dump[idump])); + if (next_dump[idump] <= ntimestep) + error->all(FLERR,"Dump every variable returned a bad timestep"); + } - // dump mode is by simulation time - // set next_time_dump and next_dump + // dump mode is by simulation time + // set next_time_dump and next_dump - } else { + } else { - bigint nextdump; - double nexttime; - double tcurrent = update->atime + - (ntimestep - update->atimestep) * update->dt; + bigint nextdump; + double nexttime; + double tcurrent = update->atime + + (ntimestep - update->atimestep) * update->dt; - if (every_time_dump[idump] > 0.0) { + if (every_time_dump[idump] > 0.0) { - // which = SETUP: nexttime = next multiple of every_time_dump - // which = WRITE: increment nexttime by every_time_dump - // which = RESET_DT: no change to previous nexttime (only timestep has changed) + // which = SETUP: nexttime = next multiple of every_time_dump + // which = WRITE: increment nexttime by every_time_dump + // which = RESET_DT: no change to previous nexttime (only timestep has changed) - switch (which) { - case SETUP: - nexttime = static_cast (tcurrent/every_time_dump[idump]) * - every_time_dump[idump] + every_time_dump[idump]; - break; + switch (which) { + case SETUP: + nexttime = static_cast (tcurrent/every_time_dump[idump]) * + every_time_dump[idump] + every_time_dump[idump]; + break; - case WRITE: - nexttime = next_time_dump[idump] + every_time_dump[idump]; - break; + case WRITE: + nexttime = next_time_dump[idump] + every_time_dump[idump]; + break; - case RESET_DT: - nexttime = next_time_dump[idump]; - break; + case RESET_DT: + nexttime = next_time_dump[idump]; + break; - default: - nexttime = 0; - error->all(FLERR,"Unexpected argument to calculate_next_dump"); - } + default: + nexttime = 0; + error->all(FLERR,"Unexpected argument to calculate_next_dump"); + } - nextdump = ntimestep + - static_cast ((nexttime - tcurrent - EPSDT*update->dt) / update->dt) + 1; + nextdump = ntimestep + + static_cast ((nexttime - tcurrent - EPSDT*update->dt) / update->dt) + 1; - // if delta is too small to reach next timestep, use multiple of delta + // if delta is too small to reach next timestep, use multiple of delta - if (nextdump == ntimestep) { - double tnext = update->atime + - (ntimestep+1 - update->atimestep) * update->dt; - int multiple = static_cast - ((tnext - nexttime) / every_time_dump[idump]); - nexttime = nexttime + (multiple+1)*every_time_dump[idump]; - nextdump = ntimestep + - static_cast ((nexttime - tcurrent - EPSDT*update->dt) / update->dt) + 1; - } + if (nextdump == ntimestep) { + double tnext = update->atime + + (ntimestep+1 - update->atimestep) * update->dt; + int multiple = static_cast + ((tnext - nexttime) / every_time_dump[idump]); + nexttime = nexttime + (multiple+1)*every_time_dump[idump]; + nextdump = ntimestep + + static_cast ((nexttime - tcurrent - EPSDT*update->dt) / update->dt) + 1; + } - } else { + } else { - // do not re-evaulate variable for which = RESET_DT, leave nexttime as-is - // unless next_time_dump < 0.0, which means variable never yet evaluated + // do not re-evaulate variable for which = RESET_DT, leave nexttime as-is + // unless next_time_dump < 0.0, which means variable never yet evaluated - if (which < RESET_DT || next_time_dump[idump] < 0.0) { - nexttime = input->variable->compute_equal(ivar_dump[idump]); - } else - nexttime = next_time_dump[idump]; + if (which < RESET_DT || next_time_dump[idump] < 0.0) { + nexttime = input->variable->compute_equal(ivar_dump[idump]); + } else + nexttime = next_time_dump[idump]; - if (nexttime <= tcurrent) - error->all(FLERR,"Dump every/time variable returned a bad time"); + if (nexttime <= tcurrent) + error->all(FLERR,"Dump every/time variable returned a bad time"); - nextdump = ntimestep + - static_cast ((nexttime - tcurrent - EPSDT*update->dt) / update->dt) + 1; - if (nextdump <= ntimestep) - error->all(FLERR,"Dump every/time variable too small for next timestep"); - } + nextdump = ntimestep + + static_cast ((nexttime - tcurrent - EPSDT*update->dt) / update->dt) + 1; + if (nextdump <= ntimestep) + error->all(FLERR,"Dump every/time variable too small for next timestep"); + } - next_time_dump[idump] = nexttime; - next_dump[idump] = nextdump; - } - } + next_time_dump[idump] = nexttime; + next_dump[idump] = nextdump; + } +} /* ---------------------------------------------------------------------- */ From 167abe9ce085fcb72e13780a28fb621ee4e86a64 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 9 Sep 2022 12:41:24 -0400 Subject: [PATCH 4/6] add preprocessor flags to select between the changed and the old code variant --- lib/gpu/geryon/nvd_device.h | 13 +++++++++++++ lib/gpu/geryon/nvd_macros.h | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/lib/gpu/geryon/nvd_device.h b/lib/gpu/geryon/nvd_device.h index 36e34cb0f1..5e2444b4d1 100644 --- a/lib/gpu/geryon/nvd_device.h +++ b/lib/gpu/geryon/nvd_device.h @@ -316,7 +316,9 @@ class UCL_Device { std::vector _cq; CUdevice _cu_device; CUcontext _context; +#if GERYON_NVD_PRIMARY_CONTEXT CUcontext _old_context; +#endif }; // Grabs the properties for all devices @@ -392,9 +394,14 @@ int UCL_Device::set_platform(const int pid) { int UCL_Device::set(int num) { clear(); _device=_properties[num].device_id; +#if GERYON_NVD_PRIMARY_CONTEXT CU_SAFE_CALL_NS(cuCtxGetCurrent(&_old_context)); CU_SAFE_CALL_NS(cuDeviceGet(&_cu_device,_device)); CUresult err=cuDevicePrimaryCtxRetain(&_context,_cu_device); +#else + CU_SAFE_CALL_NS(cuDeviceGet(&_cu_device,_device)); + CUresult err=cuCtxCreate(&_context,0,_cu_device); +#endif if (err!=CUDA_SUCCESS) { #ifndef UCL_NO_EXIT std::cerr << "UCL Error: Could not access accelerator number " << num @@ -403,17 +410,23 @@ int UCL_Device::set(int num) { #endif return UCL_ERROR; } +#if GERYON_NVD_PRIMARY_CONTEXT if (_context != _old_context) { CU_SAFE_CALL_NS(cuCtxSetCurrent(_context)); } +#endif return UCL_SUCCESS; } void UCL_Device::clear() { if (_device>-1) { for (int i=1; i Date: Fri, 9 Sep 2022 12:48:14 -0400 Subject: [PATCH 5/6] whitespace --- src/output.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/output.cpp b/src/output.cpp index fd5bde5624..9442311724 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -295,7 +295,7 @@ void Output::setup(int memflag) } else next_restart = update->laststep + 1; // print memory usage unless being called between multiple runs - + if (memflag) memory_usage(); // set next_thermo to multiple of every or variable eval if var defined @@ -303,11 +303,11 @@ void Output::setup(int memflag) // thermo may invoke computes so wrap with clear/add modify->clearstep_compute(); - + thermo->header(); thermo->compute(0); last_thermo = ntimestep; - + if (var_thermo) { next_thermo = static_cast (input->variable->compute_equal(ivar_thermo)); @@ -317,11 +317,11 @@ void Output::setup(int memflag) next_thermo = (ntimestep/thermo_every)*thermo_every + thermo_every; next_thermo = MIN(next_thermo,update->laststep); } else next_thermo = update->laststep; - + modify->addstep_compute(next_thermo); - + // next = next timestep any output will be done - + next = MIN(next_dump_any,next_restart); next = MIN(next,next_thermo); } @@ -492,7 +492,7 @@ void Output::calculate_next_dump(int which, int idump, bigint ntimestep) // which = WRITE: increment next_dump by every_dump // current step is already multiple of every_dump - if (which == SETUP) + if (which == SETUP) next_dump[idump] = (ntimestep/every_dump[idump])*every_dump[idump] + every_dump[idump]; else if (which == WRITE) next_dump[idump] += every_dump[idump]; From 935eea8219f528915dbf5f6d976e7c760d6b077a Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 9 Sep 2022 15:57:34 -0400 Subject: [PATCH 6/6] test a variety of dump output frequency and starting timesteps and run lengths --- unittest/formats/test_dump_atom.cpp | 60 +++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/unittest/formats/test_dump_atom.cpp b/unittest/formats/test_dump_atom.cpp index 0140b1a9e4..3eea4c2746 100644 --- a/unittest/formats/test_dump_atom.cpp +++ b/unittest/formats/test_dump_atom.cpp @@ -699,6 +699,66 @@ TEST_F(DumpAtomTest, binary_write_dump) delete_file(dump_file); } +TEST_F(DumpAtomTest, frequency) +{ + auto dump_file = dump_filename("frequency"); + BEGIN_HIDE_OUTPUT(); + command("dump id all atom 5 " + dump_file); + command("run 15 post no"); + command("run 12 post no"); + END_HIDE_OUTPUT(); + + // NOTE: must reset to current timestep (27) to avoid unexpected issues with following + TEST_FAILURE(".*ERROR: Cannot reset timestep with active dump - must undump first.*", + command("reset_timestep 27");); + + BEGIN_HIDE_OUTPUT(); + command("run 3 post no"); + command("undump id"); + command("reset_timestep 5"); + command("dump id all atom 10 " + dump_file); + command("dump_modify id append yes"); + command("run 20 post no"); + command("undump id"); + END_HIDE_OUTPUT(); + + std::vector expected, values; + values = extract_items(dump_file, "TIMESTEP"); + expected = {"0", "5", "10", "15", "20", "25", "30", "10", "20"}; + ASSERT_EQ(values.size(), expected.size()); + for (int i = 0; i < expected.size(); ++i) + ASSERT_THAT(values[i], Eq(expected[i])); + + BEGIN_HIDE_OUTPUT(); + command("reset_timestep 10"); + command("dump id all atom 10 " + dump_file); + command("run 20 post no"); + command("undump id"); + END_HIDE_OUTPUT(); + + values = extract_items(dump_file, "TIMESTEP"); + expected = {"10", "20", "30"}; + ASSERT_EQ(values.size(), expected.size()); + for (int i = 0; i < expected.size(); ++i) + ASSERT_THAT(values[i], Eq(expected[i])); + + BEGIN_HIDE_OUTPUT(); + command("reset_timestep 0"); + command("dump id all atom 10 " + dump_file); + command("minimize 0.0 0.0 15 30"); + command("run 20 post no"); + command("undump id"); + END_HIDE_OUTPUT(); + + values = extract_items(dump_file, "TIMESTEP"); + expected = {"0", "10", "15", "20", "30"}; + ASSERT_EQ(values.size(), expected.size()); + for (int i = 0; i < expected.size(); ++i) + ASSERT_THAT(values[i], Eq(expected[i])); + + delete_file(dump_file); +} + //------------------------------------------------------------------------------------------------- // dump_modify //-------------------------------------------------------------------------------------------------