From 48c85f77189513ebf306c132400168c37054e8f5 Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 11 Dec 2023 12:42:45 -0700 Subject: [PATCH 1/3] Port Fix temp berendsen and rescale to use Kokkos --- src/KOKKOS/Install.sh | 4 + src/KOKKOS/fix_temp_berendsen_kokkos.cpp | 132 ++++++++++++++++++++++ src/KOKKOS/fix_temp_berendsen_kokkos.h | 44 ++++++++ src/KOKKOS/fix_temp_rescale_kokkos.cpp | 138 +++++++++++++++++++++++ src/KOKKOS/fix_temp_rescale_kokkos.h | 44 ++++++++ src/fix_temp_berendsen.h | 2 +- 6 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 src/KOKKOS/fix_temp_berendsen_kokkos.cpp create mode 100644 src/KOKKOS/fix_temp_berendsen_kokkos.h create mode 100644 src/KOKKOS/fix_temp_rescale_kokkos.cpp create mode 100644 src/KOKKOS/fix_temp_rescale_kokkos.h diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 489efc55a0..4be74334c9 100755 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -177,6 +177,10 @@ action fix_shardlow_kokkos.cpp fix_shardlow.cpp action fix_shardlow_kokkos.h fix_shardlow.h action fix_spring_self_kokkos.cpp action fix_spring_self_kokkos.h +action fix_temp_berendsen_kokkos.cpp +action fix_temp_berendsen_kokkos.h +action fix_temp_rescale_kokkos.cpp +action fix_temp_rescale_kokkos.h action fix_viscous_kokkos.cpp action fix_viscous_kokkos.h action fix_wall_gran_kokkos.cpp fix_wall_gran.cpp diff --git a/src/KOKKOS/fix_temp_berendsen_kokkos.cpp b/src/KOKKOS/fix_temp_berendsen_kokkos.cpp new file mode 100644 index 0000000000..cf4be489af --- /dev/null +++ b/src/KOKKOS/fix_temp_berendsen_kokkos.cpp @@ -0,0 +1,132 @@ +// clang-format off +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "fix_temp_berendsen_kokkos.h" + +#include "atom_kokkos.h" +#include "comm.h" +#include "compute.h" +#include "error.h" +#include "force.h" +#include "group.h" +#include "input.h" +#include "modify.h" +#include "update.h" +#include "variable.h" +#include "atom_masks.h" + +#include +#include + +using namespace LAMMPS_NS; +using namespace FixConst; + +enum{NOBIAS,BIAS}; +enum{CONSTANT,EQUAL}; + +/* ---------------------------------------------------------------------- */ + +template +FixTempBerendsenKokkos::FixTempBerendsenKokkos(LAMMPS *lmp, int narg, char **arg) : + FixTempBerendsen(lmp, narg, arg) +{ + kokkosable = 1; + atomKK = (AtomKokkos *)atom; + execution_space = ExecutionSpaceFromDevice::space; + + datamask_read = EMPTY_MASK; + datamask_modify = EMPTY_MASK; +} + +/* ---------------------------------------------------------------------- */ + +template +void FixTempBerendsenKokkos::end_of_step() +{ + double t_current = temperature->compute_scalar(); + double tdof = temperature->dof; + + // there is nothing to do, if there are no degrees of freedom + + if (tdof < 1) return; + + if (t_current == 0.0) + error->all(FLERR, "Computed current temperature for fix temp/berendsen must not be 0.0"); + + double delta = update->ntimestep - update->beginstep; + if (delta != 0.0) delta /= update->endstep - update->beginstep; + + // set current t_target + // if variable temp, evaluate variable, wrap with clear/add + + if (tstyle == CONSTANT) + t_target = t_start + delta * (t_stop-t_start); + else { + modify->clearstep_compute(); + t_target = input->variable->compute_equal(tvar); + if (t_target < 0.0) + error->one(FLERR, "Fix temp/berendsen variable {} returned negative temperature", + input->variable->names[tvar]); + modify->addstep_compute(update->ntimestep + nevery); + } + + // rescale velocities by lamda + // for BIAS: + // temperature is current, so do not need to re-compute + // OK to not test returned v = 0, since lamda is multiplied by v + + double lamda = sqrt(1.0 + update->dt/t_period*(t_target/t_current - 1.0)); + double efactor = 0.5 * force->boltz * tdof; + energy += t_current * (1.0-lamda*lamda) * efactor; + + auto v = atomKK->k_v.view(); + auto mask = atomKK->k_mask.view(); + int nlocal = atom->nlocal; + auto groupbit = this->groupbit; + + if (which == NOBIAS) { + atomKK->sync(temperature->execution_space,temperature->datamask_read); + temperature->compute_scalar(); // is this needed? + temperature->remove_bias_all(); + atomKK->modified(temperature->execution_space,temperature->datamask_modify); + atomKK->sync(execution_space,temperature->datamask_modify); + } + + atomKK->sync(execution_space,V_MASK|MASK_MASK); + + Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal), LAMMPS_LAMBDA(int i) { + if (mask[i] & groupbit) { + v(i,0) *= lamda; + v(i,1) *= lamda; + v(i,2) *= lamda; + } + }); + + atomKK->modified(execution_space,V_MASK); + + if (which == NOBIAS) { + atomKK->sync(temperature->execution_space,temperature->datamask_read); + temperature->restore_bias_all(); + atomKK->modified(temperature->execution_space,temperature->datamask_modify); + atomKK->sync(execution_space,temperature->datamask_modify); + } +} +/* ---------------------------------------------------------------------- */ + +namespace LAMMPS_NS { +template class FixTempBerendsenKokkos; +#ifdef LMP_KOKKOS_GPU +template class FixTempBerendsenKokkos; +#endif +} diff --git a/src/KOKKOS/fix_temp_berendsen_kokkos.h b/src/KOKKOS/fix_temp_berendsen_kokkos.h new file mode 100644 index 0000000000..6a0aa5ce98 --- /dev/null +++ b/src/KOKKOS/fix_temp_berendsen_kokkos.h @@ -0,0 +1,44 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS +// clang-format off +FixStyle(temp/berendsen/kk,FixTempBerendsenKokkos); +FixStyle(temp/berendsen/kk/device,FixTempBerendsenKokkos); +FixStyle(temp/berendsen/kk/host,FixTempBerendsenKokkos); +// clang-format on +#else + +// clang-format off +#ifndef LMP_FIX_TEMP_BERENDSEN_KOKKOS_H +#define LMP_FIX_TEMP_BERENDSEN_KOKKOS_H + +#include "fix_temp_berendsen.h" +#include "kokkos_type.h" + +namespace LAMMPS_NS { + +template +class FixTempBerendsenKokkos : public FixTempBerendsen { + public: + typedef DeviceType device_type; + + FixTempBerendsenKokkos(class LAMMPS *, int, char **); + ~FixTempBerendsenKokkos() override {} + void end_of_step() override; +}; + +} // namespace LAMMPS_NS + +#endif +#endif diff --git a/src/KOKKOS/fix_temp_rescale_kokkos.cpp b/src/KOKKOS/fix_temp_rescale_kokkos.cpp new file mode 100644 index 0000000000..0f17eb1be1 --- /dev/null +++ b/src/KOKKOS/fix_temp_rescale_kokkos.cpp @@ -0,0 +1,138 @@ +// clang-format off +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "fix_temp_rescale_kokkos.h" + +#include "atom_kokkos.h" +#include "comm.h" +#include "compute.h" +#include "error.h" +#include "force.h" +#include "group.h" +#include "input.h" +#include "modify.h" +#include "update.h" +#include "variable.h" +#include "atom_masks.h" + +#include +#include + +using namespace LAMMPS_NS; +using namespace FixConst; + +enum{NOBIAS,BIAS}; +enum{CONSTANT,EQUAL}; + +/* ---------------------------------------------------------------------- */ + +template +FixTempRescaleKokkos::FixTempRescaleKokkos(LAMMPS *lmp, int narg, char **arg) : + FixTempRescale(lmp, narg, arg) +{ + kokkosable = 1; + atomKK = (AtomKokkos *)atom; + execution_space = ExecutionSpaceFromDevice::space; + + datamask_read = EMPTY_MASK; + datamask_modify = EMPTY_MASK; +} + +/* ---------------------------------------------------------------------- */ + +template +void FixTempRescaleKokkos::end_of_step() +{ + double t_current = temperature->compute_scalar(); + + // there is nothing to do, if there are no degrees of freedom + + if (temperature->dof < 1) return; + + // protect against division by zero + + if (t_current == 0.0) + error->all(FLERR,"Computed temperature for fix temp/rescale cannot be 0.0"); + + double delta = update->ntimestep - update->beginstep; + if (delta != 0.0) delta /= update->endstep - update->beginstep; + + // set current t_target + // if variable temp, evaluate variable, wrap with clear/add + + if (tstyle == CONSTANT) + t_target = t_start + delta * (t_stop-t_start); + else { + modify->clearstep_compute(); + t_target = input->variable->compute_equal(tvar); + if (t_target < 0.0) + error->one(FLERR, "Fix temp/rescale variable returned negative temperature"); + modify->addstep_compute(update->ntimestep + nevery); + } + + // rescale velocity of appropriate atoms if outside window + // for BIAS: + // temperature is current, so do not need to re-compute + // OK to not test returned v = 0, since factor is multiplied by v + + if (fabs(t_current-t_target) > t_window) { + t_target = t_current - fraction*(t_current-t_target); + double factor = sqrt(t_target/t_current); + double efactor = 0.5 * force->boltz * temperature->dof; + + energy += (t_current-t_target) * efactor; + + auto v = atomKK->k_v.view(); + auto mask = atomKK->k_mask.view(); + int nlocal = atom->nlocal; + auto groupbit = this->groupbit; + + if (which == NOBIAS) { + atomKK->sync(temperature->execution_space,temperature->datamask_read); + temperature->compute_scalar(); // is this needed? + temperature->remove_bias_all(); + atomKK->modified(temperature->execution_space,temperature->datamask_modify); + atomKK->sync(execution_space,temperature->datamask_modify); + } + + atomKK->sync(execution_space,V_MASK|MASK_MASK); + + Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal), LAMMPS_LAMBDA(int i) { + if (mask[i] & groupbit) { + v(i,0) *= factor; + v(i,1) *= factor; + v(i,2) *= factor; + } + }); + + atomKK->modified(execution_space,V_MASK); + + if (which == NOBIAS) { + atomKK->sync(temperature->execution_space,temperature->datamask_read); + temperature->restore_bias_all(); + atomKK->modified(temperature->execution_space,temperature->datamask_modify); + atomKK->sync(execution_space,temperature->datamask_modify); + + } + } +} + +/* ---------------------------------------------------------------------- */ + +namespace LAMMPS_NS { +template class FixTempRescaleKokkos; +#ifdef LMP_KOKKOS_GPU +template class FixTempRescaleKokkos; +#endif +} diff --git a/src/KOKKOS/fix_temp_rescale_kokkos.h b/src/KOKKOS/fix_temp_rescale_kokkos.h new file mode 100644 index 0000000000..7dd3111325 --- /dev/null +++ b/src/KOKKOS/fix_temp_rescale_kokkos.h @@ -0,0 +1,44 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS +// clang-format off +FixStyle(temp/rescale/kk,FixTempRescaleKokkos); +FixStyle(temp/rescale/kk/device,FixTempRescaleKokkos); +FixStyle(temp/rescale/kk/host,FixTempRescaleKokkos); +// clang-format on +#else + +// clang-format off +#ifndef LMP_FIX_TEMP_RESCALE_KOKKOS_H +#define LMP_FIX_TEMP_RESCALE_KOKKOS_H + +#include "fix_temp_rescale.h" +#include "kokkos_type.h" + +namespace LAMMPS_NS { + +template +class FixTempRescaleKokkos : public FixTempRescale { + public: + typedef DeviceType device_type; + + FixTempRescaleKokkos(class LAMMPS *, int, char **); + ~FixTempRescaleKokkos() override {} + void end_of_step() override; +}; + +} // namespace LAMMPS_NS + +#endif +#endif diff --git a/src/fix_temp_berendsen.h b/src/fix_temp_berendsen.h index 78ece2af22..f137830508 100644 --- a/src/fix_temp_berendsen.h +++ b/src/fix_temp_berendsen.h @@ -38,7 +38,7 @@ class FixTempBerendsen : public Fix { void restart(char *buf) override; void *extract(const char *, int &) override; - private: + protected: int which; double t_start, t_stop, t_period, t_target; double energy; From 9d51d9c0e9d9519b0faa7bc8d8537698cbc4606b Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 11 Dec 2023 12:55:06 -0700 Subject: [PATCH 2/3] Update docs --- doc/src/Commands_fix.rst | 4 ++-- doc/src/fix_temp_berendsen.rst | 7 +++++++ doc/src/fix_temp_rescale.rst | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/src/Commands_fix.rst b/doc/src/Commands_fix.rst index 3283561c41..e89e302673 100644 --- a/doc/src/Commands_fix.rst +++ b/doc/src/Commands_fix.rst @@ -239,10 +239,10 @@ OPT. * :doc:`store/force ` * :doc:`store/state ` * :doc:`tdpd/source ` - * :doc:`temp/berendsen ` + * :doc:`temp/berendsen (k) ` * :doc:`temp/csld ` * :doc:`temp/csvr ` - * :doc:`temp/rescale ` + * :doc:`temp/rescale (k) ` * :doc:`temp/rescale/eff ` * :doc:`tfmc ` * :doc:`tgnpt/drude ` diff --git a/doc/src/fix_temp_berendsen.rst b/doc/src/fix_temp_berendsen.rst index 67e496e6c5..541f3191d5 100644 --- a/doc/src/fix_temp_berendsen.rst +++ b/doc/src/fix_temp_berendsen.rst @@ -1,8 +1,11 @@ .. index:: fix temp/berendsen +.. index:: fix temp/berendsen/kk fix temp/berendsen command ========================== +Accelerator Variants: *temp/berendsen/kk* + Syntax """""" @@ -118,6 +121,10 @@ remaining thermal degrees of freedom, and the bias is added back in. ---------- +.. include:: accel_styles.rst + +---------- + Restart, fix_modify, output, run start/stop, minimize info """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" diff --git a/doc/src/fix_temp_rescale.rst b/doc/src/fix_temp_rescale.rst index bfdcaa90f8..2dd2178346 100644 --- a/doc/src/fix_temp_rescale.rst +++ b/doc/src/fix_temp_rescale.rst @@ -1,8 +1,11 @@ .. index:: fix temp/rescale +.. index:: fix temp/rescale/kk fix temp/rescale command ======================== +Accelerator Variants: *temp/rescale/kk* + Syntax """""" @@ -125,6 +128,10 @@ remaining thermal degrees of freedom, and the bias is added back in. ---------- +.. include:: accel_styles.rst + +---------- + Restart, fix_modify, output, run start/stop, minimize info """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" From e147d131fb4bb7b89f4c75ac5edb3c3dd5b3aa0b Mon Sep 17 00:00:00 2001 From: Stan Gerald Moore Date: Mon, 11 Dec 2023 12:58:15 -0700 Subject: [PATCH 3/3] Fix issue with temp compute --- src/KOKKOS/fix_temp_berendsen_kokkos.cpp | 5 ++++- src/KOKKOS/fix_temp_rescale_kokkos.cpp | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/KOKKOS/fix_temp_berendsen_kokkos.cpp b/src/KOKKOS/fix_temp_berendsen_kokkos.cpp index cf4be489af..b986b3189a 100644 --- a/src/KOKKOS/fix_temp_berendsen_kokkos.cpp +++ b/src/KOKKOS/fix_temp_berendsen_kokkos.cpp @@ -54,7 +54,11 @@ FixTempBerendsenKokkos::FixTempBerendsenKokkos(LAMMPS *lmp, int narg template void FixTempBerendsenKokkos::end_of_step() { + atomKK->sync(temperature->execution_space,temperature->datamask_read); double t_current = temperature->compute_scalar(); + atomKK->modified(temperature->execution_space,temperature->datamask_modify); + atomKK->sync(execution_space,temperature->datamask_modify); + double tdof = temperature->dof; // there is nothing to do, if there are no degrees of freedom @@ -97,7 +101,6 @@ void FixTempBerendsenKokkos::end_of_step() if (which == NOBIAS) { atomKK->sync(temperature->execution_space,temperature->datamask_read); - temperature->compute_scalar(); // is this needed? temperature->remove_bias_all(); atomKK->modified(temperature->execution_space,temperature->datamask_modify); atomKK->sync(execution_space,temperature->datamask_modify); diff --git a/src/KOKKOS/fix_temp_rescale_kokkos.cpp b/src/KOKKOS/fix_temp_rescale_kokkos.cpp index 0f17eb1be1..3a1c6ddd26 100644 --- a/src/KOKKOS/fix_temp_rescale_kokkos.cpp +++ b/src/KOKKOS/fix_temp_rescale_kokkos.cpp @@ -54,7 +54,10 @@ FixTempRescaleKokkos::FixTempRescaleKokkos(LAMMPS *lmp, int narg, ch template void FixTempRescaleKokkos::end_of_step() { + atomKK->sync(temperature->execution_space,temperature->datamask_read); double t_current = temperature->compute_scalar(); + atomKK->modified(temperature->execution_space,temperature->datamask_modify); + atomKK->sync(execution_space,temperature->datamask_modify); // there is nothing to do, if there are no degrees of freedom @@ -100,7 +103,6 @@ void FixTempRescaleKokkos::end_of_step() if (which == NOBIAS) { atomKK->sync(temperature->execution_space,temperature->datamask_read); - temperature->compute_scalar(); // is this needed? temperature->remove_bias_all(); atomKK->modified(temperature->execution_space,temperature->datamask_modify); atomKK->sync(execution_space,temperature->datamask_modify);