From ec291174092595119d72d70cdab01b810ec9ea6d Mon Sep 17 00:00:00 2001 From: vladgl Date: Fri, 6 Oct 2023 19:23:15 +0300 Subject: [PATCH 01/33] Add new fix wall/flow --- src/KOKKOS/Install.sh | 2 + src/KOKKOS/fix_wall_flow_kokkos.cpp | 295 ++++++++++++++++++++++++++++ src/KOKKOS/fix_wall_flow_kokkos.h | 129 ++++++++++++ src/fix_wall_flow.cpp | 248 +++++++++++++++++++++++ src/fix_wall_flow.h | 60 ++++++ 5 files changed, 734 insertions(+) create mode 100644 src/KOKKOS/fix_wall_flow_kokkos.cpp create mode 100644 src/KOKKOS/fix_wall_flow_kokkos.h create mode 100644 src/fix_wall_flow.cpp create mode 100644 src/fix_wall_flow.h diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 462c0cbe57..75949c35d8 100755 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -187,6 +187,8 @@ action fix_temp_rescale_kokkos.cpp action fix_temp_rescale_kokkos.h action fix_viscous_kokkos.cpp action fix_viscous_kokkos.h +action fix_wall_flow_kokkos.cpp fix_wall_flow.cpp +action fix_wall_flow_kokkos.h fix_wall_flow.h action fix_wall_gran_kokkos.cpp fix_wall_gran.cpp action fix_wall_gran_kokkos.h fix_wall_gran.h action fix_wall_gran_old.cpp fix_wall_gran.cpp diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp new file mode 100644 index 0000000000..501a1bbd3f --- /dev/null +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -0,0 +1,295 @@ +#include "fix_wall_flow_kokkos.h" +#include "atom_kokkos.h" +#include "memory_kokkos.h" +#include "math_const.h" +#include "atom_masks.h" +#include "force.h" + +#include +//#include "comm.h +// clang-format off + +using namespace LAMMPS_NS; + +template +FixWallFlowKokkos::FixWallFlowKokkos(LAMMPS *lmp, int narg, char **arg) : + FixWallFlow(lmp, narg, arg), rand_pool(rndseed + comm->me) +{ + kokkosable = 1; + exchange_comm_device = sort_device = 1; + atomKK = (AtomKokkos *) atom; + execution_space = ExecutionSpaceFromDevice::space; + datamask_read = X_MASK | RMASS_MASK | TYPE_MASK | MASK_MASK; + datamask_modify = V_MASK; + + memory->destroy(current_segment); + current_segment = nullptr; + grow_arrays(atomKK->nmax); + + d_walls = d_walls_t("FixWallFlowKokkos::walls", walls.size()); + auto h_walls = Kokkos::create_mirror_view(d_walls); + for (int i = 0; i < walls.size(); ++i) + { + h_walls(i) = walls[i]; + } + Kokkos::deep_copy(d_walls, h_walls); +} + +template +FixWallFlowKokkos::~FixWallFlowKokkos() +{ + if (copymode) return; + memoryKK->destroy_kokkos(k_current_segment, current_segment); +} + +template +void FixWallFlowKokkos::init() +{ + atomKK->sync(execution_space, datamask_read); + k_current_segment.template sync(); + d_x = atomKK->k_x.template view(); + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0, atom->nlocal), *this); + copymode = 0; + + k_current_segment.template modify(); +} + +template +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWallFlowInit, + const int &i) const +{ + double pos = d_x(i, flowax); + d_current_segment(i) = compute_current_segment_kk(pos); +} + +template +void FixWallFlowKokkos::end_of_step() +{ + atomKK->sync(execution_space, datamask_read); + k_current_segment.template sync(); + + d_x = atomKK->k_x.template view(); + d_v = atomKK->k_v.template view(); + d_type = atomKK->k_type.template view(); + d_mask = atomKK->k_mask.template view(); + d_mass = atomKK->k_mass.template view(); + d_rmass = atomKK->k_rmass.template view(); + + copymode = 1; + if (d_rmass.data()) { + Kokkos::parallel_for( + Kokkos::RangePolicy>(0, atom->nlocal), *this); + } else { + Kokkos::parallel_for( + Kokkos::RangePolicy>(0, atom->nlocal), *this); + } + copymode = 0; + atomKK->modified(execution_space, datamask_modify); + k_current_segment.template modify(); +} + +template +template +KOKKOS_INLINE_FUNCTION +void FixWallFlowKokkos::operator()(TagFixWallFlowEndOfStep, + const int &atom_i) const +{ + if (d_mask[atom_i] & groupbit) + { + double pos = d_x(atom_i, flowax); + int prev_segment = d_current_segment(atom_i); + d_current_segment(atom_i) = compute_current_segment_kk(pos); + if (prev_segment != d_current_segment(atom_i)) + { + generate_velocity_kk(atom_i); + } + } +} + +template +template +KOKKOS_INLINE_FUNCTION +void FixWallFlowKokkos::generate_velocity_kk(int atom_i) const +{ + const int newton_iteration_count = 10; + double mass = get_mass(MTag(), atom_i); + const double gamma = 1.0 / std::sqrt(2.0 * kT / mass); + double delta = gamma * flowvel; + + const double edd = std::exp(-delta * delta) / MathConst::MY_PIS + delta * std::erf(delta); + const double probability_threshold = 0.5 * (1. + delta / edd); + + double direction = 1.0; + + rand_type_t rand_gen = rand_pool.get_state(); + + if (/*random->uniform()*/ rand_gen.drand() > probability_threshold) { + delta = -delta; + direction = -direction; + } + + const double xi_0 = rand_gen.drand(); //random->uniform(); + const double F_inf = edd + delta; + const double xi = xi_0 * F_inf; + const double x_0 = (std::sqrt(delta * delta + 2) - delta) * 0.5; + double x = x_0; + for (int i = 0; i < newton_iteration_count; ++i) { + x -= (std::exp(x * x) * MathConst::MY_PIS * (xi - delta * std::erfc(x)) - 1.0) / (x + delta) * + 0.5; + } + + const double nu = x + delta; + const double v = nu / gamma; + + d_v(atom_i, flowax) = v * direction; + d_v(atom_i, (flowax + 1) % 3) = /*random->gaussian()*/ rand_gen.normal() / (gamma * MathConst::MY_SQRT2); + d_v(atom_i, (flowax + 2) % 3) = /*random->gaussian()*/ rand_gen.normal() / (gamma * MathConst::MY_SQRT2); + + rand_pool.free_state(rand_gen); +} + +template +KOKKOS_INLINE_FUNCTION +int FixWallFlowKokkos::compute_current_segment_kk(double pos) const +{ + int result = 0; + for (; result < d_walls.extent(0) - 1; ++result) { + if (pos >= d_walls[result] && pos < d_walls[result + 1]) { return result; } + } + return -1; // -1 is "out of box" region +} + + +template +void FixWallFlowKokkos::grow_arrays(int nmax) +{ + k_current_segment.template sync(); + memoryKK->grow_kokkos(k_current_segment, current_segment, nmax, "WallFlowKK::current_segment"); + k_current_segment.template modify(); + + d_current_segment = k_current_segment.template view(); + h_current_segment = k_current_segment.template view(); +} + +template +void FixWallFlowKokkos::copy_arrays(int i, int j, int) +{ + k_current_segment.template sync(); + h_current_segment(j) = h_current_segment(i); + k_current_segment.template modify(); +} + +/* ---------------------------------------------------------------------- + sort local atom-based arrays +------------------------------------------------------------------------- */ + +template +void FixWallFlowKokkos::sort_kokkos(Kokkos::BinSort &Sorter) +{ + // always sort on the device + + k_current_segment.sync_device(); + + Sorter.sort(LMPDeviceType(), k_current_segment.d_view); + + k_current_segment.modify_device(); +} + +template +int FixWallFlowKokkos::pack_exchange(int i, double *buf) +{ + k_current_segment.sync_host(); + buf[0] = static_cast(h_current_segment(i)); + return 1; +} + +template +KOKKOS_INLINE_FUNCTION +void FixWallFlowKokkos::operator()(TagFixWallFlowPackExchange, const int &mysend) const { + const int send_i = d_sendlist(mysend); + const int segment = d_current_segment(send_i); + d_buf(mysend) = static_cast(segment); + + const int copy_i = d_copylist(mysend); + if (copy_i > -1) { + d_current_segment(send_i) = d_current_segment(copy_i); + } +} + +template +int FixWallFlowKokkos::pack_exchange_kokkos( + const int &nsend, DAT::tdual_xfloat_2d &k_buf, + DAT::tdual_int_1d k_sendlist, + DAT::tdual_int_1d k_copylist, + ExecutionSpace space) +{ + k_current_segment.template sync(); + + k_buf.template sync(); + k_sendlist.template sync(); + k_copylist.template sync(); + + d_sendlist = k_sendlist.view(); + d_copylist = k_copylist.view(); + + d_buf = typename ArrayTypes::t_xfloat_1d_um( + k_buf.template view().data(), + k_buf.extent(0)*k_buf.extent(1)); + + copymode = 1; + + Kokkos::parallel_for(Kokkos::RangePolicy(0, nsend), *this); + + copymode = 0; + + k_buf.template modify(); + k_current_segment.template modify(); + + return nsend; +} + +template +int FixWallFlowKokkos::unpack_exchange(int i, double *buf) +{ + k_current_segment.sync_host(); + h_current_segment(i) = static_cast(buf[0]); + k_current_segment.modify_host(); + return 1; +} + +template +KOKKOS_INLINE_FUNCTION +void FixWallFlowKokkos::operator()(TagFixWallFlowUnpackExchange, const int &i) const +{ + int index = d_indices(i); + if (index > -1) { + d_current_segment(index) = static_cast(d_buf(i)); + } +} + +template +void FixWallFlowKokkos::unpack_exchange_kokkos( + DAT::tdual_xfloat_2d &k_buf, + DAT::tdual_int_1d &k_indices, int nrecv, + ExecutionSpace space) +{ + d_buf = typename ArrayTypes::t_xfloat_1d_um( + k_buf.template view().data(), + k_buf.extent(0)*k_buf.extent(1)); + d_indices = k_indices.view(); + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nrecv),*this); + copymode = 0; + + k_current_segment.template modify(); +} + +namespace LAMMPS_NS { +template class FixWallFlowKokkos; +#ifdef LMP_KOKKOS_GPU +template class FixWallFlowKokkos; +#endif +} // namespace LAMMPS_NS diff --git a/src/KOKKOS/fix_wall_flow_kokkos.h b/src/KOKKOS/fix_wall_flow_kokkos.h new file mode 100644 index 0000000000..1faa8f4f32 --- /dev/null +++ b/src/KOKKOS/fix_wall_flow_kokkos.h @@ -0,0 +1,129 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS +// clang-format off +FixStyle(wall/flow/kk,FixWallFlowKokkos); +FixStyle(wall/flow/kk/device,FixWallFlowKokkos); +FixStyle(wall/flow/kk/host,FixWallFlowKokkos); +// clang-format on +#else + +// clang-format off +#ifndef LMP_FIX_WALL_FLOW_KOKKOS_H +#define LMP_FIX_WALL_FLOW_KOKKOS_H + +#include "fix_wall_flow.h" +#include "kokkos_type.h" +#include "kokkos_base.h" +#include "Kokkos_Random.hpp" +#include "comm_kokkos.h" + +namespace LAMMPS_NS { + +struct TagFixWallFlowInit{}; +template +struct TagFixWallFlowEndOfStep{}; +struct TagFixWallFlowPackExchange{}; +struct TagFixWallFlowUnpackExchange{}; + +template +class FixWallFlowKokkos : public FixWallFlow, public KokkosBase { + public: + typedef DeviceType device_type; + typedef ArrayTypes AT; + struct MassTag{}; + struct RMassTag{}; + FixWallFlowKokkos(class LAMMPS *, int, char **); + ~FixWallFlowKokkos(); + + void init() override; + void end_of_step() override; + void grow_arrays(int) override; + void copy_arrays(int, int, int) override; + void sort_kokkos(Kokkos::BinSort &Sorter) override; + int pack_exchange(int, double *) override; + int unpack_exchange(int, double *) override; + + KOKKOS_INLINE_FUNCTION + void operator() (TagFixWallFlowInit, const int&) const; + + template + KOKKOS_INLINE_FUNCTION + void operator()(TagFixWallFlowEndOfStep, const int&) const; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixWallFlowPackExchange, const int&) const; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixWallFlowUnpackExchange, const int&) const; + + int pack_exchange_kokkos(const int &nsend,DAT::tdual_xfloat_2d &buf, + DAT::tdual_int_1d k_sendlist, + DAT::tdual_int_1d k_copylist, + ExecutionSpace space) override; + + void unpack_exchange_kokkos(DAT::tdual_xfloat_2d &k_buf, + DAT::tdual_int_1d &indices,int nrecv, + ExecutionSpace space) override; + protected: + typename AT::t_x_array d_x; + typename AT::t_v_array d_v; + typename AT::t_int_1d d_type; + typename AT::t_int_1d d_mask; + + typename AT::t_float_1d d_mass; + typename AT::t_float_1d d_rmass; + + typedef typename AT::t_xfloat_1d d_walls_t; + typedef Kokkos::Random_XorShift64_Pool rand_pool_t; + typedef typename rand_pool_t::generator_type rand_type_t; + + typename AT::tdual_int_1d k_current_segment; + typename AT::t_int_1d d_current_segment; + typename HAT::t_int_1d h_current_segment; + + typename AT::t_int_1d d_sendlist; + typename AT::t_xfloat_1d d_buf; + typename AT::t_int_1d d_copylist; + typename AT::t_int_1d d_indices; + + d_walls_t d_walls; + + rand_pool_t rand_pool; + + template + KOKKOS_INLINE_FUNCTION + void generate_velocity_kk(int atom_i) const; + + KOKKOS_INLINE_FUNCTION + int compute_current_segment_kk(double pos) const; + + KOKKOS_INLINE_FUNCTION + double get_mass(MassTag, int atom_i) const + { + return d_mass(d_type(atom_i)); + } + + KOKKOS_INLINE_FUNCTION + double get_mass(RMassTag, int atom_i) const + { + return d_rmass(atom_i); + } +}; + +} + +#endif +#endif + diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp new file mode 100644 index 0000000000..a6e3e38cbc --- /dev/null +++ b/src/fix_wall_flow.cpp @@ -0,0 +1,248 @@ +// clang-format off + /* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "fix_wall_flow.h" + +#include "atom.h" +#include "comm.h" +#include "domain.h" +#include "error.h" +#include "input.h" +#include "lattice.h" +#include "modify.h" +#include "update.h" +#include "variable.h" +#include "random_mars.h" +#include "memory.h" +#include "force.h" +#include "math_const.h" + +#include +#include +#include +#include +#include + +using namespace LAMMPS_NS; +using namespace FixConst; + +/* ---------------------------------------------------------------------- */ + +// fix name groupId wall/flow vel temp dim N coords... + +FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : + Fix(lmp, narg, arg), + flowax(FlowAxis::AX_X), + flowvel(0.0), + flowdir(0), + rndseed(0), + current_segment(nullptr) +{ + if (narg < 9) utils::missing_cmd_args(FLERR, "fix wall/flow", error); + + dynamic_group_allow = 1; + bool do_abort = false; + + int iarg = 3; + // parsing axis + if (strcmp(arg[iarg], "x") == 0) flowax = FlowAxis::AX_X; + else if (strcmp(arg[iarg],"y") == 0) flowax = FlowAxis::AX_Y; + else if (strcmp(arg[iarg],"z") == 0) flowax = FlowAxis::AX_Z; + else error->all(FLERR,"Illegal fix wall/flow argument: axis must by x or y or z, but {} specified", arg[iarg]); + + ++iarg; + // parsing velocity + flowvel = utils::numeric(FLERR,arg[iarg],do_abort,lmp); + if (flowvel == 0.0) error->all(FLERR,"Illegal fix wall/flow argument: velocity cannot be 0"); + if (flowvel > 0.0) flowdir = 1; + else flowdir = -1; + if(flowdir < 0) error->all(FLERR, "Negative direction is not supported yet"); + + ++iarg; + // parsing temperature + double flowtemp = utils::numeric(FLERR,arg[iarg],do_abort,lmp); + kT = lmp->force->boltz * flowtemp / force->mvv2e; + + ++iarg; + // parsing seed + rndseed = utils::inumeric(FLERR, arg[iarg],do_abort,lmp); + if(rndseed <= 0) error->all(FLERR, "Random seed must be positive!"); + + ++iarg; + // parsing wall count + int wallcount = utils::inumeric(FLERR,arg[iarg],do_abort,lmp); + if(wallcount <= 0) error->all(FLERR,"Illegal fix wall/flow argument: wall count must be positive"); + + ++iarg; + // parsing walls + if(narg - iarg != wallcount) error->all(FLERR, "Wrong fix wall/flow wall count {}," + " must be {}", + wallcount, narg - iarg); + walls.resize(wallcount + 2); + walls.front() = domain->boxlo[flowax]; + for (size_t w = 1; w <= wallcount; ++w, ++iarg) + { + walls[w] = utils::numeric(FLERR,arg[iarg],do_abort,lmp); + } + walls.back() = domain->boxhi[flowax]; + if (!std::is_sorted(walls.begin(), walls.end(), std::less_equal())) + { + error->all(FLERR, "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); + } + + memory->grow(current_segment, atom->nmax, "WallFlow::current_segment"); + atom->add_callback(Atom::GROW); + if (restart_peratom) atom->add_callback(Atom::RESTART); + + maxexchange = 1; + + random = new RanMars(lmp, rndseed + comm->me); + } + +/* ---------------------------------------------------------------------- */ + +FixWallFlow::~FixWallFlow() +{ + if (copymode) return; + atom->delete_callback(id, Atom::GROW); + if (restart_peratom) atom->delete_callback(id, Atom::RESTART); + memory->destroy(current_segment); + + delete random; +} + +/* ---------------------------------------------------------------------- */ + +int FixWallFlow::setmask() +{ + int mask = 0; + + mask |= END_OF_STEP; + + return mask; +} + +/* ---------------------------------------------------------------------- */ + +void FixWallFlow::init() +{ + int nrigid = 0; + for (int i = 0; i < modify->nfix; i++) + if (modify->fix[i]->rigid_flag) nrigid++; + + if (nrigid && comm->me == 0) + error->warning(FLERR,"FixWallFlow is not compatible with rigid bodies"); + + for (int i = 0; i < atom->nlocal; ++i) + { + double pos = atom->x[i][flowax]; + current_segment[i] = compute_current_segment(pos); + } +} + +/* ---------------------------------------------------------------------- */ + +void FixWallFlow::end_of_step() +{ + double **x = atom->x; + int *mask = atom->mask; + int nlocal = atom->nlocal; + + for (int i = 0; i < nlocal; ++i) + { + if (mask[i] & groupbit) + { + double pos = x[i][flowax]; + int prev_segment = current_segment[i]; + current_segment[i] = compute_current_segment(pos); + + if (prev_segment != current_segment[i]) + { + generate_velocity(i); + } + } + } +} + +void FixWallFlow::generate_velocity(int atom_i) +{ + const int newton_iteration_count = 10; + double *vel = atom->v[atom_i]; + double mass = atom->mass[atom->type[atom_i]]; + const double gamma = 1.0 / std::sqrt(2.0 * kT / mass); + double delta = gamma * flowvel; + + const double edd = std::exp(-delta*delta) / MathConst::MY_PIS + delta * std::erf(delta); + const double probability_threshold = 0.5f * (1.f + delta / edd); + + double direction = 1.0; + + if (random->uniform() > probability_threshold) + { + delta = -delta; + direction = -direction; + } + + const double xi_0 = random->uniform(); + const double F_inf = edd + delta; + const double xi = xi_0 * F_inf; + const double x_0 = (std::sqrt(delta*delta + 2) - delta) * 0.5; + double x = x_0; + for (int i = 0; i < newton_iteration_count; ++i) + { + x -= (std::exp(x*x) * MathConst::MY_PIS * (xi - delta * std::erfc(x)) - 1.0) / (x + delta) * 0.5; + } + + const double nu = x + delta; + const double v = nu / gamma; + + vel[flowax] = v * direction; + vel[(flowax + 1) % 3] = random->gaussian() / (gamma * MathConst::MY_SQRT2); + vel[(flowax + 2) % 3] = random->gaussian() / (gamma * MathConst::MY_SQRT2); +} + +int FixWallFlow::compute_current_segment(double pos) const +{ + int result = 0; + for (; result < walls.size()-1; ++result) + { + if (pos >= walls[result] && pos < walls[result + 1]) + { + return result; + } + } + return -1; // -1 is "out of box" region +} + +void FixWallFlow::grow_arrays(int nmax) +{ + memory->grow(current_segment, nmax, "WallFlow::current_segment"); +} + +void FixWallFlow::copy_arrays(int i, int j, int) +{ + current_segment[j] = current_segment[i]; +} + +int FixWallFlow::pack_exchange(int i, double* buf) +{ + buf[0] = static_cast(current_segment[i]); + return 1; +} + +int FixWallFlow::unpack_exchange(int i, double* buf) +{ + current_segment[i] = static_cast(buf[0]); + return 1; +} diff --git a/src/fix_wall_flow.h b/src/fix_wall_flow.h new file mode 100644 index 0000000000..ec9e4a4601 --- /dev/null +++ b/src/fix_wall_flow.h @@ -0,0 +1,60 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS +// clang-format off +FixStyle(wall/flow,FixWallFlow); +// clang-format on +#else + +#ifndef LMP_FIX_WALL_FLOW_H +#define LMP_FIX_WALL_FLOW_H + +#include "fix.h" +#include +namespace LAMMPS_NS { + +class FixWallFlow : public Fix { + public: + enum FlowAxis {AX_X = 0, AX_Y = 1, AX_Z = 2}; + + FixWallFlow(class LAMMPS *, int, char **); + ~FixWallFlow() override; + int setmask() override; + void init() override; + void end_of_step() override; + + void grow_arrays(int) override; + void copy_arrays(int, int, int) override; + + int pack_exchange(int, double *) override; + int unpack_exchange(int, double *) override; + + protected: + FlowAxis flowax; + double flowvel; + double kT; + std::vector walls; + + int flowdir; + int rndseed; + class RanMars *random; + int *current_segment; + int compute_current_segment(double pos) const; + void generate_velocity(int i); +}; + +} // namespace LAMMPS_NS + +#endif +#endif From b775085189d3b66d193cc4b8e96cb3599f9c5db0 Mon Sep 17 00:00:00 2001 From: vladgl Date: Mon, 9 Oct 2023 13:29:15 +0300 Subject: [PATCH 02/33] Add licensing info --- src/KOKKOS/fix_wall_flow_kokkos.cpp | 19 +++++++++++++++++++ src/fix_wall_flow.cpp | 8 ++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp index 501a1bbd3f..46805f07d6 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.cpp +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -1,3 +1,22 @@ +// clang-format off +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + https://www.lammps.org/, Sandia National Laboratories + LAMMPS development team: developers@lammps.org + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing authors: Vladislav Galigerov (HSE), + Daniil Pavlov (MIPT) +------------------------------------------------------------------------- */ + #include "fix_wall_flow_kokkos.h" #include "atom_kokkos.h" #include "memory_kokkos.h" diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index a6e3e38cbc..adaf7412cc 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -1,5 +1,5 @@ // clang-format off - /* ---------------------------------------------------------------------- +/* ---------------------------------------------------------------------- LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator https://www.lammps.org/, Sandia National Laboratories LAMMPS development team: developers@lammps.org @@ -12,8 +12,12 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ -#include "fix_wall_flow.h" +/* ---------------------------------------------------------------------- + Contributing authors: Vladislav Galigerov (HSE), + Daniil Pavlov (MIPT) +------------------------------------------------------------------------- */ +#include "fix_wall_flow.h" #include "atom.h" #include "comm.h" #include "domain.h" From 70cc1039fd304a2e1b64897e37ad4efbe14915d4 Mon Sep 17 00:00:00 2001 From: vladgl Date: Mon, 9 Oct 2023 17:30:57 +0300 Subject: [PATCH 03/33] Some cleaning --- src/KOKKOS/fix_wall_flow_kokkos.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp index 46805f07d6..b0efee0941 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.cpp +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -25,7 +25,7 @@ #include "force.h" #include -//#include "comm.h + // clang-format off using namespace LAMMPS_NS; From 19e45187f8d4bf7663ffb3f24591ce66f9153bdd Mon Sep 17 00:00:00 2001 From: vladgl Date: Mon, 9 Oct 2023 17:32:59 +0300 Subject: [PATCH 04/33] Add cite string --- src/fix_wall_flow.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index adaf7412cc..d6dbafd64c 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -19,6 +19,7 @@ #include "fix_wall_flow.h" #include "atom.h" +#include "citeme.h" #include "comm.h" #include "domain.h" #include "error.h" @@ -43,7 +44,16 @@ using namespace FixConst; /* ---------------------------------------------------------------------- */ -// fix name groupId wall/flow vel temp dim N coords... +static const char cite_fix_wall_flow_c[] = + "fix wall/flow command: doi:{tba}\n\n" + "@Article{Pavlov-etal-IJHPCA-2023,\n" + " author = {Daniil Pavlov and Vladislav Galigerov and Daniil Kolotinskii and Vsevolod Nikolskiy and Vladimir Stegailov},\n" + " title = {GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence},\n" + " journal = {International Journal of High Performance Computing Applications},\n" + " year = 2023,\n" + " volume = {tba},\n" + " pages = {tba}\n” + "}\n\n"; FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : Fix(lmp, narg, arg), @@ -53,6 +63,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : rndseed(0), current_segment(nullptr) { + if (lmp->citeme) lmp->citeme->add(cite_fix_wall_flow_c); if (narg < 9) utils::missing_cmd_args(FLERR, "fix wall/flow", error); dynamic_group_allow = 1; From d7f7306b7d14ecb067b08c0fbf17aaf26ee83852 Mon Sep 17 00:00:00 2001 From: vladgl Date: Fri, 13 Oct 2023 14:26:05 +0300 Subject: [PATCH 05/33] Fix character --- src/fix_wall_flow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index d6dbafd64c..ff4d4078dd 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -52,7 +52,7 @@ static const char cite_fix_wall_flow_c[] = " journal = {International Journal of High Performance Computing Applications},\n" " year = 2023,\n" " volume = {tba},\n" - " pages = {tba}\n” + " pages = {tba}\n" "}\n\n"; FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : From 249f3b0af4149d90ea68d3a008ed3b3148e9b5c1 Mon Sep 17 00:00:00 2001 From: vladgl Date: Sat, 14 Oct 2023 13:10:08 +0300 Subject: [PATCH 06/33] Add units command --- src/fix_wall_flow.cpp | 44 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index ff4d4078dd..e5602b7af9 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -82,7 +82,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : if (flowvel == 0.0) error->all(FLERR,"Illegal fix wall/flow argument: velocity cannot be 0"); if (flowvel > 0.0) flowdir = 1; else flowdir = -1; - if(flowdir < 0) error->all(FLERR, "Negative direction is not supported yet"); + if(flowdir < 0) error->all(FLERR, "Illegal fix wall/flow argument: negative direction is not supported yet"); ++iarg; // parsing temperature @@ -92,29 +92,59 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : ++iarg; // parsing seed rndseed = utils::inumeric(FLERR, arg[iarg],do_abort,lmp); - if(rndseed <= 0) error->all(FLERR, "Random seed must be positive!"); + if(rndseed <= 0) error->all(FLERR, "Illegal fix wall/flow argument: random seed must be positive integer"); ++iarg; // parsing wall count int wallcount = utils::inumeric(FLERR,arg[iarg],do_abort,lmp); - if(wallcount <= 0) error->all(FLERR,"Illegal fix wall/flow argument: wall count must be positive"); + if(wallcount <= 0) error->all(FLERR,"Illegal fix wall/flow argument: wall count must be positive integer"); ++iarg; // parsing walls - if(narg - iarg != wallcount) error->all(FLERR, "Wrong fix wall/flow wall count {}," - " must be {}", - wallcount, narg - iarg); + if(narg - iarg != wallcount && narg - iarg != wallcount + 2) error->all(FLERR, "Wrong fix wall/flow wall count"); + auto getscale = [&]() -> double { + switch (flowax) + { + case FlowAxis::AX_X: + return domain->lattice->xlattice; + case FlowAxis::AX_Y: + return domain->lattice->ylattice; + case FlowAxis::AX_Z: + return domain->lattice->zlattice; + default: return 0.0; + } + return 0.0; + }; + double scale = getscale(); + + if (narg - iarg == wallcount + 2) + { + if(strcmp(arg[narg - 2], "units") != 0) error->all(FLERR, "Wrong fix wall/flow units command"); + if (strcmp(arg[narg - 1], "box") == 0) scale = 1.0; + else if (strcmp(arg[narg - 1], "lattice") == 0) + { + scale = getscale(); + } + else error->all(FLERR, "Wrong fix wall/flow units command"); + } + walls.resize(wallcount + 2); walls.front() = domain->boxlo[flowax]; for (size_t w = 1; w <= wallcount; ++w, ++iarg) { - walls[w] = utils::numeric(FLERR,arg[iarg],do_abort,lmp); + walls[w] = utils::numeric(FLERR,arg[iarg],do_abort,lmp) * scale; } walls.back() = domain->boxhi[flowax]; if (!std::is_sorted(walls.begin(), walls.end(), std::less_equal())) { error->all(FLERR, "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); } + std::cout << "Walls:\n" + for (auto w : walls) + { + std::cout << w << " "; + } + std::cout << std::endl; memory->grow(current_segment, atom->nmax, "WallFlow::current_segment"); atom->add_callback(Atom::GROW); From 381330c3c25aa1e04819925e42ad8202b0615072 Mon Sep 17 00:00:00 2001 From: vladgl Date: Sat, 14 Oct 2023 13:11:06 +0300 Subject: [PATCH 07/33] Fix typo --- src/fix_wall_flow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index e5602b7af9..beaaa474eb 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -139,7 +139,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : { error->all(FLERR, "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); } - std::cout << "Walls:\n" + std::cout << "Walls:\n"; for (auto w : walls) { std::cout << w << " "; From c53847ad882f282ecf737040bedf3968682c656f Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Wed, 31 Jan 2024 15:35:04 +0300 Subject: [PATCH 08/33] Update citation --- src/fix_wall_flow.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index beaaa474eb..591f640279 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -45,14 +45,15 @@ using namespace FixConst; /* ---------------------------------------------------------------------- */ static const char cite_fix_wall_flow_c[] = - "fix wall/flow command: doi:{tba}\n\n" - "@Article{Pavlov-etal-IJHPCA-2023,\n" + "fix wall/flow command: doi:10.1177/10943420231213013\n\n" + "@Article{Pavlov-etal-IJHPCA-2024,\n" " author = {Daniil Pavlov and Vladislav Galigerov and Daniil Kolotinskii and Vsevolod Nikolskiy and Vladimir Stegailov},\n" - " title = {GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence},\n" - " journal = {International Journal of High Performance Computing Applications},\n" - " year = 2023,\n" - " volume = {tba},\n" - " pages = {tba}\n" + " title = {GPU-based molecular dynamics of fluid flows: Reaching for turbulence},\n" + " journal = {The International Journal of High Performance Computing Applications},\n" + " year = 2024,\n" + " volume = 38,\n" + " number = 1,\n" + " pages = 34-49\n" "}\n\n"; FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : From 90105fd58a0a5e55ee43993c9b9c7415376ea5ca Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Sun, 4 Feb 2024 16:02:53 +0300 Subject: [PATCH 09/33] Add wall_flow documentation --- doc/src/Bibliography.rst | 3 + doc/src/Commands_fix.rst | 1 + doc/src/fix.rst | 1 + doc/src/fix_wall_flow.rst | 123 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 doc/src/fix_wall_flow.rst diff --git a/doc/src/Bibliography.rst b/doc/src/Bibliography.rst index 4ed8e73dfe..9778340c94 100644 --- a/doc/src/Bibliography.rst +++ b/doc/src/Bibliography.rst @@ -877,6 +877,9 @@ Bibliography **(PLUMED)** G.A. Tribello, M. Bonomi, D. Branduardi, C. Camilloni and G. Bussi, Comp. Phys. Comm 185, 604 (2014) +**(Pavlov)** +D Pavlov, V Galigerov, D Kolotinskii, V Nikolskiy, V Stegailov, International Journal of High Performance Computing Applications, 38, 34-49 (2024). + **(Paquay)** Paquay and Kusters, Biophys. J., 110, 6, (2016). preprint available at `arXiv:1411.3019 `_. diff --git a/doc/src/Commands_fix.rst b/doc/src/Commands_fix.rst index e89e302673..304f54f690 100644 --- a/doc/src/Commands_fix.rst +++ b/doc/src/Commands_fix.rst @@ -262,6 +262,7 @@ OPT. * :doc:`wall/body/polyhedron ` * :doc:`wall/colloid ` * :doc:`wall/ees ` + * :doc:`wall/flow (k) ` * :doc:`wall/gran (k) ` * :doc:`wall/gran/region ` * :doc:`wall/harmonic ` diff --git a/doc/src/fix.rst b/doc/src/fix.rst index 69a7212487..4ff7fd6bd6 100644 --- a/doc/src/fix.rst +++ b/doc/src/fix.rst @@ -427,6 +427,7 @@ accelerated styles exist. * :doc:`wall/body/polyhedron ` - time integration for body particles of style :doc:`rounded/polyhedron ` * :doc:`wall/colloid ` - Lennard-Jones wall interacting with finite-size particles * :doc:`wall/ees ` - wall for ellipsoidal particles +* :doc:`wall/flow ` - flow boundary conditions * :doc:`wall/gran ` - frictional wall(s) for granular simulations * :doc:`wall/gran/region ` - :doc:`fix wall/region ` equivalent for use with granular particles * :doc:`wall/harmonic ` - harmonic spring wall diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst new file mode 100644 index 0000000000..56a2510d08 --- /dev/null +++ b/doc/src/fix_wall_flow.rst @@ -0,0 +1,123 @@ +.. index:: fix wall/flow +.. index:: fix wall/flow/kk + +fix wall/flow command +===================== + +Accelerator Variants: *wall/flow/kk* + +Syntax +"""""" + +.. code-block:: LAMMPS + + fix ID group-ID wall/flow ax vf T seed N coords... + +* ID, group-ID are documented in :doc:`fix ` command +* wall/flow = style name of this fix command +* ax = flow axis (*x*, *y*, or *z* character) +* vf = *ax* component of generated flow velocity +* T = flow temperature (temperature units) +* seed = random seed for stochasticity (positive integer) +* N = number of walls (positive integer) +* coords = set of N wall coordinates (box units) along *ax* axis arranged in ascending order. Note that an additional implicit wall is introduced at the boundary of the simulation domain, so the resulting system always has N+1 walls. + +Examples +"""""""" + +.. code-block:: LAMMPS + + fix 1 g_flow wall/flow x ${VFLOW} ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} + fix 2 all wall/flow 0.4 0.2 3 1 400 + +Description +""""""""""" + +This fix implements flow boundary conditions (FBC) introduced in :ref:`(Pavlov) ` and :ref:`(Pavlov) `. +The goal is to generate a stationary flow with a shifted Maxwell velocity distribution: + +.. math:: + + f_z(v_z) \propto \exp{\left(-\frac{m (v_z-v_{\text{flow}})^2}{2 k T}\right)} + +This is achieved by reassigning the velocity of each particle that passes a wall. +Such reassigning represents an emission of a new particle into the system with +simultaneous removal of a particle with the same position. +The parallel velocity components parallel to the wall are re-assigned according +to the Maxwell velocity distribution. The perpendicular component is assigned +according to the following velocity distribution: + +.. math:: + + f_{\text{z generated}}(v_z) \propto v_z f_z(v_z) + +It can be shown that in an ideal-gas scenario this makes the velocity +distribution of particles between walls exactly as desired. + +Since in most cases simulated systems are not ideal gas, +the need for multiple walls might arise, as a single wall may not be +sufficient for maintaining a stationary flow without congestions +manifesting as areas with increased density located upstream from static obstacles. + +For the same reason, the actual temperature and velocity of the generated +flow may differ from ones requested. The degree of such discrepancy is determined +by how different from the ideal gas the simulated system is. Therefore, a calibration procedure is required for each system as described in :ref:`(Pavlov) `. + +The interactions between particles on different sides of a wall are not disabled or neglected and the +particle positions aren't affected by the velocity reassignment. +This removes the need to modify the force field to work correctly in cases when a particle is close +to a wall (for example, if particle positions were uniformly redistributed across the surface of the wall, +two particles could end up too close to each other, potentially causing the simulation to explode). +However due to this compromise, some collective phenomena such as areas with increased/decreased density +or collective movements are not fully removed when particles cross a wall. +This unwanted consequence can also be potentially mitigated by using more than one wall. + + +---------- + +Note that when high flow velocity is reached, a lost atoms error may +occur (see :doc:`error messages `). +If this message appears when using this fix, you can, for example, reduce the frequency of the +neighbor list rebuild via :doc:`neigh_modify ` command. + +Restart, fix_modify, output, run start/stop, minimize info +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +No information about this fix is written to :doc:`binary restart files `. + +None of the :doc:`fix_modify ` options are relevant to +this fix. + +No global or per-atom quantities are stored by this fix for access by +various :doc:`output commands `. + +No parameter of this fix can be used with the *start/stop* keywords of +the :doc:`run ` command. + +This fix is not invoked during :doc:`energy minimization `. + +Restrictions +"""""""""""" + +Flow boundary conditions should not be used with rigid bodies such as those +defined by a "fix rigid" command. + +Related commands +"""""""""""""""" + +:doc:`fix wall/reflect ` command + +Default +""""""" + +none + +---------- + +.. _fbc-Pavlov1: + +**(Pavlov)** Pavlov, Kolotinskii, Stegailov, “GPU-Based Molecular Dynamics of Turbulent Liquid Flows with OpenMM”, In: Proceedings of PPAM-2022, LNCS (Springer), vol. 13826, pp. 346–358 (2023) + +.. _fbc-Pavlov2: + +**(Pavlov)** Pavlov, Galigerov, Kolotinskii, Nikolskiy, Stegailov, "GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence”, Int. J. High Perf. Comp. Appl., (2024) \ No newline at end of file From 4ad5a9d3eb01a7ad2a3771d43c678b8c821e5e24 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Tue, 6 Feb 2024 21:00:37 +0300 Subject: [PATCH 10/33] Fix doc --- doc/src/fix_wall_flow.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index 56a2510d08..e0f5424658 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -11,7 +11,7 @@ Syntax .. code-block:: LAMMPS - fix ID group-ID wall/flow ax vf T seed N coords... + fix ID group-ID wall/flow ax vf T seed N coords ... keyword value * ID, group-ID are documented in :doc:`fix ` command * wall/flow = style name of this fix command @@ -22,6 +22,15 @@ Syntax * N = number of walls (positive integer) * coords = set of N wall coordinates (box units) along *ax* axis arranged in ascending order. Note that an additional implicit wall is introduced at the boundary of the simulation domain, so the resulting system always has N+1 walls. +* zero or more keyword/value pairs may be appended +* keyword = *units* + + .. parsed-literal:: + + *units* value = *lattice* or *box* + *lattice* = the wall positions are defined in lattice units + *box* = the wall positions are defined in simulation box units + Examples """""""" @@ -110,7 +119,7 @@ Related commands Default """"""" -none +The default for the units keyword is lattice. ---------- From 0f5436de99607f000a0b4c2291443baf2d5e1c46 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Tue, 6 Feb 2024 22:43:56 +0300 Subject: [PATCH 11/33] Fix spelling --- doc/src/fix_wall_flow.rst | 10 +++++----- doc/utils/sphinx-config/false_positives.txt | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index e0f5424658..f6f5a242ac 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -42,7 +42,7 @@ Examples Description """"""""""" -This fix implements flow boundary conditions (FBC) introduced in :ref:`(Pavlov) ` and :ref:`(Pavlov) `. +This fix implements flow boundary conditions (FBC) introduced in :ref:`(Pavlov1) ` and :ref:`(Pavlov2) `. The goal is to generate a stationary flow with a shifted Maxwell velocity distribution: .. math:: @@ -52,7 +52,7 @@ The goal is to generate a stationary flow with a shifted Maxwell velocity distri This is achieved by reassigning the velocity of each particle that passes a wall. Such reassigning represents an emission of a new particle into the system with simultaneous removal of a particle with the same position. -The parallel velocity components parallel to the wall are re-assigned according +The velocity components parallel to the wall are re-assigned according to the Maxwell velocity distribution. The perpendicular component is assigned according to the following velocity distribution: @@ -65,7 +65,7 @@ distribution of particles between walls exactly as desired. Since in most cases simulated systems are not ideal gas, the need for multiple walls might arise, as a single wall may not be -sufficient for maintaining a stationary flow without congestions +sufficient for maintaining a stationary flow without congestion manifesting as areas with increased density located upstream from static obstacles. For the same reason, the actual temperature and velocity of the generated @@ -125,8 +125,8 @@ The default for the units keyword is lattice. .. _fbc-Pavlov1: -**(Pavlov)** Pavlov, Kolotinskii, Stegailov, “GPU-Based Molecular Dynamics of Turbulent Liquid Flows with OpenMM”, In: Proceedings of PPAM-2022, LNCS (Springer), vol. 13826, pp. 346–358 (2023) +**(Pavlov1)** Pavlov, Kolotinskii, Stegailov, "GPU-Based Molecular Dynamics of Turbulent Liquid Flows with OpenMM", Proceedings of PPAM-2022, LNCS (Springer), vol. 13826, pp. 346-358 (2023) .. _fbc-Pavlov2: -**(Pavlov)** Pavlov, Galigerov, Kolotinskii, Nikolskiy, Stegailov, "GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence”, Int. J. High Perf. Comp. Appl., (2024) \ No newline at end of file +**(Pavlov2)** Pavlov, Galigerov, Kolotinskii, Nikolskiy, Stegailov, "GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence", Int. J. High Perf. Comp. Appl., (2024) \ No newline at end of file diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 4f5fe6fdaf..6106a1638c 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -1770,6 +1770,7 @@ Kolafa Kollman kolmogorov Kolmogorov +Kolotinskii Kondor konglt Koning @@ -2774,6 +2775,7 @@ PEigenDense Peng peptide peratom +Perf Pergamon pergrid peri @@ -3884,6 +3886,7 @@ Verlet versa Verstraelen ves +vf vflag vfrac vhi From a3a054cc152f08006f8b6bb8870869f4d4f5626c Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Tue, 6 Feb 2024 23:19:47 +0300 Subject: [PATCH 12/33] Another spell fix --- doc/src/fix_wall_flow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index f6f5a242ac..d89e0b6d89 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -73,7 +73,7 @@ flow may differ from ones requested. The degree of such discrepancy is determine by how different from the ideal gas the simulated system is. Therefore, a calibration procedure is required for each system as described in :ref:`(Pavlov) `. The interactions between particles on different sides of a wall are not disabled or neglected and the -particle positions aren't affected by the velocity reassignment. +particle positions are not affected by the velocity reassignment. This removes the need to modify the force field to work correctly in cases when a particle is close to a wall (for example, if particle positions were uniformly redistributed across the surface of the wall, two particles could end up too close to each other, potentially causing the simulation to explode). From 5bab14d31e9b2b8ec9622778de2ec78088b1125d Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Tue, 6 Feb 2024 23:23:11 +0300 Subject: [PATCH 13/33] Sync unpack_exchange_kokkos with develop --- src/KOKKOS/fix_wall_flow_kokkos.cpp | 1 + src/KOKKOS/fix_wall_flow_kokkos.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp index b0efee0941..daf73e7ef2 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.cpp +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -292,6 +292,7 @@ template void FixWallFlowKokkos::unpack_exchange_kokkos( DAT::tdual_xfloat_2d &k_buf, DAT::tdual_int_1d &k_indices, int nrecv, + int /*nrecv1*/, int /*nextrarecv1*/, ExecutionSpace space) { d_buf = typename ArrayTypes::t_xfloat_1d_um( diff --git a/src/KOKKOS/fix_wall_flow_kokkos.h b/src/KOKKOS/fix_wall_flow_kokkos.h index 1faa8f4f32..10bfed962d 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.h +++ b/src/KOKKOS/fix_wall_flow_kokkos.h @@ -75,6 +75,7 @@ class FixWallFlowKokkos : public FixWallFlow, public KokkosBase { void unpack_exchange_kokkos(DAT::tdual_xfloat_2d &k_buf, DAT::tdual_int_1d &indices,int nrecv, + int /*nrecv1*/, int /*nextrarecv1*/, ExecutionSpace space) override; protected: typename AT::t_x_array d_x; From df7662162e29cbcb40e8eecb26736950535c50f8 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Wed, 7 Feb 2024 13:53:40 +0300 Subject: [PATCH 14/33] Add example --- examples/wall/in.wall.flow | 79 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 examples/wall/in.wall.flow diff --git a/examples/wall/in.wall.flow b/examples/wall/in.wall.flow new file mode 100644 index 0000000000..59d1b99eda --- /dev/null +++ b/examples/wall/in.wall.flow @@ -0,0 +1,79 @@ +variable nrun equal 10000 +variable dump_count equal 10 + +variable nwall equal 4 +variable w1 equal 67 +variable w2 equal 71 +variable w3 equal 75 +variable w4 equal 79 + +variable x_cylinder equal 20 +variable y_cylinder equal 17 +variable r_cylinder equal 4 + +variable MASS equal 1 +variable TEMP equal 0.4 +variable VFLOW equal 0.5 + +units lj +atom_style atomic + +lattice fcc 0.3 +region sim_box block 0 84 0 34 0 10 + +boundary p p p + +create_box 2 sim_box +region reg_cylinder cylinder z ${x_cylinder} ${y_cylinder} ${r_cylinder} EDGE EDGE + +create_atoms 1 box + +## setup obstacle ## +group g_obst region reg_cylinder +group g_flow subtract all g_obst +set group g_obst type 2 + +mass 1 ${MASS} +mass 2 ${MASS} + +velocity g_flow create ${TEMP} 4928459 rot yes dist gaussian +velocity g_obst set 0.0 0.0 0.0 + +pair_style lj/cut 1.122462 +pair_coeff 1 1 1.0 1.0 +pair_coeff 1 2 1.0 1.0 +pair_coeff 2 2 1.0 1.0 +pair_modify shift yes + +neighbor 0.3 bin +neigh_modify delay 0 every 20 check no + +fix 1 g_flow nve +fix 2 g_flow wall/flow x ${VFLOW} ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} + +variable dump_every equal ${nrun}/${dump_count} +variable thermo_every equal ${dump_every} +variable restart_every equal ${nrun}/10 + +##### uncomment for grid aggregation ##### +#variable gr_Nx equal 42 +#variable gr_Ny equal 17 +#variable gr_Nz equal 1 +#variable gr_Nevery equal ${dump_every} +#variable gr_Nrepeat equal 1 +#variable gr_Nfreq equal ${dump_every} +#fix 3 g_flow ave/grid ${gr_Nevery} ${gr_Nrepeat} ${gr_Nfreq} ${gr_Nx} ${gr_Ny} ${gr_Nz} vx vy vz density/mass norm all ave one +#compute ct_gridId g_flow property/grid ${gr_Nx} ${gr_Ny} ${gr_Nz} id +#dump dmp_grid g_flow grid ${dump_every} grid.lammpstrj c_ct_gridId:grid:data f_3:grid:data[*] +########################################## + +#dump dmp_coord all atom ${dump_every} dump.lammpstrj + +#compute ct_Temp g_flow temp/com +#thermo_style custom step temp epair emol etotal press c_ct_Temp + +#restart ${restart_every} flow.restart + +timestep 0.005 +thermo ${thermo_every} +run ${nrun} From e33590b2fc101813d0beef963daf31c0dd40a264 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Wed, 7 Feb 2024 16:03:32 +0300 Subject: [PATCH 15/33] Whitespace --- doc/src/fix_wall_flow.rst | 2 +- src/KOKKOS/fix_wall_flow_kokkos.cpp | 32 ++++++++++++++--------------- src/KOKKOS/fix_wall_flow_kokkos.h | 6 +++--- src/fix_wall_flow.cpp | 4 ++-- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index d89e0b6d89..3b8d7300e6 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -129,4 +129,4 @@ The default for the units keyword is lattice. .. _fbc-Pavlov2: -**(Pavlov2)** Pavlov, Galigerov, Kolotinskii, Nikolskiy, Stegailov, "GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence", Int. J. High Perf. Comp. Appl., (2024) \ No newline at end of file +**(Pavlov2)** Pavlov, Galigerov, Kolotinskii, Nikolskiy, Stegailov, "GPU-based Molecular Dynamics of Fluid Flows: Reaching for Turbulence", Int. J. High Perf. Comp. Appl., (2024) diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp index daf73e7ef2..a143578fdc 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.cpp +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -48,7 +48,7 @@ FixWallFlowKokkos::FixWallFlowKokkos(LAMMPS *lmp, int narg, char **a d_walls = d_walls_t("FixWallFlowKokkos::walls", walls.size()); auto h_walls = Kokkos::create_mirror_view(d_walls); for (int i = 0; i < walls.size(); ++i) - { + { h_walls(i) = walls[i]; } Kokkos::deep_copy(d_walls, h_walls); @@ -61,7 +61,7 @@ FixWallFlowKokkos::~FixWallFlowKokkos() memoryKK->destroy_kokkos(k_current_segment, current_segment); } -template +template void FixWallFlowKokkos::init() { atomKK->sync(execution_space, datamask_read); @@ -83,10 +83,10 @@ KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWall d_current_segment(i) = compute_current_segment_kk(pos); } -template -void FixWallFlowKokkos::end_of_step() +template +void FixWallFlowKokkos::end_of_step() { - atomKK->sync(execution_space, datamask_read); + atomKK->sync(execution_space, datamask_read); k_current_segment.template sync(); d_x = atomKK->k_x.template view(); @@ -111,17 +111,17 @@ void FixWallFlowKokkos::end_of_step() template template -KOKKOS_INLINE_FUNCTION +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWallFlowEndOfStep, const int &atom_i) const { - if (d_mask[atom_i] & groupbit) - { + if (d_mask[atom_i] & groupbit) + { double pos = d_x(atom_i, flowax); int prev_segment = d_current_segment(atom_i); d_current_segment(atom_i) = compute_current_segment_kk(pos); - if (prev_segment != d_current_segment(atom_i)) - { + if (prev_segment != d_current_segment(atom_i)) + { generate_velocity_kk(atom_i); } } @@ -129,7 +129,7 @@ void FixWallFlowKokkos::operator()(TagFixWallFlowEndOfStep, template template -KOKKOS_INLINE_FUNCTION +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::generate_velocity_kk(int atom_i) const { const int newton_iteration_count = 10; @@ -170,7 +170,7 @@ void FixWallFlowKokkos::generate_velocity_kk(int atom_i) const } template -KOKKOS_INLINE_FUNCTION +KOKKOS_INLINE_FUNCTION int FixWallFlowKokkos::compute_current_segment_kk(double pos) const { int result = 0; @@ -181,8 +181,8 @@ int FixWallFlowKokkos::compute_current_segment_kk(double pos) const } -template -void FixWallFlowKokkos::grow_arrays(int nmax) +template +void FixWallFlowKokkos::grow_arrays(int nmax) { k_current_segment.template sync(); memoryKK->grow_kokkos(k_current_segment, current_segment, nmax, "WallFlowKK::current_segment"); @@ -193,7 +193,7 @@ void FixWallFlowKokkos::grow_arrays(int nmax) } template -void FixWallFlowKokkos::copy_arrays(int i, int j, int) +void FixWallFlowKokkos::copy_arrays(int i, int j, int) { k_current_segment.template sync(); h_current_segment(j) = h_current_segment(i); @@ -252,7 +252,7 @@ int FixWallFlowKokkos::pack_exchange_kokkos( d_sendlist = k_sendlist.view(); d_copylist = k_copylist.view(); - + d_buf = typename ArrayTypes::t_xfloat_1d_um( k_buf.template view().data(), k_buf.extent(0)*k_buf.extent(1)); diff --git a/src/KOKKOS/fix_wall_flow_kokkos.h b/src/KOKKOS/fix_wall_flow_kokkos.h index 10bfed962d..8de0eded0a 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.h +++ b/src/KOKKOS/fix_wall_flow_kokkos.h @@ -107,16 +107,16 @@ class FixWallFlowKokkos : public FixWallFlow, public KokkosBase { KOKKOS_INLINE_FUNCTION void generate_velocity_kk(int atom_i) const; - KOKKOS_INLINE_FUNCTION + KOKKOS_INLINE_FUNCTION int compute_current_segment_kk(double pos) const; - KOKKOS_INLINE_FUNCTION + KOKKOS_INLINE_FUNCTION double get_mass(MassTag, int atom_i) const { return d_mass(d_type(atom_i)); } - KOKKOS_INLINE_FUNCTION + KOKKOS_INLINE_FUNCTION double get_mass(RMassTag, int atom_i) const { return d_rmass(atom_i); diff --git a/src/fix_wall_flow.cpp b/src/fix_wall_flow.cpp index 591f640279..695b93aea8 100644 --- a/src/fix_wall_flow.cpp +++ b/src/fix_wall_flow.cpp @@ -99,7 +99,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : // parsing wall count int wallcount = utils::inumeric(FLERR,arg[iarg],do_abort,lmp); if(wallcount <= 0) error->all(FLERR,"Illegal fix wall/flow argument: wall count must be positive integer"); - + ++iarg; // parsing walls if(narg - iarg != wallcount && narg - iarg != wallcount + 2) error->all(FLERR, "Wrong fix wall/flow wall count"); @@ -173,7 +173,7 @@ FixWallFlow::~FixWallFlow() int FixWallFlow::setmask() { int mask = 0; - + mask |= END_OF_STEP; return mask; From 93fcf3cc754e4e77aba34ce839633a57ebe45146 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Wed, 7 Feb 2024 15:07:11 -0500 Subject: [PATCH 16/33] move fix to EXTRA-FIX package, update docs, add reference logs to example --- doc/src/fix_wall_flow.rst | 6 + examples/wall/in.wall.flow | 2 +- examples/wall/log.7Feb24.wall.flow.g++.1 | 182 +++++++++++++++++++++++ examples/wall/log.7Feb24.wall.flow.g++.4 | 182 +++++++++++++++++++++++ src/.gitignore | 2 + src/{ => EXTRA-FIX}/fix_wall_flow.cpp | 0 src/{ => EXTRA-FIX}/fix_wall_flow.h | 0 7 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 examples/wall/log.7Feb24.wall.flow.g++.1 create mode 100644 examples/wall/log.7Feb24.wall.flow.g++.4 rename src/{ => EXTRA-FIX}/fix_wall_flow.cpp (100%) rename src/{ => EXTRA-FIX}/fix_wall_flow.h (100%) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index 3b8d7300e6..464021ff52 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -42,6 +42,8 @@ Examples Description """"""""""" +.. versionadded:: TBD + This fix implements flow boundary conditions (FBC) introduced in :ref:`(Pavlov1) ` and :ref:`(Pavlov2) `. The goal is to generate a stationary flow with a shifted Maxwell velocity distribution: @@ -108,6 +110,10 @@ This fix is not invoked during :doc:`energy minimization `. Restrictions """""""""""" +Fix *wall_flow* is part of the EXTRA-FIX package. It is only enabled +if LAMMPS was built with that package. See the :doc:`Build package +` page for more info. + Flow boundary conditions should not be used with rigid bodies such as those defined by a "fix rigid" command. diff --git a/examples/wall/in.wall.flow b/examples/wall/in.wall.flow index 59d1b99eda..9dfe001a55 100644 --- a/examples/wall/in.wall.flow +++ b/examples/wall/in.wall.flow @@ -1,4 +1,4 @@ -variable nrun equal 10000 +variable nrun equal 1000 variable dump_count equal 10 variable nwall equal 4 diff --git a/examples/wall/log.7Feb24.wall.flow.g++.1 b/examples/wall/log.7Feb24.wall.flow.g++.1 new file mode 100644 index 0000000000..75e8b66fe1 --- /dev/null +++ b/examples/wall/log.7Feb24.wall.flow.g++.1 @@ -0,0 +1,182 @@ +LAMMPS (21 Nov 2023 - Development - patch_21Nov2023-758-ge33590b2fc-modified) + using 1 OpenMP thread(s) per MPI task +variable nrun equal 1000 +variable dump_count equal 10 + +variable nwall equal 4 +variable w1 equal 67 +variable w2 equal 71 +variable w3 equal 75 +variable w4 equal 79 + +variable x_cylinder equal 20 +variable y_cylinder equal 17 +variable r_cylinder equal 4 + +variable MASS equal 1 +variable TEMP equal 0.4 +variable VFLOW equal 0.5 + +units lj +atom_style atomic + +lattice fcc 0.3 +Lattice spacing in x,y,z = 2.3712622 2.3712622 2.3712622 +region sim_box block 0 84 0 34 0 10 + +boundary p p p + +create_box 2 sim_box +Created orthogonal box = (0 0 0) to (199.18603 80.622915 23.712622) + 1 by 1 by 1 MPI processor grid +region reg_cylinder cylinder z ${x_cylinder} ${y_cylinder} ${r_cylinder} EDGE EDGE +region reg_cylinder cylinder z 20 ${y_cylinder} ${r_cylinder} EDGE EDGE +region reg_cylinder cylinder z 20 17 ${r_cylinder} EDGE EDGE +region reg_cylinder cylinder z 20 17 4 EDGE EDGE + +create_atoms 1 box +Created 114240 atoms + using lattice units in orthogonal box = (0 0 0) to (199.18603 80.622915 23.712622) + create_atoms CPU = 0.010 seconds + +## setup obstacle ## +group g_obst region reg_cylinder +1950 atoms in group g_obst +group g_flow subtract all g_obst +112290 atoms in group g_flow +set group g_obst type 2 +Setting atom values ... + 1950 settings made for type + +mass 1 ${MASS} +mass 1 1 +mass 2 ${MASS} +mass 2 1 + +velocity g_flow create ${TEMP} 4928459 rot yes dist gaussian +velocity g_flow create 0.4 4928459 rot yes dist gaussian +velocity g_obst set 0.0 0.0 0.0 + +pair_style lj/cut 1.122462 +pair_coeff 1 1 1.0 1.0 +pair_coeff 1 2 1.0 1.0 +pair_coeff 2 2 1.0 1.0 +pair_modify shift yes + +neighbor 0.3 bin +neigh_modify delay 0 every 20 check no + +fix 1 g_flow nve +fix 2 g_flow wall/flow x ${VFLOW} ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 ${nwall} ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 71 ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 71 75 ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 71 75 79 + +variable dump_every equal ${nrun}/${dump_count} +variable dump_every equal 1000/${dump_count} +variable dump_every equal 1000/10 +variable thermo_every equal ${dump_every} +variable thermo_every equal 100 +variable restart_every equal ${nrun}/10 +variable restart_every equal 1000/10 + +##### uncomment for grid aggregation ##### +#variable gr_Nx equal 42 +#variable gr_Ny equal 17 +#variable gr_Nz equal 1 +#variable gr_Nevery equal ${dump_every} +#variable gr_Nrepeat equal 1 +#variable gr_Nfreq equal ${dump_every} +#fix 3 g_flow ave/grid ${gr_Nevery} ${gr_Nrepeat} ${gr_Nfreq} ${gr_Nx} ${gr_Ny} ${gr_Nz} vx vy vz density/mass norm all ave one +#compute ct_gridId g_flow property/grid ${gr_Nx} ${gr_Ny} ${gr_Nz} id +#dump dmp_grid g_flow grid ${dump_every} grid.lammpstrj c_ct_gridId:grid:data f_3:grid:data[*] +########################################## + +#dump dmp_coord all atom ${dump_every} dump.lammpstrj + +#compute ct_Temp g_flow temp/com +#thermo_style custom step temp epair emol etotal press c_ct_Temp + +#restart ${restart_every} flow.restart + +timestep 0.005 +thermo ${thermo_every} +thermo 100 +run ${nrun} +run 1000 + +CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE + +Your simulation uses code contributions which should be cited: + +- fix wall/flow command: doi:10.1177/10943420231213013 + +@Article{Pavlov-etal-IJHPCA-2024, + author = {Daniil Pavlov and Vladislav Galigerov and Daniil Kolotinskii and Vsevolod Nikolskiy and Vladimir Stegailov}, + title = {GPU-based molecular dynamics of fluid flows: Reaching for turbulence}, + journal = {The International Journal of High Performance Computing Applications}, + year = 2024, + volume = 38, + number = 1, + pages = 34-49 +} + +CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE + +Generated 0 of 1 mixed pair_coeff terms from geometric mixing rule +Neighbor list info ... + update: every = 20 steps, delay = 0 steps, check = no + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 1.422462 + ghost atom cutoff = 1.422462 + binsize = 0.711231, bins = 281 114 34 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 26.69 | 26.69 | 26.69 Mbytes + Step Temp E_pair E_mol TotEng Press + 0 0.39317221 0 0 0.58975315 0.11795063 + 100 0.3671684 0.045118445 0 0.59586622 0.27378331 + 200 0.3732041 0.036897471 0 0.59669873 0.24917809 + 300 0.37432305 0.036501844 0 0.5979815 0.24715194 + 400 0.37603886 0.035350565 0 0.59940392 0.24480762 + 500 0.37617142 0.036949771 0 0.60120196 0.24862985 + 600 0.37751983 0.036484268 0 0.60275905 0.24784635 + 700 0.37787831 0.037327783 0 0.60414029 0.25060427 + 800 0.37959242 0.036206184 0 0.60558983 0.2476903 + 900 0.38019033 0.036874395 0 0.6071549 0.24984211 + 1000 0.38070666 0.037068948 0 0.60812395 0.25041936 +Loop time of 5.61598 on 1 procs for 1000 steps with 114240 atoms + +Performance: 76923.319 tau/day, 178.063 timesteps/s, 20.342 Matom-step/s +99.7% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.6351 | 2.6351 | 2.6351 | 0.0 | 46.92 +Neigh | 1.2994 | 1.2994 | 1.2994 | 0.0 | 23.14 +Comm | 0.26576 | 0.26576 | 0.26576 | 0.0 | 4.73 +Output | 0.0030531 | 0.0030531 | 0.0030531 | 0.0 | 0.05 +Modify | 1.3019 | 1.3019 | 1.3019 | 0.0 | 23.18 +Other | | 0.1107 | | | 1.97 + +Nlocal: 114240 ave 114240 max 114240 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 20119 ave 20119 max 20119 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 164018 ave 164018 max 164018 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 164018 +Ave neighs/atom = 1.4357318 +Neighbor list builds = 50 +Dangerous builds not checked +Total wall time: 0:00:05 diff --git a/examples/wall/log.7Feb24.wall.flow.g++.4 b/examples/wall/log.7Feb24.wall.flow.g++.4 new file mode 100644 index 0000000000..1efe7bb28e --- /dev/null +++ b/examples/wall/log.7Feb24.wall.flow.g++.4 @@ -0,0 +1,182 @@ +LAMMPS (21 Nov 2023 - Development - patch_21Nov2023-758-ge33590b2fc-modified) + using 1 OpenMP thread(s) per MPI task +variable nrun equal 1000 +variable dump_count equal 10 + +variable nwall equal 4 +variable w1 equal 67 +variable w2 equal 71 +variable w3 equal 75 +variable w4 equal 79 + +variable x_cylinder equal 20 +variable y_cylinder equal 17 +variable r_cylinder equal 4 + +variable MASS equal 1 +variable TEMP equal 0.4 +variable VFLOW equal 0.5 + +units lj +atom_style atomic + +lattice fcc 0.3 +Lattice spacing in x,y,z = 2.3712622 2.3712622 2.3712622 +region sim_box block 0 84 0 34 0 10 + +boundary p p p + +create_box 2 sim_box +Created orthogonal box = (0 0 0) to (199.18603 80.622915 23.712622) + 4 by 1 by 1 MPI processor grid +region reg_cylinder cylinder z ${x_cylinder} ${y_cylinder} ${r_cylinder} EDGE EDGE +region reg_cylinder cylinder z 20 ${y_cylinder} ${r_cylinder} EDGE EDGE +region reg_cylinder cylinder z 20 17 ${r_cylinder} EDGE EDGE +region reg_cylinder cylinder z 20 17 4 EDGE EDGE + +create_atoms 1 box +Created 114240 atoms + using lattice units in orthogonal box = (0 0 0) to (199.18603 80.622915 23.712622) + create_atoms CPU = 0.003 seconds + +## setup obstacle ## +group g_obst region reg_cylinder +1950 atoms in group g_obst +group g_flow subtract all g_obst +112290 atoms in group g_flow +set group g_obst type 2 +Setting atom values ... + 1950 settings made for type + +mass 1 ${MASS} +mass 1 1 +mass 2 ${MASS} +mass 2 1 + +velocity g_flow create ${TEMP} 4928459 rot yes dist gaussian +velocity g_flow create 0.4 4928459 rot yes dist gaussian +velocity g_obst set 0.0 0.0 0.0 + +pair_style lj/cut 1.122462 +pair_coeff 1 1 1.0 1.0 +pair_coeff 1 2 1.0 1.0 +pair_coeff 2 2 1.0 1.0 +pair_modify shift yes + +neighbor 0.3 bin +neigh_modify delay 0 every 20 check no + +fix 1 g_flow nve +fix 2 g_flow wall/flow x ${VFLOW} ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 ${nwall} ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 ${w1} ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 ${w2} ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 71 ${w3} ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 71 75 ${w4} +fix 2 g_flow wall/flow x 0.5 0.4 123 4 67 71 75 79 + +variable dump_every equal ${nrun}/${dump_count} +variable dump_every equal 1000/${dump_count} +variable dump_every equal 1000/10 +variable thermo_every equal ${dump_every} +variable thermo_every equal 100 +variable restart_every equal ${nrun}/10 +variable restart_every equal 1000/10 + +##### uncomment for grid aggregation ##### +#variable gr_Nx equal 42 +#variable gr_Ny equal 17 +#variable gr_Nz equal 1 +#variable gr_Nevery equal ${dump_every} +#variable gr_Nrepeat equal 1 +#variable gr_Nfreq equal ${dump_every} +#fix 3 g_flow ave/grid ${gr_Nevery} ${gr_Nrepeat} ${gr_Nfreq} ${gr_Nx} ${gr_Ny} ${gr_Nz} vx vy vz density/mass norm all ave one +#compute ct_gridId g_flow property/grid ${gr_Nx} ${gr_Ny} ${gr_Nz} id +#dump dmp_grid g_flow grid ${dump_every} grid.lammpstrj c_ct_gridId:grid:data f_3:grid:data[*] +########################################## + +#dump dmp_coord all atom ${dump_every} dump.lammpstrj + +#compute ct_Temp g_flow temp/com +#thermo_style custom step temp epair emol etotal press c_ct_Temp + +#restart ${restart_every} flow.restart + +timestep 0.005 +thermo ${thermo_every} +thermo 100 +run ${nrun} +run 1000 + +CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE + +Your simulation uses code contributions which should be cited: + +- fix wall/flow command: doi:10.1177/10943420231213013 + +@Article{Pavlov-etal-IJHPCA-2024, + author = {Daniil Pavlov and Vladislav Galigerov and Daniil Kolotinskii and Vsevolod Nikolskiy and Vladimir Stegailov}, + title = {GPU-based molecular dynamics of fluid flows: Reaching for turbulence}, + journal = {The International Journal of High Performance Computing Applications}, + year = 2024, + volume = 38, + number = 1, + pages = 34-49 +} + +CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE-CITE + +Generated 0 of 1 mixed pair_coeff terms from geometric mixing rule +Neighbor list info ... + update: every = 20 steps, delay = 0 steps, check = no + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 1.422462 + ghost atom cutoff = 1.422462 + binsize = 0.711231, bins = 281 114 34 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 8.496 | 8.496 | 8.496 Mbytes + Step Temp E_pair E_mol TotEng Press + 0 0.39317221 0 0 0.58975315 0.11795063 + 100 0.36726398 0.045386014 0 0.59627716 0.27402111 + 200 0.37384538 0.036574547 0 0.5973377 0.24836729 + 300 0.37487455 0.036519645 0 0.59882654 0.24691726 + 400 0.37591417 0.036405755 0 0.60027207 0.24700641 + 500 0.37654714 0.037008829 0 0.60182459 0.24883444 + 600 0.3778008 0.03663706 0 0.6033333 0.24874392 + 700 0.37851338 0.036714175 0 0.60447928 0.24881829 + 800 0.37984876 0.036237049 0 0.6060052 0.24843003 + 900 0.38022763 0.036847615 0 0.60718407 0.24987198 + 1000 0.38084717 0.037139994 0 0.60840575 0.25070072 +Loop time of 2.20347 on 4 procs for 1000 steps with 114240 atoms + +Performance: 196054.093 tau/day, 453.829 timesteps/s, 51.845 Matom-step/s +95.6% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.67927 | 0.70882 | 0.73473 | 2.4 | 32.17 +Neigh | 0.32928 | 0.34467 | 0.36084 | 2.0 | 15.64 +Comm | 0.3211 | 0.36609 | 0.40741 | 6.1 | 16.61 +Output | 0.0017748 | 0.0032465 | 0.0046508 | 2.1 | 0.15 +Modify | 0.71135 | 0.74424 | 0.76001 | 2.3 | 33.78 +Other | | 0.03641 | | | 1.65 + +Nlocal: 28560 ave 29169 max 27884 min +Histogram: 1 0 0 0 0 2 0 0 0 1 +Nghost: 6452.25 ave 6546 max 6368 min +Histogram: 1 0 0 0 2 0 0 0 0 1 +Neighs: 40893 ave 42032 max 39445 min +Histogram: 1 0 0 0 1 0 0 1 0 1 + +Total # of neighbors = 163572 +Ave neighs/atom = 1.4318277 +Neighbor list builds = 50 +Dangerous builds not checked +Total wall time: 0:00:02 diff --git a/src/.gitignore b/src/.gitignore index 1e4c5b9ddb..41f4e7b614 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1023,6 +1023,8 @@ /fix_wall_colloid.h /fix_wall_ees.cpp /fix_wall_ees.h +/fix_wall_flow.cpp +/fix_wall_flow.h /fix_wall_region_ees.cpp /fix_wall_region_ees.h /fix_wall_reflect_stochastic.cpp diff --git a/src/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp similarity index 100% rename from src/fix_wall_flow.cpp rename to src/EXTRA-FIX/fix_wall_flow.cpp diff --git a/src/fix_wall_flow.h b/src/EXTRA-FIX/fix_wall_flow.h similarity index 100% rename from src/fix_wall_flow.h rename to src/EXTRA-FIX/fix_wall_flow.h From 022cedeff0b3b4ed6653a35b8da763372d9ceee0 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Fri, 9 Feb 2024 12:08:36 +0300 Subject: [PATCH 17/33] Remove debug output --- src/EXTRA-FIX/fix_wall_flow.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 695b93aea8..6b42c34731 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -34,10 +34,8 @@ #include "math_const.h" #include -#include #include #include -#include using namespace LAMMPS_NS; using namespace FixConst; @@ -140,12 +138,6 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : { error->all(FLERR, "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); } - std::cout << "Walls:\n"; - for (auto w : walls) - { - std::cout << w << " "; - } - std::cout << std::endl; memory->grow(current_segment, atom->nmax, "WallFlow::current_segment"); atom->add_callback(Atom::GROW); From 5839b67d278485b9c4c8cdda2a76132d8da4049b Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Fri, 9 Feb 2024 12:26:40 +0300 Subject: [PATCH 18/33] Simplify the code and remove redundant include --- src/EXTRA-FIX/fix_wall_flow.cpp | 11 +++++------ src/EXTRA-FIX/fix_wall_flow.h | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 6b42c34731..0bd9f162a2 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -114,17 +114,16 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : } return 0.0; }; - double scale = getscale(); + double scale = 0.0; + if(flowax == FlowAxis::AX_X) scale = domain->lattice->xlattice; + else if(flowax == FlowAxis::AX_Y) scale = domain->lattice->ylattice; + else if(flowax == FlowAxis::AX_Z) scale = domain->lattice->zlattice; if (narg - iarg == wallcount + 2) { if(strcmp(arg[narg - 2], "units") != 0) error->all(FLERR, "Wrong fix wall/flow units command"); if (strcmp(arg[narg - 1], "box") == 0) scale = 1.0; - else if (strcmp(arg[narg - 1], "lattice") == 0) - { - scale = getscale(); - } - else error->all(FLERR, "Wrong fix wall/flow units command"); + else if (strcmp(arg[narg - 1], "lattice") != 0) error->all(FLERR, "Wrong fix wall/flow units command"); } walls.resize(wallcount + 2); diff --git a/src/EXTRA-FIX/fix_wall_flow.h b/src/EXTRA-FIX/fix_wall_flow.h index ec9e4a4601..f4a4a69b1c 100644 --- a/src/EXTRA-FIX/fix_wall_flow.h +++ b/src/EXTRA-FIX/fix_wall_flow.h @@ -21,7 +21,7 @@ FixStyle(wall/flow,FixWallFlow); #define LMP_FIX_WALL_FLOW_H #include "fix.h" -#include + namespace LAMMPS_NS { class FixWallFlow : public Fix { From cba3c91b510debb4bd0e79bb5ce31d3dc5828cb2 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Fri, 9 Feb 2024 12:29:10 +0300 Subject: [PATCH 19/33] Remove lambda --- src/EXTRA-FIX/fix_wall_flow.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 0bd9f162a2..0357d14773 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -101,19 +101,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : ++iarg; // parsing walls if(narg - iarg != wallcount && narg - iarg != wallcount + 2) error->all(FLERR, "Wrong fix wall/flow wall count"); - auto getscale = [&]() -> double { - switch (flowax) - { - case FlowAxis::AX_X: - return domain->lattice->xlattice; - case FlowAxis::AX_Y: - return domain->lattice->ylattice; - case FlowAxis::AX_Z: - return domain->lattice->zlattice; - default: return 0.0; - } - return 0.0; - }; + double scale = 0.0; if(flowax == FlowAxis::AX_X) scale = domain->lattice->xlattice; else if(flowax == FlowAxis::AX_Y) scale = domain->lattice->ylattice; From 6aad6177b0d52f2d6a6ad02763ef09275b657783 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Fri, 9 Feb 2024 12:31:19 +0300 Subject: [PATCH 20/33] Remove accessing internal data of the Modify class --- src/EXTRA-FIX/fix_wall_flow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 0357d14773..704a8d854d 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -163,8 +163,9 @@ int FixWallFlow::setmask() void FixWallFlow::init() { int nrigid = 0; - for (int i = 0; i < modify->nfix; i++) - if (modify->fix[i]->rigid_flag) nrigid++; + + for (auto ifix : modify->get_fix_list()) + if (ifix->rigid_flag) nrigid++; if (nrigid && comm->me == 0) error->warning(FLERR,"FixWallFlow is not compatible with rigid bodies"); From be742253e2d1bbf040b4b6a4069a217ec7335caa Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Fri, 9 Feb 2024 13:05:45 +0300 Subject: [PATCH 21/33] Fix bug with the masses of the atoms --- src/EXTRA-FIX/fix_wall_flow.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 704a8d854d..bbd5d39ce3 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -205,7 +205,13 @@ void FixWallFlow::generate_velocity(int atom_i) { const int newton_iteration_count = 10; double *vel = atom->v[atom_i]; - double mass = atom->mass[atom->type[atom_i]]; + + double *prmass = atom->rmass; + double *pmass = atom->mass; + double mass = 0.0; + if(prmass) mass = prmass[atom_i]; + else mass = pmass[atom->type[atom_i]]; + const double gamma = 1.0 / std::sqrt(2.0 * kT / mass); double delta = gamma * flowvel; From c7831b29c0fb8b1f43ba27f70b0621e606710f84 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Fri, 9 Feb 2024 13:28:35 +0300 Subject: [PATCH 22/33] Fix formatting issues --- src/EXTRA-FIX/fix_wall_flow.cpp | 173 ++++++++++++++-------------- src/EXTRA-FIX/fix_wall_flow.h | 2 +- src/KOKKOS/fix_wall_flow_kokkos.cpp | 122 ++++++++------------ 3 files changed, 138 insertions(+), 159 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index bbd5d39ce3..bc2ddcd137 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -1,4 +1,3 @@ -// clang-format off /* ---------------------------------------------------------------------- LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator https://www.lammps.org/, Sandia National Laboratories @@ -23,18 +22,18 @@ #include "comm.h" #include "domain.h" #include "error.h" +#include "force.h" #include "input.h" #include "lattice.h" +#include "math_const.h" +#include "memory.h" #include "modify.h" +#include "random_mars.h" #include "update.h" #include "variable.h" -#include "random_mars.h" -#include "memory.h" -#include "force.h" -#include "math_const.h" -#include #include +#include #include using namespace LAMMPS_NS; @@ -43,24 +42,21 @@ using namespace FixConst; /* ---------------------------------------------------------------------- */ static const char cite_fix_wall_flow_c[] = - "fix wall/flow command: doi:10.1177/10943420231213013\n\n" - "@Article{Pavlov-etal-IJHPCA-2024,\n" - " author = {Daniil Pavlov and Vladislav Galigerov and Daniil Kolotinskii and Vsevolod Nikolskiy and Vladimir Stegailov},\n" - " title = {GPU-based molecular dynamics of fluid flows: Reaching for turbulence},\n" - " journal = {The International Journal of High Performance Computing Applications},\n" - " year = 2024,\n" - " volume = 38,\n" - " number = 1,\n" - " pages = 34-49\n" - "}\n\n"; + "fix wall/flow command: doi:10.1177/10943420231213013\n\n" + "@Article{Pavlov-etal-IJHPCA-2024,\n" + " author = {Daniil Pavlov and Vladislav Galigerov and Daniil Kolotinskii and Vsevolod " + "Nikolskiy and Vladimir Stegailov},\n" + " title = {GPU-based molecular dynamics of fluid flows: Reaching for turbulence},\n" + " journal = {The International Journal of High Performance Computing Applications},\n" + " year = 2024,\n" + " volume = 38,\n" + " number = 1,\n" + " pages = 34-49\n" + "}\n\n"; FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : - Fix(lmp, narg, arg), - flowax(FlowAxis::AX_X), - flowvel(0.0), - flowdir(0), - rndseed(0), - current_segment(nullptr) + Fix(lmp, narg, arg), flowax(FlowAxis::AX_X), flowvel(0.0), flowdir(0), rndseed(0), + current_segment(nullptr) { if (lmp->citeme) lmp->citeme->add(cite_fix_wall_flow_c); if (narg < 9) utils::missing_cmd_args(FLERR, "fix wall/flow", error); @@ -70,60 +66,74 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : int iarg = 3; // parsing axis - if (strcmp(arg[iarg], "x") == 0) flowax = FlowAxis::AX_X; - else if (strcmp(arg[iarg],"y") == 0) flowax = FlowAxis::AX_Y; - else if (strcmp(arg[iarg],"z") == 0) flowax = FlowAxis::AX_Z; - else error->all(FLERR,"Illegal fix wall/flow argument: axis must by x or y or z, but {} specified", arg[iarg]); + if (strcmp(arg[iarg], "x") == 0) + flowax = FlowAxis::AX_X; + else if (strcmp(arg[iarg], "y") == 0) + flowax = FlowAxis::AX_Y; + else if (strcmp(arg[iarg], "z") == 0) + flowax = FlowAxis::AX_Z; + else + error->all(FLERR, "Illegal fix wall/flow argument: axis must by x or y or z, but {} specified", + arg[iarg]); ++iarg; // parsing velocity - flowvel = utils::numeric(FLERR,arg[iarg],do_abort,lmp); - if (flowvel == 0.0) error->all(FLERR,"Illegal fix wall/flow argument: velocity cannot be 0"); - if (flowvel > 0.0) flowdir = 1; - else flowdir = -1; - if(flowdir < 0) error->all(FLERR, "Illegal fix wall/flow argument: negative direction is not supported yet"); + flowvel = utils::numeric(FLERR, arg[iarg], do_abort, lmp); + if (flowvel == 0.0) error->all(FLERR, "Illegal fix wall/flow argument: velocity cannot be 0"); + if (flowvel > 0.0) + flowdir = 1; + else + flowdir = -1; + if (flowdir < 0) + error->all(FLERR, "Illegal fix wall/flow argument: negative direction is not supported yet"); ++iarg; // parsing temperature - double flowtemp = utils::numeric(FLERR,arg[iarg],do_abort,lmp); + double flowtemp = utils::numeric(FLERR, arg[iarg], do_abort, lmp); kT = lmp->force->boltz * flowtemp / force->mvv2e; ++iarg; // parsing seed - rndseed = utils::inumeric(FLERR, arg[iarg],do_abort,lmp); - if(rndseed <= 0) error->all(FLERR, "Illegal fix wall/flow argument: random seed must be positive integer"); + rndseed = utils::inumeric(FLERR, arg[iarg], do_abort, lmp); + if (rndseed <= 0) + error->all(FLERR, "Illegal fix wall/flow argument: random seed must be positive integer"); ++iarg; // parsing wall count - int wallcount = utils::inumeric(FLERR,arg[iarg],do_abort,lmp); - if(wallcount <= 0) error->all(FLERR,"Illegal fix wall/flow argument: wall count must be positive integer"); + int wallcount = utils::inumeric(FLERR, arg[iarg], do_abort, lmp); + if (wallcount <= 0) + error->all(FLERR, "Illegal fix wall/flow argument: wall count must be positive integer"); ++iarg; // parsing walls - if(narg - iarg != wallcount && narg - iarg != wallcount + 2) error->all(FLERR, "Wrong fix wall/flow wall count"); + if (narg - iarg != wallcount && narg - iarg != wallcount + 2) + error->all(FLERR, "Wrong fix wall/flow wall count"); double scale = 0.0; - if(flowax == FlowAxis::AX_X) scale = domain->lattice->xlattice; - else if(flowax == FlowAxis::AX_Y) scale = domain->lattice->ylattice; - else if(flowax == FlowAxis::AX_Z) scale = domain->lattice->zlattice; + if (flowax == FlowAxis::AX_X) + scale = domain->lattice->xlattice; + else if (flowax == FlowAxis::AX_Y) + scale = domain->lattice->ylattice; + else if (flowax == FlowAxis::AX_Z) + scale = domain->lattice->zlattice; - if (narg - iarg == wallcount + 2) - { - if(strcmp(arg[narg - 2], "units") != 0) error->all(FLERR, "Wrong fix wall/flow units command"); - if (strcmp(arg[narg - 1], "box") == 0) scale = 1.0; - else if (strcmp(arg[narg - 1], "lattice") != 0) error->all(FLERR, "Wrong fix wall/flow units command"); + if (narg - iarg == wallcount + 2) { + if (strcmp(arg[narg - 2], "units") != 0) error->all(FLERR, "Wrong fix wall/flow units command"); + if (strcmp(arg[narg - 1], "box") == 0) + scale = 1.0; + else if (strcmp(arg[narg - 1], "lattice") != 0) + error->all(FLERR, "Wrong fix wall/flow units command"); } walls.resize(wallcount + 2); walls.front() = domain->boxlo[flowax]; - for (size_t w = 1; w <= wallcount; ++w, ++iarg) - { - walls[w] = utils::numeric(FLERR,arg[iarg],do_abort,lmp) * scale; + for (size_t w = 1; w <= wallcount; ++w, ++iarg) { + walls[w] = utils::numeric(FLERR, arg[iarg], do_abort, lmp) * scale; } walls.back() = domain->boxhi[flowax]; - if (!std::is_sorted(walls.begin(), walls.end(), std::less_equal())) - { - error->all(FLERR, "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); + if (!std::is_sorted(walls.begin(), walls.end(), std::less_equal())) { + error->all(FLERR, + "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); } memory->grow(current_segment, atom->nmax, "WallFlow::current_segment"); @@ -133,7 +143,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : maxexchange = 1; random = new RanMars(lmp, rndseed + comm->me); - } +} /* ---------------------------------------------------------------------- */ @@ -168,10 +178,9 @@ void FixWallFlow::init() if (ifix->rigid_flag) nrigid++; if (nrigid && comm->me == 0) - error->warning(FLERR,"FixWallFlow is not compatible with rigid bodies"); + error->warning(FLERR, "FixWallFlow is not compatible with rigid bodies"); - for (int i = 0; i < atom->nlocal; ++i) - { + for (int i = 0; i < atom->nlocal; ++i) { double pos = atom->x[i][flowax]; current_segment[i] = compute_current_segment(pos); } @@ -185,18 +194,13 @@ void FixWallFlow::end_of_step() int *mask = atom->mask; int nlocal = atom->nlocal; - for (int i = 0; i < nlocal; ++i) - { - if (mask[i] & groupbit) - { - double pos = x[i][flowax]; - int prev_segment = current_segment[i]; - current_segment[i] = compute_current_segment(pos); + for (int i = 0; i < nlocal; ++i) { + if (mask[i] & groupbit) { + double pos = x[i][flowax]; + int prev_segment = current_segment[i]; + current_segment[i] = compute_current_segment(pos); - if (prev_segment != current_segment[i]) - { - generate_velocity(i); - } + if (prev_segment != current_segment[i]) { generate_velocity(i); } } } } @@ -209,31 +213,32 @@ void FixWallFlow::generate_velocity(int atom_i) double *prmass = atom->rmass; double *pmass = atom->mass; double mass = 0.0; - if(prmass) mass = prmass[atom_i]; - else mass = pmass[atom->type[atom_i]]; + if (prmass) + mass = prmass[atom_i]; + else + mass = pmass[atom->type[atom_i]]; const double gamma = 1.0 / std::sqrt(2.0 * kT / mass); double delta = gamma * flowvel; - const double edd = std::exp(-delta*delta) / MathConst::MY_PIS + delta * std::erf(delta); + const double edd = std::exp(-delta * delta) / MathConst::MY_PIS + delta * std::erf(delta); const double probability_threshold = 0.5f * (1.f + delta / edd); double direction = 1.0; - if (random->uniform() > probability_threshold) - { - delta = -delta; - direction = -direction; + if (random->uniform() > probability_threshold) { + delta = -delta; + direction = -direction; } const double xi_0 = random->uniform(); const double F_inf = edd + delta; const double xi = xi_0 * F_inf; - const double x_0 = (std::sqrt(delta*delta + 2) - delta) * 0.5; + const double x_0 = (std::sqrt(delta * delta + 2) - delta) * 0.5; double x = x_0; - for (int i = 0; i < newton_iteration_count; ++i) - { - x -= (std::exp(x*x) * MathConst::MY_PIS * (xi - delta * std::erfc(x)) - 1.0) / (x + delta) * 0.5; + for (int i = 0; i < newton_iteration_count; ++i) { + x -= (std::exp(x * x) * MathConst::MY_PIS * (xi - delta * std::erfc(x)) - 1.0) / (x + delta) * + 0.5; } const double nu = x + delta; @@ -247,14 +252,10 @@ void FixWallFlow::generate_velocity(int atom_i) int FixWallFlow::compute_current_segment(double pos) const { int result = 0; - for (; result < walls.size()-1; ++result) - { - if (pos >= walls[result] && pos < walls[result + 1]) - { - return result; - } + for (; result < walls.size() - 1; ++result) { + if (pos >= walls[result] && pos < walls[result + 1]) { return result; } } - return -1; // -1 is "out of box" region + return -1; // -1 is "out of box" region } void FixWallFlow::grow_arrays(int nmax) @@ -267,13 +268,13 @@ void FixWallFlow::copy_arrays(int i, int j, int) current_segment[j] = current_segment[i]; } -int FixWallFlow::pack_exchange(int i, double* buf) +int FixWallFlow::pack_exchange(int i, double *buf) { buf[0] = static_cast(current_segment[i]); return 1; } -int FixWallFlow::unpack_exchange(int i, double* buf) +int FixWallFlow::unpack_exchange(int i, double *buf) { current_segment[i] = static_cast(buf[0]); return 1; diff --git a/src/EXTRA-FIX/fix_wall_flow.h b/src/EXTRA-FIX/fix_wall_flow.h index f4a4a69b1c..8e16a850b1 100644 --- a/src/EXTRA-FIX/fix_wall_flow.h +++ b/src/EXTRA-FIX/fix_wall_flow.h @@ -26,7 +26,7 @@ namespace LAMMPS_NS { class FixWallFlow : public Fix { public: - enum FlowAxis {AX_X = 0, AX_Y = 1, AX_Z = 2}; + enum FlowAxis { AX_X = 0, AX_Y = 1, AX_Z = 2 }; FixWallFlow(class LAMMPS *, int, char **); ~FixWallFlow() override; diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp index a143578fdc..f9d11efe81 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.cpp +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -1,4 +1,3 @@ -// clang-format off /* ---------------------------------------------------------------------- LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator https://www.lammps.org/, Sandia National Laboratories @@ -19,18 +18,16 @@ #include "fix_wall_flow_kokkos.h" #include "atom_kokkos.h" -#include "memory_kokkos.h" -#include "math_const.h" #include "atom_masks.h" #include "force.h" +#include "math_const.h" +#include "memory_kokkos.h" #include -// clang-format off - using namespace LAMMPS_NS; -template +template FixWallFlowKokkos::FixWallFlowKokkos(LAMMPS *lmp, int narg, char **arg) : FixWallFlow(lmp, narg, arg), rand_pool(rndseed + comm->me) { @@ -47,22 +44,17 @@ FixWallFlowKokkos::FixWallFlowKokkos(LAMMPS *lmp, int narg, char **a d_walls = d_walls_t("FixWallFlowKokkos::walls", walls.size()); auto h_walls = Kokkos::create_mirror_view(d_walls); - for (int i = 0; i < walls.size(); ++i) - { - h_walls(i) = walls[i]; - } + for (int i = 0; i < walls.size(); ++i) { h_walls(i) = walls[i]; } Kokkos::deep_copy(d_walls, h_walls); } -template -FixWallFlowKokkos::~FixWallFlowKokkos() +template FixWallFlowKokkos::~FixWallFlowKokkos() { if (copymode) return; memoryKK->destroy_kokkos(k_current_segment, current_segment); } -template -void FixWallFlowKokkos::init() +template void FixWallFlowKokkos::init() { atomKK->sync(execution_space, datamask_read); k_current_segment.template sync(); @@ -83,8 +75,7 @@ KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWall d_current_segment(i) = compute_current_segment_kk(pos); } -template -void FixWallFlowKokkos::end_of_step() +template void FixWallFlowKokkos::end_of_step() { atomKK->sync(execution_space, datamask_read); k_current_segment.template sync(); @@ -111,26 +102,20 @@ void FixWallFlowKokkos::end_of_step() template template -KOKKOS_INLINE_FUNCTION -void FixWallFlowKokkos::operator()(TagFixWallFlowEndOfStep, +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWallFlowEndOfStep, const int &atom_i) const { - if (d_mask[atom_i] & groupbit) - { + if (d_mask[atom_i] & groupbit) { double pos = d_x(atom_i, flowax); int prev_segment = d_current_segment(atom_i); d_current_segment(atom_i) = compute_current_segment_kk(pos); - if (prev_segment != d_current_segment(atom_i)) - { - generate_velocity_kk(atom_i); - } + if (prev_segment != d_current_segment(atom_i)) { generate_velocity_kk(atom_i); } } } -template -template -KOKKOS_INLINE_FUNCTION -void FixWallFlowKokkos::generate_velocity_kk(int atom_i) const +template +template +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::generate_velocity_kk(int atom_i) const { const int newton_iteration_count = 10; double mass = get_mass(MTag(), atom_i); @@ -163,15 +148,17 @@ void FixWallFlowKokkos::generate_velocity_kk(int atom_i) const const double v = nu / gamma; d_v(atom_i, flowax) = v * direction; - d_v(atom_i, (flowax + 1) % 3) = /*random->gaussian()*/ rand_gen.normal() / (gamma * MathConst::MY_SQRT2); - d_v(atom_i, (flowax + 2) % 3) = /*random->gaussian()*/ rand_gen.normal() / (gamma * MathConst::MY_SQRT2); + d_v(atom_i, (flowax + 1) % 3) = + /*random->gaussian()*/ rand_gen.normal() / (gamma * MathConst::MY_SQRT2); + d_v(atom_i, (flowax + 2) % 3) = + /*random->gaussian()*/ rand_gen.normal() / (gamma * MathConst::MY_SQRT2); rand_pool.free_state(rand_gen); } template -KOKKOS_INLINE_FUNCTION -int FixWallFlowKokkos::compute_current_segment_kk(double pos) const +KOKKOS_INLINE_FUNCTION int +FixWallFlowKokkos::compute_current_segment_kk(double pos) const { int result = 0; for (; result < d_walls.extent(0) - 1; ++result) { @@ -180,9 +167,7 @@ int FixWallFlowKokkos::compute_current_segment_kk(double pos) const return -1; // -1 is "out of box" region } - -template -void FixWallFlowKokkos::grow_arrays(int nmax) +template void FixWallFlowKokkos::grow_arrays(int nmax) { k_current_segment.template sync(); memoryKK->grow_kokkos(k_current_segment, current_segment, nmax, "WallFlowKK::current_segment"); @@ -192,8 +177,7 @@ void FixWallFlowKokkos::grow_arrays(int nmax) h_current_segment = k_current_segment.template view(); } -template -void FixWallFlowKokkos::copy_arrays(int i, int j, int) +template void FixWallFlowKokkos::copy_arrays(int i, int j, int) { k_current_segment.template sync(); h_current_segment(j) = h_current_segment(i); @@ -204,7 +188,7 @@ void FixWallFlowKokkos::copy_arrays(int i, int j, int) sort local atom-based arrays ------------------------------------------------------------------------- */ -template +template void FixWallFlowKokkos::sort_kokkos(Kokkos::BinSort &Sorter) { // always sort on the device @@ -216,33 +200,31 @@ void FixWallFlowKokkos::sort_kokkos(Kokkos::BinSort -int FixWallFlowKokkos::pack_exchange(int i, double *buf) +template int FixWallFlowKokkos::pack_exchange(int i, double *buf) { k_current_segment.sync_host(); buf[0] = static_cast(h_current_segment(i)); return 1; } -template -KOKKOS_INLINE_FUNCTION -void FixWallFlowKokkos::operator()(TagFixWallFlowPackExchange, const int &mysend) const { +template +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWallFlowPackExchange, + const int &mysend) const +{ const int send_i = d_sendlist(mysend); const int segment = d_current_segment(send_i); d_buf(mysend) = static_cast(segment); const int copy_i = d_copylist(mysend); - if (copy_i > -1) { - d_current_segment(send_i) = d_current_segment(copy_i); - } + if (copy_i > -1) { d_current_segment(send_i) = d_current_segment(copy_i); } } template -int FixWallFlowKokkos::pack_exchange_kokkos( - const int &nsend, DAT::tdual_xfloat_2d &k_buf, - DAT::tdual_int_1d k_sendlist, - DAT::tdual_int_1d k_copylist, - ExecutionSpace space) +int FixWallFlowKokkos::pack_exchange_kokkos(const int &nsend, + DAT::tdual_xfloat_2d &k_buf, + DAT::tdual_int_1d k_sendlist, + DAT::tdual_int_1d k_copylist, + ExecutionSpace space) { k_current_segment.template sync(); @@ -253,13 +235,13 @@ int FixWallFlowKokkos::pack_exchange_kokkos( d_sendlist = k_sendlist.view(); d_copylist = k_copylist.view(); - d_buf = typename ArrayTypes::t_xfloat_1d_um( - k_buf.template view().data(), - k_buf.extent(0)*k_buf.extent(1)); + d_buf = typename ArrayTypes::t_xfloat_1d_um(k_buf.template view().data(), + k_buf.extent(0) * k_buf.extent(1)); copymode = 1; - Kokkos::parallel_for(Kokkos::RangePolicy(0, nsend), *this); + Kokkos::parallel_for(Kokkos::RangePolicy(0, nsend), + *this); copymode = 0; @@ -269,8 +251,7 @@ int FixWallFlowKokkos::pack_exchange_kokkos( return nsend; } -template -int FixWallFlowKokkos::unpack_exchange(int i, double *buf) +template int FixWallFlowKokkos::unpack_exchange(int i, double *buf) { k_current_segment.sync_host(); h_current_segment(i) = static_cast(buf[0]); @@ -278,30 +259,27 @@ int FixWallFlowKokkos::unpack_exchange(int i, double *buf) return 1; } -template -KOKKOS_INLINE_FUNCTION -void FixWallFlowKokkos::operator()(TagFixWallFlowUnpackExchange, const int &i) const +template +KOKKOS_INLINE_FUNCTION void FixWallFlowKokkos::operator()(TagFixWallFlowUnpackExchange, + const int &i) const { int index = d_indices(i); - if (index > -1) { - d_current_segment(index) = static_cast(d_buf(i)); - } + if (index > -1) { d_current_segment(index) = static_cast(d_buf(i)); } } template -void FixWallFlowKokkos::unpack_exchange_kokkos( - DAT::tdual_xfloat_2d &k_buf, - DAT::tdual_int_1d &k_indices, int nrecv, - int /*nrecv1*/, int /*nextrarecv1*/, - ExecutionSpace space) +void FixWallFlowKokkos::unpack_exchange_kokkos(DAT::tdual_xfloat_2d &k_buf, + DAT::tdual_int_1d &k_indices, int nrecv, + int /*nrecv1*/, int /*nextrarecv1*/, + ExecutionSpace space) { - d_buf = typename ArrayTypes::t_xfloat_1d_um( - k_buf.template view().data(), - k_buf.extent(0)*k_buf.extent(1)); + d_buf = typename ArrayTypes::t_xfloat_1d_um(k_buf.template view().data(), + k_buf.extent(0) * k_buf.extent(1)); d_indices = k_indices.view(); copymode = 1; - Kokkos::parallel_for(Kokkos::RangePolicy(0,nrecv),*this); + Kokkos::parallel_for(Kokkos::RangePolicy(0, nrecv), + *this); copymode = 0; k_current_segment.template modify(); From fb1e6610edbdf63702d81a720f3c94eb41926f2f Mon Sep 17 00:00:00 2001 From: Steve Plimpton Date: Mon, 19 Feb 2024 10:36:15 -0700 Subject: [PATCH 23/33] edits of doc page for clarity --- doc/src/fix_wall_flow.rst | 118 ++++++++++++++++++++------------ src/EXTRA-FIX/fix_wall_flow.cpp | 15 +++- src/EXTRA-FIX/fix_wall_flow.h | 1 + 3 files changed, 89 insertions(+), 45 deletions(-) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index 464021ff52..5fc9f5367c 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -11,24 +11,23 @@ Syntax .. code-block:: LAMMPS - fix ID group-ID wall/flow ax vf T seed N coords ... keyword value + fix ID group-ID wall/flow axis vflow T seed N coords ... keyword value * ID, group-ID are documented in :doc:`fix ` command * wall/flow = style name of this fix command -* ax = flow axis (*x*, *y*, or *z* character) -* vf = *ax* component of generated flow velocity +* axis = flow axis (*x*, *y*, or *z*) +* vflow = generated flow velocity in *axis* direction (velocity units) * T = flow temperature (temperature units) * seed = random seed for stochasticity (positive integer) -* N = number of walls (positive integer) -* coords = set of N wall coordinates (box units) along *ax* axis arranged in ascending order. Note that an additional implicit wall is introduced at the boundary of the simulation domain, so the resulting system always has N+1 walls. - +* N = number of walls +* coords = list of N wall positions along the *axis* direction in ascending order (distance units) * zero or more keyword/value pairs may be appended * keyword = *units* .. parsed-literal:: *units* value = *lattice* or *box* - *lattice* = the wall positions are defined in lattice units + *lattice* = wall positions are defined in lattice units *box* = the wall positions are defined in simulation box units Examples @@ -36,65 +35,96 @@ Examples .. code-block:: LAMMPS - fix 1 g_flow wall/flow x ${VFLOW} ${TEMP} 123 ${nwall} ${w1} ${w2} ${w3} ${w4} - fix 2 all wall/flow 0.4 0.2 3 1 400 + fix 1 all wall/flow x 0.4 1.5 593894 4 2.0 4.0 6.0 8.0 Description """"""""""" .. versionadded:: TBD -This fix implements flow boundary conditions (FBC) introduced in :ref:`(Pavlov1) ` and :ref:`(Pavlov2) `. -The goal is to generate a stationary flow with a shifted Maxwell velocity distribution: +This fix implements flow boundary conditions (FBC) introduced in +:ref:`(Pavlov1) ` and :ref:`(Pavlov2) `. +The goal is to generate a stationary flow with a shifted Maxwell +velocity distribution: .. math:: - f_z(v_z) \propto \exp{\left(-\frac{m (v_z-v_{\text{flow}})^2}{2 k T}\right)} + f_a(v_a) \propto \exp{\left(-\frac{m (v_a-v_{\text{flow}})^2}{2 kB T}\right)} -This is achieved by reassigning the velocity of each particle that passes a wall. -Such reassigning represents an emission of a new particle into the system with -simultaneous removal of a particle with the same position. -The velocity components parallel to the wall are re-assigned according -to the Maxwell velocity distribution. The perpendicular component is assigned -according to the following velocity distribution: +where :math:`v_a` is the component of velocity along the specified +*axis* argument (a = x,y,z), :math:`v_{\text{flow}}` is the flow +velocity specified as the *vflow* argument, *T* is the specified flow +temperature, *m* is the particle mass, and *kB* is the Boltzmann +constant. + +This is achieved by defining a series of *N* transparent walls along +the flow *axis* direction. Each wall is at the specified position +listed in the *coords* argument. Note that an additional transparent +wall is defined by the code at the boundary of the (periodic) +simulation domain in the *axis* direction. So there are effectively +N+1 walls. + +Each time a particle in the specified group passes through one of the +transparent walls, its velocity is re-assigned. Particles not in the +group do not interact with the wall. + +Conceptually, the velocity re-assignment represents creation of a new +particle within the system with simultaneous removal of the particle +which passed through the wall. The velocity components in directions +parallel to the wall are re-assigned according to the standard Maxwell +velocity distribution for the specified temperature *T*. The velocity +component perpendicular to the wall is re-assigned according to the +shifted Maxwell distribution defined above: .. math:: - f_{\text{z generated}}(v_z) \propto v_z f_z(v_z) + f_{\text{a generated}}(v_a) \propto v_a f_a(v_a) -It can be shown that in an ideal-gas scenario this makes the velocity -distribution of particles between walls exactly as desired. +It can be shown that for an ideal-gas scenario this procedure makes +the velocity distribution of particles between walls exactly as +desired. -Since in most cases simulated systems are not ideal gas, -the need for multiple walls might arise, as a single wall may not be -sufficient for maintaining a stationary flow without congestion -manifesting as areas with increased density located upstream from static obstacles. +Since in most cases simulated systems are not an ideal gas, multiple +walls can be defined, since a single wall may not be sufficient for +maintaining a stationary flow without "congestion" which can manifest +itself as regions in the flow with increased particle density located +upstream from static obstacles. -For the same reason, the actual temperature and velocity of the generated -flow may differ from ones requested. The degree of such discrepancy is determined -by how different from the ideal gas the simulated system is. Therefore, a calibration procedure is required for each system as described in :ref:`(Pavlov) `. +For the same reason, the actual temperature and velocity of the +generated flow may differ from what is requested. The degree of +discrepancy is determined by how different from an ideal gas the +simulated system is. Therefore, a calibration procedure may be +required for such a system as described in :ref:`(Pavlov) +`. -The interactions between particles on different sides of a wall are not disabled or neglected and the -particle positions are not affected by the velocity reassignment. -This removes the need to modify the force field to work correctly in cases when a particle is close -to a wall (for example, if particle positions were uniformly redistributed across the surface of the wall, -two particles could end up too close to each other, potentially causing the simulation to explode). -However due to this compromise, some collective phenomena such as areas with increased/decreased density -or collective movements are not fully removed when particles cross a wall. -This unwanted consequence can also be potentially mitigated by using more than one wall. +Note that the interactions between particles on different sides of a +transparent wall are not disabled or neglected. Likewise particle +positions are not altered by the velocity reassignment. This removes +the need to modify the force field to work correctly in cases when a +particle is close to a wall. +For example, if particle positions were uniformly redistributed across +the surface of a wall, two particles could end up too close to each +other, potentially causing the simulation to explode. However due to +this compromise, some collective phenomena such as regions with +increased/decreased density or collective movements are not fully +removed when particles cross a wall. This unwanted consequence can +also be potentially mitigated by using more multiple walls. ----------- +.. note:: -Note that when high flow velocity is reached, a lost atoms error may -occur (see :doc:`error messages `). -If this message appears when using this fix, you can, for example, reduce the frequency of the -neighbor list rebuild via :doc:`neigh_modify ` command. + When the specified flow has a high velocity, a lost atoms error can + occur (see :doc:`error messages `). If this + happens, you should ensure the checks for neighbor list rebuilds, + set via the :doc:`neigh_modify ` command, are as + conservative as possible (every timestep if needed). Those are the + default settings. Restart, fix_modify, output, run start/stop, minimize info """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" -No information about this fix is written to :doc:`binary restart files `. +No information about this fix is written to :doc:`binary restart files +`. None of the :doc:`fix_modify ` options are relevant to this fix. @@ -114,8 +144,8 @@ Fix *wall_flow* is part of the EXTRA-FIX package. It is only enabled if LAMMPS was built with that package. See the :doc:`Build package ` page for more info. -Flow boundary conditions should not be used with rigid bodies such as those -defined by a "fix rigid" command. +Flow boundary conditions should not be used with rigid bodies such as +those defined by a "fix rigid" command. Related commands """""""""""""""" diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index bc2ddcd137..afa0a4e4bd 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -200,11 +200,14 @@ void FixWallFlow::end_of_step() int prev_segment = current_segment[i]; current_segment[i] = compute_current_segment(pos); - if (prev_segment != current_segment[i]) { generate_velocity(i); } + if (prev_segment != current_segment[i]) + generate_velocity(i); } } } +/* ---------------------------------------------------------------------- */ + void FixWallFlow::generate_velocity(int atom_i) { const int newton_iteration_count = 10; @@ -249,6 +252,8 @@ void FixWallFlow::generate_velocity(int atom_i) vel[(flowax + 2) % 3] = random->gaussian() / (gamma * MathConst::MY_SQRT2); } +/* ---------------------------------------------------------------------- */ + int FixWallFlow::compute_current_segment(double pos) const { int result = 0; @@ -258,22 +263,30 @@ int FixWallFlow::compute_current_segment(double pos) const return -1; // -1 is "out of box" region } +/* ---------------------------------------------------------------------- */ + void FixWallFlow::grow_arrays(int nmax) { memory->grow(current_segment, nmax, "WallFlow::current_segment"); } +/* ---------------------------------------------------------------------- */ + void FixWallFlow::copy_arrays(int i, int j, int) { current_segment[j] = current_segment[i]; } +/* ---------------------------------------------------------------------- */ + int FixWallFlow::pack_exchange(int i, double *buf) { buf[0] = static_cast(current_segment[i]); return 1; } +/* ---------------------------------------------------------------------- */ + int FixWallFlow::unpack_exchange(int i, double *buf) { current_segment[i] = static_cast(buf[0]); diff --git a/src/EXTRA-FIX/fix_wall_flow.h b/src/EXTRA-FIX/fix_wall_flow.h index 8e16a850b1..0379c03783 100644 --- a/src/EXTRA-FIX/fix_wall_flow.h +++ b/src/EXTRA-FIX/fix_wall_flow.h @@ -50,6 +50,7 @@ class FixWallFlow : public Fix { int rndseed; class RanMars *random; int *current_segment; + int compute_current_segment(double pos) const; void generate_velocity(int i); }; From ec0535172c55c7bfac812ab5cab123bb5e11be90 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Mon, 19 Feb 2024 12:54:59 -0500 Subject: [PATCH 24/33] whitespace --- src/EXTRA-FIX/fix_wall_flow.cpp | 4 ++-- src/EXTRA-FIX/fix_wall_flow.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index afa0a4e4bd..586cfa7382 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -17,6 +17,7 @@ ------------------------------------------------------------------------- */ #include "fix_wall_flow.h" + #include "atom.h" #include "citeme.h" #include "comm.h" @@ -200,8 +201,7 @@ void FixWallFlow::end_of_step() int prev_segment = current_segment[i]; current_segment[i] = compute_current_segment(pos); - if (prev_segment != current_segment[i]) - generate_velocity(i); + if (prev_segment != current_segment[i]) generate_velocity(i); } } } diff --git a/src/EXTRA-FIX/fix_wall_flow.h b/src/EXTRA-FIX/fix_wall_flow.h index 0379c03783..6a662f3d94 100644 --- a/src/EXTRA-FIX/fix_wall_flow.h +++ b/src/EXTRA-FIX/fix_wall_flow.h @@ -50,7 +50,7 @@ class FixWallFlow : public Fix { int rndseed; class RanMars *random; int *current_segment; - + int compute_current_segment(double pos) const; void generate_velocity(int i); }; From e20ea968b930184bec9eb0d273caee48026f88fc Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Wed, 21 Feb 2024 20:49:19 +0300 Subject: [PATCH 25/33] Add some clarifications to the doc --- doc/src/fix_wall_flow.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/src/fix_wall_flow.rst b/doc/src/fix_wall_flow.rst index 5fc9f5367c..b40ba9697f 100644 --- a/doc/src/fix_wall_flow.rst +++ b/doc/src/fix_wall_flow.rst @@ -66,7 +66,10 @@ N+1 walls. Each time a particle in the specified group passes through one of the transparent walls, its velocity is re-assigned. Particles not in the -group do not interact with the wall. +group do not interact with the wall. This can be used, for example, to +add obstacles composed of atoms, or to simulate a solution of complex +molecules in a one-atom liquid (note that the fix has been tested for +one-atom systems only). Conceptually, the velocity re-assignment represents creation of a new particle within the system with simultaneous removal of the particle @@ -147,6 +150,10 @@ if LAMMPS was built with that package. See the :doc:`Build package Flow boundary conditions should not be used with rigid bodies such as those defined by a "fix rigid" command. +This fix can only be used with periodic boundary conditions along the +flow axis. The size of the box in this direction must not change. Also, +the fix is designed to work only in an orthogonal simulation box. + Related commands """""""""""""""" From 0f34c1ec6e49dbc34c398b142db88259b550f701 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Wed, 21 Feb 2024 21:16:21 +0300 Subject: [PATCH 26/33] Add check for triclinic and periodic boundary --- src/EXTRA-FIX/fix_wall_flow.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 586cfa7382..15b82b45b8 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -62,6 +62,9 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : if (lmp->citeme) lmp->citeme->add(cite_fix_wall_flow_c); if (narg < 9) utils::missing_cmd_args(FLERR, "fix wall/flow", error); + if (domain->triclinic != 0) + error->all(FLERR, "Fix wall/flow cannot be used with triclinic simulation box"); + dynamic_group_allow = 1; bool do_abort = false; @@ -77,6 +80,10 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : error->all(FLERR, "Illegal fix wall/flow argument: axis must by x or y or z, but {} specified", arg[iarg]); + if (domain->periodicity[flowax] != 1) + error->all(FLERR, + "Fix wall/flow cannot be used with a non-periodic boundary along the flow axis"); + ++iarg; // parsing velocity flowvel = utils::numeric(FLERR, arg[iarg], do_abort, lmp); @@ -137,6 +144,12 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : "Wrong fix wall/flow wall ordering or some walls are outside simulation domain"); } + if (std::adjacent_find(walls.begin(), walls.end()) != walls.end()) { + error->all(FLERR, + "Wrong fix wall/flow wall coordinates: some walls have the same coordinates or lie " + "on the boundary"); + } + memory->grow(current_segment, atom->nmax, "WallFlow::current_segment"); atom->add_callback(Atom::GROW); if (restart_peratom) atom->add_callback(Atom::RESTART); From 6ac24c297a43124f0d0c5ddf97b3a37768bfb0b1 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Thu, 22 Feb 2024 11:47:53 +0300 Subject: [PATCH 27/33] Add check for varying box size --- src/EXTRA-FIX/fix_wall_flow.cpp | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 15b82b45b8..629d2ad452 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -187,12 +187,34 @@ int FixWallFlow::setmask() void FixWallFlow::init() { int nrigid = 0; - - for (auto ifix : modify->get_fix_list()) + int box_change_flowax = 0; + for (auto ifix : modify->get_fix_list()) { if (ifix->rigid_flag) nrigid++; + switch (flowax) { + case FlowAxis::AX_X: + if (ifix->box_change & Fix::BOX_CHANGE_X) box_change_flowax++; + if (ifix->box_change & Fix::BOX_CHANGE_XY) box_change_flowax++; + if (ifix->box_change & Fix::BOX_CHANGE_XZ) box_change_flowax++; + break; + case FlowAxis::AX_Y: + if (ifix->box_change & Fix::BOX_CHANGE_Y) box_change_flowax++; + if (ifix->box_change & Fix::BOX_CHANGE_YZ) box_change_flowax++; + if (ifix->box_change & Fix::BOX_CHANGE_XY) box_change_flowax++; + break; + case FlowAxis::AX_Z: + if (ifix->box_change & Fix::BOX_CHANGE_Z) box_change_flowax++; + if (ifix->box_change & Fix::BOX_CHANGE_YZ) box_change_flowax++; + if (ifix->box_change & Fix::BOX_CHANGE_XZ) box_change_flowax++; + break; + } + } if (nrigid && comm->me == 0) error->warning(FLERR, "FixWallFlow is not compatible with rigid bodies"); + if (box_change_flowax && comm->me == 0) + error->warning( + FLERR, + "FixWallFlow is not compatible with simulation box size changing along flow direction"); for (int i = 0; i < atom->nlocal; ++i) { double pos = atom->x[i][flowax]; From f1a5dd0479cecfa3ff8e84b2f3379b957224a905 Mon Sep 17 00:00:00 2001 From: Vladislav Galigerov Date: Thu, 22 Feb 2024 12:28:23 +0300 Subject: [PATCH 28/33] Change warnings to errors --- src/EXTRA-FIX/fix_wall_flow.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 629d2ad452..49a8a2eb9b 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -210,9 +210,9 @@ void FixWallFlow::init() } if (nrigid && comm->me == 0) - error->warning(FLERR, "FixWallFlow is not compatible with rigid bodies"); + error->all(FLERR, "FixWallFlow is not compatible with rigid bodies"); if (box_change_flowax && comm->me == 0) - error->warning( + error->all( FLERR, "FixWallFlow is not compatible with simulation box size changing along flow direction"); From 86d306b50239d9619cb5f8179956e376b18659d6 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Thu, 22 Feb 2024 04:40:55 -0500 Subject: [PATCH 29/33] avoid excessive copies --- src/EXTRA-FIX/fix_wall_flow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index 49a8a2eb9b..add0b51ca4 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -188,7 +188,7 @@ void FixWallFlow::init() { int nrigid = 0; int box_change_flowax = 0; - for (auto ifix : modify->get_fix_list()) { + for (const auto &ifix : modify->get_fix_list()) { if (ifix->rigid_flag) nrigid++; switch (flowax) { case FlowAxis::AX_X: From 6af4dac2e572e76a74a0acde88e1514e5242431c Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Thu, 22 Feb 2024 04:41:16 -0500 Subject: [PATCH 30/33] Error::all() must be called by all MPI ranks --- src/EXTRA-FIX/fix_wall_flow.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index add0b51ca4..b6634aa87c 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -209,12 +209,11 @@ void FixWallFlow::init() } } - if (nrigid && comm->me == 0) - error->all(FLERR, "FixWallFlow is not compatible with rigid bodies"); - if (box_change_flowax && comm->me == 0) + if (nrigid) error->all(FLERR, "Fix wall/flow is not compatible with rigid bodies"); + if (box_change_flowax) error->all( FLERR, - "FixWallFlow is not compatible with simulation box size changing along flow direction"); + "Fix wall/flow is not compatible with simulation box size changing along flow direction"); for (int i = 0; i < atom->nlocal; ++i) { double pos = atom->x[i][flowax]; From 5cadcbfd6849fdbd160d029e53fd6fb22a6bf4b7 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Thu, 22 Feb 2024 04:51:35 -0500 Subject: [PATCH 31/33] silence compiler warnings --- src/EXTRA-FIX/fix_wall_flow.cpp | 4 ++-- src/KOKKOS/fix_wall_flow_kokkos.cpp | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index b6634aa87c..f194a4076a 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -135,7 +135,7 @@ FixWallFlow::FixWallFlow(LAMMPS *lmp, int narg, char **arg) : walls.resize(wallcount + 2); walls.front() = domain->boxlo[flowax]; - for (size_t w = 1; w <= wallcount; ++w, ++iarg) { + for (int w = 1; w <= wallcount; ++w, ++iarg) { walls[w] = utils::numeric(FLERR, arg[iarg], do_abort, lmp) * scale; } walls.back() = domain->boxhi[flowax]; @@ -291,7 +291,7 @@ void FixWallFlow::generate_velocity(int atom_i) int FixWallFlow::compute_current_segment(double pos) const { int result = 0; - for (; result < walls.size() - 1; ++result) { + for (; result < (int)walls.size() - 1; ++result) { if (pos >= walls[result] && pos < walls[result + 1]) { return result; } } return -1; // -1 is "out of box" region diff --git a/src/KOKKOS/fix_wall_flow_kokkos.cpp b/src/KOKKOS/fix_wall_flow_kokkos.cpp index f9d11efe81..b6b3f7c096 100644 --- a/src/KOKKOS/fix_wall_flow_kokkos.cpp +++ b/src/KOKKOS/fix_wall_flow_kokkos.cpp @@ -23,8 +23,6 @@ #include "math_const.h" #include "memory_kokkos.h" -#include - using namespace LAMMPS_NS; template @@ -44,7 +42,7 @@ FixWallFlowKokkos::FixWallFlowKokkos(LAMMPS *lmp, int narg, char **a d_walls = d_walls_t("FixWallFlowKokkos::walls", walls.size()); auto h_walls = Kokkos::create_mirror_view(d_walls); - for (int i = 0; i < walls.size(); ++i) { h_walls(i) = walls[i]; } + for (int i = 0; i < (int) walls.size(); ++i) h_walls(i) = walls[i]; Kokkos::deep_copy(d_walls, h_walls); } @@ -161,7 +159,7 @@ KOKKOS_INLINE_FUNCTION int FixWallFlowKokkos::compute_current_segment_kk(double pos) const { int result = 0; - for (; result < d_walls.extent(0) - 1; ++result) { + for (; result < (int) d_walls.extent(0) - 1; ++result) { if (pos >= d_walls[result] && pos < d_walls[result + 1]) { return result; } } return -1; // -1 is "out of box" region @@ -224,7 +222,7 @@ int FixWallFlowKokkos::pack_exchange_kokkos(const int &nsend, DAT::tdual_xfloat_2d &k_buf, DAT::tdual_int_1d k_sendlist, DAT::tdual_int_1d k_copylist, - ExecutionSpace space) + ExecutionSpace /*space*/) { k_current_segment.template sync(); @@ -271,7 +269,7 @@ template void FixWallFlowKokkos::unpack_exchange_kokkos(DAT::tdual_xfloat_2d &k_buf, DAT::tdual_int_1d &k_indices, int nrecv, int /*nrecv1*/, int /*nextrarecv1*/, - ExecutionSpace space) + ExecutionSpace /*space*/) { d_buf = typename ArrayTypes::t_xfloat_1d_um(k_buf.template view().data(), k_buf.extent(0) * k_buf.extent(1)); From 71e0d506e71ab7453ba328984b9b0c534d37266c Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Thu, 22 Feb 2024 04:56:27 -0500 Subject: [PATCH 32/33] spelling --- doc/utils/sphinx-config/false_positives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 6106a1638c..2ec10d0e80 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -3888,6 +3888,7 @@ Verstraelen ves vf vflag +vflow vfrac vhi vibrational From b7153eebee475d25bd9851ff9fea7ae153c856fc Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 23 Feb 2024 19:04:14 -0500 Subject: [PATCH 33/33] simplify and check for triclinic again, since somebody may have used change_box --- src/EXTRA-FIX/fix_wall_flow.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/EXTRA-FIX/fix_wall_flow.cpp b/src/EXTRA-FIX/fix_wall_flow.cpp index f194a4076a..1f3dcfca5b 100644 --- a/src/EXTRA-FIX/fix_wall_flow.cpp +++ b/src/EXTRA-FIX/fix_wall_flow.cpp @@ -186,6 +186,9 @@ int FixWallFlow::setmask() void FixWallFlow::init() { + if (domain->triclinic != 0) + error->all(FLERR, "Fix wall/flow cannot be used with triclinic simulation box"); + int nrigid = 0; int box_change_flowax = 0; for (const auto &ifix : modify->get_fix_list()) { @@ -193,18 +196,12 @@ void FixWallFlow::init() switch (flowax) { case FlowAxis::AX_X: if (ifix->box_change & Fix::BOX_CHANGE_X) box_change_flowax++; - if (ifix->box_change & Fix::BOX_CHANGE_XY) box_change_flowax++; - if (ifix->box_change & Fix::BOX_CHANGE_XZ) box_change_flowax++; break; case FlowAxis::AX_Y: if (ifix->box_change & Fix::BOX_CHANGE_Y) box_change_flowax++; - if (ifix->box_change & Fix::BOX_CHANGE_YZ) box_change_flowax++; - if (ifix->box_change & Fix::BOX_CHANGE_XY) box_change_flowax++; break; case FlowAxis::AX_Z: if (ifix->box_change & Fix::BOX_CHANGE_Z) box_change_flowax++; - if (ifix->box_change & Fix::BOX_CHANGE_YZ) box_change_flowax++; - if (ifix->box_change & Fix::BOX_CHANGE_XZ) box_change_flowax++; break; } }