ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
cf2e800aaaeaee58eda5e42b402d9ecfe7f4a799
lammps/src/INTEL/pppm_electrode_intel.cpp
Axel Kohlmeyer 482a6632e9 consolidate kspace communication enumerators in kspace.h
2024-10-10 05:35:01 -04:00

1218 lines
43 KiB
C++
Raw Blame History

/* ----------------------------------------------------------------------
LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
https://www.lammps.org/, Sandia National Laboratories
LAMMPS development team: developers@lammps.org
Copyright (2003) Sandia Corporation. Under the terms of Contract
DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
certain rights in this software. This software is distributed under
the GNU General Public License.
See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */
/* ----------------------------------------------------------------------
Contributing authors: William McDoniel (RWTH Aachen University)
Rodrigo Canales (RWTH Aachen University)
Markus Hoehnerbach (RWTH Aachen University)
W. Michael Brown (Intel)
------------------------------------------------------------------------- */
#include "pppm_electrode_intel.h"
#include "angle.h"
#include "atom.h"
#include "bond.h"
#include "citeme.h"
#include "comm.h"
#include "domain.h"
#include "error.h"
#include "fft3d_wrap.h"
#include "force.h"
#include "grid3d.h"
#include "math_const.h"
#include "math_special.h"
#include "memory.h"
#include "neighbor.h"
#include "omp_compat.h"
#include "pair.h"
#include "pppm_intel.h"
#include "remap_wrap.h"
#include "slab_dipole.h"
#include "update.h"
#include "wire_dipole.h"
#include <algorithm>
#include <cmath>
#include <cstring>
using namespace LAMMPS_NS;
using namespace std;
static constexpr int OFFSET = 16384;
enum : bool { ELECTRODE = true, ELECTROLYTE = false };
static constexpr FFT_SCALAR ZEROF = 0.0;
static const char cite_pppm_electrode[] =
"kspace_style pppm/electrode command:\n\n"
"@article{Ahrens2021,\n"
"author = {Ahrens-Iwers, Ludwig J.V. and Mei{\\ss}ner, Robert H.},\n"
"doi = {10.1063/5.0063381},\n"
"title = {{Constant potential simulations on a mesh}},\n"
"journal = {Journal of Chemical Physics},\n"
"year = {2021}\n"
"volume = {155},\n"
"pages = {104104},\n"
"}\n";
PPPMElectrodeIntel::PPPMElectrodeIntel(LAMMPS *lmp) :
PPPMIntel(lmp), ElectrodeKSpace(), electrolyte_density_brick(nullptr),
electrolyte_density_fft(nullptr), boundcorr(nullptr)
{
if (lmp->citeme) lmp->citeme->add(cite_pppm_electrode);
group_group_enable = 0;
electrolyte_density_brick = nullptr;
electrolyte_density_fft = nullptr;
compute_vector_called = false;
last_source_grpbit = 1 << 0; // default to "all"
last_invert_source = false;
}
PPPMElectrodeIntel::~PPPMElectrodeIntel()
{
memory->destroy3d_offset(electrolyte_density_brick, nzlo_out, nylo_out, nxlo_out);
memory->destroy(electrolyte_density_fft);
if ((differentiation_flag != 1) && !peratom_allocate_flag)
memory->destroy3d_offset(u_brick, nzlo_out, nylo_out, nxlo_out);
}
void PPPMElectrodeIntel::init()
{
PPPMIntel::init();
// PPPM/electrode/intel - specific checks
if (slabflag == 3)
error->all(FLERR, "Cannot (yet) use PPPM/electrode/intel with 'kspace_modify slab ew2d'");
triclinic_check();
triclinic = domain->triclinic;
if (triclinic) error->all(FLERR, "Cannot (yet) use PPPM/electrode with triclinic box ");
if (domain->dimension == 2) error->all(FLERR, "Cannot use PPPM/electrode with 2d simulation");
if (!atom->q_flag) error->all(FLERR, "KSpace style requires atom attribute q");
if (slabflag == 0 && wireflag == 0 && domain->nonperiodic > 0)
error->all(FLERR, "Cannot use non-periodic boundaries with PPPM/electrode");
if (slabflag) {
if (domain->xperiodic != 1 || domain->yperiodic != 1 || domain->boundary[2][0] != 1 ||
domain->boundary[2][1] != 1)
error->all(FLERR, "Incorrect boundaries with slab PPPM/electrode");
} else if (wireflag) {
if (domain->zperiodic != 1 || domain->boundary[0][0] != 1 || domain->boundary[0][1] != 1 ||
domain->boundary[1][0] != 1 || domain->boundary[1][1] != 1)
error->all(FLERR, "Incorrect boundaries with wire PPPM/electrode");
}
compute_step = -1;
}
/* ----------------------------------------------------------------------
adjust PPPM coeffs, called initially and whenever volume has changed
------------------------------------------------------------------------- */
void PPPMElectrodeIntel::setup()
{
// PPPM/electrode/intel - specific checks
if (slabflag == 0 && wireflag == 0 && domain->nonperiodic > 0)
error->all(FLERR, "Cannot use non-periodic boundaries with PPPM/electrode");
if (slabflag) {
if (domain->xperiodic != 1 || domain->yperiodic != 1 || domain->boundary[2][0] != 1 ||
domain->boundary[2][1] != 1)
error->all(FLERR, "Incorrect boundaries with slab PPPM/electrode");
} else if (wireflag) {
if (domain->zperiodic != 1 || domain->boundary[0][0] != 1 || domain->boundary[0][1] != 1 ||
domain->boundary[1][0] != 1 || domain->boundary[1][1] != 1)
error->all(FLERR, "Incorrect boundaries with wire PPPM/electrode");
}
double *prd = domain->prd;
// volume-dependent factors
// adjust z dimension for 2d slab PPPM
// z dimension for 3d PPPM is zprd since slab_volfactor = 1.0
double xprd = prd[0];
double yprd = prd[1];
double zprd = prd[2];
double xprd_wire = xprd * wire_volfactor;
double yprd_wire = yprd * wire_volfactor;
double zprd_slab = zprd * slab_volfactor;
volume = xprd_wire * yprd_wire * zprd_slab;
// wire_volfactor hacks to reuse compute_gf code
prd[0] *= wire_volfactor;
prd[1] *= wire_volfactor;
PPPMIntel::setup();
prd[0] /= wire_volfactor;
prd[1] /= wire_volfactor;
}
void PPPMElectrodeIntel::compute(int eflag, int vflag)
{
#ifdef _LMP_INTEL_OFFLOAD
if (_use_base) {
error->all(FLERR, "Cannot use pppm/electrode/intel with offload");
// PPPM::compute(eflag, vflag);
// would work if the above line referred to PPPMElectrode
// but the required multiple inheritances would be insane
}
#endif
ev_init(eflag, vflag);
if (evflag_atom && !peratom_allocate_flag) allocate_peratom();
// update qsum and qsqsum
qsum_qsq();
// return if there are no charges
if (qsqsum == 0.0) return;
// convert atoms from box to lamda coords
if (triclinic == 0)
boxlo = domain->boxlo;
else {
boxlo = domain->boxlo_lamda;
domain->x2lamda(atom->nlocal);
}
start_compute();
// find grid points for all my particles
// map my particle charge onto my local 3d density grid
// optimized versions can only be used for orthogonal boxes
if (compute_vector_called) {
// electrolyte_density_brick is filled, so we can
// grab only electrode atoms
switch (fix->precision()) {
case FixIntel::PREC_MODE_MIXED:
make_rho_in_brick<float, double>(fix->get_mixed_buffers(), last_source_grpbit,
density_brick, !last_invert_source);
break;
case FixIntel::PREC_MODE_DOUBLE:
make_rho_in_brick<double, double>(fix->get_double_buffers(), last_source_grpbit,
density_brick, !last_invert_source);
break;
default:
make_rho_in_brick<float, float>(fix->get_single_buffers(), last_source_grpbit,
density_brick, !last_invert_source);
}
gc->reverse_comm(Grid3d::KSPACE, this, REVERSE_RHO, 1, sizeof(FFT_SCALAR), gc_buf1, gc_buf2,
MPI_FFT_SCALAR);
for (int nz = nzlo_out; nz <= nzhi_out; nz++)
for (int ny = nylo_out; ny <= nyhi_out; ny++)
for (int nx = nxlo_out; nx <= nxhi_out; nx++) {
density_brick[nz][ny][nx] += electrolyte_density_brick[nz][ny][nx];
}
} else {
switch (fix->precision()) {
case FixIntel::PREC_MODE_MIXED:
PPPMIntel::make_rho<float, double>(fix->get_mixed_buffers());
break;
case FixIntel::PREC_MODE_DOUBLE:
PPPMIntel::make_rho<double, double>(fix->get_double_buffers());
break;
default:
PPPMIntel::make_rho<float, float>(fix->get_single_buffers());
}
// all procs communicate density values from their ghost cells
// to fully sum contribution in their 3d bricks
// remap from 3d decomposition to FFT decomposition
gc->reverse_comm(Grid3d::KSPACE, this, REVERSE_RHO, 1, sizeof(FFT_SCALAR), gc_buf1, gc_buf2,
MPI_FFT_SCALAR);
}
brick2fft();
// compute potential gradient on my FFT grid and
// portion of e_long on this proc's FFT grid
// return gradients (electric fields) in 3d brick decomposition
// also performs per-atom calculations via poisson_peratom()
if (differentiation_flag == 1)
poisson_ad();
else
poisson_ik();
// all procs communicate E-field values
// to fill ghost cells surrounding their 3d bricks
if (differentiation_flag == 1)
gc->forward_comm(Grid3d::KSPACE, this, FORWARD_AD, 1, sizeof(FFT_SCALAR), gc_buf1, gc_buf2,
MPI_FFT_SCALAR);
else
gc->forward_comm(Grid3d::KSPACE, this, FORWARD_IK, 3, sizeof(FFT_SCALAR), gc_buf1, gc_buf2,
MPI_FFT_SCALAR);
// extra per-atom energy/virial communication
if (evflag_atom) {
if (differentiation_flag == 1 && vflag_atom)
gc->forward_comm(Grid3d::KSPACE, this, FORWARD_AD_PERATOM, 6, sizeof(FFT_SCALAR), gc_buf1,
gc_buf2, MPI_FFT_SCALAR);
else if (differentiation_flag == 0)
gc->forward_comm(Grid3d::KSPACE, this, FORWARD_IK_PERATOM, 7, sizeof(FFT_SCALAR), gc_buf1,
gc_buf2, MPI_FFT_SCALAR);
}
int tempslabflag = slabflag;
slabflag = 0; // bypass compute_second's slabcorr()
PPPMIntel::compute_second(eflag, vflag);
slabflag = tempslabflag;
boundcorr->compute_corr(qsum, eflag_atom, eflag_global, energy, eatom);
compute_vector_called = false;
}
void PPPMElectrodeIntel::start_compute()
{
if (compute_step < update->ntimestep) {
if (compute_step == -1) setup();
boxlo = domain->boxlo;
// extend size of per-atom arrays if necessary
if (atom->nmax > nmax) {
memory->destroy(part2grid);
if (differentiation_flag == 1) {
memory->destroy(particle_ekx);
memory->destroy(particle_eky);
memory->destroy(particle_ekz);
}
nmax = atom->nmax;
memory->create(part2grid, nmax, 3, "pppm:part2grid");
if (differentiation_flag == 1) {
memory->create(particle_ekx, nmax, "pppmintel:pekx");
memory->create(particle_eky, nmax, "pppmintel:peky");
memory->create(particle_ekz, nmax, "pppmintel:pekz");
}
}
switch (fix->precision()) {
case FixIntel::PREC_MODE_MIXED:
PPPMIntel::particle_map<float, double>(fix->get_mixed_buffers());
break;
case FixIntel::PREC_MODE_DOUBLE:
PPPMIntel::particle_map<double, double>(fix->get_double_buffers());
break;
default:
PPPMIntel::particle_map<float, float>(fix->get_single_buffers());
}
compute_step = update->ntimestep;
}
}
/* ----------------------------------------------------------------------
------------------------------------------------------------------------- */
void PPPMElectrodeIntel::compute_vector(double *vec, int sensor_grpbit, int source_grpbit,
bool invert_source)
{
start_compute();
last_source_grpbit = source_grpbit;
last_invert_source = invert_source;
// temporarily store and switch pointers so we can use brick2fft() for
// electrolyte density (without writing an additional function)
FFT_SCALAR ***density_brick_real = density_brick;
FFT_SCALAR *density_fft_real = density_fft;
if (neighbor->ago != 0) pack_buffers(); // since midstep positions may be outdated
switch (fix->precision()) {
case FixIntel::PREC_MODE_MIXED:
make_rho_in_brick<float, double>(fix->get_mixed_buffers(), source_grpbit,
electrolyte_density_brick, invert_source);
break;
case FixIntel::PREC_MODE_DOUBLE:
make_rho_in_brick<double, double>(fix->get_double_buffers(), source_grpbit,
electrolyte_density_brick, invert_source);
break;
default:
make_rho_in_brick<float, float>(fix->get_single_buffers(), source_grpbit,
electrolyte_density_brick, invert_source);
}
density_brick = electrolyte_density_brick;
density_fft = electrolyte_density_fft;
gc->reverse_comm(Grid3d::KSPACE, this, REVERSE_RHO, 1, sizeof(FFT_SCALAR), gc_buf1, gc_buf2,
MPI_FFT_SCALAR);
brick2fft();
// switch back pointers
density_brick = density_brick_real;
density_fft = density_fft_real;
// transform electrolyte charge density (r -> k) (complex conjugate)
for (int i = 0, n = 0; i < nfft; i++) {
work1[n++] = electrolyte_density_fft[i];
work1[n++] = ZEROF;
}
fft1->compute(work1, work1, -1);
// k->r FFT of Green's * electrolyte density = u_brick
for (int i = 0, n = 0; i < nfft; i++) {
work2[n] = work1[n] * greensfn[i];
n++;
work2[n] = work1[n] * greensfn[i];
n++;
}
fft2->compute(work2, work2, 1);
for (int k = nzlo_in, n = 0; k <= nzhi_in; k++)
for (int j = nylo_in; j <= nyhi_in; j++)
for (int i = nxlo_in; i <= nxhi_in; i++) {
u_brick[k][j][i] = work2[n];
n += 2;
}
gc->forward_comm(Grid3d::KSPACE, this, FORWARD_AD, 1, sizeof(FFT_SCALAR), gc_buf1, gc_buf2,
MPI_FFT_SCALAR);
switch (fix->precision()) {
case FixIntel::PREC_MODE_MIXED:
project_psi<float, double>(fix->get_mixed_buffers(), vec, sensor_grpbit);
break;
case FixIntel::PREC_MODE_DOUBLE:
project_psi<double, double>(fix->get_double_buffers(), vec, sensor_grpbit);
break;
default:
project_psi<float, float>(fix->get_single_buffers(), vec, sensor_grpbit);
}
compute_vector_called = true;
}
// project u_brick with weight matrix
template <class flt_t, class acc_t, int use_table>
void PPPMElectrodeIntel::project_psi(IntelBuffers<flt_t, acc_t> *buffers, double *vec,
int sensor_grpbit)
{
ATOM_T *_noalias const x = buffers->get_x(0);
int const nlocal = atom->nlocal;
int nthr;
if (_use_lrt)
nthr = 1;
else
nthr = comm->nthreads;
#if defined(_OPENMP)
#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(nthr, vec, sensor_grpbit) if (!_use_lrt)
#endif
{
int *mask = atom->mask;
const bigint ngridtotal = (bigint) nx_pppm * ny_pppm * nz_pppm;
const flt_t scaleinv = 1.0 / ngridtotal;
const flt_t lo0 = boxlo[0];
const flt_t lo1 = boxlo[1];
const flt_t lo2 = boxlo[2];
const flt_t xi = delxinv;
const flt_t yi = delyinv;
const flt_t zi = delzinv;
const flt_t fshiftone = shiftone;
int ifrom, ito, tid;
IP_PRE_omp_range_id(ifrom, ito, tid, nlocal, nthr);
for (int i = ifrom; i < ito; i++) {
if (!(mask[i] & sensor_grpbit)) continue;
double v = 0.;
// (nx,ny,nz) = global coords of grid pt to "lower left" of charge
// (dx,dy,dz) = distance to "lower left" grid pt
// (mx,my,mz) = global coords of moving stencil pt
int nx = part2grid[i][0];
int ny = part2grid[i][1];
int nz = part2grid[i][2];
FFT_SCALAR dx = nx + fshiftone - (x[i].x - lo0) * xi;
FFT_SCALAR dy = ny + fshiftone - (x[i].y - lo1) * yi;
FFT_SCALAR dz = nz + fshiftone - (x[i].z - lo2) * zi;
_alignvar(flt_t rho[3][INTEL_P3M_ALIGNED_MAXORDER], 64) = {0};
if (use_table) {
dx = dx * half_rho_scale + half_rho_scale_plus;
int idx = dx;
dy = dy * half_rho_scale + half_rho_scale_plus;
int idy = dy;
dz = dz * half_rho_scale + half_rho_scale_plus;
int idz = dz;
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = 0; k < INTEL_P3M_ALIGNED_MAXORDER; k++) {
rho[0][k] = rho_lookup[idx][k];
rho[1][k] = rho_lookup[idy][k];
rho[2][k] = rho_lookup[idz][k];
}
} else {
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = nlower; k <= nupper; k++) {
FFT_SCALAR r1, r2, r3;
r1 = r2 = r3 = ZEROF;
for (int l = order - 1; l >= 0; l--) {
r1 = rho_coeff[l][k] + r1 * dx;
r2 = rho_coeff[l][k] + r2 * dy;
r3 = rho_coeff[l][k] + r3 * dz;
}
rho[0][k - nlower] = r1;
rho[1][k - nlower] = r2;
rho[2][k - nlower] = r3;
}
}
#if defined(LMP_SIMD_COMPILER)
#pragma loop_count min(2), max(INTEL_P3M_ALIGNED_MAXORDER), avg(7)
#endif
for (int n = 0; n < order; n++) {
int miz = nlower + n + nz;
FFT_SCALAR z0 = rho[2][n];
#if defined(LMP_SIMD_COMPILER)
#pragma loop_count min(2), max(INTEL_P3M_ALIGNED_MAXORDER), avg(7)
#endif
for (int m = 0; m < order; m++) {
int miy = nlower + m + ny;
FFT_SCALAR y0 = z0 * rho[1][m];
#if defined(LMP_SIMD_COMPILER)
#pragma loop_count min(2), max(INTEL_P3M_ALIGNED_MAXORDER), avg(7)
#pragma simd
#endif
for (int l = 0; l < order; l++) {
int mix = nlower + l + nx;
v += y0 * rho[0][l] * u_brick[miz][miy][mix];
}
}
}
vec[i] += v * scaleinv;
}
}
}
/* ----------------------------------------------------------------------
--------------------------------------------------------------------- */
void PPPMElectrodeIntel::compute_matrix(bigint *imat, double **matrix, bool timer_flag)
{
// TODO replace compute with required setup
compute(1, 0);
// fft green's function k -> r
double *greens_real;
memory->create(greens_real, nz_pppm * ny_pppm * nx_pppm, "pppm/electrode:greens_real");
memset(greens_real, 0, nz_pppm * ny_pppm * nx_pppm * sizeof(double));
for (int i = 0, n = 0; i < nfft; i++) {
work2[n++] = greensfn[i];
work2[n++] = ZEROF;
}
fft2->compute(work2, work2, -1);
for (int k = nzlo_in, n = 0; k <= nzhi_in; k++)
for (int j = nylo_in; j <= nyhi_in; j++)
for (int i = nxlo_in; i <= nxhi_in; i++) {
greens_real[ny_pppm * nx_pppm * k + nx_pppm * j + i] = work2[n];
n += 2;
}
MPI_Allreduce(MPI_IN_PLACE, greens_real, nz_pppm * ny_pppm * nx_pppm, MPI_DOUBLE, MPI_SUM, world);
int const nlocal = atom->nlocal;
int nmat = std::count_if(&imat[0], &imat[nlocal], [](int x) {
return x >= 0;
});
MPI_Allreduce(MPI_IN_PLACE, &nmat, 1, MPI_INT, MPI_SUM, world);
double **x_ele;
memory->create(x_ele, nmat, 3, "pppm/electrode:x_ele");
memset(&(x_ele[0][0]), 0, nmat * 3 * sizeof(double));
double **x = atom->x;
for (int i = 0; i < nlocal; i++) {
int ipos = imat[i];
if (ipos < 0) continue;
for (int dim = 0; dim < 3; dim++) x_ele[ipos][dim] = x[i][dim];
}
MPI_Allreduce(MPI_IN_PLACE, &(x_ele[0][0]), nmat * 3, MPI_DOUBLE, MPI_SUM, world);
if (conp_one_step)
one_step_multiplication(imat, greens_real, x_ele, matrix, nmat, timer_flag);
else
two_step_multiplication(imat, greens_real, x_ele, matrix, nmat, timer_flag);
memory->destroy(greens_real);
memory->destroy(x_ele);
}
/* ----------------------------------------------------------------------*/
void PPPMElectrodeIntel::one_step_multiplication(bigint *imat, double *greens_real, double **x_ele,
double **matrix, int const nmat, bool timer_flag)
{
// map green's function in real space from mesh to particle positions
// with matrix multiplication 'W^T G W' in one steps. Uses less memory than
// two_step_multiplication
//
int const nlocal = atom->nlocal;
double **x = atom->x;
MPI_Barrier(world);
double step1_time = MPI_Wtime();
// precalculate rho_1d for local electrode
std::vector<int> j_list;
for (int j = 0; j < nlocal; j++) {
int jpos = imat[j];
if (jpos < 0) continue;
j_list.push_back(j);
}
int const nj_local = j_list.size();
FFT_SCALAR ***rho1d_j;
memory->create(rho1d_j, nj_local, 3, order, "pppm/electrode:rho1d_j");
_alignvar(FFT_SCALAR rho[3][INTEL_P3M_ALIGNED_MAXORDER], 64) = {0};
for (int jlist_pos = 0; jlist_pos < nj_local; jlist_pos++) {
int j = j_list[jlist_pos];
int njx = part2grid[j][0];
int njy = part2grid[j][1];
int njz = part2grid[j][2];
FFT_SCALAR const djx = njx + shiftone - (x[j][0] - boxlo[0]) * delxinv;
FFT_SCALAR const djy = njy + shiftone - (x[j][1] - boxlo[1]) * delyinv;
FFT_SCALAR const djz = njz + shiftone - (x[j][2] - boxlo[2]) * delzinv;
if (_use_table) {
int idx = (int) (djx * half_rho_scale + half_rho_scale_plus);
int idy = (int) (djy * half_rho_scale + half_rho_scale_plus);
int idz = (int) (djz * half_rho_scale + half_rho_scale_plus);
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = 0; k < INTEL_P3M_ALIGNED_MAXORDER; k++) {
rho[0][k] = rho_lookup[idx][k];
rho[1][k] = rho_lookup[idy][k];
rho[2][k] = rho_lookup[idz][k];
}
} else {
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = nlower; k <= nupper; k++) {
FFT_SCALAR r1, r2, r3;
r1 = r2 = r3 = ZEROF;
for (int l = order - 1; l >= 0; l--) {
r1 = rho_coeff[l][k] + r1 * djx;
r2 = rho_coeff[l][k] + r2 * djy;
r3 = rho_coeff[l][k] + r3 * djz;
}
rho[0][k - nlower] = r1;
rho[1][k - nlower] = r2;
rho[2][k - nlower] = r3;
}
}
for (int dim = 0; dim < 3; dim++) {
for (int oi = 0; oi < order; oi++) { rho1d_j[jlist_pos][dim][oi] = rho[dim][oi]; }
}
}
// nested loops over weights of electrode atoms i and j
// (nx,ny,nz) = global coords of grid pt to "lower left" of charge
// (dx,dy,dz) = distance to "lower left" grid pt
// (mx,my,mz) = global coords of moving stencil pt
int const order2 = INTEL_P3M_ALIGNED_MAXORDER * INTEL_P3M_ALIGNED_MAXORDER;
int const order6 = order2 * order2 * order2;
_alignvar(double amesh[order6], 64) = {0};
for (int ipos = 0; ipos < nmat; ipos++) {
double *_noalias xi_ele = x_ele[ipos];
// new calculation for nx, ny, nz because part2grid available for nlocal,
// only
int nix = static_cast<int>((xi_ele[0] - boxlo[0]) * delxinv + shift) - OFFSET;
int niy = static_cast<int>((xi_ele[1] - boxlo[1]) * delyinv + shift) - OFFSET;
int niz = static_cast<int>((xi_ele[2] - boxlo[2]) * delzinv + shift) - OFFSET;
FFT_SCALAR dix = nix + shiftone - (xi_ele[0] - boxlo[0]) * delxinv;
FFT_SCALAR diy = niy + shiftone - (xi_ele[1] - boxlo[1]) * delyinv;
FFT_SCALAR diz = niz + shiftone - (xi_ele[2] - boxlo[2]) * delzinv;
if (_use_table) {
int idx = (int) (dix * half_rho_scale + half_rho_scale_plus);
int idy = (int) (diy * half_rho_scale + half_rho_scale_plus);
int idz = (int) (diz * half_rho_scale + half_rho_scale_plus);
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = 0; k < INTEL_P3M_ALIGNED_MAXORDER; k++) {
rho[0][k] = rho_lookup[idx][k];
rho[1][k] = rho_lookup[idy][k];
rho[2][k] = rho_lookup[idz][k];
}
} else {
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = nlower; k <= nupper; k++) {
FFT_SCALAR r1, r2, r3;
r1 = r2 = r3 = ZEROF;
for (int l = order - 1; l >= 0; l--) {
r1 = rho_coeff[l][k] + r1 * dix;
r2 = rho_coeff[l][k] + r2 * diy;
r3 = rho_coeff[l][k] + r3 * diz;
}
rho[0][k - nlower] = r1;
rho[1][k - nlower] = r2;
rho[2][k - nlower] = r3;
}
}
int njx = -1;
int njy = -1;
int njz = -1; // force initial build_amesh
for (int jlist_pos = 0; jlist_pos < nj_local; jlist_pos++) {
int j = j_list[jlist_pos];
int jpos = imat[j];
if ((ipos < jpos) == !((ipos - jpos) % 2)) continue;
double aij = 0.;
if (njx != part2grid[j][0] || njy != part2grid[j][1] || njz != part2grid[j][2]) {
njx = part2grid[j][0];
njy = part2grid[j][1];
njz = part2grid[j][2];
build_amesh(njx - nix, njy - niy, njz - niz, amesh, greens_real);
}
int ind_amesh = 0;
for (int ni = 0; ni < order; ni++) {
FFT_SCALAR const iz0 = rho[2][ni];
for (int nj = 0; nj < order; nj++) {
FFT_SCALAR const jz0 = rho1d_j[jlist_pos][2][nj];
for (int mi = 0; mi < order; mi++) {
FFT_SCALAR const iy0 = iz0 * rho[1][mi];
for (int mj = 0; mj < order; mj++) {
FFT_SCALAR const jy0 = jz0 * rho1d_j[jlist_pos][1][mj];
for (int li = 0; li < order; li++) {
FFT_SCALAR const ix0 = iy0 * rho[0][li];
double aij_xscan = 0.;
for (int lj = 0; lj < order; lj++) {
aij_xscan += amesh[ind_amesh] * rho1d_j[jlist_pos][0][lj];
ind_amesh++;
}
aij += (double) ix0 * jy0 * aij_xscan;
}
}
}
}
}
matrix[ipos][jpos] += aij / volume;
if (ipos != jpos) matrix[jpos][ipos] += aij / volume;
}
}
MPI_Barrier(world);
memory->destroy(rho1d_j);
if (timer_flag && (comm->me == 0))
utils::logmesg(lmp, "Single step time: {:.4g} s\n", MPI_Wtime() - step1_time);
}
/* ----------------------------------------------------------------------*/
void PPPMElectrodeIntel::build_amesh(const int dx, // = njx - nix
const int dy, // = njy - niy
const int dz, // = njz - niz
double *amesh, double *const greens_real)
{
auto fmod = [](int x, int n) { // fast unsigned mod
int r = abs(x);
while (r >= n) r -= n;
return r;
};
int ind_amesh = 0;
for (int iz = 0; iz < order; iz++)
for (int jz = 0; jz < order; jz++) {
int const mz = fmod(dz + jz - iz, nz_pppm) * nx_pppm * ny_pppm;
for (int iy = 0; iy < order; iy++)
for (int jy = 0; jy < order; jy++) {
int const my = fmod(dy + jy - iy, ny_pppm) * nx_pppm;
for (int ix = 0; ix < order; ix++)
for (int jx = 0; jx < order; jx++) {
int const mx = fmod(dx + jx - ix, nx_pppm);
amesh[ind_amesh] = greens_real[mz + my + mx];
ind_amesh++;
}
}
}
}
/* ----------------------------------------------------------------------*/
void PPPMElectrodeIntel::two_step_multiplication(bigint *imat, double *greens_real, double **x_ele,
double **matrix, int const nmat, bool timer_flag)
{
// map green's function in real space from mesh to particle positions
// with matrix multiplication 'W^T G W' in two steps. gw is result of
// first multiplication.
int const nlocal = atom->nlocal;
MPI_Barrier(world);
double step1_time = MPI_Wtime();
int nx_ele = nxhi_out - nxlo_out + 1; // nx_pppm + order + 1;
int ny_ele = nyhi_out - nylo_out + 1; // ny_pppm + order + 1;
int nz_ele = nzhi_out - nzlo_out + 1; // nz_pppm + order + 1;
int nxyz = nx_ele * ny_ele * nz_ele;
vector<vector<double>> gw(nmat, vector<double>(nxyz, 0.));
_alignvar(FFT_SCALAR rho[3][INTEL_P3M_ALIGNED_MAXORDER], 64) = {0};
// loops over weights of electrode atoms and weights of complete grid
// (nx,ny,nz) = global coords of grid pt to "lower left" of charge
// (dx,dy,dz) = distance to "lower left" grid pt
// (mx,my,mz) = global coords of moving stencil pt
for (int ipos = 0; ipos < nmat; ipos++) {
double *_noalias xi_ele = x_ele[ipos];
// new calculation for nx, ny, nz because part2grid available for
// nlocal, only
int nix = static_cast<int>((xi_ele[0] - boxlo[0]) * delxinv + shift) - OFFSET;
int niy = static_cast<int>((xi_ele[1] - boxlo[1]) * delyinv + shift) - OFFSET;
int niz = static_cast<int>((xi_ele[2] - boxlo[2]) * delzinv + shift) - OFFSET;
FFT_SCALAR dx = nix + shiftone - (xi_ele[0] - boxlo[0]) * delxinv;
FFT_SCALAR dy = niy + shiftone - (xi_ele[1] - boxlo[1]) * delyinv;
FFT_SCALAR dz = niz + shiftone - (xi_ele[2] - boxlo[2]) * delzinv;
if (_use_table) {
dx = dx * half_rho_scale + half_rho_scale_plus;
int idx = dx;
dy = dy * half_rho_scale + half_rho_scale_plus;
int idy = dy;
dz = dz * half_rho_scale + half_rho_scale_plus;
int idz = dz;
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = 0; k < INTEL_P3M_ALIGNED_MAXORDER; k++) {
rho[0][k] = rho_lookup[idx][k];
rho[1][k] = rho_lookup[idy][k];
rho[2][k] = rho_lookup[idz][k];
}
} else {
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = nlower; k <= nupper; k++) {
FFT_SCALAR r1, r2, r3;
r1 = r2 = r3 = ZEROF;
for (int l = order - 1; l >= 0; l--) {
r1 = rho_coeff[l][k] + r1 * dx;
r2 = rho_coeff[l][k] + r2 * dy;
r3 = rho_coeff[l][k] + r3 * dz;
}
rho[0][k - nlower] = r1;
rho[1][k - nlower] = r2;
rho[2][k - nlower] = r3;
}
}
for (int ni = nlower; ni <= nupper; ni++) {
double iz0 = rho[2][ni - nlower];
int miz = ni + niz;
for (int mi = nlower; mi <= nupper; mi++) {
double iy0 = iz0 * rho[1][mi - nlower];
int miy = mi + niy;
for (int li = nlower; li <= nupper; li++) {
int mix = li + nix;
double ix0 = iy0 * rho[0][li - nlower];
for (int mjz = nzlo_out; mjz <= nzhi_out; mjz++) {
int mz = abs(mjz - miz) % nz_pppm;
for (int mjy = nylo_out; mjy <= nyhi_out; mjy++) {
int my = abs(mjy - miy) % ny_pppm;
for (int mjx = nxlo_out; mjx <= nxhi_out; mjx++) {
int mx = abs(mjx - mix) % nx_pppm;
gw[ipos][nx_ele * ny_ele * (mjz - nzlo_out) + nx_ele * (mjy - nylo_out) +
(mjx - nxlo_out)] +=
ix0 * greens_real[mz * nx_pppm * ny_pppm + my * nx_pppm + mx];
}
}
}
}
}
}
}
MPI_Barrier(world);
if (timer_flag && (comm->me == 0))
utils::logmesg(lmp, "step 1 time: {:.4g} s\n", MPI_Wtime() - step1_time);
// nested loop over electrode atoms i and j and stencil of i
double step2_time = MPI_Wtime();
double **x = atom->x;
for (int i = 0; i < nlocal; i++) {
int ipos = imat[i];
if (ipos < 0) continue;
int nix = part2grid[i][0];
int niy = part2grid[i][1];
int niz = part2grid[i][2];
FFT_SCALAR dix = nix + shiftone - (x[i][0] - boxlo[0]) * delxinv;
FFT_SCALAR diy = niy + shiftone - (x[i][1] - boxlo[1]) * delyinv;
FFT_SCALAR diz = niz + shiftone - (x[i][2] - boxlo[2]) * delzinv;
if (_use_table) {
dix = dix * half_rho_scale + half_rho_scale_plus;
int idx = dix;
diy = diy * half_rho_scale + half_rho_scale_plus;
int idy = diy;
diz = diz * half_rho_scale + half_rho_scale_plus;
int idz = diz;
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = 0; k < INTEL_P3M_ALIGNED_MAXORDER; k++) {
rho[0][k] = (double) rho_lookup[idx][k];
rho[1][k] = (double) rho_lookup[idy][k];
rho[2][k] = (double) rho_lookup[idz][k];
}
} else {
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = nlower; k <= nupper; k++) {
FFT_SCALAR r1, r2, r3;
r1 = r2 = r3 = ZEROF;
for (int l = order - 1; l >= 0; l--) {
r1 = rho_coeff[l][k] + r1 * dix;
r2 = rho_coeff[l][k] + r2 * diy;
r3 = rho_coeff[l][k] + r3 * diz;
}
rho[0][k - nlower] = (double) r1;
rho[1][k - nlower] = (double) r2;
rho[2][k - nlower] = (double) r3;
}
}
for (int jpos = 0; jpos < nmat; jpos++) {
double aij = 0.;
for (int ni = nlower; ni <= nupper; ni++) {
double iz0 = rho[2][ni - nlower];
int miz = ni + niz;
for (int mi = nlower; mi <= nupper; mi++) {
double iy0 = iz0 * rho[1][mi - nlower];
int miy = mi + niy;
for (int li = nlower; li <= nupper; li++) {
int mix = li + nix;
double ix0 = iy0 * rho[0][li - nlower];
int miz0 = miz - nzlo_out;
int miy0 = miy - nylo_out;
int mix0 = mix - nxlo_out;
aij += ix0 * gw[jpos][nx_ele * ny_ele * miz0 + nx_ele * miy0 + mix0];
}
}
}
matrix[ipos][jpos] += aij / volume;
}
}
MPI_Barrier(world);
if (timer_flag && (comm->me == 0))
utils::logmesg(lmp, "step 2 time: {:.4g} s\n", MPI_Wtime() - step2_time);
}
template <class flt_t, class acc_t, int use_table>
void PPPMElectrodeIntel::make_rho_in_brick(IntelBuffers<flt_t, acc_t> *buffers, int source_grpbit,
FFT_SCALAR ***scratch_brick, bool invert_source)
{
FFT_SCALAR *_noalias global_density = &(scratch_brick[nzlo_out][nylo_out][nxlo_out]);
ATOM_T *_noalias const x = buffers->get_x(0);
flt_t *_noalias const q = buffers->get_q(0);
int nlocal = atom->nlocal;
int nthr;
if (_use_lrt)
nthr = 1;
else
nthr = comm->nthreads;
#if defined(_OPENMP)
#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(nthr, nlocal, global_density, source_grpbit, \
invert_source) if (!_use_lrt)
#endif
{
int *mask = atom->mask;
const int nix = nxhi_out - nxlo_out + 1;
const int niy = nyhi_out - nylo_out + 1;
const flt_t lo0 = boxlo[0];
const flt_t lo1 = boxlo[1];
const flt_t lo2 = boxlo[2];
const flt_t xi = delxinv;
const flt_t yi = delyinv;
const flt_t zi = delzinv;
const flt_t fshiftone = shiftone;
const flt_t fdelvolinv = delvolinv;
int ifrom, ito, tid;
IP_PRE_omp_range_id(ifrom, ito, tid, nlocal, nthr);
FFT_SCALAR *_noalias my_density = tid == 0 ? global_density : perthread_density[tid - 1];
// clear 3d density array
memset(my_density, 0, ngrid * sizeof(FFT_SCALAR));
for (int i = ifrom; i < ito; i++) {
bool const i_in_source = !!(mask[i] & source_grpbit) ^ invert_source;
if (!i_in_source) continue;
int nx = part2grid[i][0];
int ny = part2grid[i][1];
int nz = part2grid[i][2];
int nysum = nlower + ny - nylo_out;
int nxsum = nlower + nx - nxlo_out;
int nzsum = (nlower + nz - nzlo_out) * nix * niy + nysum * nix + nxsum;
FFT_SCALAR dx = nx + fshiftone - (x[i].x - lo0) * xi;
FFT_SCALAR dy = ny + fshiftone - (x[i].y - lo1) * yi;
FFT_SCALAR dz = nz + fshiftone - (x[i].z - lo2) * zi;
_alignvar(flt_t rho[3][INTEL_P3M_ALIGNED_MAXORDER], 64) = {0};
if (use_table) {
dx = dx * half_rho_scale + half_rho_scale_plus;
int idx = dx;
dy = dy * half_rho_scale + half_rho_scale_plus;
int idy = dy;
dz = dz * half_rho_scale + half_rho_scale_plus;
int idz = dz;
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = 0; k < INTEL_P3M_ALIGNED_MAXORDER; k++) {
rho[0][k] = rho_lookup[idx][k];
rho[1][k] = rho_lookup[idy][k];
rho[2][k] = rho_lookup[idz][k];
}
} else {
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int k = nlower; k <= nupper; k++) {
FFT_SCALAR r1, r2, r3;
r1 = r2 = r3 = ZEROF;
for (int l = order - 1; l >= 0; l--) {
r1 = rho_coeff[l][k] + r1 * dx;
r2 = rho_coeff[l][k] + r2 * dy;
r3 = rho_coeff[l][k] + r3 * dz;
}
rho[0][k - nlower] = r1;
rho[1][k - nlower] = r2;
rho[2][k - nlower] = r3;
}
}
FFT_SCALAR z0 = fdelvolinv * q[i];
#if defined(LMP_SIMD_COMPILER)
#pragma loop_count min(2), max(INTEL_P3M_ALIGNED_MAXORDER), avg(7)
#endif
for (int n = 0; n < order; n++) {
int mz = n * nix * niy + nzsum;
FFT_SCALAR y0 = z0 * rho[2][n];
#if defined(LMP_SIMD_COMPILER)
#pragma loop_count min(2), max(INTEL_P3M_ALIGNED_MAXORDER), avg(7)
#endif
for (int m = 0; m < order; m++) {
int mzy = m * nix + mz;
FFT_SCALAR x0 = y0 * rho[1][m];
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int l = 0; l < INTEL_P3M_ALIGNED_MAXORDER; l++) {
int mzyx = l + mzy;
my_density[mzyx] += x0 * rho[0][l];
}
}
}
}
}
// reduce all the perthread_densities into global_density
if (nthr > 1) {
#if defined(_OPENMP)
#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(nthr, global_density) if (!_use_lrt)
#endif
{
int ifrom, ito, tid;
IP_PRE_omp_range_id(ifrom, ito, tid, ngrid, nthr);
#if defined(LMP_SIMD_COMPILER)
#pragma simd
#endif
for (int i = ifrom; i < ito; i++) {
for (int j = 1; j < nthr; j++) { global_density[i] += perthread_density[j - 1][i]; }
}
}
}
}
void PPPMElectrodeIntel::compute_group_group(int /*groupbit_A*/, int /*groupbit_B*/,
int /*AA_flag*/)
{
error->all(FLERR, "group group interaction not implemented in pppm/electrode yet");
}
void PPPMElectrodeIntel::compute_matrix_corr(bigint *imat, double **matrix)
{
boundcorr->matrix_corr(imat, matrix);
}
void PPPMElectrodeIntel::compute_vector_corr(double *vec, int sensor_grpbit, int source_grpbit,
bool invert_source)
{
boundcorr->vector_corr(vec, sensor_grpbit, source_grpbit, invert_source);
}
void PPPMElectrodeIntel::allocate()
{
if (slabflag == 1) {
// EW3Dc dipole correction
boundcorr = new SlabDipole(lmp);
} else if (wireflag == 1) {
// EW3Dc wire correction
boundcorr = new WireDipole(lmp);
} else {
// base BoundaryCorrection -- used for ffield
boundcorr = new BoundaryCorrection(lmp);
}
PPPM::allocate();
/* ----------------------------------------------------------------------
Allocate density_brick with extra padding for vector writes
------------------------------------------------------------------------- */
PPPMIntel::create3d_offset(electrolyte_density_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out,
nxlo_out, nxhi_out, "pppm/electrode:electrolyte_density_brick");
memory->create(electrolyte_density_fft, nfft_both, "pppm/electrode:electrolyte_density_fft");
memory->destroy3d_offset(density_brick, nzlo_out, nylo_out, nxlo_out);
PPPMIntel::create3d_offset(density_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out,
nxhi_out, "pppm:density_brick");
if ((differentiation_flag == 1) || u_brick != nullptr) {
memory->destroy3d_offset(u_brick, nzlo_out, nylo_out, nxlo_out);
}
if (differentiation_flag != 1) {
memory->destroy3d_offset(vdx_brick, nzlo_out, nylo_out, nxlo_out);
memory->destroy3d_offset(vdy_brick, nzlo_out, nylo_out, nxlo_out);
memory->destroy3d_offset(vdz_brick, nzlo_out, nylo_out, nxlo_out);
PPPMIntel::create3d_offset(vdx_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out,
nxhi_out, "pppm:vdx_brick");
PPPMIntel::create3d_offset(vdy_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out,
nxhi_out, "pppm:vdy_brick");
PPPMIntel::create3d_offset(vdz_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out,
nxhi_out, "pppm:vdz_brick");
}
PPPMIntel::create3d_offset(u_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:u_brick"); // use u_brick in project_psi
}
void PPPMElectrodeIntel::allocate_peratom()
{
// duplicated to avoid reallocating u_brick
peratom_allocate_flag = 1;
memory->create3d_offset(v0_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:v0_brick");
memory->create3d_offset(v1_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:v1_brick");
memory->create3d_offset(v2_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:v2_brick");
memory->create3d_offset(v3_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:v3_brick");
memory->create3d_offset(v4_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:v4_brick");
memory->create3d_offset(v5_brick, nzlo_out, nzhi_out, nylo_out, nyhi_out, nxlo_out, nxhi_out,
"pppm:v5_brick");
// use same GC ghost grid object for peratom grid communication
// but need to reallocate a larger gc_buf1 and gc_buf2
if (differentiation_flag)
npergrid = 6;
else
npergrid = 7;
memory->destroy(gc_buf1);
memory->destroy(gc_buf2);
memory->create(gc_buf1, npergrid * ngc_buf1, "pppm:gc_buf1");
memory->create(gc_buf2, npergrid * ngc_buf2, "pppm:gc_buf2");
}
void PPPMElectrodeIntel::deallocate()
{
if (boundcorr != nullptr) delete boundcorr;
// duplicated to always deallocate u_brick
memory->destroy3d_offset(density_brick, nzlo_out, nylo_out, nxlo_out);
memory->destroy3d_offset(u_brick, nzlo_out, nylo_out, nxlo_out);
if (differentiation_flag == 1) {
memory->destroy(sf_precoeff1);
memory->destroy(sf_precoeff2);
memory->destroy(sf_precoeff3);
memory->destroy(sf_precoeff4);
memory->destroy(sf_precoeff5);
memory->destroy(sf_precoeff6);
} else {
memory->destroy3d_offset(vdx_brick, nzlo_out, nylo_out, nxlo_out);
memory->destroy3d_offset(vdy_brick, nzlo_out, nylo_out, nxlo_out);
memory->destroy3d_offset(vdz_brick, nzlo_out, nylo_out, nxlo_out);
}
memory->destroy(density_fft);
memory->destroy(greensfn);
memory->destroy(work1);
memory->destroy(work2);
memory->destroy(vg);
if (triclinic == 0) {
memory->destroy1d_offset(fkx, nxlo_fft);
memory->destroy1d_offset(fky, nylo_fft);
memory->destroy1d_offset(fkz, nzlo_fft);
} else {
memory->destroy(fkx);
memory->destroy(fky);
memory->destroy(fkz);
}
memory->destroy(gf_b);
if (stagger_flag) gf_b = nullptr;
memory->destroy2d_offset(rho1d, -order_allocated / 2);
memory->destroy2d_offset(drho1d, -order_allocated / 2);
memory->destroy2d_offset(rho_coeff, (1 - order_allocated) / 2);
memory->destroy2d_offset(drho_coeff, (1 - order_allocated) / 2);
delete fft1;
delete fft2;
delete remap;
delete gc;
memory->destroy(gc_buf1);
memory->destroy(gc_buf2);
}
/* ----------------------------------------------------------------------
Pack charge data into intel package buffers after updates
------------------------------------------------------------------------- */
void PPPMElectrodeIntel::pack_buffers_q()
{
fix->start_watch(TIME_PACK);
int packthreads;
if (comm->nthreads > INTEL_HTHREADS)
packthreads = comm->nthreads;
else
packthreads = 1;
#if defined(_OPENMP)
#pragma omp parallel if (packthreads > 1)
#endif
{
int ifrom, ito, tid;
IP_PRE_omp_range_id_align(ifrom, ito, tid, atom->nlocal + atom->nghost, packthreads,
sizeof(IntelBuffers<float, double>::atom_t));
if (fix->precision() == FixIntel::PREC_MODE_MIXED)
fix->get_mixed_buffers()->thr_pack_q(ifrom, ito);
else if (fix->precision() == FixIntel::PREC_MODE_DOUBLE)
fix->get_double_buffers()->thr_pack_q(ifrom, ito);
else
fix->get_single_buffers()->thr_pack_q(ifrom, ito);
}
fix->stop_watch(TIME_PACK);
}
Reference in New Issue View Git Blame Copy Permalink