ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more debugging

Browse Source
This commit is contained in:
Steve Plimpton
2022-10-26 15:14:06 -06:00
parent a4a10e970e
commit 7bf4c8d54a
9 changed files with 264 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
src/grid2d.cpp
View File

@ -26,8 +26,6 @@
using namespace LAMMPS_NS;
enum{BRICK,TILED};
#define DELTA 16
static constexpr int OFFSET = 16384;
@ -73,9 +71,7 @@ Grid2d::Grid2d(LAMMPS *lmp, MPI_Comm gcomm,
ny = gny;
ngrid[0] = nx; ngrid[1] = ny;
if (comm->layout == Comm::LAYOUT_TILED) layout = TILED;
else layout = BRICK;
layout = comm->layout;
// partition global grid across procs
// i xyz lo/hi = lower/upper bounds of global grid this proc owns
@ -146,7 +142,7 @@ Grid2d::Grid2d(LAMMPS *lmp, MPI_Comm gcomm,
// store grid bounds and proc neighs
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
int (*procneigh)[2] = comm->procneigh;
store(ixlo,ixhi,iylo,iyhi,
oxlo,oxhi,oylo,oyhi,
@ -189,13 +185,11 @@ Grid2d::Grid2d(LAMMPS *lmp, MPI_Comm gcomm,
ny = gny;
ngrid[0] = nx; ngrid[1] = ny;
if (comm->layout == Comm::LAYOUT_TILED) layout = TILED;
else layout = BRICK;
layout = comm->layout;
// store grid bounds and proc neighs
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
int (*procneigh)[2] = comm->procneigh;
store(ixlo,ixhi,iylo,iyhi,
oxlo,oxhi,oylo,oyhi,
@ -240,14 +234,12 @@ Grid2d::Grid2d(LAMMPS *lmp, MPI_Comm gcomm, int flag,
ny = gny;
ngrid[0] = nx; ngrid[1] = ny;
if (comm->layout == Comm::LAYOUT_TILED) layout = TILED;
else layout = BRICK;
layout = comm->layout;
// store grid bounds and proc neighs
if (flag == 1) {
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
// this assumes gcomm = world
int (*procneigh)[2] = comm->procneigh;
store(ixlo,ixhi,iylo,iyhi,
@ -263,7 +255,7 @@ Grid2d::Grid2d(LAMMPS *lmp, MPI_Comm gcomm, int flag,
}
} else if (flag == 2) {
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
store(ixlo,ixhi,iylo,iyhi,
oxlo,oxhi,oylo,oyhi,
oxlo,oxhi,oylo,oyhi,
@ -286,6 +278,11 @@ Grid2d::~Grid2d()
}
memory->sfree(swap);
delete [] xsplit;
delete [] ysplit;
delete [] zsplit;
memory->destroy(grid2proc);
// tiled comm data structs
for (int i = 0; i < nsend; i++)
@ -338,31 +335,6 @@ void Grid2d::store(int ixlo, int ixhi, int iylo, int iyhi,
fullylo = fylo;
fullyhi = fyhi;
// for BRICK layout, proc xy lohi = my 4 neighbor procs in this MPI_Comm
if (layout == BRICK) {
procxlo = pxlo;
procxhi = pxhi;
procylo = pylo;
procyhi = pyhi;
}
// for TILED layout, create RCB tree of cut info for grid decomp
// access CommTiled to get cut dimension
// cut = this proc's inlo in that dim
// dim is -1 for proc 0, but never accessed
if (layout == TILED) {
rcbinfo = (RCBinfo *)
memory->smalloc(nprocs*sizeof(RCBinfo),"grid3d:rcbinfo");
RCBinfo rcbone;
rcbone.dim = comm->rcbcutdim;
if (rcbone.dim <= 0) rcbone.cut = inxlo;
else if (rcbone.dim == 1) rcbone.cut = inylo;
MPI_Allgather(&rcbone,sizeof(RCBinfo),MPI_CHAR,
rcbinfo,sizeof(RCBinfo),MPI_CHAR,gridcomm);
}
// internal data initializations
nswap = maxswap = 0;
@ -373,8 +345,53 @@ void Grid2d::store(int ixlo, int ixhi, int iylo, int iyhi,
recv = nullptr;
copy = nullptr;
requests = nullptr;
xsplit = ysplit = zsplit = nullptr;
grid2proc = nullptr;
rcbinfo = nullptr;
nsend_remap = nrecv_remap = self_remap = 0;
send_remap = nullptr;
recv_remap = nullptr;
// for non TILED layout:
// proc xyz lohi = my 6 neighbor procs in this MPI_Comm
// xyz split = copy of 1d vectors in Comm
// grid2proc = copy of 3d array in Comm
if (layout != Comm::LAYOUT_TILED) {
procxlo = pxlo;
procxhi = pxhi;
procylo = pylo;
procyhi = pyhi;
xsplit = new double[comm->procgrid[0]+1];
ysplit = new double[comm->procgrid[1]+1];
memcpy(xsplit,comm->xsplit,(comm->procgrid[0]+1)*sizeof(double));
memcpy(ysplit,comm->ysplit,(comm->procgrid[1]+1)*sizeof(double));
memory->create(grid2proc,comm->procgrid[0],comm->procgrid[1],comm->procgrid[2],
"grid3d:grid2proc");
memcpy(&grid2proc[0][0][0],&comm->grid2proc[0][0][0],
comm->procgrid[0]*comm->procgrid[1]*comm->procgrid[2]*sizeof(int));
}
// for TILED layout:
// create RCB tree of cut info for grid decomp
// access CommTiled to get cut dimension
// cut = this proc's inlo in that dim
// dim is -1 for proc 0, but never accessed
if (layout == Comm::LAYOUT_TILED) {
rcbinfo = (RCBinfo *)
memory->smalloc(nprocs*sizeof(RCBinfo),"grid3d:rcbinfo");
RCBinfo rcbone;
rcbone.dim = comm->rcbcutdim;
if (rcbone.dim <= 0) rcbone.cut = inxlo;
else if (rcbone.dim == 1) rcbone.cut = inylo;
MPI_Allgather(&rcbone,sizeof(RCBinfo),MPI_CHAR,
rcbinfo,sizeof(RCBinfo),MPI_CHAR,gridcomm);
}
}
/* ---------------------------------------------------------------------- */
@ -446,7 +463,7 @@ void Grid2d::get_bounds_ghost(int &xlo, int &xhi, int &ylo, int &yhi)
void Grid2d::setup(int &nbuf1, int &nbuf2)
{
if (layout == BRICK) setup_brick(nbuf1,nbuf2);
if (layout != Comm::LAYOUT_TILED) setup_brick(nbuf1,nbuf2);
else setup_tiled(nbuf1,nbuf2);
}
@ -865,7 +882,7 @@ void Grid2d::setup_tiled(int &nbuf1, int &nbuf2)
int Grid2d::ghost_adjacent()
{
if (layout == BRICK) return ghost_adjacent_brick();
if (layout != Comm::LAYOUT_TILED) return ghost_adjacent_brick();
return ghost_adjacent_tiled();
}
@ -911,7 +928,7 @@ int Grid2d::ghost_adjacent_tiled()
void Grid2d::forward_comm(int caller, void *ptr, int nper, int nbyte, int which,
void *buf1, void *buf2, MPI_Datatype datatype)
{
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
if (caller == KSPACE)
forward_comm_brick<KSpace>((KSpace *) ptr,nper,nbyte,which,
buf1,buf2,datatype);
@ -1016,7 +1033,7 @@ forward_comm_tiled(T *ptr, int nper, int nbyte, int which,
void Grid2d::reverse_comm(int caller, void *ptr, int nper, int nbyte, int which,
void *buf1, void *buf2, MPI_Datatype datatype)
{
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
if (caller == KSPACE)
reverse_comm_brick<KSpace>((KSpace *) ptr,nper,nbyte,which,
buf1,buf2,datatype);
@ -1130,28 +1147,18 @@ reverse_comm_tiled(T *ptr, int nper, int nbyte, int which,
caller converts them to message size for grid data it stores
------------------------------------------------------------------------- */
/* ------------------------------------------------------------------------- */
void Grid2d::setup_remap(Grid2d *old, int &nremap_buf1, int &nremap_buf2)
{
if (layout == BRICK) setup_remap_brick(old,nremap_buf1,nremap_buf2);
else setup_remap_tiled(old,nremap_buf1,nremap_buf2);
}
/* ------------------------------------------------------------------------- */
void Grid2d::setup_remap_brick(Grid2d *old, int &nremap_buf1, int &nremap_buf2)
{
nremap_buf1 = 0;
nremap_buf2 = 0;
}
/* ------------------------------------------------------------------------- */
void Grid2d::setup_remap_tiled(Grid2d *old, int &nremap_buf1, int &nremap_buf2)
{
int m;
int pbc[2];
int *box;
// deallocated existing remap data structs
deallocate_remap();
// compute overlaps of old decomp owned box with all owned boxes in new decomp
// noverlap_old = # of overlaps, including self
// overlap_old = vector of overlap info in Overlap data struct
@ -1206,7 +1213,7 @@ void Grid2d::setup_remap_tiled(Grid2d *old, int &nremap_buf1, int &nremap_buf2)
nrecv_remap = 0;
for (m = 0; m < noverlap_new; m++)
if (overlap_old[m].proc != me) nrecv_remap++;
if (overlap_new[m].proc != me) nrecv_remap++;
recv_remap = new Recv[nrecv_remap];
@ -1438,12 +1445,12 @@ void Grid2d::write_file_style(T *ptr, int which,
/* ----------------------------------------------------------------------
compute list of overlaps between box and the owned grid boxes of all procs
for brick decomp, done using Comm::grid2proc data struct
for tiled decomp, done via recursive box drop on RCB tree
for brick decomp of Grid, done using xyz split + grid2proc copied from Comm
for tiled decomp of Grid, done via recursive box drop on RCB tree
box = 4 integers = (xlo,xhi,ylo,yhi)
box can be owned cells or owned + ghost cells
pbc = flags for grid periodicity in each dim
if box includes ghost cells, it can overlap PBCs
if box includes ghost cells, it can overlap PBCs (only for setup_tiled)
each lo/hi value may extend beyond 0 to N-1 into another periodic image
return # of overlaps including with self
return list of overlaps
@ -1455,7 +1462,7 @@ int Grid2d::compute_overlap(int *box, int *pbc, Overlap *&overlap)
noverlap_list = maxoverlap_list = 0;
overlap_list = nullptr;
if (layout == BRICK) {
if (layout != Comm::LAYOUT_TILED) {
// find comm->procgrid indices in each dim for box bounds
@ -1464,42 +1471,22 @@ int Grid2d::compute_overlap(int *box, int *pbc, Overlap *&overlap)
int jproclo = find_proc_index(box[2],ngrid[1],1,comm->ysplit);
int jprochi = find_proc_index(box[3],ngrid[1],1,comm->ysplit);
// save comm->myloc values so can overwrite them k,j,i triple loop
// b/c comm->partition_grid uses comm->myloc
int obox[4];
int save_myloc[3];
save_myloc[0] = comm->myloc[0];
save_myloc[1] = comm->myloc[1];
save_myloc[2] = comm->myloc[2];
for (int j = jproclo; j <= jprochi; j++)
for (int i = iproclo; i <= iprochi; i++) {
find_proc_box(i,j,obox);
int obox[6];
for (int k = 0; k <= 0; k++)
for (int j = jproclo; j <= jprochi; j++)
for (int i = iproclo; i <= iprochi; i++) {
comm->myloc[0] = i;
comm->myloc[1] = j;
comm->myloc[2] = k;
comm->partition_grid(ngrid[0],ngrid[1],1,0.0,
obox[0],obox[1],obox[2],obox[3],obox[4],obox[5]);
if (noverlap_list == maxoverlap_list) grow_overlap();
overlap[noverlap_list].proc = comm->grid2proc[i][j][k];
overlap[noverlap_list].box[0] = MAX(box[0],obox[0]);
overlap[noverlap_list].box[1] = MIN(box[1],obox[1]);
overlap[noverlap_list].box[2] = MAX(box[2],obox[2]);
overlap[noverlap_list].box[3] = MIN(box[3],obox[3]);
noverlap_list++;
}
if (noverlap_list == maxoverlap_list) grow_overlap();
overlap_list[noverlap_list].proc = grid2proc[i][j][0];
overlap_list[noverlap_list].box[0] = MAX(box[0],obox[0]);
overlap_list[noverlap_list].box[1] = MIN(box[1],obox[1]);
overlap_list[noverlap_list].box[2] = MAX(box[2],obox[2]);
overlap_list[noverlap_list].box[3] = MIN(box[3],obox[3]);
noverlap_list++;
}
// restore comm->myloc values
comm->myloc[0] = save_myloc[0];
comm->myloc[1] = save_myloc[1];
comm->myloc[2] = save_myloc[2];
} else if (layout == TILED) {
} else if (layout == Comm::LAYOUT_TILED) {
box_drop(box,pbc);
}
@ -1727,3 +1714,34 @@ int Grid2d::find_proc_index(int igrid, int n, int dim, double *split)
return m;
}
/* ----------------------------------------------------------------------
find the grid box for proc with grid indices i,j
i,j,k = grid index (0 to N-1) in each dim
return lo/hi bounds of box in 2 dims
computation is same as Comm::partition_grid()
------------------------------------------------------------------------- */
void Grid2d::find_proc_box(int i, int j, int *box)
{
int lo,hi;
double fraclo,frachi;
fraclo = xsplit[i];
frachi = xsplit[i+1];
lo = static_cast<int> (fraclo * ngrid[0]);
if (1.0*lo != fraclo*ngrid[0]) lo++;
hi = static_cast<int> (frachi * ngrid[0]);
if (1.0*hi == frachi*ngrid[0]) hi--;
box[0] = lo;
box[1] = hi;
fraclo = ysplit[j];
frachi = ysplit[j+1];
lo = static_cast<int> (fraclo * ngrid[1]);
if (1.0*lo != fraclo*ngrid[1]) lo++;
hi = static_cast<int> (frachi * ngrid[1]);
if (1.0*hi == frachi*ngrid[1]) hi--;
box[2] = lo;
box[3] = hi;
}