From 6bd64d5aabc706835f9d62c4a457f1b9cdec380e Mon Sep 17 00:00:00 2001 From: sjplimp Date: Wed, 6 Aug 2014 16:23:15 +0000 Subject: [PATCH] git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@12256 f3b2605a-c512-4ea7-a41b-209d697bcdaa --- src/comm_brick.cpp | 18 +++++---- src/comm_tiled.cpp | 98 ++++++++++++++++++++++++++++------------------ src/dump.h | 4 +- src/dump_image.cpp | 7 ++-- src/dump_image.h | 4 +- 5 files changed, 79 insertions(+), 52 deletions(-) diff --git a/src/comm_brick.cpp b/src/comm_brick.cpp index db3205d2b3..90f4bd5339 100644 --- a/src/comm_brick.cpp +++ b/src/comm_brick.cpp @@ -1232,29 +1232,31 @@ void CommBrick::forward_comm_dump(Dump *dump) MPI_Request request; MPI_Status status; + int nsize = dump->comm_forward; + for (iswap = 0; iswap < nswap; iswap++) { // pack buffer - n = dump->pack_comm(sendnum[iswap],sendlist[iswap], - buf_send,pbc_flag[iswap],pbc[iswap]); + n = dump->pack_forward_comm(sendnum[iswap],sendlist[iswap], + buf_send,pbc_flag[iswap],pbc[iswap]); // exchange with another proc // if self, set recv buffer to send buffer if (sendproc[iswap] != me) { if (recvnum[iswap]) - MPI_Irecv(buf_recv,n*recvnum[iswap],MPI_DOUBLE,recvproc[iswap],0, + MPI_Irecv(buf_recv,nsize*recvnum[iswap],MPI_DOUBLE,recvproc[iswap],0, world,&request); if (sendnum[iswap]) - MPI_Send(buf_send,n*sendnum[iswap],MPI_DOUBLE,sendproc[iswap],0,world); + MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap],0,world); if (recvnum[iswap]) MPI_Wait(&request,&status); buf = buf_recv; } else buf = buf_send; // unpack buffer - dump->unpack_comm(recvnum[iswap],firstrecv[iswap],buf); + dump->unpack_forward_comm(recvnum[iswap],firstrecv[iswap],buf); } } @@ -1270,6 +1272,8 @@ void CommBrick::reverse_comm_dump(Dump *dump) MPI_Request request; MPI_Status status; + int nsize = dump->comm_reverse; + for (iswap = nswap-1; iswap >= 0; iswap--) { // pack buffer @@ -1281,10 +1285,10 @@ void CommBrick::reverse_comm_dump(Dump *dump) if (sendproc[iswap] != me) { if (sendnum[iswap]) - MPI_Irecv(buf_recv,n*sendnum[iswap],MPI_DOUBLE,sendproc[iswap],0, + MPI_Irecv(buf_recv,nsize*sendnum[iswap],MPI_DOUBLE,sendproc[iswap],0, world,&request); if (recvnum[iswap]) - MPI_Send(buf_send,n*recvnum[iswap],MPI_DOUBLE,recvproc[iswap],0,world); + MPI_Send(buf_send,n,MPI_DOUBLE,recvproc[iswap],0,world); if (sendnum[iswap]) MPI_Wait(&request,&status); buf = buf_recv; } else buf = buf_send; diff --git a/src/comm_tiled.cpp b/src/comm_tiled.cpp index 4c71e4ded6..092b8e091f 100644 --- a/src/comm_tiled.cpp +++ b/src/comm_tiled.cpp @@ -1578,31 +1578,43 @@ void CommTiled::reverse_comm_compute(Compute *compute) void CommTiled::forward_comm_dump(Dump *dump) { - int i,irecv,n; + int i,irecv,n,nsend,nrecv; MPI_Status status; - for (int iswap = 0; iswap < nswap; iswap++) { - if (sendproc[iswap][0] != me) { - for (i = 0; i < nrecvproc[iswap]; i++) - MPI_Irecv(&buf_recv[forward_recv_offset[iswap][i]], - size_forward_recv[iswap][i], - MPI_DOUBLE,recvproc[iswap][i],0,world,&requests[i]); - for (i = 0; i < nsendproc[iswap]; i++) { - n = dump->pack_comm(sendnum[iswap][i],sendlist[iswap][i], - buf_send,pbc_flag[iswap][i],pbc[iswap][i]); - MPI_Send(buf_send,n*sendnum[iswap][i],MPI_DOUBLE, - sendproc[iswap][i],0,world); - } - for (i = 0; i < nrecvproc[iswap]; i++) { - MPI_Waitany(nrecvproc[iswap],requests,&irecv,&status); - dump->unpack_comm(recvnum[iswap][irecv],firstrecv[iswap][irecv], - &buf_recv[forward_recv_offset[iswap][irecv]]); - } + int nsize = dump->comm_forward; - } else { - n = dump->pack_comm(sendnum[iswap][0],sendlist[iswap][0], - buf_send,pbc_flag[iswap][0],pbc[iswap][0]); - dump->unpack_comm(recvnum[iswap][0],firstrecv[iswap][0],buf_send); + for (int iswap = 0; iswap < nswap; iswap++) { + nsend = nsendproc[iswap] - sendself[iswap]; + nrecv = nrecvproc[iswap] - sendself[iswap]; + + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) + MPI_Irecv(&buf_recv[nsize*forward_recv_offset[iswap][i]], + nsize*recvnum[iswap][i], + MPI_DOUBLE,recvproc[iswap][i],0,world,&requests[i]); + } + if (sendother[iswap]) { + for (i = 0; i < nsendproc[iswap]; i++) { + n = dump->pack_forward_comm(sendnum[iswap][i],sendlist[iswap][i], + buf_send,pbc_flag[iswap][i], + pbc[iswap][i]); + MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap][i],0,world); + } + } + if (sendself[iswap]) { + dump->pack_forward_comm(sendnum[iswap][nsend],sendlist[iswap][nsend], + buf_send,pbc_flag[iswap][nsend], + pbc[iswap][nsend]); + dump->unpack_forward_comm(recvnum[iswap][nrecv], + firstrecv[iswap][nrecv],buf_send); + } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { + MPI_Waitany(nrecv,requests,&irecv,&status); + dump->unpack_forward_comm(recvnum[iswap][irecv],firstrecv[iswap][irecv], + &buf_recv[nsize* + forward_recv_offset[iswap][irecv]]); + } } } } @@ -1614,31 +1626,41 @@ void CommTiled::forward_comm_dump(Dump *dump) void CommTiled::reverse_comm_dump(Dump *dump) { - int i,irecv,n; + int i,irecv,n,nsend,nrecv; MPI_Status status; + int nsize = dump->comm_reverse; + for (int iswap = nswap-1; iswap >= 0; iswap--) { - if (sendproc[iswap][0] != me) { - for (i = 0; i < nsendproc[iswap]; i++) - MPI_Irecv(&buf_recv[reverse_recv_offset[iswap][i]], - size_reverse_recv[iswap][i],MPI_DOUBLE, + nsend = nsendproc[iswap] - sendself[iswap]; + nrecv = nrecvproc[iswap] - sendself[iswap]; + + if (sendother[iswap]) { + for (i = 0; i < nsend; i++) + MPI_Irecv(&buf_recv[nsize*reverse_recv_offset[iswap][i]], + nsize*sendnum[iswap][i],MPI_DOUBLE, sendproc[iswap][i],0,world,&requests[i]); - for (i = 0; i < nrecvproc[iswap]; i++) { + } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { n = dump->pack_reverse_comm(recvnum[iswap][i],firstrecv[iswap][i], buf_send); - MPI_Send(buf_send,n*recvnum[iswap][i],MPI_DOUBLE, - recvproc[iswap][i],0,world); + MPI_Send(buf_send,n,MPI_DOUBLE,recvproc[iswap][i],0,world); } - for (i = 0; i < nsendproc[iswap]; i++) { - MPI_Waitany(nsendproc[iswap],requests,&irecv,&status); + } + if (sendself[iswap]) { + dump->pack_reverse_comm(recvnum[iswap][nrecv],firstrecv[iswap][nrecv], + buf_send); + dump->unpack_reverse_comm(sendnum[iswap][nsend],sendlist[iswap][nsend], + buf_send); + } + if (sendother[iswap]) { + for (i = 0; i < nsend; i++) { + MPI_Waitany(nsend,requests,&irecv,&status); dump->unpack_reverse_comm(sendnum[iswap][irecv],sendlist[iswap][irecv], - &buf_recv[reverse_recv_offset[iswap][irecv]]); + &buf_recv[nsize* + reverse_recv_offset[iswap][irecv]]); } - - } else { - n = dump->pack_reverse_comm(recvnum[iswap][0],firstrecv[iswap][0], - buf_send); - dump->unpack_reverse_comm(sendnum[iswap][0],sendlist[iswap][0],buf_send); } } } diff --git a/src/dump.h b/src/dump.h index b14a858d56..37b8ec3323 100644 --- a/src/dump.h +++ b/src/dump.h @@ -41,8 +41,8 @@ class Dump : protected Pointers { void init(); virtual void write(); - virtual int pack_comm(int, int *, double *, int, int *) {return 0;} - virtual void unpack_comm(int, int, double *) {} + virtual int pack_forward_comm(int, int *, double *, int, int *) {return 0;} + virtual void unpack_forward_comm(int, int, double *) {} virtual int pack_reverse_comm(int, int, double *) {return 0;} virtual void unpack_reverse_comm(int, int *, double *) {} diff --git a/src/dump_image.cpp b/src/dump_image.cpp index d5b613c162..bc8256817c 100644 --- a/src/dump_image.cpp +++ b/src/dump_image.cpp @@ -949,7 +949,8 @@ void DumpImage::create_image() /* ---------------------------------------------------------------------- */ -int DumpImage::pack_comm(int n, int *list, double *buf, int pbc_flag, int *pbc) +int DumpImage::pack_forward_comm(int n, int *list, double *buf, + int pbc_flag, int *pbc) { int i,j,m; @@ -969,12 +970,12 @@ int DumpImage::pack_comm(int n, int *list, double *buf, int pbc_flag, int *pbc) } } - return comm_forward; + return m; } /* ---------------------------------------------------------------------- */ -void DumpImage::unpack_comm(int n, int first, double *buf) +void DumpImage::unpack_forward_comm(int n, int first, double *buf) { int i,m,last; diff --git a/src/dump_image.h b/src/dump_image.h index 72d54e5f9c..b7558e7adf 100644 --- a/src/dump_image.h +++ b/src/dump_image.h @@ -30,8 +30,8 @@ class DumpImage : public DumpCustom { DumpImage(class LAMMPS *, int, char**); virtual ~DumpImage(); - int pack_comm(int, int *, double *, int, int *); - void unpack_comm(int, int, double *); + int pack_forward_comm(int, int *, double *, int, int *); + void unpack_forward_comm(int, int, double *); protected: int filetype;