/* ---------------------------------------------------------------------- LIGGGHTS - LAMMPS Improved for General Granular and Granular Heat Transfer Simulations LIGGGHTS is part of the CFDEMproject www.liggghts.com | www.cfdem.com Christoph Kloss, christoph.kloss@cfdem.com Copyright 2009-2012 JKU Linz Copyright 2012- DCS Computing GmbH, Linz LIGGGHTS is based on LAMMPS LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator http://lammps.sandia.gov, Sandia National Laboratories Steve Plimpton, sjplimp@sandia.gov This software is distributed under the GNU General Public License. See the README file in the top-level directory. ------------------------------------------------------------------------- */ #ifndef LMP_MULTI_NODE_MESH_PARALLEL_BUFFER_I_H #define LMP_MULTI_NODE_MESH_PARALLEL_BUFFER_I_H /* ---------------------------------------------------------------------- push / pop for exchange ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::pushExchange(int dim) { //NP do NOT make a local copy of buf_send_ as this fct calls re-allocation!! // scale translate rotate not needed here bool dummy = false; double checklo,checkhi; //NP consistent with Domain::is_in_subdomain() checklo = this->domain->sublo[dim]; if(this->domain->subhi[dim] == this->domain->boxhi[dim]) checkhi = this->domain->boxhi[dim] + SMALL_DMBRDR; else checkhi = this->domain->subhi[dim]; int nsend = 0, nsend_this = 0; int i = 0; while(i < nLocal_) { if(!(this->center_(i)[dim] >= checklo && this->center_(i)[dim] < checkhi)) { nsend_this = pushElemToBuffer(i,&(buf_send_[nsend+1]),OPERATION_COMM_EXCHANGE,dummy,dummy,dummy); buf_send_[nsend] = static_cast(nsend_this+1); nsend += (nsend_this+1); /*NL*/// fprintf(this->screen,"step %d: proc %d pushes element id %d with center %f %f %f\n", /*NL*/// this->update->ntimestep,this->comm->me,this->id_slow(i),this->center_(i)[0],this->center_(i)[1],this->center_(i)[2]); if (nsend > maxsend_) grow_send(nsend,1); this->deleteElement(i); //NP deleteElement() decreases nLocal } else i++; } return nsend; } template void MultiNodeMeshParallel::popExchange(int nrecv,int dim,double *buf) { double center_elem[3]; double checklo,checkhi; int m = 0, nrecv_this; // scale translate rotate not needed here bool dummy = false; //NP consistent with Domain::is_in_subdomain() checklo = this->domain->sublo[dim]; if(this->domain->subhi[dim] == this->domain->boxhi[dim]) checkhi = this->domain->boxhi[dim] + SMALL_DMBRDR; else checkhi = this->domain->subhi[dim]; while (m < nrecv) { // number of values is first in buffer nrecv_this = static_cast(buf[m]); // center is next in buffer, test it vectorCopy3D(&(buf[m+1]),center_elem); /*NL*/// fprintf(this->screen,"proc %d: nrecv_this %d center %f %f %f xlo %f xhi %f\n", /*NL*/// this->comm->me,nrecv_this,center_elem[0],center_elem[1],center_elem[2],this->domain->sublo[0],this->domain->subhi[0]); //NP do not ask if the center is completely in, just ask for center_elem[dim] //NP this makes elements go around the corner if(center_elem[dim] >= checklo && center_elem[dim] < checkhi) { popElemFromBuffer(&(buf[m+1]),OPERATION_COMM_EXCHANGE,dummy,dummy,dummy); nLocal_++; /*NL*/// fprintf(this->screen,"proc %d pops element id %d with center %f %f %f\n", /*NL*/// this->comm->me,this->id_slow(nLocal_-1),center_elem[0],center_elem[1],center_elem[2]); } /*NL*/// else /*NL*/// fprintf(this->screen,"proc %d DID NOT pop one with center %f %f %f\n", /*NL*/// this->comm->me,center_elem[0],center_elem[1],center_elem[2]); /*NL*/// this->error->one(FLERR,"end"); m += nrecv_this; } } /* ---------------------------------------------------------------------- restart functionality - write all required data into restart buffer executed on all processes, but only proc 0 writes into writebuf ------------------------------------------------------------------------- */ template void MultiNodeMeshParallel::writeRestart(FILE *fp) { int size_this; // # elements int nlocal = this->sizeLocal(); int nglobal = sizeGlobal(); // buffer sizes int sizeMesh, sizeElements, sizeElements_all; sizeMesh = sizeRestartMesh(); sizeElements = nlocal * (sizeRestartElement() + 1); //NP need 1 extra space for pre-element buffer length double *bufMesh = NULL, *sendbufElems = NULL, *recvbufElems = NULL; bool dummy = false; /*NL*/ //fprintf(this->screen,"buffersize mesh %d buffersize elems local %d\n",sizeMesh,sizeElements); // pack global data into buffer // do this only on proc 0 if(this->comm->me == 0) { this->memory->create(bufMesh,sizeMesh,"MultiNodeMeshParallel::writeRestart:bufMesh"); pushMeshPropsToBuffer(bufMesh, OPERATION_RESTART,dummy,dummy,dummy); } // allocate send buffer and pack element data // all local elements are in list this->memory->create(sendbufElems,sizeElements,"MultiNodeMeshParallel::writeRestart:sendbuf"); sizeElements = 0; for(int i = 0; i < nlocal; i++) { size_this = pushElemToBuffer(i,&(sendbufElems[sizeElements+1]),OPERATION_RESTART,dummy,dummy,dummy); sendbufElems[sizeElements] = static_cast(size_this+1); sizeElements += (size_this+1); } // gather the pre-element data //NP send from all to proc 0 sizeElements_all = MPI_Gather0_Vector(sendbufElems,sizeElements,recvbufElems,this->world); /*NL*/ //fprintf(this->screen,"buffersize mesh %d buffersize elems local %d, buffersize elems global %d\n",sizeMesh,sizeElements, sizeElements_all); // actually write data to restart file // do this only on proc 0 if(this->comm->me == 0) { double nG = static_cast(nglobal); // for error check double sE = static_cast(sizeRestartElement()); double sM = static_cast(sizeRestartMesh()); /*NL*///fprintf(this->screen,"nglobal %f, sE %f, sM %f\n",nG,sE,sM); // size with 3 extra values int size = (sizeMesh+sizeElements_all+3) * sizeof(double); // write size fwrite(&size,sizeof(int),1,fp); // write 3 extra values fwrite(&nG,sizeof(double),1,fp); fwrite(&sE,sizeof(double),1,fp); fwrite(&sM,sizeof(double),1,fp); // write per-element and mesh data fwrite(recvbufElems,sizeof(double),sizeElements_all,fp); fwrite(bufMesh,sizeof(double),sizeMesh,fp); } // free mem if(bufMesh) this->memory->destroy(bufMesh); this->memory->destroy(sendbufElems); //NP need to use simple delete [] b/c MPI_Gather0_Vector uses new if(recvbufElems) delete []recvbufElems; } /* ---------------------------------------------------------------------- restart functionality - write all required data into restart buffer executed on all processes, but only proc 0 writes into writebuf ------------------------------------------------------------------------- */ template void MultiNodeMeshParallel::restart(double *list) { int m, nglobal, nrecv_this, nO, sE, sM; bool dummy = false; m = 0; nglobal = static_cast (list[m++]); sE = static_cast (list[m++]); sM = static_cast (list[m++]); /*NL*///fprintf(this->screen,"nglobal %d, sE %d, sM %d, sizeRestartElement() %d, sizeRestartMesh() %d\n",nglobal,sE,sM,sizeRestartElement(),sizeRestartMesh()); if(sE != sizeRestartElement() || sM != sizeRestartMesh()) this->error->all(FLERR,"Incompatible mesh restart file - mesh has different properties in restarted simulation"); for(int i = 0; i < nglobal; i++) { nrecv_this = static_cast(list[m]); /*NL*///fprintf(this->screen,"nrecv_this %d\n",nrecv_this); popElemFromBuffer(&(list[m+1]),OPERATION_RESTART,dummy,dummy,dummy); m += nrecv_this; } popMeshPropsFromBuffer(&list[m],OPERATION_RESTART,dummy,dummy,dummy); } /* ---------------------------------------------------------------------- size of restart data ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::sizeRestartMesh() { //NP get info about mesh movement to know what to communicate bool dummy = false; return meshPropsBufSize(OPERATION_RESTART,dummy,dummy,dummy); } template int MultiNodeMeshParallel::sizeRestartElement() { //NP get info about mesh movement to know what to communicate bool dummy = false; return elemBufSize(OPERATION_RESTART,dummy,dummy,dummy); } /*NP ---------------------------------------------------------------------- manual communication for this class, so lots of if / else statements here to evaluate operation flag need not do that for derived classes that implement their properties via TrackingMesh, for these cases the operation flag is evaluated in the container class on OPERATION_BORDERS, communicate center and rbound so dont have to refresh them ------------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- return required buffer size for a list of elements for borders(),forwardComm() must match push / pop implementation depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::elemListBufSize(int n,int operation,bool scale,bool translate,bool rotate) { return n*elemBufSize(operation,scale,translate,rotate); } /* ---------------------------------------------------------------------- push a list of elements for borders(), forwardComm() depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::pushElemListToBuffer(int n, int *list, double *buf,int operation,bool scale,bool translate,bool rotate) { /*NL*/// fprintf(this->screen,"pushListToBuffer n %d\n",n); /*NL*/// for(int ii = 0; ii < n; ii++) /*NL*/// fprintf(this->screen," list[%d] %d\n",ii,list[ii]); int nsend = 0; if(operation == OPERATION_COMM_EXCHANGE || operation == OPERATION_COMM_BORDERS) { //NP push center first to test against nsend += MultiNodeMesh::center_.pushElemListToBuffer(n,list,&(buf[nsend]),operation); nsend += MultiNodeMesh::node_.pushElemListToBuffer(n,list,&(buf[nsend]),operation); nsend += MultiNodeMesh::rBound_.pushElemListToBuffer(n,list,&(buf[nsend]),operation); if(this->node_orig_) nsend += this->node_orig_->pushElemListToBuffer(n,list,&(buf[nsend]),operation); return nsend; } if(operation == OPERATION_COMM_FORWARD) { /*NL*///fprintf(this->screen,"comm forward, translate is %s\n",translate?"yes":"no"); //NP node_orig cannot change during a run //NP currently proc updates owned and ghost elements for moving mesh //NP if(translate || rotate || scale) //NP nsend += MultiNodeMesh::node_.pushListToBuffer(n,list,&(buf[nsend]),operation); return nsend; } //NP OPERATION_RESTART not implemented, is per-element operation, not a list operation //NP OPERATION_COMM_REVERSE is implemented in pushElemListToBufferReverse this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::pushElemToBuffer"); return 0; } /* ---------------------------------------------------------------------- pop a list of elements for borders(), forwardComm() depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::popElemListFromBuffer(int first, int n, double *buf,int operation,bool scale,bool translate,bool rotate) { int nrecv = 0; //NP OPERATION_RESTART not implemented, is per-element operation, not a list operation if(operation == OPERATION_COMM_EXCHANGE || operation == OPERATION_COMM_BORDERS) { nrecv += MultiNodeMesh::center_.popElemListFromBuffer(first,n,&(buf[nrecv]),operation); nrecv += MultiNodeMesh::node_.popElemListFromBuffer(first,n,&(buf[nrecv]),operation); nrecv += MultiNodeMesh::rBound_.popElemListFromBuffer(first,n,&(buf[nrecv]),operation); if(MultiNodeMesh::node_orig_) nrecv += MultiNodeMesh::node_orig_->popElemListFromBuffer(first,n,&(buf[nrecv]),operation); return nrecv; } if(operation == OPERATION_COMM_FORWARD) { //NP node_orig cannot change during a run //NP currently proc updates owned and ghost elements for moving mesh //NP if(translate || rotate || scale) // nrecv += MultiNodeMesh::node_.popListFromBuffer(first,n,&(buf[nrecv]),operation); return nrecv; } this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::popElemFromBuffer"); return 0; } /* ---------------------------------------------------------------------- push a list of elements for reverseComm() depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::pushElemListToBufferReverse(int first, int n, double *buf,int operation,bool scale,bool translate,bool rotate) { int nsend = 0; if(operation == OPERATION_COMM_REVERSE) { //NP no reverse comm here return nsend; } //NP other stuff implemented in pushElemListToBuffer this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::popElemFromBuffer"); return 0; } /* ---------------------------------------------------------------------- pop a list of elements for reverseComm() depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::popElemListFromBufferReverse(int n, int *list, double *buf,int operation,bool scale,bool translate,bool rotate) { int nrecv = 0; if(operation == OPERATION_COMM_REVERSE) { //NP no reverse comm here return nrecv; } //NP other stuff implemented in pushElemListToBuffer this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::popElemFromBuffer"); return 0; } /* ---------------------------------------------------------------------- return required buffer size for one element for exchange() must match push / pop implementation depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::elemBufSize(int operation,bool scale,bool translate,bool rotate) { int size_buf = 0; //NP need to implement for per-element and per-list calls here //NP since per-list = n * per_element if(operation == OPERATION_RESTART) { size_buf += MultiNodeMesh::node_.elemBufSize(); return size_buf; } if(operation == OPERATION_COMM_EXCHANGE || operation == OPERATION_COMM_BORDERS) { size_buf += MultiNodeMesh::center_.elemBufSize(); size_buf += MultiNodeMesh::node_.elemBufSize(); size_buf += MultiNodeMesh::rBound_.elemBufSize(); if(MultiNodeMesh::node_orig_) size_buf += MultiNodeMesh::node_orig_->elemBufSize(); return size_buf; } //NP OPERATION_COMM_FORWARD, OPERATION_COMM_REVSERSE are list operations, not per-element operations //NP need to implement this here since elemListBufSize() refers to here if(operation == OPERATION_COMM_FORWARD) { //NP node_orig cannot change during a run //NP if(translate || rotate || scale) //NP size_buf += MultiNodeMesh::node_.elemBufSize(); return size_buf; } if(operation == OPERATION_COMM_REVERSE) { //NP nothing todo here //NP need to implement this return size_buf; } this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::elemBufSize"); return 0; } /* ---------------------------------------------------------------------- push one element for exchange() depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::pushElemToBuffer(int i, double *buf,int operation,bool scale,bool translate,bool rotate) { int nsend = 0; //NP OPERATION_COMM_REVERSE is a list operation, not per-element operation if(operation == OPERATION_RESTART) { nsend += MultiNodeMesh::node_.pushElemToBuffer(i,&(buf[nsend]),operation); return nsend; } if(operation == OPERATION_COMM_EXCHANGE || operation == OPERATION_COMM_BORDERS) { //NP push center first to test against nsend += MultiNodeMesh::center_.pushElemToBuffer(i,&(buf[nsend]),operation); nsend += MultiNodeMesh::node_.pushElemToBuffer(i,&(buf[nsend]),operation); nsend += MultiNodeMesh::rBound_.pushElemToBuffer(i,&(buf[nsend]),operation); if(this->node_orig_) nsend += this->node_orig_->pushElemToBuffer(i,&(buf[nsend]),operation); return nsend; } //NP OPERATION_COMM_FORWARD, OPERATION_COMM_REVSERSE are list operations, not per-element operations this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::pushElemToBuffer"); return 0; } /* ---------------------------------------------------------------------- pop one element for exchange() depending on operation and if mesh scales, translates or rotates, different properties are communicated ------------------------------------------------------------------------- */ template int MultiNodeMeshParallel::popElemFromBuffer(double *buf,int operation,bool scale,bool translate,bool rotate) { int nrecv = 0; //NP OPERATION_COMM_REVERSE is a list operation, not per-element operation if(operation == OPERATION_RESTART) { MultiVectorContainer nodeTmp; //NP pop node info from buffer and call addElement so everything is initialized correctly nrecv += nodeTmp.popElemFromBuffer(&(buf[nrecv]),operation); this->addElement(nodeTmp.begin()[0]); //NP this->addElement() creates properties, calculates everything //NP also adds custom values for which restart data is available //NP these are now cleared and re-created with correct restart data //NP in TrackingMesh::popElemFromBuffer this->prop().deleteRestartElement(nLocal_-1,false,false,false); return nrecv; } if(operation == OPERATION_COMM_EXCHANGE || operation == OPERATION_COMM_BORDERS) { nrecv += MultiNodeMesh::center_.popElemFromBuffer(&(buf[nrecv]),operation); nrecv += MultiNodeMesh::node_.popElemFromBuffer(&(buf[nrecv]),operation); nrecv += MultiNodeMesh::rBound_.popElemFromBuffer(&(buf[nrecv]),operation); if(MultiNodeMesh::node_orig_) nrecv += MultiNodeMesh::node_orig_->popElemFromBuffer(&(buf[nrecv]),operation); return nrecv; } //NP OPERATION_COMM_FORWARD, OPERATION_COMM_REVSERSE are list operations, not per-element operations this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel::popElemFromBuffer"); return 0; } #endif