mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
mapDistribute non-blocking of non-contiguous data
This commit is contained in:
@ -34,7 +34,7 @@ Description
|
||||
#include "fvMesh.H"
|
||||
#include "volFields.H"
|
||||
#include "Time.H"
|
||||
#include "mapDistribute.H"
|
||||
//#include "mapDistribute.H"
|
||||
#include "OFstream.H"
|
||||
#include "meshTools.H"
|
||||
//#include "FECCellToFaceStencil.H"
|
||||
|
||||
@ -23,19 +23,23 @@ License
|
||||
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
Application
|
||||
icoFoam
|
||||
parallelTest
|
||||
|
||||
Description
|
||||
Incompressible laminar CFD code.
|
||||
Test for various parallel routines.
|
||||
|
||||
\*---------------------------------------------------------------------------*/
|
||||
|
||||
#include "List.H"
|
||||
#include "mapDistribute.H"
|
||||
#include "argList.H"
|
||||
#include "Time.H"
|
||||
#include "IPstream.H"
|
||||
#include "OPstream.H"
|
||||
#include "vector.H"
|
||||
#include "IOstreams.H"
|
||||
#include "Random.H"
|
||||
#include "Tuple2.H"
|
||||
|
||||
using namespace Foam;
|
||||
|
||||
@ -47,6 +51,99 @@ int main(int argc, char *argv[])
|
||||
# include "setRootCase.H"
|
||||
# include "createTime.H"
|
||||
|
||||
|
||||
// Test mapDistribute
|
||||
// ~~~~~~~~~~~~~~~~~~
|
||||
|
||||
if (false)
|
||||
{
|
||||
Random rndGen(43544*Pstream::myProcNo());
|
||||
|
||||
// Generate random data.
|
||||
List<Tuple2<label, List<scalar> > > complexData(100);
|
||||
forAll(complexData, i)
|
||||
{
|
||||
complexData[i].first() = rndGen.integer(0, Pstream::nProcs()-1);
|
||||
complexData[i].second().setSize(3);
|
||||
complexData[i].second()[0] = 1;
|
||||
complexData[i].second()[1] = 2;
|
||||
complexData[i].second()[2] = 3;
|
||||
}
|
||||
|
||||
// Send all ones to processor indicated by .first()
|
||||
|
||||
|
||||
// Count how many to send
|
||||
labelList nSend(Pstream::nProcs(), 0);
|
||||
forAll(complexData, i)
|
||||
{
|
||||
label procI = complexData[i].first();
|
||||
nSend[procI]++;
|
||||
}
|
||||
|
||||
// Sync how many to send
|
||||
labelListList allNTrans(Pstream::nProcs());
|
||||
allNTrans[Pstream::myProcNo()] = nSend;
|
||||
combineReduce(allNTrans, mapDistribute::listEq());
|
||||
|
||||
// Collect items to be sent
|
||||
labelListList sendMap(Pstream::nProcs());
|
||||
forAll(sendMap, procI)
|
||||
{
|
||||
sendMap[procI].setSize(nSend[procI]);
|
||||
}
|
||||
nSend = 0;
|
||||
forAll(complexData, i)
|
||||
{
|
||||
label procI = complexData[i].first();
|
||||
sendMap[procI][nSend[procI]++] = i;
|
||||
}
|
||||
|
||||
// Collect items to be received
|
||||
labelListList recvMap(Pstream::nProcs());
|
||||
forAll(recvMap, procI)
|
||||
{
|
||||
recvMap[procI].setSize(allNTrans[procI][Pstream::myProcNo()]);
|
||||
}
|
||||
|
||||
label constructSize = 0;
|
||||
// Construct with my own elements first
|
||||
forAll(recvMap[Pstream::myProcNo()], i)
|
||||
{
|
||||
recvMap[Pstream::myProcNo()][i] = constructSize++;
|
||||
}
|
||||
// Construct from other processors
|
||||
forAll(recvMap, procI)
|
||||
{
|
||||
if (procI != Pstream::myProcNo())
|
||||
{
|
||||
forAll(recvMap[procI], i)
|
||||
{
|
||||
recvMap[procI][i] = constructSize++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Construct distribute map (destructively)
|
||||
mapDistribute map(constructSize, sendMap.xfer(), recvMap.xfer());
|
||||
|
||||
// Distribute complexData
|
||||
mapDistribute::distribute
|
||||
(
|
||||
Pstream::nonBlocking,
|
||||
List<labelPair>(),
|
||||
map.constructSize(),
|
||||
map.subMap(),
|
||||
map.constructMap(),
|
||||
complexData
|
||||
);
|
||||
|
||||
Pout<< "complexData:" << complexData << endl;
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
||||
|
||||
Perr<< "\nStarting transfers\n" << endl;
|
||||
@ -60,13 +157,13 @@ int main(int argc, char *argv[])
|
||||
{
|
||||
Perr<< "slave sending to master "
|
||||
<< Pstream::masterNo() << endl;
|
||||
OPstream toMaster(Pstream::masterNo());
|
||||
OPstream toMaster(Pstream::blocking, Pstream::masterNo());
|
||||
toMaster << data;
|
||||
}
|
||||
|
||||
Perr<< "slave receiving from master "
|
||||
<< Pstream::masterNo() << endl;
|
||||
IPstream fromMaster(Pstream::masterNo());
|
||||
IPstream fromMaster(Pstream::blocking, Pstream::masterNo());
|
||||
fromMaster >> data;
|
||||
|
||||
Perr<< data << endl;
|
||||
@ -81,7 +178,7 @@ int main(int argc, char *argv[])
|
||||
)
|
||||
{
|
||||
Perr << "master receiving from slave " << slave << endl;
|
||||
IPstream fromSlave(slave);
|
||||
IPstream fromSlave(Pstream::blocking, slave);
|
||||
fromSlave >> data;
|
||||
|
||||
Perr<< data << endl;
|
||||
@ -95,7 +192,7 @@ int main(int argc, char *argv[])
|
||||
)
|
||||
{
|
||||
Perr << "master sending to slave " << slave << endl;
|
||||
OPstream toSlave(slave);
|
||||
OPstream toSlave(Pstream::blocking, slave);
|
||||
toSlave << data;
|
||||
}
|
||||
}
|
||||
|
||||
@ -172,8 +172,8 @@ const Foam::List<Foam::labelPair>& Foam::mapDistribute::schedule() const
|
||||
Foam::mapDistribute::mapDistribute
|
||||
(
|
||||
const label constructSize,
|
||||
const labelListList& subMap,
|
||||
const labelListList& constructMap
|
||||
const Xfer<labelListList>& subMap,
|
||||
const Xfer<labelListList>& constructMap
|
||||
)
|
||||
:
|
||||
constructSize_(constructSize),
|
||||
@ -183,22 +183,6 @@ Foam::mapDistribute::mapDistribute
|
||||
{}
|
||||
|
||||
|
||||
//- (optionally destructively) construct from components
|
||||
Foam::mapDistribute::mapDistribute
|
||||
(
|
||||
const label constructSize,
|
||||
labelListList& subMap,
|
||||
labelListList& constructMap,
|
||||
const bool reUse // clone or reuse
|
||||
)
|
||||
:
|
||||
constructSize_(constructSize),
|
||||
subMap_(subMap, reUse),
|
||||
constructMap_(constructMap, reUse),
|
||||
schedulePtr_()
|
||||
{}
|
||||
|
||||
|
||||
Foam::mapDistribute::mapDistribute
|
||||
(
|
||||
const labelList& sendProcs,
|
||||
|
||||
@ -83,23 +83,36 @@ class mapDistribute
|
||||
|
||||
public:
|
||||
|
||||
// Public classes
|
||||
|
||||
//- combineReduce operator for lists. Used for counting.
|
||||
class listEq
|
||||
{
|
||||
|
||||
public:
|
||||
|
||||
template<class T>
|
||||
void operator()(T& x, const T& y) const
|
||||
{
|
||||
forAll(y, i)
|
||||
{
|
||||
if (y[i].size())
|
||||
{
|
||||
x[i] = y[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Constructors
|
||||
|
||||
//- Construct from components
|
||||
mapDistribute
|
||||
(
|
||||
const label constructSize,
|
||||
const labelListList& subMap,
|
||||
const labelListList& constructMap
|
||||
);
|
||||
|
||||
//- (optionally destructively) construct from components
|
||||
mapDistribute
|
||||
(
|
||||
const label constructSize,
|
||||
labelListList& subMap,
|
||||
labelListList& constructMap,
|
||||
const bool reUse // clone or reuse
|
||||
const Xfer<labelListList>& subMap,
|
||||
const Xfer<labelListList>& constructMap
|
||||
);
|
||||
|
||||
//- Construct from reverse addressing: per data item the send
|
||||
@ -205,11 +218,7 @@ public:
|
||||
template<class T>
|
||||
void distribute(List<T>& fld) const
|
||||
{
|
||||
if
|
||||
(
|
||||
Pstream::defaultCommsType == Pstream::nonBlocking
|
||||
&& contiguous<T>()
|
||||
)
|
||||
if (Pstream::defaultCommsType == Pstream::nonBlocking)
|
||||
{
|
||||
distribute
|
||||
(
|
||||
|
||||
@ -68,35 +68,15 @@ public:
|
||||
mapDistributeLagrangian
|
||||
(
|
||||
const label nNewParticles,
|
||||
const labelListList& subParticleMap,
|
||||
const labelListList& constructParticleMap,
|
||||
const labelListList& constructCellLabels
|
||||
const Xfer<labelListList>& subParticleMap,
|
||||
const Xfer<labelListList>& constructParticleMap,
|
||||
const Xfer<labelListList>& constructCellLabels
|
||||
)
|
||||
:
|
||||
particleMap_(nNewParticles, subParticleMap, constructParticleMap),
|
||||
constructCellLabels_(constructCellLabels)
|
||||
{}
|
||||
|
||||
//- Construct from components and steal storage
|
||||
mapDistributeLagrangian
|
||||
(
|
||||
const label nNewParticles,
|
||||
labelListList& subParticleMap,
|
||||
labelListList& constructParticleMap,
|
||||
labelListList& constructCellLabels,
|
||||
const bool reUse
|
||||
)
|
||||
:
|
||||
particleMap_
|
||||
(
|
||||
nNewParticles,
|
||||
subParticleMap,
|
||||
constructParticleMap,
|
||||
reUse
|
||||
),
|
||||
constructCellLabels_(constructCellLabels, reUse)
|
||||
{}
|
||||
|
||||
|
||||
// Member Functions
|
||||
|
||||
|
||||
@ -66,27 +66,27 @@ Foam::mapDistributePolyMesh::mapDistributePolyMesh
|
||||
const label nOldPoints,
|
||||
const label nOldFaces,
|
||||
const label nOldCells,
|
||||
const labelList& oldPatchStarts,
|
||||
const labelList& oldPatchNMeshPoints,
|
||||
const Xfer<labelList>& oldPatchStarts,
|
||||
const Xfer<labelList>& oldPatchNMeshPoints,
|
||||
|
||||
// how to subset pieces of mesh to send across
|
||||
const labelListList& subPointMap,
|
||||
const labelListList& subFaceMap,
|
||||
const labelListList& subCellMap,
|
||||
const labelListList& subPatchMap,
|
||||
const Xfer<labelListList>& subPointMap,
|
||||
const Xfer<labelListList>& subFaceMap,
|
||||
const Xfer<labelListList>& subCellMap,
|
||||
const Xfer<labelListList>& subPatchMap,
|
||||
|
||||
// how to reconstruct received mesh
|
||||
const labelListList& constructPointMap,
|
||||
const labelListList& constructFaceMap,
|
||||
const labelListList& constructCellMap,
|
||||
const labelListList& constructPatchMap
|
||||
const Xfer<labelListList>& constructPointMap,
|
||||
const Xfer<labelListList>& constructFaceMap,
|
||||
const Xfer<labelListList>& constructCellMap,
|
||||
const Xfer<labelListList>& constructPatchMap
|
||||
)
|
||||
:
|
||||
mesh_(mesh),
|
||||
nOldPoints_(nOldPoints),
|
||||
nOldFaces_(nOldFaces),
|
||||
nOldCells_(nOldCells),
|
||||
oldPatchSizes_(oldPatchStarts.size()),
|
||||
oldPatchSizes_(oldPatchStarts().size()),
|
||||
oldPatchStarts_(oldPatchStarts),
|
||||
oldPatchNMeshPoints_(oldPatchNMeshPoints),
|
||||
pointMap_(mesh.nPoints(), subPointMap, constructPointMap),
|
||||
@ -98,44 +98,6 @@ Foam::mapDistributePolyMesh::mapDistributePolyMesh
|
||||
}
|
||||
|
||||
|
||||
//- (optionally destructively) construct from components
|
||||
Foam::mapDistributePolyMesh::mapDistributePolyMesh
|
||||
(
|
||||
const polyMesh& mesh,
|
||||
const label nOldPoints,
|
||||
const label nOldFaces,
|
||||
const label nOldCells,
|
||||
labelList& oldPatchStarts,
|
||||
labelList& oldPatchNMeshPoints,
|
||||
|
||||
labelListList& subPointMap,
|
||||
labelListList& subFaceMap,
|
||||
labelListList& subCellMap,
|
||||
labelListList& subPatchMap,
|
||||
labelListList& constructPointMap,
|
||||
labelListList& constructFaceMap,
|
||||
labelListList& constructCellMap,
|
||||
labelListList& constructPatchMap,
|
||||
const bool reUse // clone or reuse
|
||||
)
|
||||
:
|
||||
mesh_(mesh),
|
||||
nOldPoints_(nOldPoints),
|
||||
nOldFaces_(nOldFaces),
|
||||
nOldCells_(nOldCells),
|
||||
oldPatchSizes_(oldPatchStarts.size()),
|
||||
oldPatchStarts_(oldPatchStarts, reUse),
|
||||
oldPatchNMeshPoints_(oldPatchNMeshPoints, reUse),
|
||||
|
||||
pointMap_(mesh.nPoints(), subPointMap, constructPointMap, reUse),
|
||||
faceMap_(mesh.nFaces(), subFaceMap, constructFaceMap, reUse),
|
||||
cellMap_(mesh.nCells(), subCellMap, constructCellMap, reUse),
|
||||
patchMap_(mesh.boundaryMesh().size(), subPatchMap, constructPatchMap, reUse)
|
||||
{
|
||||
calcPatchSizes();
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
|
||||
|
||||
void Foam::mapDistributePolyMesh::distributePointIndices(labelList& lst) const
|
||||
|
||||
@ -120,42 +120,20 @@ public:
|
||||
const label nOldPoints,
|
||||
const label nOldFaces,
|
||||
const label nOldCells,
|
||||
const labelList& oldPatchStarts,
|
||||
const labelList& oldPatchNMeshPoints,
|
||||
const Xfer<labelList>& oldPatchStarts,
|
||||
const Xfer<labelList>& oldPatchNMeshPoints,
|
||||
|
||||
// how to subset pieces of mesh to send across
|
||||
const labelListList& subPointMap,
|
||||
const labelListList& subFaceMap,
|
||||
const labelListList& subCellMap,
|
||||
const labelListList& subPatchMap,
|
||||
const Xfer<labelListList>& subPointMap,
|
||||
const Xfer<labelListList>& subFaceMap,
|
||||
const Xfer<labelListList>& subCellMap,
|
||||
const Xfer<labelListList>& subPatchMap,
|
||||
|
||||
// how to reconstruct received mesh
|
||||
const labelListList& constructPointMap,
|
||||
const labelListList& constructFaceMap,
|
||||
const labelListList& constructCellMap,
|
||||
const labelListList& constructPatchMap
|
||||
);
|
||||
|
||||
//- (optionally destructively) construct from components
|
||||
// Note that mesh has to be changed already!
|
||||
mapDistributePolyMesh
|
||||
(
|
||||
const polyMesh& mesh,
|
||||
const label nOldPoints,
|
||||
const label nOldFaces,
|
||||
const label nOldCells,
|
||||
labelList& oldPatchStarts,
|
||||
labelList& oldPatchNMeshPoints,
|
||||
|
||||
labelListList& subPointMap,
|
||||
labelListList& subFaceMap,
|
||||
labelListList& subCellMap,
|
||||
labelListList& subPatchMap,
|
||||
labelListList& constructPointMap,
|
||||
labelListList& constructFaceMap,
|
||||
labelListList& constructCellMap,
|
||||
labelListList& constructPatchMap,
|
||||
const bool reUse // clone or reuse
|
||||
const Xfer<labelListList>& constructPointMap,
|
||||
const Xfer<labelListList>& constructFaceMap,
|
||||
const Xfer<labelListList>& constructCellMap,
|
||||
const Xfer<labelListList>& constructPatchMap
|
||||
);
|
||||
|
||||
|
||||
|
||||
@ -25,6 +25,7 @@ License
|
||||
\*---------------------------------------------------------------------------*/
|
||||
|
||||
#include "Pstream.H"
|
||||
#include "PstreamCombineReduceOps.H"
|
||||
|
||||
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
|
||||
|
||||
@ -183,6 +184,122 @@ void Foam::mapDistribute::distribute
|
||||
else if (commsType == Pstream::nonBlocking)
|
||||
{
|
||||
if (!contiguous<T>())
|
||||
{
|
||||
// 1. convert to contiguous buffer
|
||||
// 2. send buffer
|
||||
// 3. receive buffer
|
||||
// 4. read from buffer into List<T>
|
||||
|
||||
List<List<char> > sendFields(Pstream::nProcs());
|
||||
labelListList allNTrans(Pstream::nProcs());
|
||||
labelList& nsTransPs = allNTrans[Pstream::myProcNo()];
|
||||
nsTransPs.setSize(Pstream::nProcs(), 0);
|
||||
|
||||
// Stream data into sendField buffers
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = subMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
// Put data into send buffer
|
||||
OPstream toDomain(Pstream::nonBlocking, domain);
|
||||
toDomain << UIndirectList<T>(field, map);
|
||||
|
||||
// Store the size
|
||||
nsTransPs[domain] = toDomain.bufPosition();
|
||||
|
||||
// Transfer buffer out
|
||||
sendFields[domain].transfer(toDomain.buf());
|
||||
toDomain.bufPosition() = 0;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Send sizes across
|
||||
combineReduce(allNTrans, listEq());
|
||||
|
||||
// Start sending buffers
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = subMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
OPstream::write
|
||||
(
|
||||
Pstream::nonBlocking,
|
||||
domain,
|
||||
reinterpret_cast<const char*>
|
||||
(
|
||||
sendFields[domain].begin()
|
||||
),
|
||||
nsTransPs[domain]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Set up receives from neighbours
|
||||
|
||||
PtrList<IPstream> fromSlave(Pstream::nProcs());
|
||||
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = constructMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
// Start receiving
|
||||
fromSlave.set
|
||||
(
|
||||
domain,
|
||||
new IPstream
|
||||
(
|
||||
Pstream::nonBlocking,
|
||||
domain,
|
||||
allNTrans[domain][Pstream::myProcNo()]
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
// Set up 'send' to myself
|
||||
const labelList& mySubMap = subMap[Pstream::myProcNo()];
|
||||
List<T> mySubField(mySubMap.size());
|
||||
forAll(mySubMap, i)
|
||||
{
|
||||
mySubField[i] = field[mySubMap[i]];
|
||||
}
|
||||
// Combine bits. Note that can reuse field storage
|
||||
field.setSize(constructSize);
|
||||
// Receive sub field from myself
|
||||
{
|
||||
const labelList& map = constructMap[Pstream::myProcNo()];
|
||||
|
||||
forAll(map, i)
|
||||
{
|
||||
field[map[i]] = mySubField[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Wait till all finished
|
||||
IPstream::waitRequests();
|
||||
OPstream::waitRequests();
|
||||
|
||||
// Consume
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = constructMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
List<T> recvField(fromSlave[domain]);
|
||||
|
||||
if (recvField.size() != map.size())
|
||||
{
|
||||
FatalErrorIn
|
||||
(
|
||||
@ -196,10 +313,24 @@ void Foam::mapDistribute::distribute
|
||||
" const labelListList& constructMap,\n"
|
||||
" List<T>& field\n"
|
||||
")\n"
|
||||
) << "Non-blocking only supported for contiguous data."
|
||||
<< exit(FatalError);
|
||||
) << "Expected from processor " << domain
|
||||
<< " " << map.size() << " but received "
|
||||
<< recvField.size() << " elements."
|
||||
<< abort(FatalError);
|
||||
}
|
||||
|
||||
forAll(map, i)
|
||||
{
|
||||
field[map[i]] = recvField[i];
|
||||
}
|
||||
|
||||
// Delete receive buffer
|
||||
fromSlave.set(domain, NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Set up sends to neighbours
|
||||
|
||||
List<List<T > > sendFields(Pstream::nProcs());
|
||||
@ -222,7 +353,7 @@ void Foam::mapDistribute::distribute
|
||||
Pstream::nonBlocking,
|
||||
domain,
|
||||
reinterpret_cast<const char*>(subField.begin()),
|
||||
subField.size()*sizeof(T)
|
||||
subField.byteSize()
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -243,7 +374,7 @@ void Foam::mapDistribute::distribute
|
||||
Pstream::nonBlocking,
|
||||
domain,
|
||||
reinterpret_cast<char*>(recvFields[domain].begin()),
|
||||
recvFields[domain].size()*sizeof(T)
|
||||
recvFields[domain].byteSize()
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -320,6 +451,7 @@ void Foam::mapDistribute::distribute
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
FatalErrorIn("mapDistribute::distribute(..)")
|
||||
@ -487,6 +619,117 @@ void Foam::mapDistribute::distribute
|
||||
else if (commsType == Pstream::nonBlocking)
|
||||
{
|
||||
if (!contiguous<T>())
|
||||
{
|
||||
// 1. convert to contiguous buffer
|
||||
// 2. send buffer
|
||||
// 3. receive buffer
|
||||
// 4. read from buffer into List<T>
|
||||
|
||||
List<List<char> > sendFields(Pstream::nProcs());
|
||||
labelListList allNTrans(Pstream::nProcs());
|
||||
labelList& nsTransPs = allNTrans[Pstream::myProcNo()];
|
||||
nsTransPs.setSize(Pstream::nProcs());
|
||||
|
||||
// Stream data into sendField buffers
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = subMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
// Put data into send buffer
|
||||
OPstream toDomain(Pstream::nonBlocking, domain);
|
||||
toDomain << UIndirectList<T>(field, map);
|
||||
|
||||
// Store the size
|
||||
nsTransPs[domain] = toDomain.bufPosition();
|
||||
|
||||
// Transfer buffer out
|
||||
sendFields[domain].transfer(toDomain.buf());
|
||||
toDomain.bufPosition() = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Send sizes across
|
||||
combineReduce(allNTrans, listEq());
|
||||
|
||||
// Start sending buffers
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = subMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
OPstream::write
|
||||
(
|
||||
Pstream::nonBlocking,
|
||||
domain,
|
||||
reinterpret_cast<const char*>
|
||||
(
|
||||
sendFields[domain].begin()
|
||||
),
|
||||
nsTransPs[domain]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Set up receives from neighbours
|
||||
|
||||
PtrList<IPstream> fromSlave(Pstream::nProcs());
|
||||
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = constructMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
// Start receiving
|
||||
fromSlave.set
|
||||
(
|
||||
domain,
|
||||
new IPstream
|
||||
(
|
||||
Pstream::nonBlocking,
|
||||
domain,
|
||||
allNTrans[domain][Pstream::myProcNo()]
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
// Set up 'send' to myself
|
||||
List<T> mySubField(field, subMap[Pstream::myProcNo()]);
|
||||
// Combine bits. Note that can reuse field storage
|
||||
field.setSize(constructSize);
|
||||
field = nullValue;
|
||||
// Receive sub field from myself
|
||||
{
|
||||
const labelList& map = constructMap[Pstream::myProcNo()];
|
||||
|
||||
forAll(map, i)
|
||||
{
|
||||
cop(field[map[i]], mySubField[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Wait till all finished
|
||||
IPstream::waitRequests();
|
||||
OPstream::waitRequests();
|
||||
|
||||
// Consume
|
||||
for (label domain = 0; domain < Pstream::nProcs(); domain++)
|
||||
{
|
||||
const labelList& map = constructMap[domain];
|
||||
|
||||
if (domain != Pstream::myProcNo() && map.size())
|
||||
{
|
||||
List<T> recvField(fromSlave[domain]);
|
||||
|
||||
if (recvField.size() != map.size())
|
||||
{
|
||||
FatalErrorIn
|
||||
(
|
||||
@ -500,10 +743,24 @@ void Foam::mapDistribute::distribute
|
||||
" const labelListList& constructMap,\n"
|
||||
" List<T>& field\n"
|
||||
")\n"
|
||||
) << "Non-blocking only supported for contiguous data."
|
||||
<< exit(FatalError);
|
||||
) << "Expected from processor " << domain
|
||||
<< " " << map.size() << " but received "
|
||||
<< recvField.size() << " elements."
|
||||
<< abort(FatalError);
|
||||
}
|
||||
|
||||
forAll(map, i)
|
||||
{
|
||||
cop(field[map[i]], recvField[i]);
|
||||
}
|
||||
|
||||
// Delete receive buffer
|
||||
fromSlave.set(domain, NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Set up sends to neighbours
|
||||
|
||||
List<List<T > > sendFields(Pstream::nProcs());
|
||||
@ -623,6 +880,7 @@ void Foam::mapDistribute::distribute
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
FatalErrorIn("mapDistribute::distribute(..)")
|
||||
|
||||
@ -28,12 +28,9 @@ License
|
||||
#include "fvMesh.H"
|
||||
#include "Time.H"
|
||||
#include "boundBox.H"
|
||||
#include "globalIndex.H"
|
||||
#include "wallPolyPatch.H"
|
||||
#include "mapDistributePolyMesh.H"
|
||||
#include "cellSet.H"
|
||||
#include "syncTools.H"
|
||||
#include "motionSmoother.H"
|
||||
#include "refinementParameters.H"
|
||||
#include "snapParameters.H"
|
||||
#include "layerParameters.H"
|
||||
|
||||
@ -28,8 +28,6 @@ License
|
||||
#include "meshRefinement.H"
|
||||
#include "fvMesh.H"
|
||||
#include "Time.H"
|
||||
#include "boundBox.H"
|
||||
#include "mapDistributePolyMesh.H"
|
||||
#include "cellSet.H"
|
||||
#include "syncTools.H"
|
||||
#include "refinementParameters.H"
|
||||
|
||||
@ -1418,18 +1418,18 @@ Foam::autoPtr<Foam::mapDistributePolyMesh> Foam::fvMeshDistribute::distribute
|
||||
nOldPoints,
|
||||
nOldFaces,
|
||||
nOldCells,
|
||||
oldPatchStarts,
|
||||
oldPatchNMeshPoints,
|
||||
oldPatchStarts.xfer(),
|
||||
oldPatchNMeshPoints.xfer(),
|
||||
|
||||
labelListList(1, identity(mesh_.nPoints())),//subPointMap
|
||||
labelListList(1, identity(mesh_.nFaces())), //subFaceMap
|
||||
labelListList(1, identity(mesh_.nCells())), //subCellMap
|
||||
labelListList(1, identity(patches.size())), //subPatchMap
|
||||
labelListList(1, identity(mesh_.nPoints())).xfer(),//subPointMap
|
||||
labelListList(1, identity(mesh_.nFaces())).xfer(), //subFaceMap
|
||||
labelListList(1, identity(mesh_.nCells())).xfer(), //subCellMap
|
||||
labelListList(1, identity(patches.size())).xfer(), //subPatchMap
|
||||
|
||||
labelListList(1, identity(mesh_.nPoints())),//constructPointMap
|
||||
labelListList(1, identity(mesh_.nFaces())), //constructFaceMap
|
||||
labelListList(1, identity(mesh_.nCells())), //constructCellMap
|
||||
labelListList(1, identity(patches.size())) //constructPatchMap
|
||||
labelListList(1, identity(mesh_.nPoints())).xfer(),//pointMap
|
||||
labelListList(1, identity(mesh_.nFaces())).xfer(), //faceMap
|
||||
labelListList(1, identity(mesh_.nCells())).xfer(), //cellMap
|
||||
labelListList(1, identity(patches.size())).xfer() //patchMap
|
||||
)
|
||||
);
|
||||
}
|
||||
@ -2207,19 +2207,18 @@ Foam::autoPtr<Foam::mapDistributePolyMesh> Foam::fvMeshDistribute::distribute
|
||||
nOldPoints,
|
||||
nOldFaces,
|
||||
nOldCells,
|
||||
oldPatchStarts,
|
||||
oldPatchNMeshPoints,
|
||||
oldPatchStarts.xfer(),
|
||||
oldPatchNMeshPoints.xfer(),
|
||||
|
||||
subPointMap,
|
||||
subFaceMap,
|
||||
subCellMap,
|
||||
subPatchMap,
|
||||
subPointMap.xfer(),
|
||||
subFaceMap.xfer(),
|
||||
subCellMap.xfer(),
|
||||
subPatchMap.xfer(),
|
||||
|
||||
constructPointMap,
|
||||
constructFaceMap,
|
||||
constructCellMap,
|
||||
constructPatchMap,
|
||||
true // reuse storage
|
||||
constructPointMap.xfer(),
|
||||
constructFaceMap.xfer(),
|
||||
constructCellMap.xfer(),
|
||||
constructPatchMap.xfer()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@ -280,9 +280,8 @@ Foam::extendedCellToFaceStencil::calcDistributeMap
|
||||
new mapDistribute
|
||||
(
|
||||
nCompact,
|
||||
sendCompact,
|
||||
recvCompact,
|
||||
true // reuse send/recv maps.
|
||||
sendCompact.xfer(),
|
||||
recvCompact.xfer()
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
@ -340,9 +340,8 @@ Foam::distributedTriSurfaceMesh::distributeSegments
|
||||
new mapDistribute
|
||||
(
|
||||
segmentI, // size after construction
|
||||
sendMap,
|
||||
constructMap,
|
||||
true // reuse storage
|
||||
sendMap.xfer(),
|
||||
constructMap.xfer()
|
||||
)
|
||||
);
|
||||
}
|
||||
@ -642,9 +641,8 @@ Foam::distributedTriSurfaceMesh::calcLocalQueries
|
||||
new mapDistribute
|
||||
(
|
||||
segmentI, // size after construction
|
||||
sendMap,
|
||||
constructMap,
|
||||
true // reuse storage
|
||||
sendMap.xfer(),
|
||||
constructMap.xfer()
|
||||
)
|
||||
);
|
||||
const mapDistribute& map = mapPtr();
|
||||
@ -806,9 +804,8 @@ Foam::distributedTriSurfaceMesh::calcLocalQueries
|
||||
new mapDistribute
|
||||
(
|
||||
segmentI, // size after construction
|
||||
sendMap,
|
||||
constructMap,
|
||||
true // reuse storage
|
||||
sendMap.xfer(),
|
||||
constructMap.xfer()
|
||||
)
|
||||
);
|
||||
return mapPtr;
|
||||
@ -2399,9 +2396,8 @@ void Foam::distributedTriSurfaceMesh::distribute
|
||||
new mapDistribute
|
||||
(
|
||||
allTris.size(),
|
||||
faceSendMap,
|
||||
faceConstructMap,
|
||||
true
|
||||
faceSendMap.xfer(),
|
||||
faceConstructMap.xfer()
|
||||
)
|
||||
);
|
||||
pointMap.reset
|
||||
@ -2409,9 +2405,8 @@ void Foam::distributedTriSurfaceMesh::distribute
|
||||
new mapDistribute
|
||||
(
|
||||
allPoints.size(),
|
||||
pointSendMap,
|
||||
pointConstructMap,
|
||||
true
|
||||
pointSendMap.xfer(),
|
||||
pointConstructMap.xfer()
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user