zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: avoid prior communication when using mapDistribute

Browse Source 
- in most cases can simply construct mapDistribute with the sendMap
  and have it take care of communication and addressing for the
  corresponding constructMap.

  This removes code duplication, which in some cases was also using
  much less efficient mechanisms (eg, combineReduce on list of
  lists, or an allGatherList on the send sizes etc) and also
  reduces the number of places where Pstream::exchange/exchangeSizes
  is being called.

ENH: reduce communication in turbulentDFSEMInlet

- was doing an allGatherList to populate a mapDistribute.
  Now simply use PstreamBuffers mechanisms directly.
This commit is contained in:
Mark Olesen
2023-02-14 11:24:22 +01:00
parent 1ce7a62209
commit fb69a54bc3
12 changed files with 105 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
applications/test/parallel/Test-parallel.C
View File

@ -67,54 +67,24 @@ void testMapDistribute()
labelList nSend(Pstream::nProcs(), Zero);
forAll(complexData, i)
{
const label procI = complexData[i].first();
nSend[procI]++;
const label proci = complexData[i].first();
nSend[proci]++;
}
// Collect items to be sent
labelListList sendMap(Pstream::nProcs());
forAll(sendMap, procI)
forAll(sendMap, proci)
{
sendMap[procI].setSize(nSend[procI]);
sendMap[proci].resize_nocopy(nSend[proci]);
nSend[proci] = 0;
}
nSend = 0;
forAll(complexData, i)
{
const label procI = complexData[i].first();
sendMap[procI][nSend[procI]++] = i;
const label proci = complexData[i].first();
sendMap[proci][nSend[proci]++] = i;
}
// Sync how many to send
labelList nRecv;
Pstream::exchangeSizes(sendMap, nRecv);
// Collect items to be received
labelListList recvMap(Pstream::nProcs());
forAll(recvMap, procI)
{
recvMap[procI].setSize(nRecv[procI]);
}
label constructSize = 0;
// Construct with my own elements first
forAll(recvMap[Pstream::myProcNo()], i)
{
recvMap[Pstream::myProcNo()][i] = constructSize++;
}
// Construct from other processors
forAll(recvMap, procI)
{
if (procI != Pstream::myProcNo())
{
forAll(recvMap[procI], i)
{
recvMap[procI][i] = constructSize++;
}
}
}
// Construct distribute map (destructively)
mapDistribute map(constructSize, std::move(sendMap), std::move(recvMap));
mapDistribute map(std::move(sendMap));
// Distribute complexData
map.distribute(complexData);

45
applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/DelaunayMesh/DistributedDelaunayMesh.C
View File

@ -54,7 +54,6 @@ Foam::DistributedDelaunayMesh<Triangulation>::buildMap
forAll(toProc, i)
{
label proci = toProc[i];
nSend[proci]++;
}
@ -64,8 +63,7 @@ Foam::DistributedDelaunayMesh<Triangulation>::buildMap
forAll(nSend, proci)
{
sendMap[proci].setSize(nSend[proci]);
sendMap[proci].resize_nocopy(nSend[proci]);
nSend[proci] = 0;
}
@ -73,49 +71,10 @@ Foam::DistributedDelaunayMesh<Triangulation>::buildMap
forAll(toProc, i)
{
label proci = toProc[i];
sendMap[proci][nSend[proci]++] = i;
}
// 4. Send over how many I need to receive
labelList recvSizes;
Pstream::exchangeSizes(sendMap, recvSizes);
// Determine receive map
// ~~~~~~~~~~~~~~~~~~~~~
labelListList constructMap(Pstream::nProcs());
// Local transfers first
constructMap[Pstream::myProcNo()] = identity
(
sendMap[Pstream::myProcNo()].size()
);
label constructSize = constructMap[Pstream::myProcNo()].size();
forAll(constructMap, proci)
{
if (proci != Pstream::myProcNo())
{
label nRecv = recvSizes[proci];
constructMap[proci].setSize(nRecv);
for (label i = 0; i < nRecv; i++)
{
constructMap[proci][i] = constructSize++;
}
}
}
return autoPtr<mapDistribute>::New
(
constructSize,
std::move(sendMap),
std::move(constructMap)
);
return autoPtr<mapDistribute>::New(std::move(sendMap));
}

45
applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/backgroundMeshDecomposition/backgroundMeshDecomposition.C
View File

@ -59,7 +59,6 @@ Foam::autoPtr<Foam::mapDistribute> Foam::backgroundMeshDecomposition::buildMap
forAll(toProc, i)
{
label proci = toProc[i];
nSend[proci]++;
}
@ -69,8 +68,7 @@ Foam::autoPtr<Foam::mapDistribute> Foam::backgroundMeshDecomposition::buildMap
forAll(nSend, proci)
{
sendMap[proci].setSize(nSend[proci]);
sendMap[proci].resize_nocopy(nSend[proci]);
nSend[proci] = 0;
}
@ -78,49 +76,10 @@ Foam::autoPtr<Foam::mapDistribute> Foam::backgroundMeshDecomposition::buildMap
forAll(toProc, i)
{
label proci = toProc[i];
sendMap[proci][nSend[proci]++] = i;
}
// 4. Send over how many I need to receive
labelList recvSizes;
Pstream::exchangeSizes(sendMap, recvSizes);
// Determine receive map
// ~~~~~~~~~~~~~~~~~~~~~
labelListList constructMap(Pstream::nProcs());
// Local transfers first
constructMap[Pstream::myProcNo()] = identity
(
sendMap[Pstream::myProcNo()].size()
);
label constructSize = constructMap[Pstream::myProcNo()].size();
forAll(constructMap, proci)
{
if (proci != Pstream::myProcNo())
{
label nRecv = recvSizes[proci];
constructMap[proci].setSize(nRecv);
for (label i = 0; i < nRecv; i++)
{
constructMap[proci][i] = constructSize++;
}
}
}
return autoPtr<mapDistribute>::New
(
constructSize,
std::move(sendMap),
std::move(constructMap)
);
return autoPtr<mapDistribute>::New(std::move(sendMap));
}

36
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C
View File

@ -146,7 +146,6 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
labelListList subMap;
// Allocate transfer buffers
PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
@ -283,33 +282,24 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
lpi.rename(cloudName);
}
// Work the send indices (subMap) into a mapDistributeBase
labelListList sizes(Pstream::nProcs());
labelList& nsTransPs = sizes[Pstream::myProcNo()];
nsTransPs.setSize(Pstream::nProcs());
forAll(subMap, sendProcI)
{
nsTransPs[sendProcI] = subMap[sendProcI].size();
}
// Send sizes across. Note: blocks.
Pstream::combineReduce(sizes, Pstream::listEq());
// Until now (FEB-2023) we have always used processor ordering for the
// construct map (whereas mapDistribute has local transfers first),
// so we'll stick with that for now, but can likely just use the subMap
// directly with mapDistribute and have it determine the constructMap.
labelList recvSizes;
Pstream::exchangeSizes(subMap, recvSizes);
labelListList constructMap(Pstream::nProcs());
label constructSize = 0;
forAll(constructMap, procI)
labelListList constructMap(Pstream::nProcs());
forAll(constructMap, proci)
{
const label nRecv = sizes[procI][UPstream::myProcNo()];
labelList& map = constructMap[procI];
map.setSize(nRecv);
forAll(map, i)
{
map[i] = constructSize++;
}
const label len = recvSizes[proci];
constructMap[proci] = identity(len, constructSize);
constructSize += len;
}
return autoPtr<mapDistributeBase>::New
(
constructSize,