From fb69a54bc309f5893e2bb49de4741749b8d51c0e Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 14 Feb 2023 11:24:22 +0100 Subject: [PATCH] ENH: avoid prior communication when using mapDistribute - in most cases can simply construct mapDistribute with the sendMap and have it take care of communication and addressing for the corresponding constructMap. This removes code duplication, which in some cases was also using much less efficient mechanisms (eg, combineReduce on list of lists, or an allGatherList on the send sizes etc) and also reduces the number of places where Pstream::exchange/exchangeSizes is being called. ENH: reduce communication in turbulentDFSEMInlet - was doing an allGatherList to populate a mapDistribute. Now simply use PstreamBuffers mechanisms directly. --- applications/test/parallel/Test-parallel.C | 46 +---- .../DelaunayMesh/DistributedDelaunayMesh.C | 45 +---- .../backgroundMeshDecomposition.C | 45 +---- .../parLagrangianDistributor.C | 36 ++-- .../turbulentDFSEMInletFvPatchVectorField.C | 105 +++--------- .../basic/InteractionLists/InteractionLists.C | 47 +----- .../advancingFrontAMIParallelOps.C | 43 +---- .../nearestFaceAMI/nearestFaceAMI.C | 1 - src/meshTools/processorLOD/box/box.C | 63 +------ src/meshTools/processorLOD/box/box.H | 10 -- .../distributedTriSurfaceMesh.C | 159 +++++------------- .../meshToMesh/meshToMeshParallelOps.C | 39 +---- 12 files changed, 105 insertions(+), 534 deletions(-) diff --git a/applications/test/parallel/Test-parallel.C b/applications/test/parallel/Test-parallel.C index b6ac4f7e28..580ae453a8 100644 --- a/applications/test/parallel/Test-parallel.C +++ b/applications/test/parallel/Test-parallel.C @@ -67,54 +67,24 @@ void testMapDistribute() labelList nSend(Pstream::nProcs(), Zero); forAll(complexData, i) { - const label procI = complexData[i].first(); - nSend[procI]++; + const label proci = complexData[i].first(); + nSend[proci]++; } // Collect items to be sent labelListList sendMap(Pstream::nProcs()); - forAll(sendMap, procI) + forAll(sendMap, proci) { - sendMap[procI].setSize(nSend[procI]); + sendMap[proci].resize_nocopy(nSend[proci]); + nSend[proci] = 0; } - nSend = 0; forAll(complexData, i) { - const label procI = complexData[i].first(); - sendMap[procI][nSend[procI]++] = i; + const label proci = complexData[i].first(); + sendMap[proci][nSend[proci]++] = i; } - // Sync how many to send - labelList nRecv; - Pstream::exchangeSizes(sendMap, nRecv); - - // Collect items to be received - labelListList recvMap(Pstream::nProcs()); - forAll(recvMap, procI) - { - recvMap[procI].setSize(nRecv[procI]); - } - - label constructSize = 0; - // Construct with my own elements first - forAll(recvMap[Pstream::myProcNo()], i) - { - recvMap[Pstream::myProcNo()][i] = constructSize++; - } - // Construct from other processors - forAll(recvMap, procI) - { - if (procI != Pstream::myProcNo()) - { - forAll(recvMap[procI], i) - { - recvMap[procI][i] = constructSize++; - } - } - } - - // Construct distribute map (destructively) - mapDistribute map(constructSize, std::move(sendMap), std::move(recvMap)); + mapDistribute map(std::move(sendMap)); // Distribute complexData map.distribute(complexData); diff --git a/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/DelaunayMesh/DistributedDelaunayMesh.C b/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/DelaunayMesh/DistributedDelaunayMesh.C index 3000cf0aed..01c14c7060 100644 --- a/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/DelaunayMesh/DistributedDelaunayMesh.C +++ b/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/DelaunayMesh/DistributedDelaunayMesh.C @@ -54,7 +54,6 @@ Foam::DistributedDelaunayMesh::buildMap forAll(toProc, i) { label proci = toProc[i]; - nSend[proci]++; } @@ -64,8 +63,7 @@ Foam::DistributedDelaunayMesh::buildMap forAll(nSend, proci) { - sendMap[proci].setSize(nSend[proci]); - + sendMap[proci].resize_nocopy(nSend[proci]); nSend[proci] = 0; } @@ -73,49 +71,10 @@ Foam::DistributedDelaunayMesh::buildMap forAll(toProc, i) { label proci = toProc[i]; - sendMap[proci][nSend[proci]++] = i; } - // 4. Send over how many I need to receive - labelList recvSizes; - Pstream::exchangeSizes(sendMap, recvSizes); - - - // Determine receive map - // ~~~~~~~~~~~~~~~~~~~~~ - - labelListList constructMap(Pstream::nProcs()); - - // Local transfers first - constructMap[Pstream::myProcNo()] = identity - ( - sendMap[Pstream::myProcNo()].size() - ); - - label constructSize = constructMap[Pstream::myProcNo()].size(); - - forAll(constructMap, proci) - { - if (proci != Pstream::myProcNo()) - { - label nRecv = recvSizes[proci]; - - constructMap[proci].setSize(nRecv); - - for (label i = 0; i < nRecv; i++) - { - constructMap[proci][i] = constructSize++; - } - } - } - - return autoPtr::New - ( - constructSize, - std::move(sendMap), - std::move(constructMap) - ); + return autoPtr::New(std::move(sendMap)); } diff --git a/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/backgroundMeshDecomposition/backgroundMeshDecomposition.C b/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/backgroundMeshDecomposition/backgroundMeshDecomposition.C index a224a42ebd..3589c67ba8 100644 --- a/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/backgroundMeshDecomposition/backgroundMeshDecomposition.C +++ b/applications/utilities/mesh/generation/foamyMesh/conformalVoronoiMesh/backgroundMeshDecomposition/backgroundMeshDecomposition.C @@ -59,7 +59,6 @@ Foam::autoPtr Foam::backgroundMeshDecomposition::buildMap forAll(toProc, i) { label proci = toProc[i]; - nSend[proci]++; } @@ -69,8 +68,7 @@ Foam::autoPtr Foam::backgroundMeshDecomposition::buildMap forAll(nSend, proci) { - sendMap[proci].setSize(nSend[proci]); - + sendMap[proci].resize_nocopy(nSend[proci]); nSend[proci] = 0; } @@ -78,49 +76,10 @@ Foam::autoPtr Foam::backgroundMeshDecomposition::buildMap forAll(toProc, i) { label proci = toProc[i]; - sendMap[proci][nSend[proci]++] = i; } - // 4. Send over how many I need to receive - labelList recvSizes; - Pstream::exchangeSizes(sendMap, recvSizes); - - - // Determine receive map - // ~~~~~~~~~~~~~~~~~~~~~ - - labelListList constructMap(Pstream::nProcs()); - - // Local transfers first - constructMap[Pstream::myProcNo()] = identity - ( - sendMap[Pstream::myProcNo()].size() - ); - - label constructSize = constructMap[Pstream::myProcNo()].size(); - - forAll(constructMap, proci) - { - if (proci != Pstream::myProcNo()) - { - label nRecv = recvSizes[proci]; - - constructMap[proci].setSize(nRecv); - - for (label i = 0; i < nRecv; i++) - { - constructMap[proci][i] = constructSize++; - } - } - } - - return autoPtr::New - ( - constructSize, - std::move(sendMap), - std::move(constructMap) - ); + return autoPtr::New(std::move(sendMap)); } diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C index ef20c1cec0..0b7dbb462e 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C @@ -146,7 +146,6 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions labelListList subMap; - // Allocate transfer buffers PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); @@ -283,33 +282,24 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions lpi.rename(cloudName); } - // Work the send indices (subMap) into a mapDistributeBase - labelListList sizes(Pstream::nProcs()); - labelList& nsTransPs = sizes[Pstream::myProcNo()]; - nsTransPs.setSize(Pstream::nProcs()); - forAll(subMap, sendProcI) - { - nsTransPs[sendProcI] = subMap[sendProcI].size(); - } - // Send sizes across. Note: blocks. - Pstream::combineReduce(sizes, Pstream::listEq()); + // Until now (FEB-2023) we have always used processor ordering for the + // construct map (whereas mapDistribute has local transfers first), + // so we'll stick with that for now, but can likely just use the subMap + // directly with mapDistribute and have it determine the constructMap. + + labelList recvSizes; + Pstream::exchangeSizes(subMap, recvSizes); - labelListList constructMap(Pstream::nProcs()); label constructSize = 0; - forAll(constructMap, procI) + labelListList constructMap(Pstream::nProcs()); + + forAll(constructMap, proci) { - const label nRecv = sizes[procI][UPstream::myProcNo()]; - - labelList& map = constructMap[procI]; - - map.setSize(nRecv); - forAll(map, i) - { - map[i] = constructSize++; - } + const label len = recvSizes[proci]; + constructMap[proci] = identity(len, constructSize); + constructSize += len; } - return autoPtr::New ( constructSize, diff --git a/src/finiteVolume/fields/fvPatchFields/derived/turbulentDFSEMInlet/turbulentDFSEMInletFvPatchVectorField.C b/src/finiteVolume/fields/fvPatchFields/derived/turbulentDFSEMInlet/turbulentDFSEMInletFvPatchVectorField.C index 8d39187889..1e7508dc47 100644 --- a/src/finiteVolume/fields/fvPatchFields/derived/turbulentDFSEMInlet/turbulentDFSEMInletFvPatchVectorField.C +++ b/src/finiteVolume/fields/fvPatchFields/derived/turbulentDFSEMInlet/turbulentDFSEMInletFvPatchVectorField.C @@ -450,13 +450,16 @@ void Foam::turbulentDFSEMInletFvPatchVectorField::convectEddies } } - reduce(nRecycled, sumOp