From 1348cd7e7b9e01ee390c259f79a6f11a520af26e Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 22 Feb 2022 13:40:07 +0100 Subject: [PATCH] ENH: use broadcasting Pstreams for one-to-all sends --- .../redistributePar/loadOrCreateMesh.C | 30 +++---- .../polyMesh/globalMeshData/globalMeshData.C | 85 ++++++++----------- .../mapDistribute/mapDistributeBase.C | 28 ++---- .../polyMesh/syncTools/syncToolsTemplates.C | 52 +++++------- src/dynamicMesh/fvMeshTools/fvMeshTools.C | 29 +++---- 5 files changed, 93 insertions(+), 131 deletions(-) diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C index 23969883b3..520d2796b2 100644 --- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C +++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2012-2017 OpenFOAM Foundation - Copyright (C) 2015-2021 OpenCFD Ltd. + Copyright (C) 2015-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -88,25 +88,23 @@ Foam::autoPtr Foam::loadOrCreateMesh ); Pstream::parRun(oldParRun); + } - // Send patches - for (const int slave : Pstream::subProcs()) + if (Pstream::parRun()) + { + if (Pstream::master()) { - OPstream toSlave(Pstream::commsTypes::scheduled, slave); - toSlave << patchEntries; + // Broadcast: send patches + OPBstream toAll(Pstream::masterNo()); + toAll << patchEntries; + } + else + { + // Broadcast: receive patches + IPBstream fromMaster(Pstream::masterNo()); + fromMaster >> patchEntries; } } - else - { - // Receive patches - IPstream fromMaster - ( - Pstream::commsTypes::scheduled, - Pstream::masterNo() - ); - fromMaster >> patchEntries; - } - // Dummy meshes diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C index 177f57c66b..5469f8680d 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2021 OpenCFD Ltd. + Copyright (C) 2015-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -388,20 +388,20 @@ void Foam::globalMeshData::calcSharedEdges() const } countSharedEdges(localShared, globalShared, sharedEdgeI); - // Receive data from slaves and insert + // Receive data and insert if (Pstream::parRun()) { - for (const int slave : Pstream::subProcs()) + for (const int proci : Pstream::subProcs()) { // Receive the edges using shared points from the slave. - IPstream fromSlave(Pstream::commsTypes::blocking, slave); - EdgeMap procSharedEdges(fromSlave); + IPstream fromProc(Pstream::commsTypes::blocking, proci); + EdgeMap procSharedEdges(fromProc); if (debug) { Pout<< "globalMeshData::calcSharedEdges : " << "Merging in from proc" - << Foam::name(slave) << " : " << procSharedEdges.size() + << proci << " : " << procSharedEdges.size() << endl; } countSharedEdges(procSharedEdges, globalShared, sharedEdgeI); @@ -432,36 +432,32 @@ void Foam::globalMeshData::calcSharedEdges() const } - // Send back to slaves. + // Broadcast: send back to all if (Pstream::parRun()) { - for (const int slave : Pstream::subProcs()) - { - // Receive the edges using shared points from the slave. - OPstream toSlave(Pstream::commsTypes::blocking, slave); - toSlave << globalShared; - } + OPBstream toAll(Pstream::masterNo()); // == worldComm + toAll << globalShared; } } else { - // Send local edges to master + if (Pstream::parRun()) { - OPstream toMaster - ( - Pstream::commsTypes::blocking, - Pstream::masterNo() - ); - toMaster << localShared; - } - // Receive merged edges from master. - { - IPstream fromMaster - ( - Pstream::commsTypes::blocking, - Pstream::masterNo() - ); - fromMaster >> globalShared; + // Send local edges to master + { + OPstream toMaster + ( + Pstream::commsTypes::blocking, + Pstream::masterNo() + ); + toMaster << localShared; + } + + // Broadcast: receive merged edges from master + { + IPBstream fromMaster(Pstream::masterNo()); // == worldComm + fromMaster >> globalShared; + } } } @@ -1896,14 +1892,14 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const sharedPoints[sharedPointi] = mesh_.points()[pointLabels[i]]; } - // Receive data from slaves and insert - for (const int slave : Pstream::subProcs()) + // Receive data and insert + for (const int proci : Pstream::subProcs()) { - IPstream fromSlave(Pstream::commsTypes::blocking, slave); + IPstream fromProc(Pstream::commsTypes::blocking, proci); labelList nbrSharedPointAddr; pointField nbrSharedPoints; - fromSlave >> nbrSharedPointAddr >> nbrSharedPoints; + fromProc >> nbrSharedPointAddr >> nbrSharedPoints; forAll(nbrSharedPointAddr, i) { @@ -1913,22 +1909,15 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const } } - // Send back - for (const int slave : Pstream::subProcs()) + // Broadcast: send back { - OPstream toSlave - ( - Pstream::commsTypes::blocking, - slave, - sharedPoints.size_bytes() - ); - toSlave << sharedPoints; + OPBstream toAll(Pstream::masterNo()); // == worldComm + toAll << sharedPoints; } } else { - // Slave: - // send points + // Send address and points { OPstream toMaster ( @@ -1940,13 +1929,9 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const << UIndirectList(mesh_.points(), pointLabels)(); } - // Receive sharedPoints + // Broadcast: receive sharedPoints { - IPstream fromMaster - ( - Pstream::commsTypes::blocking, - Pstream::masterNo() - ); + IPBstream fromMaster(Pstream::masterNo()); // == worldComm fromMaster >> sharedPoints; } } diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C index d72d9e445b..d028ad6b1b 100644 --- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C +++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2015-2017 OpenFOAM Foundation - Copyright (C) 2015-2021 OpenCFD Ltd. + Copyright (C) 2015-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -106,18 +106,11 @@ Foam::List Foam::mapDistributeBase::schedule } } } - // Send back - for (const int slave : Pstream::subProcs(comm)) + + // Broadcast: send merged to all { - OPstream toSlave - ( - Pstream::commsTypes::scheduled, - slave, - 0, - tag, - comm - ); - toSlave << allComms; + OPBstream toAll(Pstream::masterNo(), comm); + toAll << allComms; } } else @@ -133,15 +126,10 @@ Foam::List Foam::mapDistributeBase::schedule ); toMaster << allComms; } + + // Broadcast: receive merged { - IPstream fromMaster - ( - Pstream::commsTypes::scheduled, - Pstream::masterNo(), - 0, - tag, - comm - ); + IPBstream fromMaster(Pstream::masterNo(), comm); fromMaster >> allComms; } } diff --git a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C index aca1207b15..7491722996 100644 --- a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2021 OpenCFD Ltd. + Copyright (C) 2015-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -286,11 +286,11 @@ void Foam::syncTools::syncPointMap { if (Pstream::master()) { - // Receive the edges using shared points from the slave. - for (const int slave : Pstream::subProcs()) + // Receive the edges using shared points from other procs + for (const int proci : Pstream::subProcs()) { - IPstream fromSlave(Pstream::commsTypes::scheduled, slave); - Map nbrValues(fromSlave); + IPstream fromProc(Pstream::commsTypes::scheduled, proci); + Map nbrValues(fromProc); // Merge neighbouring values with my values forAllConstIters(nbrValues, iter) @@ -305,16 +305,15 @@ void Foam::syncTools::syncPointMap } } - // Send back - for (const int slave : Pstream::subProcs()) + // Broadcast: send merged values to all { - OPstream toSlave(Pstream::commsTypes::scheduled, slave); - toSlave << sharedPointValues; + OPBstream toAll(Pstream::masterNo()); // == worldComm + toAll << sharedPointValues; } } else { - // Slave: send to master + // Send to master { OPstream toMaster ( @@ -323,13 +322,10 @@ void Foam::syncTools::syncPointMap ); toMaster << sharedPointValues; } - // Receive merged values + + // Broadcast: receive merged values { - IPstream fromMaster - ( - Pstream::commsTypes::scheduled, - Pstream::masterNo() - ); + IPBstream fromMaster(Pstream::masterNo()); // == worldComm fromMaster >> sharedPointValues; } } @@ -640,11 +636,11 @@ void Foam::syncTools::syncEdgeMap { if (Pstream::master()) { - // Receive the edges using shared points from the slave. - for (const int slave : Pstream::subProcs()) + // Receive the edges using shared points from other procs + for (const int proci : Pstream::subProcs()) { - IPstream fromSlave(Pstream::commsTypes::scheduled, slave); - EdgeMap nbrValues(fromSlave); + IPstream fromProc(Pstream::commsTypes::scheduled, proci); + EdgeMap nbrValues(fromProc); // Merge neighbouring values with my values forAllConstIters(nbrValues, iter) @@ -659,11 +655,10 @@ void Foam::syncTools::syncEdgeMap } } - // Send back - for (const int slave : Pstream::subProcs()) + // Broadcast: send merged values to all { - OPstream toSlave(Pstream::commsTypes::scheduled, slave); - toSlave << sharedEdgeValues; + OPBstream toAll(Pstream::masterNo()); // == worldComm + toAll << sharedEdgeValues; } } else @@ -677,13 +672,10 @@ void Foam::syncTools::syncEdgeMap ); toMaster << sharedEdgeValues; } - // Receive merged values + + // Broadcast: receive merged values { - IPstream fromMaster - ( - Pstream::commsTypes::scheduled, - Pstream::masterNo() - ); + IPBstream fromMaster(Pstream::masterNo()); // == worldComm fromMaster >> sharedEdgeValues; } } diff --git a/src/dynamicMesh/fvMeshTools/fvMeshTools.C b/src/dynamicMesh/fvMeshTools/fvMeshTools.C index 4ca422e9df..07fbec6a71 100644 --- a/src/dynamicMesh/fvMeshTools/fvMeshTools.C +++ b/src/dynamicMesh/fvMeshTools/fvMeshTools.C @@ -478,23 +478,22 @@ Foam::autoPtr Foam::fvMeshTools::newMesh ) ); Pstream::parRun(oldParRun); - - // Send patches - for (const int slave : Pstream::subProcs()) - { - OPstream toSlave(Pstream::commsTypes::scheduled, slave); - toSlave << patchEntries; - } } - else + + if (Pstream::parRun()) { - // Receive patches - IPstream fromMaster - ( - Pstream::commsTypes::scheduled, - Pstream::masterNo() - ); - fromMaster >> patchEntries; + if (Pstream::master()) + { + // Broadcast: send patches to all + OPBstream toAll(Pstream::masterNo()); // == worldComm + toAll << patchEntries; + } + else + { + // Broadcast: receive patches + IPBstream fromMaster(Pstream::masterNo()); // == worldComm + fromMaster >> patchEntries; + } } Pstream::scatter(facesInstance);