From 74fd94dff307e5af671fd06a043dc74523298dd4 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Fri, 10 Feb 2023 09:25:20 +0100 Subject: [PATCH] REGRESSION: overly aggressive handling of MPI groups - attempted reduction in bookkeeping (commit: 068ab8ccc797) meant that the worldComm didn't have a group from which sub-communicators could be spun off. - do not force reset of PstreamBuffers positions STYLE: UPstream::globalComm instead of '0' --- src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C | 3 --- src/Pstream/mpi/UPstream.C | 8 ++++++-- src/functionObjects/utilities/syncObjects/syncObjects.C | 5 ++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C index ff37820401..1979e80374 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C @@ -41,7 +41,6 @@ void Foam::PstreamBuffers::finalExchange // Could also check that it is not called twice // but that is used for overlapping send/recv (eg, overset) finishedSendsCalled_ = true; - recvPositions_ = Zero; if (commsType_ == UPstream::commsTypes::nonBlocking) { @@ -102,7 +101,6 @@ void Foam::PstreamBuffers::finalExchange // Could also check that it is not called twice // but that is used for overlapping send/recv (eg, overset) finishedSendsCalled_ = true; - recvPositions_ = Zero; if (commsType_ == UPstream::commsTypes::nonBlocking) { @@ -140,7 +138,6 @@ void Foam::PstreamBuffers::finalExchangeGatherScatter // Could also check that it is not called twice // but that is used for overlapping send/recv (eg, overset) finishedSendsCalled_ = true; - recvPositions_ = Zero; if (commsType_ == UPstream::commsTypes::nonBlocking) { diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index 005707be61..61667ce2b0 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -535,12 +535,16 @@ void Foam::UPstream::allocatePstreamCommunicator PstreamGlobals::pendingMPIFree_[index] = PstreamGlobals::NonePending; PstreamGlobals::MPICommunicators_[index] = MPI_COMM_WORLD; - PstreamGlobals::MPIGroups_[index] = MPI_GROUP_NULL; // TBD: MPI_Comm_dup(MPI_COMM_WORLD, ...); // with pendingMPIFree_[index] = CommPending ... // Note: freePstreamCommunicator may need an update + MPI_Comm_group + ( + PstreamGlobals::MPICommunicators_[index], + &PstreamGlobals::MPIGroups_[index] + ); MPI_Comm_rank ( PstreamGlobals::MPICommunicators_[index], @@ -565,8 +569,8 @@ void Foam::UPstream::allocatePstreamCommunicator PstreamGlobals::pendingMPIFree_[index] = PstreamGlobals::NonePending; PstreamGlobals::MPICommunicators_[index] = MPI_COMM_SELF; - PstreamGlobals::MPIGroups_[index] = MPI_GROUP_NULL; + MPI_Comm_group(MPI_COMM_SELF, &PstreamGlobals::MPIGroups_[index]); MPI_Comm_rank(MPI_COMM_SELF, &myProcNo_[index]); // Number of ranks is always 1 (self communicator) diff --git a/src/functionObjects/utilities/syncObjects/syncObjects.C b/src/functionObjects/utilities/syncObjects/syncObjects.C index 88f613da6b..06776eae3b 100644 --- a/src/functionObjects/utilities/syncObjects/syncObjects.C +++ b/src/functionObjects/utilities/syncObjects/syncObjects.C @@ -90,9 +90,8 @@ void Foam::functionObjects::syncObjects::sync() // Note provision of explicit all-world communicator PstreamBuffers pBufs ( - Pstream::commsTypes::nonBlocking, - UPstream::msgType(), - 0 + UPstream::globalComm, + UPstream::commsTypes::nonBlocking );