From 06f479fbd4100d4e07649b8ca5ea4ceb2bb534c0 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 11 Jan 2023 19:01:31 +0100 Subject: [PATCH] ENH: improve handling of wait/finished requests - now simply a no-op for out-of-range values (instead of an error), which simplifies the calling code. Previously ========== if (request_ >= 0 && request_ < UPstream::nRequests()) { UPstream::waitRequest(request_); } Updated ======= UPstream::waitRequest(request_); - when 'recycling' freed request indices, ensure they are actually within the currently addressable range - MPI finalization now checks outstanding requests against MPI_REQUEST_NULL to verify that they have been waited or tested on. Previously simply checked against freed request indices ENH: consistent initialisation of send/receive bookkeeping --- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 19 +- .../lduCalculatedProcessorField.C | 88 +++------ .../lduCalculatedProcessorField.H | 18 +- .../processorGAMGInterfaceField.C | 40 ++-- .../processorGAMGInterfaceField.H | 14 +- src/Pstream/dummy/Make/files | 1 + src/Pstream/dummy/UPstream.C | 24 +-- src/Pstream/dummy/UPstreamRequest.C | 43 ++++ src/Pstream/mpi/Make/files | 1 + src/Pstream/mpi/PstreamGlobals.C | 7 +- src/Pstream/mpi/PstreamGlobals.H | 41 ++-- src/Pstream/mpi/UPstream.C | 174 +--------------- src/Pstream/mpi/UPstreamRequest.C | 185 ++++++++++++++++++ .../calculatedProcessorFvPatchField.C | 121 ++++-------- .../calculatedProcessorFvPatchField.H | 19 +- .../processor/processorFvPatchField.C | 172 ++++++---------- .../processor/processorFvPatchField.H | 17 +- .../calculatedProcessorGAMGInterfaceField.C | 40 ++-- .../calculatedProcessorGAMGInterfaceField.H | 16 +- 19 files changed, 473 insertions(+), 567 deletions(-) create mode 100644 src/Pstream/dummy/UPstreamRequest.C create mode 100644 src/Pstream/mpi/UPstreamRequest.C diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index a5111d019e..ea1b63a7fe 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -447,21 +447,26 @@ public: //- Number of outstanding requests static label nRequests() noexcept; - //- Truncate outstanding requests to given length + //- Truncate outstanding requests to given length, which is + //- expected to be in the range 0 to nRequests. + // A no-op for out-of-range values. static void resetRequests(const label n); //- Wait until all requests (from start onwards) have finished. - // A no-op if parRun() == false + // A no-op if parRun() == false, if there are no pending requests + // or if the start is out-of-range (0 to nRequests) static void waitRequests(const label start = 0); //- Wait until request i has finished. - // A no-op if parRun() == false - // or for placeholder (negative) request indices + // A no-op if parRun() == false, + // there are no pending requests, + // or if the index is out-of-range (0 to nRequests) static void waitRequest(const label i); //- Non-blocking comms: has request i finished? - // A no-op and returns true if parRun() == false - // or for placeholder (negative) request indices + // A no-op and returns true if parRun() == false, + // there are no pending requests, + // or if the index is out-of-range (0 to nRequests) static bool finishedRequest(const label i); static int allocateTag(const char* const msg = nullptr); diff --git a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.C b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.C index b72ad7e7b8..0212005b24 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.C +++ b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -37,12 +37,8 @@ Foam::lduCalculatedProcessorField::lduCalculatedProcessorField : LduInterfaceField(interface), procInterface_(refCast(interface)), - sendBuf_(procInterface_.faceCells().size()), - receiveBuf_(procInterface_.faceCells().size()), - scalarSendBuf_(procInterface_.faceCells().size()), - scalarReceiveBuf_(procInterface_.faceCells().size()), - outstandingSendRequest_(-1), - outstandingRecvRequest_(-1) + sendRequest_(-1), + recvRequest_(-1) {} @@ -54,12 +50,8 @@ Foam::lduCalculatedProcessorField::lduCalculatedProcessorField : LduInterfaceField(refCast(ptf)), procInterface_(ptf.procInterface_), - sendBuf_(procInterface_.faceCells().size()), - receiveBuf_(procInterface_.faceCells().size()), - scalarSendBuf_(procInterface_.faceCells().size()), - scalarReceiveBuf_(procInterface_.faceCells().size()), - outstandingSendRequest_(-1), - outstandingRecvRequest_(-1) + sendRequest_(-1), + recvRequest_(-1) {} @@ -68,31 +60,11 @@ Foam::lduCalculatedProcessorField::lduCalculatedProcessorField template bool Foam::lduCalculatedProcessorField::ready() const { - if - ( - this->outstandingSendRequest_ >= 0 - && this->outstandingSendRequest_ < UPstream::nRequests() - ) - { - if (!UPstream::finishedRequest(this->outstandingSendRequest_)) - { - return false; - } - } - this->outstandingSendRequest_ = -1; + if (!UPstream::finishedRequest(this->sendRequest_)) return false; + this->sendRequest_ = -1; - if - ( - this->outstandingRecvRequest_ >= 0 - && this->outstandingRecvRequest_ < UPstream::nRequests() - ) - { - if (!UPstream::finishedRequest(this->outstandingRecvRequest_)) - { - return false; - } - } - this->outstandingRecvRequest_ = -1; + if (!UPstream::finishedRequest(this->recvRequest_)) return false; + this->recvRequest_ = -1; return true; } @@ -111,6 +83,13 @@ void Foam::lduCalculatedProcessorField::initInterfaceMatrixUpdate const Pstream::commsTypes commsType ) const { + if (!this->ready()) + { + FatalErrorInFunction + << "Outstanding request." + << abort(FatalError); + } + // Bypass patchInternalField since uses fvPatch addressing const labelList& fc = lduAddr.patchAddr(patchId); @@ -120,21 +99,12 @@ void Foam::lduCalculatedProcessorField::initInterfaceMatrixUpdate scalarSendBuf_[i] = psiInternal[fc[i]]; } - if (!this->ready()) - { - FatalErrorInFunction - << "On patch " - << " outstanding request." - << abort(FatalError); - } - - scalarReceiveBuf_.setSize(scalarSendBuf_.size()); - outstandingRecvRequest_ = UPstream::nRequests(); + recvRequest_ = UPstream::nRequests(); UIPstream::read ( - Pstream::commsTypes::nonBlocking, + UPstream::commsTypes::nonBlocking, procInterface_.neighbProcNo(), scalarReceiveBuf_.data_bytes(), scalarReceiveBuf_.size_bytes(), @@ -142,11 +112,10 @@ void Foam::lduCalculatedProcessorField::initInterfaceMatrixUpdate procInterface_.comm() ); - outstandingSendRequest_ = UPstream::nRequests(); - + sendRequest_ = UPstream::nRequests(); UOPstream::write ( - Pstream::commsTypes::nonBlocking, + UPstream::commsTypes::nonBlocking, procInterface_.neighbProcNo(), scalarSendBuf_.cdata_bytes(), scalarSendBuf_.size_bytes(), @@ -207,19 +176,12 @@ void Foam::lduCalculatedProcessorField::updateInterfaceMatrix return; } - if - ( - outstandingRecvRequest_ >= 0 - && outstandingRecvRequest_ < UPstream::nRequests() - ) - { - UPstream::waitRequest(outstandingRecvRequest_); - } - // Recv finished so assume sending finished as well. - outstandingSendRequest_ = -1; - outstandingRecvRequest_ = -1; + // Treat send as finished when recv is done + UPstream::waitRequest(recvRequest_); + recvRequest_ = -1; + sendRequest_ = -1; - // Consume straight from scalarReceiveBuf_. Note use of our own + // Consume straight from receive buffer. Note use of our own // helper to avoid using fvPatch addressing addToInternalField(result, !add, coeffs, scalarReceiveBuf_); diff --git a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.H b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.H index 60c188626e..1bb956585a 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.H +++ b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.H @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -42,8 +42,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef lduCalculatedProcessorField_H -#define lduCalculatedProcessorField_H +#ifndef Foam_lduCalculatedProcessorField_H +#define Foam_lduCalculatedProcessorField_H #include "lduPrimitiveProcessorInterface.H" #include "processorLduInterfaceField.H" @@ -74,6 +74,12 @@ protected: // Sending and receiving + //- Current (non-blocking) send request + mutable label sendRequest_; + + //- Current (non-blocking) recv request + mutable label recvRequest_; + //- Send buffer mutable Field sendBuf_; @@ -86,12 +92,6 @@ protected: //- Scalar receive buffer mutable solveScalarField scalarReceiveBuf_; - //- Outstanding request - mutable label outstandingSendRequest_; - - //- Outstanding request - mutable label outstandingRecvRequest_; - // Protected Member Functions diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C index bcc476d840..9e75568ace 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2019-2021 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -61,10 +61,11 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField GAMGInterfaceField(GAMGCp, fineInterface), procInterface_(refCast(GAMGCp)), doTransform_(false), - rank_(0) + rank_(0), + sendRequest_(-1), + recvRequest_(-1) { - const processorLduInterfaceField& p = - refCast(fineInterface); + const auto& p = refCast(fineInterface); doTransform_ = p.doTransform(); rank_ = p.rank(); @@ -81,7 +82,9 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField GAMGInterfaceField(GAMGCp, doTransform, rank), procInterface_(refCast(GAMGCp)), doTransform_(doTransform), - rank_(rank) + rank_(rank), + sendRequest_(-1), + recvRequest_(-1) {} @@ -109,10 +112,11 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate { // Fast path. scalarReceiveBuf_.setSize(scalarSendBuf_.size()); - outstandingRecvRequest_ = UPstream::nRequests(); + + recvRequest_ = UPstream::nRequests(); UIPstream::read ( - Pstream::commsTypes::nonBlocking, + UPstream::commsTypes::nonBlocking, procInterface_.neighbProcNo(), scalarReceiveBuf_.data_bytes(), scalarReceiveBuf_.size_bytes(), @@ -120,10 +124,10 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate comm() ); - outstandingSendRequest_ = UPstream::nRequests(); + sendRequest_ = UPstream::nRequests(); UOPstream::write ( - Pstream::commsTypes::nonBlocking, + UPstream::commsTypes::nonBlocking, procInterface_.neighbProcNo(), scalarSendBuf_.cdata_bytes(), scalarSendBuf_.size_bytes(), @@ -165,20 +169,12 @@ void Foam::processorGAMGInterfaceField::updateInterfaceMatrix && !Pstream::floatTransfer ) { - // Fast path. - if - ( - outstandingRecvRequest_ >= 0 - && outstandingRecvRequest_ < UPstream::nRequests() - ) - { - UPstream::waitRequest(outstandingRecvRequest_); - } - // Recv finished so assume sending finished as well. - outstandingSendRequest_ = -1; - outstandingRecvRequest_ = -1; + // Fast path: consume straight from receive buffer - // Consume straight from scalarReceiveBuf_ + // Treat send as finished when recv is done + UPstream::waitRequest(recvRequest_); + recvRequest_ = -1; + sendRequest_ = -1; // Transform according to the transformation tensor transformCoupleField(scalarReceiveBuf_, cmpt); diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H index c4f9457427..ee07b013d9 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2014 OpenFOAM Foundation - Copyright (C) 2019 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -35,8 +35,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef processorGAMGInterfaceField_H -#define processorGAMGInterfaceField_H +#ifndef Foam_processorGAMGInterfaceField_H +#define Foam_processorGAMGInterfaceField_H #include "GAMGInterfaceField.H" #include "processorGAMGInterface.H" @@ -70,11 +70,11 @@ class processorGAMGInterfaceField // Sending and receiving - //- Outstanding request - mutable label outstandingSendRequest_; + //- Current (non-blocking) send request + mutable label sendRequest_; - //- Outstanding request - mutable label outstandingRecvRequest_; + //- Current (non-blocking) recv request + mutable label recvRequest_; //- Scalar send buffer mutable solveScalarField scalarSendBuf_; diff --git a/src/Pstream/dummy/Make/files b/src/Pstream/dummy/Make/files index 7ba370e470..d5e68ac408 100644 --- a/src/Pstream/dummy/Make/files +++ b/src/Pstream/dummy/Make/files @@ -3,6 +3,7 @@ UPstreamAllToAll.C UPstreamBroadcast.C UPstreamGatherScatter.C UPstreamReduce.C +UPstreamRequest.C UIPstreamRead.C UOPstreamWrite.C diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C index a586c1ef1a..db934c00e3 100644 --- a/src/Pstream/dummy/UPstream.C +++ b/src/Pstream/dummy/UPstream.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2018 OpenFOAM Foundation - Copyright (C) 2016-2022 OpenCFD Ltd. + Copyright (C) 2016-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -26,7 +26,7 @@ License \*---------------------------------------------------------------------------*/ -#include "Pstream.H" +#include "UPstream.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -87,28 +87,12 @@ void Foam::UPstream::freePstreamCommunicator(const label) {} -Foam::label Foam::UPstream::nRequests() noexcept -{ - return 0; -} +int Foam::UPstream::allocateTag(const char* const msg) { return 0; } -void Foam::UPstream::resetRequests(const label n) +void Foam::UPstream::freeTag(const int tag, const char* const msg) {} -void Foam::UPstream::waitRequests(const label start) -{} - - -void Foam::UPstream::waitRequest(const label i) -{} - - -bool Foam::UPstream::finishedRequest(const label i) -{ - return true; -} - // ************************************************************************* // diff --git a/src/Pstream/dummy/UPstreamRequest.C b/src/Pstream/dummy/UPstreamRequest.C new file mode 100644 index 0000000000..ab104e0b3e --- /dev/null +++ b/src/Pstream/dummy/UPstreamRequest.C @@ -0,0 +1,43 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2023 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "UPstream.H" + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +Foam::label Foam::UPstream::nRequests() noexcept { return 0; } + +void Foam::UPstream::resetRequests(const label n) {} + +void Foam::UPstream::waitRequests(const label start) {} + +void Foam::UPstream::waitRequest(const label i) {} + +bool Foam::UPstream::finishedRequest(const label i) { return true; } + + +// ************************************************************************* // diff --git a/src/Pstream/mpi/Make/files b/src/Pstream/mpi/Make/files index 217347cc29..48a99bac40 100644 --- a/src/Pstream/mpi/Make/files +++ b/src/Pstream/mpi/Make/files @@ -4,6 +4,7 @@ UPstreamAllToAll.C UPstreamBroadcast.C UPstreamGatherScatter.C UPstreamReduce.C +UPstreamRequest.C UIPstreamRead.C UOPstreamWrite.C diff --git a/src/Pstream/mpi/PstreamGlobals.C b/src/Pstream/mpi/PstreamGlobals.C index f6b09aa4cd..2d6e65aa10 100644 --- a/src/Pstream/mpi/PstreamGlobals.C +++ b/src/Pstream/mpi/PstreamGlobals.C @@ -29,16 +29,15 @@ License // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // +Foam::DynamicList Foam::PstreamGlobals::MPICommunicators_; +Foam::DynamicList Foam::PstreamGlobals::MPIGroups_; + Foam::DynamicList Foam::PstreamGlobals::outstandingRequests_; Foam::DynamicList Foam::PstreamGlobals::freedRequests_; int Foam::PstreamGlobals::nTags_ = 0; - Foam::DynamicList Foam::PstreamGlobals::freedTags_; -Foam::DynamicList Foam::PstreamGlobals::MPICommunicators_; -Foam::DynamicList Foam::PstreamGlobals::MPIGroups_; - // * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * // diff --git a/src/Pstream/mpi/PstreamGlobals.H b/src/Pstream/mpi/PstreamGlobals.H index 33287c6c9a..859d0aa64b 100644 --- a/src/Pstream/mpi/PstreamGlobals.H +++ b/src/Pstream/mpi/PstreamGlobals.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2013-2015 OpenFOAM Foundation - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -49,6 +49,13 @@ namespace Foam namespace PstreamGlobals { +// Current communicators, which may be allocated or predefined +// (eg, MPI_COMM_SELF, MPI_COMM_WORLD) +extern DynamicList MPICommunicators_; + +// Groups associated with the currrent communicators. +extern DynamicList MPIGroups_; + //- Outstanding non-blocking operations. extern DynamicList outstandingRequests_; extern DynamicList