From 0b51b764d621a97d28880065fd594bf8a72c0c44 Mon Sep 17 00:00:00 2001 From: mattijs Date: Tue, 25 Aug 2009 23:03:33 +0100 Subject: [PATCH] bundle outstanding send and receive requests --- src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H | 6 -- src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H | 6 -- src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H | 6 ++ .../GeometricField/GeometricBoundaryField.C | 3 +- .../mapPolyMesh/mapDistribute/mapDistribute.C | 3 +- .../mapDistribute/mapDistributeTemplates.C | 12 ++-- src/Pstream/dummy/IPread.C | 13 ---- src/Pstream/dummy/OPwrite.C | 13 ---- src/Pstream/dummy/Pstream.C | 13 +++- src/Pstream/mpi/IPread.C | 51 +------------- src/Pstream/mpi/Make/files | 1 + src/Pstream/mpi/OPwrite.C | 56 +-------------- src/Pstream/mpi/Pstream.C | 52 ++++++++++++++ src/Pstream/mpi/PstreamGlobals.C | 45 ++++++++++++ src/Pstream/mpi/PstreamGlobals.H | 69 +++++++++++++++++++ 15 files changed, 196 insertions(+), 153 deletions(-) create mode 100644 src/Pstream/mpi/PstreamGlobals.C create mode 100644 src/Pstream/mpi/PstreamGlobals.H diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H index 09db5931b8..db40ccfd4f 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H @@ -117,12 +117,6 @@ public: const std::streamsize bufSize ); - //- Non-blocking receives: wait until all have finished. - static void waitRequests(); - - //- Non-blocking receives: has request i finished? - static bool finishedRequest(const label i); - //- Return next token from stream Istream& read(token&); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H index 6db9a827b1..bccb5b08ef 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H @@ -115,12 +115,6 @@ public: const std::streamsize bufSize ); - //- Non-blocking writes: wait until all have finished. - static void waitRequests(); - - //- Non-blocking writes: has request i finished? - static bool finishedRequest(const label i); - //- Write next token to stream Ostream& write(const token&); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index 5e8e5cbbb7..f54f02b271 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -264,6 +264,12 @@ public: // Spawns slave processes and initialises inter-communication static bool init(int& argc, char**& argv); + //- Non-blocking comms: wait until all have finished. + static void waitRequests(); + + //- Non-blocking comms: has request i finished? + static bool finishedRequest(const label i); + //- Is this a parallel run? static bool parRun() { diff --git a/src/OpenFOAM/fields/GeometricFields/GeometricField/GeometricBoundaryField.C b/src/OpenFOAM/fields/GeometricFields/GeometricField/GeometricBoundaryField.C index b3b6bb1d72..b2aa0d93e3 100644 --- a/src/OpenFOAM/fields/GeometricFields/GeometricField/GeometricBoundaryField.C +++ b/src/OpenFOAM/fields/GeometricFields/GeometricField/GeometricBoundaryField.C @@ -300,8 +300,7 @@ evaluate() // Block for any outstanding requests if (Pstream::defaultCommsType == Pstream::nonBlocking) { - IPstream::waitRequests(); - OPstream::waitRequests(); + Pstream::waitRequests(); } forAll(*this, patchi) diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C index e2c2dbf773..0c524920af 100644 --- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C +++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C @@ -339,8 +339,7 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed) // Wait for all to finish - OPstream::waitRequests(); - IPstream::waitRequests(); + Pstream::waitRequests(); // Compact out all submap entries that are referring to unused elements diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C index da1ed19d1e..bd3e6b744e 100644 --- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C @@ -287,8 +287,7 @@ void Foam::mapDistribute::distribute // Wait till all finished - IPstream::waitRequests(); - OPstream::waitRequests(); + Pstream::waitRequests(); // Consume for (label domain = 0; domain < Pstream::nProcs(); domain++) @@ -413,8 +412,7 @@ void Foam::mapDistribute::distribute // Wait for all to finish - OPstream::waitRequests(); - IPstream::waitRequests(); + Pstream::waitRequests(); // Collect neighbour fields @@ -717,8 +715,7 @@ void Foam::mapDistribute::distribute // Wait till all finished - IPstream::waitRequests(); - OPstream::waitRequests(); + Pstream::waitRequests(); // Consume for (label domain = 0; domain < Pstream::nProcs(); domain++) @@ -842,8 +839,7 @@ void Foam::mapDistribute::distribute // Wait for all to finish - OPstream::waitRequests(); - IPstream::waitRequests(); + Pstream::waitRequests(); // Collect neighbour fields diff --git a/src/Pstream/dummy/IPread.C b/src/Pstream/dummy/IPread.C index 6ab24bc152..8e3d64982b 100644 --- a/src/Pstream/dummy/IPread.C +++ b/src/Pstream/dummy/IPread.C @@ -86,17 +86,4 @@ int Foam::IPstream::read } -void Foam::IPstream::waitRequests() -{} - - -bool Foam::IPstream::finishedRequest(const label) -{ - notImplemented("IPstream::finishedRequest()"); - return false; -} - - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - // ************************************************************************* // diff --git a/src/Pstream/dummy/OPwrite.C b/src/Pstream/dummy/OPwrite.C index 1b7326bdd8..ad12d738fd 100644 --- a/src/Pstream/dummy/OPwrite.C +++ b/src/Pstream/dummy/OPwrite.C @@ -65,17 +65,4 @@ bool Foam::OPstream::write } -void Foam::OPstream::waitRequests() -{} - - -bool Foam::OPstream::finishedRequest(const label) -{ - notImplemented("OPstream::finishedRequest()"); - return false; -} - - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - // ************************************************************************* // diff --git a/src/Pstream/dummy/Pstream.C b/src/Pstream/dummy/Pstream.C index 2c3825eac4..bd22701cfe 100644 --- a/src/Pstream/dummy/Pstream.C +++ b/src/Pstream/dummy/Pstream.C @@ -61,6 +61,17 @@ void Foam::Pstream::abort() void Foam::reduce(scalar&, const sumOp&) {} -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + + +void Foam::Pstream::waitRequests() +{} + + +bool Foam::Pstream::finishedRequest(const label i) +{ + notImplemented("Pstream::finishedRequest()"); + return false; +} + // ************************************************************************* // diff --git a/src/Pstream/mpi/IPread.C b/src/Pstream/mpi/IPread.C index c08c6b56ab..966ae7ddc9 100644 --- a/src/Pstream/mpi/IPread.C +++ b/src/Pstream/mpi/IPread.C @@ -30,6 +30,7 @@ Description #include "mpi.h" #include "IPstream.H" +#include "PstreamGlobals.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -37,7 +38,7 @@ Description // Outstanding non-blocking operations. //! @cond fileScope -Foam::DynamicList IPstream_outstandingRequests_; +//Foam::DynamicList IPstream_outstandingRequests_; //! @endcond fileScope // * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * // @@ -185,7 +186,7 @@ Foam::label Foam::IPstream::read return 0; } - IPstream_outstandingRequests_.append(request); + PstreamGlobals::outstandingRequests_.append(request); // Assume the message is completely received. return bufSize; @@ -204,52 +205,6 @@ Foam::label Foam::IPstream::read } -void Foam::IPstream::waitRequests() -{ - if (IPstream_outstandingRequests_.size()) - { - if - ( - MPI_Waitall - ( - IPstream_outstandingRequests_.size(), - IPstream_outstandingRequests_.begin(), - MPI_STATUSES_IGNORE - ) - ) - { - FatalErrorIn - ( - "IPstream::waitRequests()" - ) << "MPI_Waitall returned with error" << endl; - } - - IPstream_outstandingRequests_.clear(); - } -} - - -bool Foam::IPstream::finishedRequest(const label i) -{ - if (i >= IPstream_outstandingRequests_.size()) - { - FatalErrorIn - ( - "IPstream::finishedRequest(const label)" - ) << "There are " << IPstream_outstandingRequests_.size() - << " outstanding send requests and you are asking for i=" << i - << nl - << "Maybe you are mixing blocking/non-blocking comms?" - << Foam::abort(FatalError); - } - - int flag; - MPI_Test(&IPstream_outstandingRequests_[i], &flag, MPI_STATUS_IGNORE); - - return flag != 0; -} - - // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // ************************************************************************* // diff --git a/src/Pstream/mpi/Make/files b/src/Pstream/mpi/Make/files index 65016b83f8..6b1d3f4c57 100644 --- a/src/Pstream/mpi/Make/files +++ b/src/Pstream/mpi/Make/files @@ -1,5 +1,6 @@ OPwrite.C IPread.C Pstream.C +PstreamGlobals.C LIB = $(FOAM_MPI_LIBBIN)/libPstream diff --git a/src/Pstream/mpi/OPwrite.C b/src/Pstream/mpi/OPwrite.C index 4836b92c8b..1e963ec13f 100644 --- a/src/Pstream/mpi/OPwrite.C +++ b/src/Pstream/mpi/OPwrite.C @@ -30,13 +30,7 @@ Description #include "mpi.h" #include "OPstream.H" - -// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // - -// Outstanding non-blocking operations. -//! @cond fileScope -Foam::DynamicList OPstream_outstandingRequests_; -//! @endcond fileScope +#include "PstreamGlobals.H" // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // @@ -126,7 +120,7 @@ bool Foam::OPstream::write &request ); - OPstream_outstandingRequests_.append(request); + PstreamGlobals::outstandingRequests_.append(request); } else { @@ -142,52 +136,6 @@ bool Foam::OPstream::write } -void Foam::OPstream::waitRequests() -{ - if (OPstream_outstandingRequests_.size()) - { - if - ( - MPI_Waitall - ( - OPstream_outstandingRequests_.size(), - OPstream_outstandingRequests_.begin(), - MPI_STATUSES_IGNORE - ) - ) - { - FatalErrorIn - ( - "OPstream::waitRequests()" - ) << "MPI_Waitall returned with error" << Foam::endl; - } - - OPstream_outstandingRequests_.clear(); - } -} - - -bool Foam::OPstream::finishedRequest(const label i) -{ - if (i >= OPstream_outstandingRequests_.size()) - { - FatalErrorIn - ( - "OPstream::finishedRequest(const label)" - ) << "There are " << OPstream_outstandingRequests_.size() - << " outstanding send requests and you are asking for i=" << i - << nl - << "Maybe you are mixing blocking/non-blocking comms?" - << Foam::abort(FatalError); - } - - int flag; - MPI_Test(&OPstream_outstandingRequests_[i], &flag, MPI_STATUS_IGNORE); - - return flag != 0; -} - - // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // ************************************************************************* // diff --git a/src/Pstream/mpi/Pstream.C b/src/Pstream/mpi/Pstream.C index ba4629d754..074ba1aa28 100644 --- a/src/Pstream/mpi/Pstream.C +++ b/src/Pstream/mpi/Pstream.C @@ -29,6 +29,7 @@ License #include "Pstream.H" #include "PstreamReduceOps.H" #include "OSspecific.H" +#include "PstreamGlobals.H" #include #include @@ -422,6 +423,57 @@ void Foam::reduce(scalar& Value, const sumOp& bop) } +void Foam::Pstream::waitRequests() +{ + if (PstreamGlobals::outstandingRequests_.size()) + { + if + ( + MPI_Waitall + ( + PstreamGlobals::outstandingRequests_.size(), + PstreamGlobals::outstandingRequests_.begin(), + MPI_STATUSES_IGNORE + ) + ) + { + FatalErrorIn + ( + "Pstream::waitRequests()" + ) << "MPI_Waitall returned with error" << Foam::endl; + } + + PstreamGlobals::outstandingRequests_.clear(); + } +} + + +bool Foam::Pstream::finishedRequest(const label i) +{ + if (i >= PstreamGlobals::outstandingRequests_.size()) + { + FatalErrorIn + ( + "Pstream::finishedRequest(const label)" + ) << "There are " << PstreamGlobals::outstandingRequests_.size() + << " outstanding send requests and you are asking for i=" << i + << nl + << "Maybe you are mixing blocking/non-blocking comms?" + << Foam::abort(FatalError); + } + + int flag; + MPI_Test + ( + &PstreamGlobals::outstandingRequests_[i], + &flag, + MPI_STATUS_IGNORE + ); + + return flag != 0; +} + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // ************************************************************************* // diff --git a/src/Pstream/mpi/PstreamGlobals.C b/src/Pstream/mpi/PstreamGlobals.C new file mode 100644 index 0000000000..f9271e5fe4 --- /dev/null +++ b/src/Pstream/mpi/PstreamGlobals.C @@ -0,0 +1,45 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM; if not, write to the Free Software Foundation, + Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +\*---------------------------------------------------------------------------*/ + +#include "PstreamGlobals.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // + +// Outstanding non-blocking operations. +//! @cond fileScope +DynamicList PstreamGlobals::outstandingRequests_; +//! @endcond fileScope + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace Foam + +// ************************************************************************* // diff --git a/src/Pstream/mpi/PstreamGlobals.H b/src/Pstream/mpi/PstreamGlobals.H new file mode 100644 index 0000000000..07aa35a48f --- /dev/null +++ b/src/Pstream/mpi/PstreamGlobals.H @@ -0,0 +1,69 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2 of the License, or (at your + option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM; if not, write to the Free Software Foundation, + Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Namespace + Foam::PstreamGlobals + +Description + Global functions and variables for working with parallel streams, + but principally for gamma/mpi + +SourceFiles + PstreamGlobals.C + +\*---------------------------------------------------------------------------*/ + +#ifndef PstreamGlobals_H +#define PstreamGlobals_H + +#include "mpi.h" + +#include "DynamicList.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +/*---------------------------------------------------------------------------*\ + Class PstreamGlobals Declaration +\*---------------------------------------------------------------------------*/ + +namespace PstreamGlobals +{ + +extern DynamicList outstandingRequests_; + +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* //