bundle outstanding send and receive requests

src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H

@@ -117,12 +117,6 @@ public:
                 const std::streamsize bufSize
             );
 
-            //- Non-blocking receives: wait until all have finished.
-            static void waitRequests();
-
-            //- Non-blocking receives: has request i finished?
-            static bool finishedRequest(const label i);
-
             //- Return next token from stream
             Istream& read(token&);
 

src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H

@@ -115,12 +115,6 @@ public:
                 const std::streamsize bufSize
             );
 
-            //- Non-blocking writes: wait until all have finished.
-            static void waitRequests();
-
-            //- Non-blocking writes: has request i finished?
-            static bool finishedRequest(const label i);
-
             //- Write next token to stream
             Ostream& write(const token&);
 

src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H

@@ -264,6 +264,12 @@ public:
         //  Spawns slave processes and initialises inter-communication
         static bool init(int& argc, char**& argv);
 
+        //- Non-blocking comms: wait until all have finished.
+        static void waitRequests();
+
+        //- Non-blocking comms: has request i finished?
+        static bool finishedRequest(const label i);
+
         //- Is this a parallel run?
         static bool parRun()
         {

src/OpenFOAM/fields/GeometricFields/GeometricField/GeometricBoundaryField.C

@@ -300,8 +300,7 @@ evaluate()
         // Block for any outstanding requests
         if (Pstream::defaultCommsType == Pstream::nonBlocking)
         {
-            IPstream::waitRequests();
+            Pstream::waitRequests();
-            OPstream::waitRequests();
         }
 
         forAll(*this, patchi)

src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C

@@ -339,8 +339,7 @@ void Foam::mapDistribute::compact(const boolList& elemIsUsed)
 
         // Wait for all to finish
 
-        OPstream::waitRequests();
+        Pstream::waitRequests();
-        IPstream::waitRequests();
 
 
         // Compact out all submap entries that are referring to unused elements

src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C

@@ -287,8 +287,7 @@ void Foam::mapDistribute::distribute
 
 
             // Wait till all finished
-            IPstream::waitRequests();
+            Pstream::waitRequests();
-            OPstream::waitRequests();
 
             // Consume
             for (label domain = 0; domain < Pstream::nProcs(); domain++)
@@ -413,8 +412,7 @@ void Foam::mapDistribute::distribute
 
             // Wait for all to finish
 
-            OPstream::waitRequests();
+            Pstream::waitRequests();
-            IPstream::waitRequests();
 
             // Collect neighbour fields
 
@@ -717,8 +715,7 @@ void Foam::mapDistribute::distribute
 
 
             // Wait till all finished
-            IPstream::waitRequests();
+            Pstream::waitRequests();
-            OPstream::waitRequests();
 
             // Consume
             for (label domain = 0; domain < Pstream::nProcs(); domain++)
@@ -842,8 +839,7 @@ void Foam::mapDistribute::distribute
 
             // Wait for all to finish
 
-            OPstream::waitRequests();
+            Pstream::waitRequests();
-            IPstream::waitRequests();
 
             // Collect neighbour fields
 

src/Pstream/dummy/IPread.C

@@ -86,17 +86,4 @@ int Foam::IPstream::read
 }
 
 
-void Foam::IPstream::waitRequests()
-{}
-
-
-bool Foam::IPstream::finishedRequest(const label)
-{
-    notImplemented("IPstream::finishedRequest()");
-    return false;
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
 // ************************************************************************* //

src/Pstream/dummy/OPwrite.C

@@ -65,17 +65,4 @@ bool Foam::OPstream::write
 }
 
 
-void Foam::OPstream::waitRequests()
-{}
-
-
-bool Foam::OPstream::finishedRequest(const label)
-{
-    notImplemented("OPstream::finishedRequest()");
-    return false;
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
 // ************************************************************************* //

src/Pstream/dummy/Pstream.C

@@ -61,6 +61,17 @@ void Foam::Pstream::abort()
 void Foam::reduce(scalar&, const sumOp<scalar>&)
 {}
 
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+
+void Foam::Pstream::waitRequests()
+{}
+
+
+bool Foam::Pstream::finishedRequest(const label i)
+{
+    notImplemented("Pstream::finishedRequest()");
+    return false;
+}
+
 
 // ************************************************************************* //

src/Pstream/mpi/IPread.C

@@ -30,6 +30,7 @@ Description
 #include "mpi.h"
 
 #include "IPstream.H"
+#include "PstreamGlobals.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -37,7 +38,7 @@ Description
 
 // Outstanding non-blocking operations.
 //! @cond fileScope
-Foam::DynamicList<MPI_Request> IPstream_outstandingRequests_;
+//Foam::DynamicList<MPI_Request> IPstream_outstandingRequests_;
 //! @endcond fileScope
 
 // * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
@@ -185,7 +186,7 @@ Foam::label Foam::IPstream::read
             return 0;
         }
 
-        IPstream_outstandingRequests_.append(request);
+        PstreamGlobals::outstandingRequests_.append(request);
 
         // Assume the message is completely received.
         return bufSize;
@@ -204,52 +205,6 @@ Foam::label Foam::IPstream::read
 }
 
 
-void Foam::IPstream::waitRequests()
-{
-    if (IPstream_outstandingRequests_.size())
-    {
-        if
-        (
-            MPI_Waitall
-            (
-                IPstream_outstandingRequests_.size(),
-                IPstream_outstandingRequests_.begin(),
-                MPI_STATUSES_IGNORE
-            )
-        )
-        {
-            FatalErrorIn
-            (
-                "IPstream::waitRequests()"
-            )   << "MPI_Waitall returned with error" << endl;
-        }
-
-        IPstream_outstandingRequests_.clear();
-    }
-}
-
-
-bool Foam::IPstream::finishedRequest(const label i)
-{
-    if (i >= IPstream_outstandingRequests_.size())
-    {
-        FatalErrorIn
-        (
-            "IPstream::finishedRequest(const label)"
-        )   << "There are " << IPstream_outstandingRequests_.size()
-            << " outstanding send requests and you are asking for i=" << i
-            << nl
-            << "Maybe you are mixing blocking/non-blocking comms?"
-            << Foam::abort(FatalError);
-    }
-
-    int flag;
-    MPI_Test(&IPstream_outstandingRequests_[i], &flag, MPI_STATUS_IGNORE);
-
-    return flag != 0;
-}
-
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 // ************************************************************************* //

src/Pstream/mpi/Make/files

@@ -1,5 +1,6 @@
 OPwrite.C
 IPread.C
 Pstream.C
+PstreamGlobals.C
 
 LIB = $(FOAM_MPI_LIBBIN)/libPstream

src/Pstream/mpi/OPwrite.C

@@ -30,13 +30,7 @@ Description
 #include "mpi.h"
 
 #include "OPstream.H"
-
+#include "PstreamGlobals.H"
-// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
-
-// Outstanding non-blocking operations.
-//! @cond fileScope
-Foam::DynamicList<MPI_Request> OPstream_outstandingRequests_;
-//! @endcond fileScope
 
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
@@ -126,7 +120,7 @@ bool Foam::OPstream::write
             &request
         );
 
-        OPstream_outstandingRequests_.append(request);
+        PstreamGlobals::outstandingRequests_.append(request);
     }
     else
     {
@@ -142,52 +136,6 @@ bool Foam::OPstream::write
 }
 
 
-void Foam::OPstream::waitRequests()
-{
-    if (OPstream_outstandingRequests_.size())
-    {
-        if
-        (
-            MPI_Waitall
-            (
-                OPstream_outstandingRequests_.size(),
-                OPstream_outstandingRequests_.begin(),
-                MPI_STATUSES_IGNORE
-            )
-        )
-        {
-            FatalErrorIn
-            (
-                "OPstream::waitRequests()"
-            )   << "MPI_Waitall returned with error" << Foam::endl;
-        }
-
-        OPstream_outstandingRequests_.clear();
-    }
-}
-
-
-bool Foam::OPstream::finishedRequest(const label i)
-{
-    if (i >= OPstream_outstandingRequests_.size())
-    {
-        FatalErrorIn
-        (
-            "OPstream::finishedRequest(const label)"
-        )   << "There are " << OPstream_outstandingRequests_.size()
-            << " outstanding send requests and you are asking for i=" << i
-            << nl
-            << "Maybe you are mixing blocking/non-blocking comms?"
-            << Foam::abort(FatalError);
-    }
-
-    int flag;
-    MPI_Test(&OPstream_outstandingRequests_[i], &flag, MPI_STATUS_IGNORE);
-
-    return flag != 0;
-}
-
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 // ************************************************************************* //

src/Pstream/mpi/Pstream.C

@@ -29,6 +29,7 @@ License
 #include "Pstream.H"
 #include "PstreamReduceOps.H"
 #include "OSspecific.H"
+#include "PstreamGlobals.H"
 
 #include <cstring>
 #include <cstdlib>
@@ -422,6 +423,57 @@ void Foam::reduce(scalar& Value, const sumOp<scalar>& bop)
 }
 
 
+void Foam::Pstream::waitRequests()
+{
+    if (PstreamGlobals::outstandingRequests_.size())
+    {
+        if
+        (
+            MPI_Waitall
+            (
+                PstreamGlobals::outstandingRequests_.size(),
+                PstreamGlobals::outstandingRequests_.begin(),
+                MPI_STATUSES_IGNORE
+            )
+        )
+        {
+            FatalErrorIn
+            (
+                "Pstream::waitRequests()"
+            )   << "MPI_Waitall returned with error" << Foam::endl;
+        }
+
+        PstreamGlobals::outstandingRequests_.clear();
+    }
+}
+
+
+bool Foam::Pstream::finishedRequest(const label i)
+{
+    if (i >= PstreamGlobals::outstandingRequests_.size())
+    {
+        FatalErrorIn
+        (
+            "Pstream::finishedRequest(const label)"
+        )   << "There are " << PstreamGlobals::outstandingRequests_.size()
+            << " outstanding send requests and you are asking for i=" << i
+            << nl
+            << "Maybe you are mixing blocking/non-blocking comms?"
+            << Foam::abort(FatalError);
+    }
+
+    int flag;
+    MPI_Test
+    (
+       &PstreamGlobals::outstandingRequests_[i],
+       &flag,
+        MPI_STATUS_IGNORE
+    );
+
+    return flag != 0;
+}
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 // ************************************************************************* //

src/Pstream/mpi/PstreamGlobals.C

@@ -0,0 +1,45 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 1991-2009 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software; you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation; either version 2 of the License, or (at your
+    option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM; if not, write to the Free Software Foundation,
+    Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+\*---------------------------------------------------------------------------*/
+
+#include "PstreamGlobals.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+// Outstanding non-blocking operations.
+//! @cond fileScope
+DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;
+//! @endcond fileScope
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// ************************************************************************* //

src/Pstream/mpi/PstreamGlobals.H

@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 1991-2009 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software; you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation; either version 2 of the License, or (at your
+    option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM; if not, write to the Free Software Foundation,
+    Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+Namespace
+    Foam::PstreamGlobals
+
+Description
+    Global functions and variables for working with parallel streams,
+    but principally for gamma/mpi
+
+SourceFiles
+    PstreamGlobals.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef PstreamGlobals_H
+#define PstreamGlobals_H
+
+#include "mpi.h"
+
+#include "DynamicList.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                           Class PstreamGlobals Declaration
+\*---------------------------------------------------------------------------*/
+
+namespace PstreamGlobals
+{
+
+extern DynamicList<MPI_Request> outstandingRequests_;
+
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //