ENH: communicators: more communicators through

src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -330,7 +330,7 @@ Foam::Istream& Foam::UIPstream::rewind()
 void Foam::UIPstream::print(Ostream& os) const
 {
     os  << "Reading from processor " << fromProcNo_
-        << " to processor " << myProcNo << " using communicator " << comm_
+        << " using communicator " << comm_
         <<  " and tag " << tag_
         << Foam::endl;
 }

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

@@ -339,6 +339,39 @@ void Foam::UPstream::freeCommunicators(const bool doPstream)
 }
 
 
+int Foam::UPstream::baseProcNo(const label myComm, const int myProcID)
+{
+    int procID = myProcID;
+    label comm = myComm;
+
+    while (parent(comm) != -1)
+    {
+        const List<int>& parentRanks = UPstream::procID(comm);
+        procID = parentRanks[procID];
+        comm = UPstream::parent(comm);
+    }
+
+    return procID;
+}
+
+
+Foam::label Foam::UPstream::myProcNo(const label myComm, const int baseProcID)
+{
+    const List<int>& parentRanks = procID(myComm);
+    label parentComm = parent(myComm);
+
+    if (parentComm == -1)
+    {
+        return findIndex(parentRanks, baseProcID);
+    }
+    else
+    {
+        label parentRank = myProcNo(parentComm, baseProcID);
+        return findIndex(parentRanks, parentRank);
+    }
+}
+
+
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 // By default this is not a parallel run

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

@@ -323,18 +323,6 @@ public:
                 comm_(allocateCommunicator(parent, subRanks, doPstream))
             {}
 
-//            communicator(const label parent)
-//            :
-//                comm_
-//                (
-//                    allocateCommunicator
-//                    (
-//                        parent,
-//                        identity(UPstream::nProcs(parent))
-//                    )
-//                )
-//            {}
-
             ~communicator()
             {
                 freeCommunicator(comm_);
@@ -346,6 +334,14 @@ public:
             }
         };
 
+        //- Return physical processor number (i.e. processor number in
+        //  worldComm) given communicator and procssor
+        static int baseProcNo(const label myComm, const int myProcID);
+
+        //- Return processor number in communicator (given physical processor
+        //  number) (= reverse of baseProcNo)
+        static label myProcNo(const label myComm, const int baseProcID);
+
 
         //- Add the valid option this type of communications library
         //  adds/requires on the command line
@@ -380,6 +376,12 @@ public:
 
             static void freeTag(const word&, const int tag);
 
+//            //- Return (index of) request to use in calls to nonblocking ops
+//            static label allocateRequest(const word&);
+//
+//            //- Free (index of) request
+//            static void freeRequest(const word&, const label request);
+
 
         //- Is this a parallel run?
         static bool& parRun()
@@ -416,7 +418,6 @@ public:
             return parentCommunicator_(communicator);
         }
 
-
 //        //- Process IDs
 //        static const List<int>& procIDs()
 //        {

src/OpenFOAM/db/error/messageStream.C

@@ -196,14 +196,7 @@ Foam::OSstream& Foam::messageStream::operator()(const label communicator)
             {
                 if (title().size())
                 {
-                    if (Pstream::parRun() && !collect)
+                    Pout<< title().c_str();
-                    {
-                        Pout<< title().c_str();
-                    }
-                    else
-                    {
-                        Sout<< title().c_str();
-                    }
                 }
 
                 if (maxErrors_)
@@ -218,14 +211,7 @@ Foam::OSstream& Foam::messageStream::operator()(const label communicator)
                     }
                 }
 
-                if (Pstream::parRun() && !collect)
+                return Pout;
-                {
-                    return Pout;
-                }
-                else
-                {
-                    return Sout;
-                }
             }
         }
     }

src/OpenFOAM/matrices/LUscalarMatrix/LUscalarMatrix.C

@@ -33,8 +33,8 @@ License
 
 Foam::LUscalarMatrix::LUscalarMatrix(const scalarSquareMatrix& matrix)
 :
-    comm_(Pstream::worldComm),
     scalarSquareMatrix(matrix),
+    comm_(Pstream::worldComm),
     pivotIndices_(n())
 {
     LUDecompose(*this, pivotIndices_);
@@ -67,11 +67,6 @@ Foam::LUscalarMatrix::LUscalarMatrix
             )
         );
 
-
-Pout<< "LUscalarMatrix :"
-    << " comm:" << comm_
-    << " master:" << Pstream::master(comm_) << endl;
-
         if (Pstream::master(comm_))
         {
             for
@@ -81,8 +76,6 @@ Pout<< "LUscalarMatrix :"
                 slave++
             )
             {
-Pout<< "Receiving from " << slave
-    << " using comm:" << comm_ << endl;
                 lduMatrices.set
                 (
                     lduMatrixi++,

src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H

@@ -108,7 +108,7 @@ public:
             Pout<< "lduPrimitiveMesh :"
                 << " nCells:" << nCells
                 << " l:" << lowerAddr_.size()
-                << " u::" << upperAddr_.size()
+                << " u:" << upperAddr_.size()
                 << " pa:" << patchAddr_.size()
                 << " interfaces:" << interfaces_.size()
                 << " comm:" << comm_
@@ -124,9 +124,11 @@ public:
                             const processorLduInterface
                         >(interfaces_[i]);
 
-                        Pout<< "    comm:" << pi.comm()
+                        Pout<< "    patch:" << i
+                            << " size:" << patchAddr_[i].size()
                             << " myProcNo:" << pi.myProcNo()
                             << " neighbProcNo:" << pi.neighbProcNo()
+                            << " comm:" << pi.comm()
                             << endl;
                     }
                 }
@@ -158,7 +160,7 @@ public:
             Pout<< "lduPrimitiveMesh :"
                 << " nCells:" << nCells
                 << " l:" << lowerAddr_.size()
-                << " u::" << upperAddr_.size()
+                << " u:" << upperAddr_.size()
                 << " pa:" << patchAddr_.size()
                 << " interfaces:" << interfaces_.size()
                 << " comm:" << comm_
@@ -174,9 +176,11 @@ public:
                             const processorLduInterface
                         >(interfaces_[i]);
 
-                        Pout<< "    comm:" << pi.comm()
+                        Pout<< "    patch:" << i
+                            << " size:" << patchAddr_[i].size()
                             << " myProcNo:" << pi.myProcNo()
                             << " neighbProcNo:" << pi.neighbProcNo()
+                            << " comm:" << pi.comm()
                             << endl;
                     }
                 }

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -31,18 +31,51 @@ Foam::globalIndex::globalIndex
 (
     const label localSize,
     const int tag,
+    const label comm,
     const bool parallel
 )
+:
+    offsets_(Pstream::nProcs(comm)+1)
+{
+    labelList localSizes(Pstream::nProcs(comm), 0);
+    localSizes[Pstream::myProcNo(comm)] = localSize;
+    if (parallel)
+    {
+        Pstream::gatherList(localSizes, tag, comm);
+        Pstream::scatterList(localSizes, tag, comm);
+    }
+
+    label offset = 0;
+    offsets_[0] = 0;
+    for (label procI = 0; procI < Pstream::nProcs(comm); procI++)
+    {
+        label oldOffset = offset;
+        offset += localSizes[procI];
+
+        if (offset < oldOffset)
+        {
+            FatalErrorIn
+            (
+                "globalIndex::globalIndex"
+                "(const label, const int, const label, const bool)"
+            )   << "Overflow : sum of sizes " << localSizes
+                << " exceeds capability of label (" << labelMax
+                << "). Please recompile with larger datatype for label."
+                << exit(FatalError);
+        }
+        offsets_[procI+1] = offset;
+    }
+}
+
+
+Foam::globalIndex::globalIndex(const label localSize)
 :
     offsets_(Pstream::nProcs()+1)
 {
     labelList localSizes(Pstream::nProcs(), 0);
     localSizes[Pstream::myProcNo()] = localSize;
-    if (parallel)
+    Pstream::gatherList(localSizes, Pstream::msgType());
-    {
+    Pstream::scatterList(localSizes, Pstream::msgType());
-        Pstream::gatherList(localSizes, tag);
-        Pstream::scatterList(localSizes, tag);
-    }
 
     label offset = 0;
     offsets_[0] = 0;
@@ -64,6 +97,12 @@ Foam::globalIndex::globalIndex
 }
 
 
+Foam::globalIndex::globalIndex(const labelList& offsets)
+:
+    offsets_(offsets)
+{}
+
+
 Foam::globalIndex::globalIndex(Istream& is)
 {
     is >> offsets_;

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndex.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -71,21 +71,29 @@ public:
 
     // Constructors
 
+
+        //- Construct from local max size
+        globalIndex(const label localSize);
+
         //- Construct from local max size
         globalIndex
         (
             const label localSize,
-            const int tag = Pstream::msgType(),
+            const int tag,
-            const bool parallel = true              // use parallel comms
+            const label comm,
+            const bool parallel     // use parallel comms
         );
 
+        //- Construct from components
+        globalIndex(const labelList& offsets);
+
         //- Construct from Istream
         globalIndex(Istream& is);
 
 
     // Member Functions
 
-        // Queries relating to my processor
+        // Queries relating to my processor (using world communicator)
 
             //- my local size
             inline label localSize() const;

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -47,7 +47,7 @@ inline Foam::label Foam::globalIndex::localSize() const
 
 inline Foam::label Foam::globalIndex::size() const
 {
-    return offsets_[Pstream::nProcs()];
+    return offsets_.last();
 }
 
 
@@ -104,7 +104,7 @@ inline Foam::label Foam::globalIndex::toLocal(const label i) const
 
 inline Foam::label Foam::globalIndex::whichProcID(const label i) const
 {
-    if (i < 0 || i >= offsets_[Pstream::nProcs()])
+    if (i < 0 || i >= offsets_.size()-1)
     {
         FatalErrorIn("globalIndex::whichProcID(const label)")
             << "Global " << i << " does not belong on any processor."

src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -61,7 +61,7 @@ Foam::processorPolyPatch::processorPolyPatch
 )
 :
     coupledPolyPatch(name, size, start, index, bm, typeName, transform),
-    comm_(comm),
+//    comm_(comm),
     myProcNo_(myProcNo),
     neighbProcNo_(neighbProcNo),
     neighbFaceCentres_(),
@@ -80,10 +80,10 @@ Foam::processorPolyPatch::processorPolyPatch
 )
 :
     coupledPolyPatch(name, dict, index, bm, patchType),
-    comm_
+//    comm_
-    (
+//    (
-        dict.lookupOrDefault("communicator", UPstream::worldComm)
+//        dict.lookupOrDefault("communicator", UPstream::worldComm)
-    ),
+//    ),
     myProcNo_(readLabel(dict.lookup("myProcNo"))),
     neighbProcNo_(readLabel(dict.lookup("neighbProcNo"))),
     neighbFaceCentres_(),
@@ -99,7 +99,7 @@ Foam::processorPolyPatch::processorPolyPatch
 )
 :
     coupledPolyPatch(pp, bm),
-    comm_(pp.comm_),
+//    comm_(pp.comm_),
     myProcNo_(pp.myProcNo_),
     neighbProcNo_(pp.neighbProcNo_),
     neighbFaceCentres_(),
@@ -118,7 +118,7 @@ Foam::processorPolyPatch::processorPolyPatch
 )
 :
     coupledPolyPatch(pp, bm, index, newSize, newStart),
-    comm_(pp.comm_),
+//    comm_(pp.comm_),
     myProcNo_(pp.myProcNo_),
     neighbProcNo_(pp.neighbProcNo_),
     neighbFaceCentres_(),
@@ -137,7 +137,7 @@ Foam::processorPolyPatch::processorPolyPatch
 )
 :
     coupledPolyPatch(pp, bm, index, mapAddressing, newStart),
-    comm_(pp.comm_),
+//    comm_(pp.comm_),
     myProcNo_(pp.myProcNo_),
     neighbProcNo_(pp.neighbProcNo_),
     neighbFaceCentres_(),
@@ -1091,11 +1091,11 @@ bool Foam::processorPolyPatch::order
 void Foam::processorPolyPatch::write(Ostream& os) const
 {
     coupledPolyPatch::write(os);
-    if (comm_ != UPstream::worldComm)
+//    if (comm_ != UPstream::worldComm)
-    {
+//    {
-        os.writeKeyword("communicator") << comm_
+//        os.writeKeyword("communicator") << comm_
-            << token::END_STATEMENT << nl;
+//            << token::END_STATEMENT << nl;
-    }
+//    }
     os.writeKeyword("myProcNo") << myProcNo_
         << token::END_STATEMENT << nl;
     os.writeKeyword("neighbProcNo") << neighbProcNo_

src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H

@@ -42,6 +42,7 @@ SourceFiles
 #include "coupledPolyPatch.H"
 #include "polyBoundaryMesh.H"
 #include "faceListFwd.H"
+#include "polyMesh.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -58,9 +59,6 @@ class processorPolyPatch
 {
     // Private data
 
-        //- Communicator to use
-        label comm_;
-
         int myProcNo_;
         int neighbProcNo_;
 
@@ -252,7 +250,8 @@ public:
         //- Return communicator used for communication
         label comm() const
         {
-            return comm_;
+            //return comm_;
+            return boundaryMesh().mesh().comm();
         }
 
         //- Return processor number
@@ -270,11 +269,11 @@ public:
 
         // For testing
 
-            //- Return communicator used for communication
+//            //- Return communicator used for communication
-            label& comm()
+//            label& comm()
-            {
+//            {
-                return comm_;
+//                return comm_;
-            }
+//            }
 
             //- Return processor number
             int& myProcNo()

src/Pstream/mpi/PstreamGlobals.C

@@ -37,6 +37,27 @@ namespace Foam
 DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;
 //! \endcond
 
+//// Max outstanding non-blocking operations.
+////! \cond fileScope
+//int PstreamGlobals::nRequests_ = 0;
+////! \endcond
+
+// Free'd non-blocking operations.
+//! \cond fileScope
+//DynamicList<label> PstreamGlobals::freedRequests_;
+//! \endcond
+
+// Max outstanding message tag operations.
+//! \cond fileScope
+int PstreamGlobals::nTags_ = 0;
+//! \endcond
+
+// Free'd message tags
+//! \cond fileScope
+DynamicList<int> PstreamGlobals::freedTags_;
+//! \endcond
+
+
 // Allocated communicators.
 //! \cond fileScope
 DynamicList<MPI_Comm> PstreamGlobals::MPICommunicators_;

src/Pstream/mpi/PstreamGlobals.H

@@ -54,6 +54,13 @@ namespace PstreamGlobals
 
 extern DynamicList<MPI_Request> outstandingRequests_;
 
+//extern int nRequests_;
+//extern DynamicList<label> freedRequests_;
+
+extern int nTags_;
+
+extern DynamicList<int> freedTags_;
+
 
 // Current communicators. First element will be MPI_COMM_WORLD
 extern DynamicList<MPI_Comm> MPICommunicators_;

src/Pstream/mpi/UIPread.C

@@ -70,7 +70,7 @@ Foam::UIPstream::UIPstream
 
         label wantedSize = externalBuf_.capacity();
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UIPstream::UIPstream : read from:" << fromProcNo
                 << " tag:" << tag << " wanted size:" << wantedSize
@@ -94,7 +94,7 @@ Foam::UIPstream::UIPstream
             externalBuf_.setCapacity(messageSize_);
             wantedSize = messageSize_;
 
-            if (debug&1)
+            if (debug)
             {
                 Pout<< "UIPstream::UIPstream : probed size:" << wantedSize
                     << Foam::endl;
@@ -151,7 +151,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
         // Message is already received into externalBuf
         messageSize_ = buffers.recvBuf_[fromProcNo].size();
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UIPstream::UIPstream PstreamBuffers :"
                 << " fromProcNo:" << fromProcNo
@@ -165,7 +165,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
 
         label wantedSize = externalBuf_.capacity();
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UIPstream::UIPstream PstreamBuffers :"
                 << " read from:" << fromProcNo
@@ -189,7 +189,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
             externalBuf_.setCapacity(messageSize_);
             wantedSize = messageSize_;
 
-            if (debug&1)
+            if (debug)
             {
                 Pout<< "UIPstream::UIPstream PstreamBuffers : probed size:"
                     << wantedSize << Foam::endl;
@@ -229,7 +229,7 @@ Foam::label Foam::UIPstream::read
     const label communicator
 )
 {
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UIPstream::read : starting read from:" << fromProcNo
             << " tag:" << tag << " comm:" << communicator
@@ -282,7 +282,7 @@ Foam::label Foam::UIPstream::read
         int messageSize;
         MPI_Get_count(&status, MPI_BYTE, &messageSize);
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UIPstream::read : finished read from:" << fromProcNo
                 << " tag:" << tag << " read size:" << label(bufSize)
@@ -332,7 +332,7 @@ Foam::label Foam::UIPstream::read
             return 0;
         }
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UIPstream::read : started read from:" << fromProcNo
                 << " tag:" << tag << " read size:" << label(bufSize)

src/Pstream/mpi/UOPwrite.C

@@ -43,7 +43,7 @@ bool Foam::UOPstream::write
     const label communicator
 )
 {
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UOPstream::write : starting write to:" << toProcNo
             << " tag:" << tag
@@ -80,7 +80,7 @@ bool Foam::UOPstream::write
             PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
         );
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UOPstream::write : finished write to:" << toProcNo
                 << " tag:" << tag << " size:" << label(bufSize)
@@ -100,7 +100,7 @@ bool Foam::UOPstream::write
             PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
         );
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UOPstream::write : finished write to:" << toProcNo
                 << " tag:" << tag << " size:" << label(bufSize)
@@ -123,7 +123,7 @@ bool Foam::UOPstream::write
             &request
         );
 
-        if (debug&1)
+        if (debug)
         {
             Pout<< "UOPstream::write : started write to:" << toProcNo
                 << " tag:" << tag << " size:" << label(bufSize)

src/Pstream/mpi/UPstream.C

@@ -68,7 +68,7 @@ bool Foam::UPstream::init(int& argc, char**& argv)
     int myRank;
     MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
 
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UPstream::init : initialised with numProcs:" << numprocs
             << " myRank:" << myRank << endl;
@@ -120,7 +120,7 @@ bool Foam::UPstream::init(int& argc, char**& argv)
 
 void Foam::UPstream::exit(int errnum)
 {
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UPstream::exit." << endl;
     }
@@ -280,6 +280,13 @@ void Foam::reduce
 
     requestID = PstreamGlobals::outstandingRequests_.size();
     PstreamGlobals::outstandingRequests_.append(request);
+
+    if (debug)
+    {
+        Pout<< "UPstream::allocateRequest for non-blocking reduce"
+            << " : request:" << requestID
+            << endl;
+
 #else
     // Non-blocking not yet implemented in mpi
     reduce(Value, bop, tag, communicator);
@@ -434,7 +441,7 @@ void Foam::UPstream::resetRequests(const label i)
 
 void Foam::UPstream::waitRequests(const label start)
 {
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UPstream::waitRequests : starting wait for "
             << PstreamGlobals::outstandingRequests_.size()-start
@@ -469,7 +476,7 @@ void Foam::UPstream::waitRequests(const label start)
         resetRequests(start);
     }
 
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UPstream::waitRequests : finished wait." << endl;
     }
@@ -478,7 +485,7 @@ void Foam::UPstream::waitRequests(const label start)
 
 void Foam::UPstream::waitRequest(const label i)
 {
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UPstream::waitRequest : starting wait for request:" << i
             << endl;
@@ -511,7 +518,7 @@ void Foam::UPstream::waitRequest(const label i)
         )   << "MPI_Wait returned with error" << Foam::endl;
     }
 
-    if (debug&1)
+    if (debug)
     {
         Pout<< "UPstream::waitRequest : finished wait for request:" << i
             << endl;
@@ -521,9 +528,9 @@ void Foam::UPstream::waitRequest(const label i)
 
 bool Foam::UPstream::finishedRequest(const label i)
 {
-    if (debug&1)
+    if (debug)
     {
-        Pout<< "UPstream::waitRequests : checking finishedRequest:" << i
+        Pout<< "UPstream::finishedRequest : checking request:" << i
             << endl;
     }
 
@@ -547,9 +554,9 @@ bool Foam::UPstream::finishedRequest(const label i)
         MPI_STATUS_IGNORE
     );
 
-    if (debug&1)
+    if (debug)
     {
-        Pout<< "UPstream::waitRequests : finished finishedRequest:" << i
+        Pout<< "UPstream::finishedRequest : finished request:" << i
             << endl;
     }
 
@@ -557,4 +564,152 @@ bool Foam::UPstream::finishedRequest(const label i)
 }
 
 
+int Foam::UPstream::allocateTag(const char* s)
+{
+    int tag;
+    if (PstreamGlobals::freedTags_.size())
+    {
+        tag = PstreamGlobals::freedTags_.remove();
+    }
+    else
+    {
+        tag = PstreamGlobals::nTags_++;
+    }
+
+    if (debug)
+    {
+        //if (UPstream::lateBlocking > 0)
+        //{
+        //    string& poutp = Pout.prefix();
+        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
+        //    Perr.prefix() = Pout.prefix();
+        //}
+        Pout<< "UPstream::allocateTag " << s
+            << " : tag:" << tag
+            << endl;
+    }
+
+    return tag;
+}
+
+
+int Foam::UPstream::allocateTag(const word& s)
+{
+    int tag;
+    if (PstreamGlobals::freedTags_.size())
+    {
+        tag = PstreamGlobals::freedTags_.remove();
+    }
+    else
+    {
+        tag = PstreamGlobals::nTags_++;
+    }
+
+    if (debug)
+    {
+        //if (UPstream::lateBlocking > 0)
+        //{
+        //    string& poutp = Pout.prefix();
+        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
+        //    Perr.prefix() = Pout.prefix();
+        //}
+        Pout<< "UPstream::allocateTag " << s
+            << " : tag:" << tag
+            << endl;
+    }
+
+    return tag;
+}
+
+
+void Foam::UPstream::freeTag(const char* s, const int tag)
+{
+    if (debug)
+    {
+        //if (UPstream::lateBlocking > 0)
+        //{
+        //    string& poutp = Pout.prefix();
+        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
+        //    Perr.prefix() = Pout.prefix();
+        //}
+        Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
+    }
+    PstreamGlobals::freedTags_.append(tag);
+}
+
+
+void Foam::UPstream::freeTag(const word& s, const int tag)
+{
+    if (debug)
+    {
+        //if (UPstream::lateBlocking > 0)
+        //{
+        //    string& poutp = Pout.prefix();
+        //    poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
+        //    Perr.prefix() = Pout.prefix();
+        //}
+        Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
+    }
+    PstreamGlobals::freedTags_.append(tag);
+}
+
+
+//Foam::label Foam::UPstream::allocateRequest(const word& s)
+//{
+//    label request;
+//    if (PstreamGlobals::freedRequests_.size())
+//    {
+//        request = PstreamGlobals::freedRequests_.remove();
+//    }
+//    else
+//    {
+//        request = PstreamGlobals::outstandingRequests_.size();
+//
+//        // Make sure outstanding requests table can hold this
+//        PstreamGlobals::outstandingRequests_.reserve(request);
+//    }
+//
+//    if (debug)
+//    {
+//        //if (UPstream::lateBlocking > 0)
+//        //{
+//        //    string& poutp = Pout.prefix();
+//        //    poutp[poutp.size()-(UPstream::lateBlocking+2)+request] = 'X';
+//        //    Perr.prefix() = Pout.prefix();
+//        //}
+//        Pout<< "UPstream::allocateRequest " << s
+//            << " : request:" << request
+//            << " out of:" << PstreamGlobals::outstandingRequests_.size()
+//            << " capacity:" << PstreamGlobals::outstandingRequests_.capacity()
+//            << endl;
+//    }
+//
+//    return request;
+//}
+//
+//
+//void Foam::UPstream::freeRequest(const word& s, const label index)
+//{
+//    if (PstreamGlobals::outstandingRequests_(index) != MPI_REQUEST_NULL)
+//    {
+//        PstreamGlobals::outstandingRequests_(index) = MPI_REQUEST_NULL;
+//        PstreamGlobals::freedRequests_.append(index);
+//
+//        if (debug)
+//        {
+//            //if (UPstream::lateBlocking > 0)
+//            //{
+//            //    string& poutp = Pout.prefix();
+//            //    poutp[poutp.size()-(UPstream::lateBlocking+2)+index] = ' ';
+//            //    Perr.prefix() = Pout.prefix();
+//            //}
+//            Pout<< "UPstream::freeRequest " << s
+//                << " : request:" << index
+//                << " now nullRequest:"
+//                << long(PstreamGlobals::outstandingRequests_(index)) << endl;
+//        }
+//    }
+//}
+
+
 // ************************************************************************* //

src/finiteVolume/fvMatrices/fvMatrix/fvMatrixSolve.C

@@ -60,7 +60,8 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solve
 {
     if (debug)
     {
-        Info<< "fvMatrix<Type>::solve(const dictionary& solverControls) : "
+        Info(this->mesh().comm())
+            << "fvMatrix<Type>::solve(const dictionary& solverControls) : "
                "solving fvMatrix<Type>"
             << endl;
     }
@@ -107,7 +108,8 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveSegregated
 {
     if (debug)
     {
-        Info<< "fvMatrix<Type>::solveSegregated"
+        Info(this->mesh().comm())
+            << "fvMatrix<Type>::solveSegregated"
                "(const dictionary& solverControls) : "
                "solving fvMatrix<Type>"
             << endl;
@@ -199,8 +201,6 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveSegregated
         )->solve(psiCmpt, sourceCmpt, cmpt);
 
         //solverPerf.print(Info);
-Pout<< "** calling solverPerformance::print with comm:" << this->mesh().comm()
-    << endl;
         solverPerf.print(Info(this->mesh().comm()));
 
         solverPerfVec = max(solverPerfVec, solverPerf);
@@ -226,7 +226,8 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveCoupled
 {
     if (debug)
     {
-        Info<< "fvMatrix<Type>::solveCoupled"
+        Info(this->mesh().comm())
+            << "fvMatrix<Type>::solveCoupled"
                "(const dictionary& solverControls) : "
                "solving fvMatrix<Type>"
             << endl;
@@ -265,8 +266,6 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveCoupled
     );
 
     //solverPerf.print(Info);
-Pout<< "** calling solverPerformance::print with comm:" << this->mesh().comm()
-    << endl;
     solverPerf.print(Info(this->mesh().comm()));
 
     psi.correctBoundaryConditions();

src/finiteVolume/fvMatrices/fvScalarMatrix/fvScalarMatrix.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -61,7 +61,8 @@ Foam::fvMatrix<Foam::scalar>::solver
 {
     if (debug)
     {
-        Info<< "fvMatrix<scalar>::solver(const dictionary& solverControls) : "
+        Info(this->mesh().comm())
+            << "fvMatrix<scalar>::solver(const dictionary& solverControls) : "
                "solver for fvMatrix<scalar>"
             << endl;
     }
@@ -117,7 +118,7 @@ Foam::solverPerformance Foam::fvMatrix<Foam::scalar>::fvSolver::solve
         totalSource
     );
 
-    solverPerf.print(Info);
+    solverPerf.print(Info(fvMat_.mesh().comm()));
 
     fvMat_.diag() = saveDiag;
 
@@ -137,7 +138,8 @@ Foam::solverPerformance Foam::fvMatrix<Foam::scalar>::solveSegregated
 {
     if (debug)
     {
-        Info<< "fvMatrix<scalar>::solveSegregated"
+        Info(this->mesh().comm())
+            << "fvMatrix<scalar>::solveSegregated"
                "(const dictionary& solverControls) : "
                "solving fvMatrix<scalar>"
             << endl;
@@ -163,7 +165,7 @@ Foam::solverPerformance Foam::fvMatrix<Foam::scalar>::solveSegregated
         solverControls
     )->solve(psi.internalField(), totalSource);
 
-    solverPerf.print(Info);
+    solverPerf.print(Info(mesh().comm()));
 
     diag() = saveDiag;
 

src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -177,7 +177,13 @@ void Foam::decompositionMethod::calcCellCells
     // Create global cell numbers
     // ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    globalIndex globalAgglom(nCoarse, Pstream::msgType(), parallel);
+    globalIndex globalAgglom
+    (
+        nCoarse,
+        Pstream::msgType(),
+        Pstream::worldComm,
+        parallel
+    );
 
 
     // Get agglomerate owner on other side of coupled faces